1 /*
2 * Copyright 2003-2023, Haiku Inc. All rights reserved.
3 * Distributed under the terms of the MIT License.
4 *
5 * Authors:
6 * Axel Dörfler <axeld@pinc-software.de>
7 * Ingo Weinhold <bonefish@cs.tu-berlin.de>
8 * François Revol <revol@free.fr>
9 * Ithamar R. Adema <ithamar@upgrade-android.com>
10 *
11 * Copyright 2001, Travis Geiselbrecht. All rights reserved.
12 * Distributed under the terms of the NewOS License.
13 */
14
15
16 #include <int.h>
17
18 #include <arch_cpu_defs.h>
19 #include <arch/smp.h>
20 #include <boot/kernel_args.h>
21 #include <device_manager.h>
22 #include <kscheduler.h>
23 #include <interrupt_controller.h>
24 #include <ksyscalls.h>
25 #include <smp.h>
26 #include <syscall_numbers.h>
27 #include <thread.h>
28 #include <timer.h>
29 #include <AutoDeleterDrivers.h>
30 #include <util/AutoLock.h>
31 #include <util/DoublyLinkedList.h>
32 #include <util/kernel_cpp.h>
33 #include <vm/vm.h>
34 #include <vm/vm_priv.h>
35 #include <vm/VMAddressSpace.h>
36 #include <algorithm>
37 #include <string.h>
38
39 #include <drivers/bus/FDT.h>
40 #include "arch_int_gicv2.h"
41 #include "soc.h"
42
43 #include "soc_pxa.h"
44 #include "soc_omap3.h"
45 #include "soc_sun4i.h"
46
47 #include "ARMVMTranslationMap.h"
48
49 //#define TRACE_ARCH_INT
50 #ifdef TRACE_ARCH_INT
51 # define TRACE(x...) dprintf(x)
52 #else
53 # define TRACE(x...) ;
54 #endif
55
56 #define VECTORPAGE_SIZE 64
57 #define USER_VECTOR_ADDR_LOW 0x00000000
58 #define USER_VECTOR_ADDR_HIGH 0xffff0000
59
60 extern int _vectors_start;
61 extern int _vectors_end;
62
63 static area_id sUserVectorPageArea;
64 static void *sUserVectorPageAddress;
65 //static fdt_module_info *sFdtModule;
66
67 // An iframe stack used in the early boot process when we don't have
68 // threads yet.
69 struct iframe_stack gBootFrameStack;
70
71
72 void
arch_int_enable_io_interrupt(int32 irq)73 arch_int_enable_io_interrupt(int32 irq)
74 {
75 TRACE("arch_int_enable_io_interrupt(%" B_PRId32 ")\n", irq);
76 InterruptController *ic = InterruptController::Get();
77 if (ic != NULL)
78 ic->EnableInterrupt(irq);
79 }
80
81
82 void
arch_int_disable_io_interrupt(int32 irq)83 arch_int_disable_io_interrupt(int32 irq)
84 {
85 TRACE("arch_int_disable_io_interrupt(%" B_PRId32 ")\n", irq);
86 InterruptController *ic = InterruptController::Get();
87 if (ic != NULL)
88 ic->DisableInterrupt(irq);
89 }
90
91
92 /* arch_int_*_interrupts() and friends are in arch_asm.S */
93
94 int32
arch_int_assign_to_cpu(int32 irq,int32 cpu)95 arch_int_assign_to_cpu(int32 irq, int32 cpu)
96 {
97 // Not yet supported.
98 return 0;
99 }
100
101
102 static void
print_iframe(const char * event,struct iframe * frame)103 print_iframe(const char *event, struct iframe *frame)
104 {
105 if (event)
106 dprintf("Exception: %s\n", event);
107
108 dprintf("R00=%08x R01=%08x R02=%08x R03=%08x\n"
109 "R04=%08x R05=%08x R06=%08x R07=%08x\n",
110 frame->r0, frame->r1, frame->r2, frame->r3,
111 frame->r4, frame->r5, frame->r6, frame->r7);
112 dprintf("R08=%08x R09=%08x R10=%08x R11=%08x\n"
113 "R12=%08x SPs=%08x LRs=%08x PC =%08x\n",
114 frame->r8, frame->r9, frame->r10, frame->r11,
115 frame->r12, frame->svc_sp, frame->svc_lr, frame->pc);
116 dprintf(" SPu=%08x LRu=%08x CPSR=%08x\n",
117 frame->usr_sp, frame->usr_lr, frame->spsr);
118 }
119
120
121 extern "C" void arm_vector_init(void);
122
123
124 status_t
arch_int_init(kernel_args * args)125 arch_int_init(kernel_args *args)
126 {
127 TRACE("arch_int_init\n");
128
129 // copy vector code to vector page
130 memcpy((void*)USER_VECTOR_ADDR_HIGH, &_vectors_start, VECTORPAGE_SIZE);
131
132 // initialize stack for vectors
133 arm_vector_init();
134
135 // enable high vectors
136 arm_set_sctlr(arm_get_sctlr() | SCTLR_HIGH_VECTORS);
137
138 return B_OK;
139 }
140
141
142 status_t
arch_int_init_post_vm(kernel_args * args)143 arch_int_init_post_vm(kernel_args *args)
144 {
145 TRACE("arch_int_init_post_vm\n");
146
147 sUserVectorPageAddress = (addr_t*)USER_VECTOR_ADDR_HIGH;
148 sUserVectorPageArea = create_area("user_vectorpage",
149 (void **)&sUserVectorPageAddress, B_EXACT_ADDRESS,
150 B_PAGE_SIZE, B_ALREADY_WIRED, B_READ_AREA | B_EXECUTE_AREA);
151
152 if (sUserVectorPageArea < 0)
153 panic("user vector page @ %p could not be created (%x)!",
154 sUserVectorPageAddress, sUserVectorPageArea);
155
156 if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2,
157 sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
158 InterruptController *ic = new(std::nothrow) GICv2InterruptController(
159 args->arch_args.interrupt_controller.regs1.start,
160 args->arch_args.interrupt_controller.regs2.start);
161 if (ic == NULL)
162 return B_NO_MEMORY;
163 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_OMAP3,
164 sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
165 InterruptController *ic = new(std::nothrow) OMAP3InterruptController(
166 args->arch_args.interrupt_controller.regs1.start);
167 if (ic == NULL)
168 return B_NO_MEMORY;
169 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_PXA,
170 sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
171 InterruptController *ic = new(std::nothrow) PXAInterruptController(
172 args->arch_args.interrupt_controller.regs1.start);
173 if (ic == NULL)
174 return B_NO_MEMORY;
175 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_SUN4I,
176 sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
177 InterruptController *ic = new(std::nothrow) Sun4iInterruptController(
178 args->arch_args.interrupt_controller.regs1.start);
179 if (ic == NULL)
180 return B_NO_MEMORY;
181 } else {
182 panic("No interrupt controllers found!\n");
183 }
184
185 return B_OK;
186 }
187
188
189 status_t
arch_int_init_io(kernel_args * args)190 arch_int_init_io(kernel_args* args)
191 {
192 return B_OK;
193 }
194
195
196 status_t
arch_int_init_post_device_manager(struct kernel_args * args)197 arch_int_init_post_device_manager(struct kernel_args *args)
198 {
199 return B_ENTRY_NOT_FOUND;
200 }
201
202
203 // Little helper class for handling the
204 // iframe stack as used by KDL.
205 class IFrameScope {
206 public:
IFrameScope(struct iframe * iframe)207 IFrameScope(struct iframe *iframe) {
208 fThread = thread_get_current_thread();
209 if (fThread)
210 arm_push_iframe(&fThread->arch_info.iframes, iframe);
211 else
212 arm_push_iframe(&gBootFrameStack, iframe);
213 }
214
~IFrameScope()215 virtual ~IFrameScope() {
216 // pop iframe
217 if (fThread)
218 arm_pop_iframe(&fThread->arch_info.iframes);
219 else
220 arm_pop_iframe(&gBootFrameStack);
221 }
222 private:
223 Thread* fThread;
224 };
225
226
227 extern "C" void
arch_arm_undefined(struct iframe * iframe)228 arch_arm_undefined(struct iframe *iframe)
229 {
230 print_iframe("Undefined Instruction", iframe);
231 IFrameScope scope(iframe); // push/pop iframe
232
233 panic("not handled!");
234 }
235
236
237 extern "C" void
arch_arm_syscall(struct iframe * iframe)238 arch_arm_syscall(struct iframe *iframe)
239 {
240 #ifdef TRACE_ARCH_INT
241 print_iframe("Software interrupt", iframe);
242 #endif
243
244 IFrameScope scope(iframe);
245
246 uint32_t syscall = *(uint32_t *)(iframe->pc-4) & 0x00ffffff;
247 TRACE("syscall number: %d\n", syscall);
248
249 uint32_t args[20];
250 if (syscall < kSyscallCount) {
251 TRACE("syscall(%s,%d)\n",
252 kExtendedSyscallInfos[syscall].name,
253 kExtendedSyscallInfos[syscall].parameter_count);
254
255 int argSize = kSyscallInfos[syscall].parameter_size;
256 memcpy(args, &iframe->r0, std::min<int>(argSize, 4 * sizeof(uint32)));
257 if (argSize > 4 * sizeof(uint32)) {
258 status_t res = user_memcpy(&args[4], (void *)iframe->usr_sp,
259 (argSize - 4 * sizeof(uint32)));
260 if (res < B_OK) {
261 dprintf("can't read syscall arguments on user stack\n");
262 iframe->r0 = res;
263 return;
264 }
265 }
266 }
267
268 thread_get_current_thread()->arch_info.userFrame = iframe;
269 thread_get_current_thread()->arch_info.oldR0 = iframe->r0;
270 thread_at_kernel_entry(system_time());
271
272 enable_interrupts();
273
274 uint64 returnValue = 0;
275 syscall_dispatcher(syscall, (void*)args, &returnValue);
276
277 TRACE("returning %" B_PRId64 "\n", returnValue);
278 iframe->r0 = returnValue;
279
280 disable_interrupts();
281 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED);
282 if ((thread_get_current_thread()->flags & (THREAD_FLAGS_SIGNALS_PENDING
283 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) {
284 enable_interrupts();
285 thread_at_kernel_exit();
286 } else {
287 thread_at_kernel_exit_no_signals();
288 }
289 if ((thread_get_current_thread()->flags & THREAD_FLAGS_RESTART_SYSCALL) != 0) {
290 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL);
291 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED);
292 iframe->r0 = thread_get_current_thread()->arch_info.oldR0;
293 iframe->pc -= 4;
294 }
295
296 thread_get_current_thread()->arch_info.userFrame = NULL;
297 }
298
299
300 static bool
arch_arm_handle_access_flag_fault(addr_t far,uint32 fsr,bool isWrite,bool isExec)301 arch_arm_handle_access_flag_fault(addr_t far, uint32 fsr, bool isWrite, bool isExec)
302 {
303 VMAddressSpacePutter addressSpace;
304 if (IS_KERNEL_ADDRESS(far))
305 addressSpace.SetTo(VMAddressSpace::GetKernel());
306 else if (IS_USER_ADDRESS(far))
307 addressSpace.SetTo(VMAddressSpace::GetCurrent());
308
309 if (!addressSpace.IsSet())
310 return false;
311
312 ARMVMTranslationMap *map = (ARMVMTranslationMap *)addressSpace->TranslationMap();
313
314 if ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_ACCESS_FLAG_FAULT) {
315 phys_addr_t physAddr;
316 uint32 pageFlags;
317
318 map->QueryInterrupt(far, &physAddr, &pageFlags);
319
320 if ((PAGE_PRESENT & pageFlags) == 0)
321 return false;
322
323 if ((pageFlags & PAGE_ACCESSED) == 0) {
324 map->SetFlags(far, PAGE_ACCESSED);
325 return true;
326 }
327 }
328
329 if (isWrite && ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_PERMISSION_FAULT_L2)) {
330 phys_addr_t physAddr;
331 uint32 pageFlags;
332
333 map->QueryInterrupt(far, &physAddr, &pageFlags);
334
335 if ((PAGE_PRESENT & pageFlags) == 0)
336 return false;
337
338 if (((pageFlags & B_KERNEL_WRITE_AREA) && ((pageFlags & PAGE_MODIFIED) == 0))) {
339 map->SetFlags(far, PAGE_MODIFIED);
340 return true;
341 }
342 }
343
344 return false;
345 }
346
347
348 static void
arch_arm_page_fault(struct iframe * frame,addr_t far,uint32 fsr,bool isWrite,bool isExec)349 arch_arm_page_fault(struct iframe *frame, addr_t far, uint32 fsr, bool isWrite, bool isExec)
350 {
351 if (arch_arm_handle_access_flag_fault(far, fsr, isWrite, isExec))
352 return;
353
354 Thread *thread = thread_get_current_thread();
355 bool isUser = (frame->spsr & CPSR_MODE_MASK) == CPSR_MODE_USR;
356 addr_t newip = 0;
357
358 #ifdef TRACE_ARCH_INT
359 print_iframe("Page Fault", frame);
360 dprintf("FAR: %08lx, FSR: %08x, isUser: %d, isWrite: %d, isExec: %d, thread: %s\n", far, fsr, isUser, isWrite, isExec, thread->name);
361 #endif
362
363 IFrameScope scope(frame);
364
365 if (debug_debugger_running()) {
366 // If this CPU or this thread has a fault handler, we're allowed to be
367 // here.
368 if (thread != NULL) {
369 cpu_ent* cpu = &gCPU[smp_get_current_cpu()];
370
371 if (cpu->fault_handler != 0) {
372 debug_set_page_fault_info(far, frame->pc,
373 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0);
374 frame->svc_sp = cpu->fault_handler_stack_pointer;
375 frame->pc = cpu->fault_handler;
376 return;
377 }
378
379 if (thread->fault_handler != 0) {
380 kprintf("ERROR: thread::fault_handler used in kernel "
381 "debugger!\n");
382 debug_set_page_fault_info(far, frame->pc,
383 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0);
384 frame->pc = reinterpret_cast<uintptr_t>(thread->fault_handler);
385 return;
386 }
387 }
388
389 // otherwise, not really
390 panic("page fault in debugger without fault handler! Touching "
391 "address %p from pc %p\n", (void *)far, (void *)frame->pc);
392 return;
393 } else if (isExec && !isUser && (far < KERNEL_BASE) &&
394 (((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L1) || ((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L2))) {
395 panic("PXN violation trying to execute user-mapped address 0x%08" B_PRIxADDR " from kernel mode\n",
396 far);
397 } else if (!isExec && ((fsr & 0x060f) == FSR_FS_ALIGNMENT_FAULT)) {
398 panic("unhandled alignment exception\n");
399 } else if ((fsr & 0x060f) == FSR_FS_ACCESS_FLAG_FAULT) {
400 panic("unhandled access flag fault\n");
401 } else if ((frame->spsr & CPSR_I) != 0) {
402 // interrupts disabled
403
404 // If a page fault handler is installed, we're allowed to be here.
405 // TODO: Now we are generally allowing user_memcpy() with interrupts
406 // disabled, which in most cases is a bug. We should add some thread
407 // flag allowing to explicitly indicate that this handling is desired.
408 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler);
409 if (thread && thread->fault_handler != 0) {
410 if (frame->pc != handler) {
411 frame->pc = handler;
412 return;
413 }
414
415 // The fault happened at the fault handler address. This is a
416 // certain infinite loop.
417 panic("page fault, interrupts disabled, fault handler loop. "
418 "Touching address %p from pc %p\n", (void*)far,
419 (void*)frame->pc);
420 }
421
422 // If we are not running the kernel startup the page fault was not
423 // allowed to happen and we must panic.
424 panic("page fault, but interrupts were disabled. Touching address "
425 "%p from pc %p\n", (void *)far, (void *)frame->pc);
426 return;
427 } else if (thread != NULL && thread->page_faults_allowed < 1) {
428 panic("page fault not allowed at this place. Touching address "
429 "%p from pc %p\n", (void *)far, (void *)frame->pc);
430 return;
431 }
432
433 enable_interrupts();
434
435 vm_page_fault(far, frame->pc, isWrite, isExec, isUser, &newip);
436
437 if (newip != 0) {
438 // the page fault handler wants us to modify the iframe to set the
439 // IP the cpu will return to to be this ip
440 frame->pc = newip;
441 }
442 }
443
444
445 extern "C" void
arch_arm_data_abort(struct iframe * frame)446 arch_arm_data_abort(struct iframe *frame)
447 {
448 addr_t dfar = arm_get_dfar();
449 uint32 dfsr = arm_get_dfsr();
450 bool isWrite = (dfsr & FSR_WNR) == FSR_WNR;
451
452 arch_arm_page_fault(frame, dfar, dfsr, isWrite, false);
453 }
454
455
456 extern "C" void
arch_arm_prefetch_abort(struct iframe * frame)457 arch_arm_prefetch_abort(struct iframe *frame)
458 {
459 addr_t ifar = arm_get_ifar();
460 uint32 ifsr = arm_get_ifsr();
461
462 arch_arm_page_fault(frame, ifar, ifsr, false, true);
463 }
464
465
466 extern "C" void
arch_arm_irq(struct iframe * iframe)467 arch_arm_irq(struct iframe *iframe)
468 {
469 IFrameScope scope(iframe);
470
471 InterruptController *ic = InterruptController::Get();
472 if (ic != NULL)
473 ic->HandleInterrupt();
474
475 Thread* thread = thread_get_current_thread();
476 cpu_status state = disable_interrupts();
477 if (thread->post_interrupt_callback != NULL) {
478 void (*callback)(void*) = thread->post_interrupt_callback;
479 void* data = thread->post_interrupt_data;
480
481 thread->post_interrupt_callback = NULL;
482 thread->post_interrupt_data = NULL;
483
484 restore_interrupts(state);
485
486 callback(data);
487 } else if (thread->cpu->invoke_scheduler) {
488 SpinLocker schedulerLocker(thread->scheduler_lock);
489 scheduler_reschedule(B_THREAD_READY);
490 schedulerLocker.Unlock();
491 restore_interrupts(state);
492 }
493 }
494
495
496 extern "C" void
arch_arm_fiq(struct iframe * iframe)497 arch_arm_fiq(struct iframe *iframe)
498 {
499 IFrameScope scope(iframe);
500
501 panic("FIQ not implemented yet!");
502 }
503