1 /* 2 * Copyright 2003-2023, Haiku Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Axel Dörfler <axeld@pinc-software.de> 7 * Ingo Weinhold <bonefish@cs.tu-berlin.de> 8 * François Revol <revol@free.fr> 9 * Ithamar R. Adema <ithamar@upgrade-android.com> 10 * 11 * Copyright 2001, Travis Geiselbrecht. All rights reserved. 12 * Distributed under the terms of the NewOS License. 13 */ 14 15 16 #include <int.h> 17 18 #include <arch_cpu_defs.h> 19 #include <arch/smp.h> 20 #include <boot/kernel_args.h> 21 #include <device_manager.h> 22 #include <kscheduler.h> 23 #include <interrupt_controller.h> 24 #include <ksyscalls.h> 25 #include <smp.h> 26 #include <syscall_numbers.h> 27 #include <thread.h> 28 #include <timer.h> 29 #include <AutoDeleterDrivers.h> 30 #include <util/AutoLock.h> 31 #include <util/DoublyLinkedList.h> 32 #include <util/kernel_cpp.h> 33 #include <vm/vm.h> 34 #include <vm/vm_priv.h> 35 #include <vm/VMAddressSpace.h> 36 #include <algorithm> 37 #include <string.h> 38 39 #include <drivers/bus/FDT.h> 40 #include "arch_int_gicv2.h" 41 #include "soc.h" 42 43 #include "soc_pxa.h" 44 #include "soc_omap3.h" 45 #include "soc_sun4i.h" 46 47 #include "ARMVMTranslationMap.h" 48 49 //#define TRACE_ARCH_INT 50 #ifdef TRACE_ARCH_INT 51 # define TRACE(x...) dprintf(x) 52 #else 53 # define TRACE(x...) ; 54 #endif 55 56 #define VECTORPAGE_SIZE 64 57 #define USER_VECTOR_ADDR_LOW 0x00000000 58 #define USER_VECTOR_ADDR_HIGH 0xffff0000 59 60 extern int _vectors_start; 61 extern int _vectors_end; 62 63 static area_id sVectorPageArea; 64 static void *sVectorPageAddress; 65 static area_id sUserVectorPageArea; 66 static void *sUserVectorPageAddress; 67 //static fdt_module_info *sFdtModule; 68 69 // An iframe stack used in the early boot process when we don't have 70 // threads yet. 71 struct iframe_stack gBootFrameStack; 72 73 74 void 75 arch_int_enable_io_interrupt(int irq) 76 { 77 TRACE("arch_int_enable_io_interrupt(%d)\n", irq); 78 InterruptController *ic = InterruptController::Get(); 79 if (ic != NULL) 80 ic->EnableInterrupt(irq); 81 } 82 83 84 void 85 arch_int_disable_io_interrupt(int irq) 86 { 87 TRACE("arch_int_disable_io_interrupt(%d)\n", irq); 88 InterruptController *ic = InterruptController::Get(); 89 if (ic != NULL) 90 ic->DisableInterrupt(irq); 91 } 92 93 94 /* arch_int_*_interrupts() and friends are in arch_asm.S */ 95 96 int32 97 arch_int_assign_to_cpu(int32 irq, int32 cpu) 98 { 99 // Not yet supported. 100 return 0; 101 } 102 103 104 static void 105 print_iframe(const char *event, struct iframe *frame) 106 { 107 if (event) 108 dprintf("Exception: %s\n", event); 109 110 dprintf("R00=%08x R01=%08x R02=%08x R03=%08x\n" 111 "R04=%08x R05=%08x R06=%08x R07=%08x\n", 112 frame->r0, frame->r1, frame->r2, frame->r3, 113 frame->r4, frame->r5, frame->r6, frame->r7); 114 dprintf("R08=%08x R09=%08x R10=%08x R11=%08x\n" 115 "R12=%08x SPs=%08x LRs=%08x PC =%08x\n", 116 frame->r8, frame->r9, frame->r10, frame->r11, 117 frame->r12, frame->svc_sp, frame->svc_lr, frame->pc); 118 dprintf(" SPu=%08x LRu=%08x CPSR=%08x\n", 119 frame->usr_sp, frame->usr_lr, frame->spsr); 120 } 121 122 123 status_t 124 arch_int_init(kernel_args *args) 125 { 126 return B_OK; 127 } 128 129 130 extern "C" void arm_vector_init(void); 131 132 133 status_t 134 arch_int_init_post_vm(kernel_args *args) 135 { 136 // create a read/write kernel area 137 sVectorPageArea = create_area("vectorpage", (void **)&sVectorPageAddress, 138 B_ANY_ADDRESS, VECTORPAGE_SIZE, B_FULL_LOCK, 139 B_KERNEL_WRITE_AREA | B_KERNEL_READ_AREA); 140 if (sVectorPageArea < 0) 141 panic("vector page could not be created!"); 142 143 // clone it at a fixed address with user read/only permissions 144 sUserVectorPageAddress = (addr_t*)USER_VECTOR_ADDR_HIGH; 145 sUserVectorPageArea = clone_area("user_vectorpage", 146 (void **)&sUserVectorPageAddress, B_EXACT_ADDRESS, 147 B_READ_AREA | B_EXECUTE_AREA, sVectorPageArea); 148 149 if (sUserVectorPageArea < 0) 150 panic("user vector page @ %p could not be created (%x)!", 151 sVectorPageAddress, sUserVectorPageArea); 152 153 // copy vectors into the newly created area 154 memcpy(sVectorPageAddress, &_vectors_start, VECTORPAGE_SIZE); 155 156 arm_vector_init(); 157 158 // see if high vectors are enabled 159 if ((arm_get_sctlr() & (1 << 13)) != 0) 160 dprintf("High vectors already enabled\n"); 161 else { 162 arm_set_sctlr(arm_get_sctlr() | (1 << 13)); 163 164 if ((arm_get_sctlr() & (1 << 13)) == 0) 165 dprintf("Unable to enable high vectors!\n"); 166 else 167 dprintf("Enabled high vectors\n"); 168 } 169 170 if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2, 171 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 172 InterruptController *ic = new(std::nothrow) GICv2InterruptController( 173 args->arch_args.interrupt_controller.regs1.start, 174 args->arch_args.interrupt_controller.regs2.start); 175 if (ic == NULL) 176 return B_NO_MEMORY; 177 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_OMAP3, 178 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 179 InterruptController *ic = new(std::nothrow) OMAP3InterruptController( 180 args->arch_args.interrupt_controller.regs1.start); 181 if (ic == NULL) 182 return B_NO_MEMORY; 183 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_PXA, 184 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 185 InterruptController *ic = new(std::nothrow) PXAInterruptController( 186 args->arch_args.interrupt_controller.regs1.start); 187 if (ic == NULL) 188 return B_NO_MEMORY; 189 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_SUN4I, 190 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 191 InterruptController *ic = new(std::nothrow) Sun4iInterruptController( 192 args->arch_args.interrupt_controller.regs1.start); 193 if (ic == NULL) 194 return B_NO_MEMORY; 195 } else { 196 panic("No interrupt controllers found!\n"); 197 } 198 199 return B_OK; 200 } 201 202 203 status_t 204 arch_int_init_io(kernel_args* args) 205 { 206 return B_OK; 207 } 208 209 210 status_t 211 arch_int_init_post_device_manager(struct kernel_args *args) 212 { 213 return B_ENTRY_NOT_FOUND; 214 } 215 216 217 // Little helper class for handling the 218 // iframe stack as used by KDL. 219 class IFrameScope { 220 public: 221 IFrameScope(struct iframe *iframe) { 222 fThread = thread_get_current_thread(); 223 if (fThread) 224 arm_push_iframe(&fThread->arch_info.iframes, iframe); 225 else 226 arm_push_iframe(&gBootFrameStack, iframe); 227 } 228 229 virtual ~IFrameScope() { 230 // pop iframe 231 if (fThread) 232 arm_pop_iframe(&fThread->arch_info.iframes); 233 else 234 arm_pop_iframe(&gBootFrameStack); 235 } 236 private: 237 Thread* fThread; 238 }; 239 240 241 extern "C" void 242 arch_arm_undefined(struct iframe *iframe) 243 { 244 print_iframe("Undefined Instruction", iframe); 245 IFrameScope scope(iframe); // push/pop iframe 246 247 panic("not handled!"); 248 } 249 250 251 extern "C" void 252 arch_arm_syscall(struct iframe *iframe) 253 { 254 #ifdef TRACE_ARCH_INT 255 print_iframe("Software interrupt", iframe); 256 #endif 257 258 IFrameScope scope(iframe); 259 260 uint32_t syscall = *(uint32_t *)(iframe->pc-4) & 0x00ffffff; 261 TRACE("syscall number: %d\n", syscall); 262 263 uint32_t args[20]; 264 if (syscall < kSyscallCount) { 265 TRACE("syscall(%s,%d)\n", 266 kExtendedSyscallInfos[syscall].name, 267 kExtendedSyscallInfos[syscall].parameter_count); 268 269 int argSize = kSyscallInfos[syscall].parameter_size; 270 memcpy(args, &iframe->r0, std::min<int>(argSize, 4 * sizeof(uint32))); 271 if (argSize > 4 * sizeof(uint32)) { 272 status_t res = user_memcpy(&args[4], (void *)iframe->usr_sp, 273 (argSize - 4 * sizeof(uint32))); 274 if (res < B_OK) { 275 dprintf("can't read syscall arguments on user stack\n"); 276 iframe->r0 = res; 277 return; 278 } 279 } 280 } 281 282 thread_get_current_thread()->arch_info.userFrame = iframe; 283 thread_get_current_thread()->arch_info.oldR0 = iframe->r0; 284 thread_at_kernel_entry(system_time()); 285 286 enable_interrupts(); 287 288 uint64 returnValue = 0; 289 syscall_dispatcher(syscall, (void*)args, &returnValue); 290 291 TRACE("returning %" B_PRId64 "\n", returnValue); 292 iframe->r0 = returnValue; 293 294 disable_interrupts(); 295 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 296 if ((thread_get_current_thread()->flags & (THREAD_FLAGS_SIGNALS_PENDING 297 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 298 enable_interrupts(); 299 thread_at_kernel_exit(); 300 } else { 301 thread_at_kernel_exit_no_signals(); 302 } 303 if ((thread_get_current_thread()->flags & THREAD_FLAGS_RESTART_SYSCALL) != 0) { 304 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL); 305 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED); 306 iframe->r0 = thread_get_current_thread()->arch_info.oldR0; 307 iframe->pc -= 4; 308 } 309 310 thread_get_current_thread()->arch_info.userFrame = NULL; 311 } 312 313 314 static bool 315 arch_arm_handle_access_flag_fault(addr_t far, uint32 fsr, bool isWrite, bool isExec) 316 { 317 VMAddressSpacePutter addressSpace; 318 if (IS_KERNEL_ADDRESS(far)) 319 addressSpace.SetTo(VMAddressSpace::GetKernel()); 320 else if (IS_USER_ADDRESS(far)) 321 addressSpace.SetTo(VMAddressSpace::GetCurrent()); 322 323 if (!addressSpace.IsSet()) 324 return false; 325 326 ARMVMTranslationMap *map = (ARMVMTranslationMap *)addressSpace->TranslationMap(); 327 328 if ((fsr & 0x060f) == FSR_FS_ACCESS_FLAG_FAULT) { 329 phys_addr_t physAddr; 330 uint32 pageFlags; 331 332 map->QueryInterrupt(far, &physAddr, &pageFlags); 333 334 if ((PAGE_PRESENT & pageFlags) == 0) 335 return false; 336 337 if ((pageFlags & PAGE_ACCESSED) == 0) { 338 map->SetFlags(far, PAGE_ACCESSED); 339 return true; 340 } 341 } 342 343 return false; 344 } 345 346 347 static void 348 arch_arm_page_fault(struct iframe *frame, addr_t far, uint32 fsr, bool isWrite, bool isExec) 349 { 350 if (arch_arm_handle_access_flag_fault(far, fsr, isWrite, isExec)) 351 return; 352 353 Thread *thread = thread_get_current_thread(); 354 bool isUser = (frame->spsr & CPSR_MODE_MASK) == CPSR_MODE_USR; 355 addr_t newip = 0; 356 357 #ifdef TRACE_ARCH_INT 358 print_iframe("Page Fault", frame); 359 dprintf("FAR: %08lx, FSR: %08x, isUser: %d, isWrite: %d, isExec: %d, thread: %s\n", far, fsr, isUser, isWrite, isExec, thread->name); 360 #endif 361 362 IFrameScope scope(frame); 363 364 if (debug_debugger_running()) { 365 // If this CPU or this thread has a fault handler, we're allowed to be 366 // here. 367 if (thread != NULL) { 368 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 369 370 if (cpu->fault_handler != 0) { 371 debug_set_page_fault_info(far, frame->pc, 372 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0); 373 frame->svc_sp = cpu->fault_handler_stack_pointer; 374 frame->pc = cpu->fault_handler; 375 return; 376 } 377 378 if (thread->fault_handler != 0) { 379 kprintf("ERROR: thread::fault_handler used in kernel " 380 "debugger!\n"); 381 debug_set_page_fault_info(far, frame->pc, 382 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0); 383 frame->pc = reinterpret_cast<uintptr_t>(thread->fault_handler); 384 return; 385 } 386 } 387 388 // otherwise, not really 389 panic("page fault in debugger without fault handler! Touching " 390 "address %p from pc %p\n", (void *)far, (void *)frame->pc); 391 return; 392 } else if (isExec && !isUser && (far < KERNEL_BASE) && 393 (((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L1) || ((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L2))) { 394 panic("PXN violation trying to execute user-mapped address 0x%08" B_PRIxADDR " from kernel mode\n", 395 far); 396 } else if (!isExec && ((fsr & 0x060f) == FSR_FS_ALIGNMENT_FAULT)) { 397 panic("unhandled alignment exception\n"); 398 } else if ((fsr & 0x060f) == FSR_FS_ACCESS_FLAG_FAULT) { 399 panic("unhandled access flag fault\n"); 400 } else if ((frame->spsr & CPSR_I) != 0) { 401 // interrupts disabled 402 403 // If a page fault handler is installed, we're allowed to be here. 404 // TODO: Now we are generally allowing user_memcpy() with interrupts 405 // disabled, which in most cases is a bug. We should add some thread 406 // flag allowing to explicitly indicate that this handling is desired. 407 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler); 408 if (thread && thread->fault_handler != 0) { 409 if (frame->pc != handler) { 410 frame->pc = handler; 411 return; 412 } 413 414 // The fault happened at the fault handler address. This is a 415 // certain infinite loop. 416 panic("page fault, interrupts disabled, fault handler loop. " 417 "Touching address %p from pc %p\n", (void*)far, 418 (void*)frame->pc); 419 } 420 421 // If we are not running the kernel startup the page fault was not 422 // allowed to happen and we must panic. 423 panic("page fault, but interrupts were disabled. Touching address " 424 "%p from pc %p\n", (void *)far, (void *)frame->pc); 425 return; 426 } else if (thread != NULL && thread->page_faults_allowed < 1) { 427 panic("page fault not allowed at this place. Touching address " 428 "%p from pc %p\n", (void *)far, (void *)frame->pc); 429 return; 430 } 431 432 enable_interrupts(); 433 434 vm_page_fault(far, frame->pc, isWrite, isExec, isUser, &newip); 435 436 if (newip != 0) { 437 // the page fault handler wants us to modify the iframe to set the 438 // IP the cpu will return to to be this ip 439 frame->pc = newip; 440 } 441 } 442 443 444 extern "C" void 445 arch_arm_data_abort(struct iframe *frame) 446 { 447 addr_t dfar = arm_get_dfar(); 448 uint32 dfsr = arm_get_dfsr(); 449 bool isWrite = (dfsr & FSR_WNR) == FSR_WNR; 450 451 arch_arm_page_fault(frame, dfar, dfsr, isWrite, false); 452 } 453 454 455 extern "C" void 456 arch_arm_prefetch_abort(struct iframe *frame) 457 { 458 addr_t ifar = arm_get_ifar(); 459 uint32 ifsr = arm_get_ifsr(); 460 461 arch_arm_page_fault(frame, ifar, ifsr, false, true); 462 } 463 464 465 extern "C" void 466 arch_arm_irq(struct iframe *iframe) 467 { 468 IFrameScope scope(iframe); 469 470 InterruptController *ic = InterruptController::Get(); 471 if (ic != NULL) 472 ic->HandleInterrupt(); 473 474 Thread* thread = thread_get_current_thread(); 475 cpu_status state = disable_interrupts(); 476 if (thread->cpu->invoke_scheduler) { 477 SpinLocker schedulerLocker(thread->scheduler_lock); 478 scheduler_reschedule(B_THREAD_READY); 479 schedulerLocker.Unlock(); 480 restore_interrupts(state); 481 } else if (thread->post_interrupt_callback != NULL) { 482 void (*callback)(void*) = thread->post_interrupt_callback; 483 void* data = thread->post_interrupt_data; 484 485 thread->post_interrupt_callback = NULL; 486 thread->post_interrupt_data = NULL; 487 488 restore_interrupts(state); 489 490 callback(data); 491 } 492 } 493 494 495 extern "C" void 496 arch_arm_fiq(struct iframe *iframe) 497 { 498 IFrameScope scope(iframe); 499 500 panic("FIQ not implemented yet!"); 501 } 502