1 /* 2 * Copyright 2003-2023, Haiku Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Axel Dörfler <axeld@pinc-software.de> 7 * Ingo Weinhold <bonefish@cs.tu-berlin.de> 8 * François Revol <revol@free.fr> 9 * Ithamar R. Adema <ithamar@upgrade-android.com> 10 * 11 * Copyright 2001, Travis Geiselbrecht. All rights reserved. 12 * Distributed under the terms of the NewOS License. 13 */ 14 15 16 #include <int.h> 17 18 #include <arch_cpu_defs.h> 19 #include <arch/smp.h> 20 #include <boot/kernel_args.h> 21 #include <device_manager.h> 22 #include <kscheduler.h> 23 #include <interrupt_controller.h> 24 #include <ksyscalls.h> 25 #include <smp.h> 26 #include <syscall_numbers.h> 27 #include <thread.h> 28 #include <timer.h> 29 #include <AutoDeleterDrivers.h> 30 #include <util/AutoLock.h> 31 #include <util/DoublyLinkedList.h> 32 #include <util/kernel_cpp.h> 33 #include <vm/vm.h> 34 #include <vm/vm_priv.h> 35 #include <vm/VMAddressSpace.h> 36 #include <algorithm> 37 #include <string.h> 38 39 #include <drivers/bus/FDT.h> 40 #include "arch_int_gicv2.h" 41 #include "soc.h" 42 43 #include "soc_pxa.h" 44 #include "soc_omap3.h" 45 #include "soc_sun4i.h" 46 47 #include "ARMVMTranslationMap.h" 48 49 //#define TRACE_ARCH_INT 50 #ifdef TRACE_ARCH_INT 51 # define TRACE(x...) dprintf(x) 52 #else 53 # define TRACE(x...) ; 54 #endif 55 56 #define VECTORPAGE_SIZE 64 57 #define USER_VECTOR_ADDR_LOW 0x00000000 58 #define USER_VECTOR_ADDR_HIGH 0xffff0000 59 60 extern int _vectors_start; 61 extern int _vectors_end; 62 63 static area_id sUserVectorPageArea; 64 static void *sUserVectorPageAddress; 65 //static fdt_module_info *sFdtModule; 66 67 // An iframe stack used in the early boot process when we don't have 68 // threads yet. 69 struct iframe_stack gBootFrameStack; 70 71 72 void 73 arch_int_enable_io_interrupt(int32 irq) 74 { 75 TRACE("arch_int_enable_io_interrupt(%" B_PRId32 ")\n", irq); 76 InterruptController *ic = InterruptController::Get(); 77 if (ic != NULL) 78 ic->EnableInterrupt(irq); 79 } 80 81 82 void 83 arch_int_disable_io_interrupt(int32 irq) 84 { 85 TRACE("arch_int_disable_io_interrupt(%" B_PRId32 ")\n", irq); 86 InterruptController *ic = InterruptController::Get(); 87 if (ic != NULL) 88 ic->DisableInterrupt(irq); 89 } 90 91 92 /* arch_int_*_interrupts() and friends are in arch_asm.S */ 93 94 int32 95 arch_int_assign_to_cpu(int32 irq, int32 cpu) 96 { 97 // Not yet supported. 98 return 0; 99 } 100 101 102 static void 103 print_iframe(const char *event, struct iframe *frame) 104 { 105 if (event) 106 dprintf("Exception: %s\n", event); 107 108 dprintf("R00=%08x R01=%08x R02=%08x R03=%08x\n" 109 "R04=%08x R05=%08x R06=%08x R07=%08x\n", 110 frame->r0, frame->r1, frame->r2, frame->r3, 111 frame->r4, frame->r5, frame->r6, frame->r7); 112 dprintf("R08=%08x R09=%08x R10=%08x R11=%08x\n" 113 "R12=%08x SPs=%08x LRs=%08x PC =%08x\n", 114 frame->r8, frame->r9, frame->r10, frame->r11, 115 frame->r12, frame->svc_sp, frame->svc_lr, frame->pc); 116 dprintf(" SPu=%08x LRu=%08x CPSR=%08x\n", 117 frame->usr_sp, frame->usr_lr, frame->spsr); 118 } 119 120 121 extern "C" void arm_vector_init(void); 122 123 124 status_t 125 arch_int_init(kernel_args *args) 126 { 127 TRACE("arch_int_init\n"); 128 129 // copy vector code to vector page 130 memcpy((void*)USER_VECTOR_ADDR_HIGH, &_vectors_start, VECTORPAGE_SIZE); 131 132 // initialize stack for vectors 133 arm_vector_init(); 134 135 // enable high vectors 136 arm_set_sctlr(arm_get_sctlr() | SCTLR_HIGH_VECTORS); 137 138 return B_OK; 139 } 140 141 142 status_t 143 arch_int_init_post_vm(kernel_args *args) 144 { 145 TRACE("arch_int_init_post_vm\n"); 146 147 sUserVectorPageAddress = (addr_t*)USER_VECTOR_ADDR_HIGH; 148 sUserVectorPageArea = create_area("user_vectorpage", 149 (void **)&sUserVectorPageAddress, B_EXACT_ADDRESS, 150 B_PAGE_SIZE, B_ALREADY_WIRED, B_READ_AREA | B_EXECUTE_AREA); 151 152 if (sUserVectorPageArea < 0) 153 panic("user vector page @ %p could not be created (%x)!", 154 sUserVectorPageAddress, sUserVectorPageArea); 155 156 if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2, 157 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 158 InterruptController *ic = new(std::nothrow) GICv2InterruptController( 159 args->arch_args.interrupt_controller.regs1.start, 160 args->arch_args.interrupt_controller.regs2.start); 161 if (ic == NULL) 162 return B_NO_MEMORY; 163 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_OMAP3, 164 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 165 InterruptController *ic = new(std::nothrow) OMAP3InterruptController( 166 args->arch_args.interrupt_controller.regs1.start); 167 if (ic == NULL) 168 return B_NO_MEMORY; 169 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_PXA, 170 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 171 InterruptController *ic = new(std::nothrow) PXAInterruptController( 172 args->arch_args.interrupt_controller.regs1.start); 173 if (ic == NULL) 174 return B_NO_MEMORY; 175 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_SUN4I, 176 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 177 InterruptController *ic = new(std::nothrow) Sun4iInterruptController( 178 args->arch_args.interrupt_controller.regs1.start); 179 if (ic == NULL) 180 return B_NO_MEMORY; 181 } else { 182 panic("No interrupt controllers found!\n"); 183 } 184 185 return B_OK; 186 } 187 188 189 status_t 190 arch_int_init_io(kernel_args* args) 191 { 192 return B_OK; 193 } 194 195 196 status_t 197 arch_int_init_post_device_manager(struct kernel_args *args) 198 { 199 return B_ENTRY_NOT_FOUND; 200 } 201 202 203 // Little helper class for handling the 204 // iframe stack as used by KDL. 205 class IFrameScope { 206 public: 207 IFrameScope(struct iframe *iframe) { 208 fThread = thread_get_current_thread(); 209 if (fThread) 210 arm_push_iframe(&fThread->arch_info.iframes, iframe); 211 else 212 arm_push_iframe(&gBootFrameStack, iframe); 213 } 214 215 virtual ~IFrameScope() { 216 // pop iframe 217 if (fThread) 218 arm_pop_iframe(&fThread->arch_info.iframes); 219 else 220 arm_pop_iframe(&gBootFrameStack); 221 } 222 private: 223 Thread* fThread; 224 }; 225 226 227 extern "C" void 228 arch_arm_undefined(struct iframe *iframe) 229 { 230 print_iframe("Undefined Instruction", iframe); 231 IFrameScope scope(iframe); // push/pop iframe 232 233 panic("not handled!"); 234 } 235 236 237 extern "C" void 238 arch_arm_syscall(struct iframe *iframe) 239 { 240 #ifdef TRACE_ARCH_INT 241 print_iframe("Software interrupt", iframe); 242 #endif 243 244 IFrameScope scope(iframe); 245 246 uint32_t syscall = *(uint32_t *)(iframe->pc-4) & 0x00ffffff; 247 TRACE("syscall number: %d\n", syscall); 248 249 uint32_t args[20]; 250 if (syscall < kSyscallCount) { 251 TRACE("syscall(%s,%d)\n", 252 kExtendedSyscallInfos[syscall].name, 253 kExtendedSyscallInfos[syscall].parameter_count); 254 255 int argSize = kSyscallInfos[syscall].parameter_size; 256 memcpy(args, &iframe->r0, std::min<int>(argSize, 4 * sizeof(uint32))); 257 if (argSize > 4 * sizeof(uint32)) { 258 status_t res = user_memcpy(&args[4], (void *)iframe->usr_sp, 259 (argSize - 4 * sizeof(uint32))); 260 if (res < B_OK) { 261 dprintf("can't read syscall arguments on user stack\n"); 262 iframe->r0 = res; 263 return; 264 } 265 } 266 } 267 268 thread_get_current_thread()->arch_info.userFrame = iframe; 269 thread_get_current_thread()->arch_info.oldR0 = iframe->r0; 270 thread_at_kernel_entry(system_time()); 271 272 enable_interrupts(); 273 274 uint64 returnValue = 0; 275 syscall_dispatcher(syscall, (void*)args, &returnValue); 276 277 TRACE("returning %" B_PRId64 "\n", returnValue); 278 iframe->r0 = returnValue; 279 280 disable_interrupts(); 281 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 282 if ((thread_get_current_thread()->flags & (THREAD_FLAGS_SIGNALS_PENDING 283 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 284 enable_interrupts(); 285 thread_at_kernel_exit(); 286 } else { 287 thread_at_kernel_exit_no_signals(); 288 } 289 if ((thread_get_current_thread()->flags & THREAD_FLAGS_RESTART_SYSCALL) != 0) { 290 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL); 291 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED); 292 iframe->r0 = thread_get_current_thread()->arch_info.oldR0; 293 iframe->pc -= 4; 294 } 295 296 thread_get_current_thread()->arch_info.userFrame = NULL; 297 } 298 299 300 static bool 301 arch_arm_handle_access_flag_fault(addr_t far, uint32 fsr, bool isWrite, bool isExec) 302 { 303 VMAddressSpacePutter addressSpace; 304 if (IS_KERNEL_ADDRESS(far)) 305 addressSpace.SetTo(VMAddressSpace::GetKernel()); 306 else if (IS_USER_ADDRESS(far)) 307 addressSpace.SetTo(VMAddressSpace::GetCurrent()); 308 309 if (!addressSpace.IsSet()) 310 return false; 311 312 ARMVMTranslationMap *map = (ARMVMTranslationMap *)addressSpace->TranslationMap(); 313 314 if ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_ACCESS_FLAG_FAULT) { 315 phys_addr_t physAddr; 316 uint32 pageFlags; 317 318 map->QueryInterrupt(far, &physAddr, &pageFlags); 319 320 if ((PAGE_PRESENT & pageFlags) == 0) 321 return false; 322 323 if ((pageFlags & PAGE_ACCESSED) == 0) { 324 map->SetFlags(far, PAGE_ACCESSED); 325 return true; 326 } 327 } 328 329 if (isWrite && ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_PERMISSION_FAULT_L2)) { 330 phys_addr_t physAddr; 331 uint32 pageFlags; 332 333 map->QueryInterrupt(far, &physAddr, &pageFlags); 334 335 if ((PAGE_PRESENT & pageFlags) == 0) 336 return false; 337 338 if (((pageFlags & B_KERNEL_WRITE_AREA) && ((pageFlags & PAGE_MODIFIED) == 0))) { 339 map->SetFlags(far, PAGE_MODIFIED); 340 return true; 341 } 342 } 343 344 return false; 345 } 346 347 348 static void 349 arch_arm_page_fault(struct iframe *frame, addr_t far, uint32 fsr, bool isWrite, bool isExec) 350 { 351 if (arch_arm_handle_access_flag_fault(far, fsr, isWrite, isExec)) 352 return; 353 354 Thread *thread = thread_get_current_thread(); 355 bool isUser = (frame->spsr & CPSR_MODE_MASK) == CPSR_MODE_USR; 356 addr_t newip = 0; 357 358 #ifdef TRACE_ARCH_INT 359 print_iframe("Page Fault", frame); 360 dprintf("FAR: %08lx, FSR: %08x, isUser: %d, isWrite: %d, isExec: %d, thread: %s\n", far, fsr, isUser, isWrite, isExec, thread->name); 361 #endif 362 363 IFrameScope scope(frame); 364 365 if (debug_debugger_running()) { 366 // If this CPU or this thread has a fault handler, we're allowed to be 367 // here. 368 if (thread != NULL) { 369 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 370 371 if (cpu->fault_handler != 0) { 372 debug_set_page_fault_info(far, frame->pc, 373 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0); 374 frame->svc_sp = cpu->fault_handler_stack_pointer; 375 frame->pc = cpu->fault_handler; 376 return; 377 } 378 379 if (thread->fault_handler != 0) { 380 kprintf("ERROR: thread::fault_handler used in kernel " 381 "debugger!\n"); 382 debug_set_page_fault_info(far, frame->pc, 383 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0); 384 frame->pc = reinterpret_cast<uintptr_t>(thread->fault_handler); 385 return; 386 } 387 } 388 389 // otherwise, not really 390 panic("page fault in debugger without fault handler! Touching " 391 "address %p from pc %p\n", (void *)far, (void *)frame->pc); 392 return; 393 } else if (isExec && !isUser && (far < KERNEL_BASE) && 394 (((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L1) || ((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L2))) { 395 panic("PXN violation trying to execute user-mapped address 0x%08" B_PRIxADDR " from kernel mode\n", 396 far); 397 } else if (!isExec && ((fsr & 0x060f) == FSR_FS_ALIGNMENT_FAULT)) { 398 panic("unhandled alignment exception\n"); 399 } else if ((fsr & 0x060f) == FSR_FS_ACCESS_FLAG_FAULT) { 400 panic("unhandled access flag fault\n"); 401 } else if ((frame->spsr & CPSR_I) != 0) { 402 // interrupts disabled 403 404 // If a page fault handler is installed, we're allowed to be here. 405 // TODO: Now we are generally allowing user_memcpy() with interrupts 406 // disabled, which in most cases is a bug. We should add some thread 407 // flag allowing to explicitly indicate that this handling is desired. 408 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler); 409 if (thread && thread->fault_handler != 0) { 410 if (frame->pc != handler) { 411 frame->pc = handler; 412 return; 413 } 414 415 // The fault happened at the fault handler address. This is a 416 // certain infinite loop. 417 panic("page fault, interrupts disabled, fault handler loop. " 418 "Touching address %p from pc %p\n", (void*)far, 419 (void*)frame->pc); 420 } 421 422 // If we are not running the kernel startup the page fault was not 423 // allowed to happen and we must panic. 424 panic("page fault, but interrupts were disabled. Touching address " 425 "%p from pc %p\n", (void *)far, (void *)frame->pc); 426 return; 427 } else if (thread != NULL && thread->page_faults_allowed < 1) { 428 panic("page fault not allowed at this place. Touching address " 429 "%p from pc %p\n", (void *)far, (void *)frame->pc); 430 return; 431 } 432 433 enable_interrupts(); 434 435 vm_page_fault(far, frame->pc, isWrite, isExec, isUser, &newip); 436 437 if (newip != 0) { 438 // the page fault handler wants us to modify the iframe to set the 439 // IP the cpu will return to to be this ip 440 frame->pc = newip; 441 } 442 } 443 444 445 extern "C" void 446 arch_arm_data_abort(struct iframe *frame) 447 { 448 addr_t dfar = arm_get_dfar(); 449 uint32 dfsr = arm_get_dfsr(); 450 bool isWrite = (dfsr & FSR_WNR) == FSR_WNR; 451 452 arch_arm_page_fault(frame, dfar, dfsr, isWrite, false); 453 } 454 455 456 extern "C" void 457 arch_arm_prefetch_abort(struct iframe *frame) 458 { 459 addr_t ifar = arm_get_ifar(); 460 uint32 ifsr = arm_get_ifsr(); 461 462 arch_arm_page_fault(frame, ifar, ifsr, false, true); 463 } 464 465 466 extern "C" void 467 arch_arm_irq(struct iframe *iframe) 468 { 469 IFrameScope scope(iframe); 470 471 InterruptController *ic = InterruptController::Get(); 472 if (ic != NULL) 473 ic->HandleInterrupt(); 474 475 Thread* thread = thread_get_current_thread(); 476 cpu_status state = disable_interrupts(); 477 if (thread->post_interrupt_callback != NULL) { 478 void (*callback)(void*) = thread->post_interrupt_callback; 479 void* data = thread->post_interrupt_data; 480 481 thread->post_interrupt_callback = NULL; 482 thread->post_interrupt_data = NULL; 483 484 restore_interrupts(state); 485 486 callback(data); 487 } else if (thread->cpu->invoke_scheduler) { 488 SpinLocker schedulerLocker(thread->scheduler_lock); 489 scheduler_reschedule(B_THREAD_READY); 490 schedulerLocker.Unlock(); 491 restore_interrupts(state); 492 } 493 } 494 495 496 extern "C" void 497 arch_arm_fiq(struct iframe *iframe) 498 { 499 IFrameScope scope(iframe); 500 501 panic("FIQ not implemented yet!"); 502 } 503