1 /* 2 * Copyright 2019-2022 Haiku, Inc. All Rights Reserved. 3 * Distributed under the terms of the MIT License. 4 */ 5 #include <int.h> 6 7 #include <arch/smp.h> 8 #include <boot/kernel_args.h> 9 #include <device_manager.h> 10 #include <kscheduler.h> 11 #include <ksyscalls.h> 12 #include <interrupt_controller.h> 13 #include <smp.h> 14 #include <thread.h> 15 #include <timer.h> 16 #include <util/AutoLock.h> 17 #include <util/DoublyLinkedList.h> 18 #include <util/kernel_cpp.h> 19 #include <vm/vm.h> 20 #include <vm/vm_priv.h> 21 #include <vm/VMAddressSpace.h> 22 #include "syscall_numbers.h" 23 #include "VMSAv8TranslationMap.h" 24 #include <string.h> 25 26 #include "soc.h" 27 #include "arch_int_gicv2.h" 28 29 #define TRACE_ARCH_INT 30 #ifdef TRACE_ARCH_INT 31 # define TRACE(x) dprintf x 32 #else 33 # define TRACE(x) ; 34 #endif 35 36 //#define TRACE_ARCH_INT_IFRAMES 37 38 // An iframe stack used in the early boot process when we don't have 39 // threads yet. 40 struct iframe_stack gBootFrameStack; 41 42 // In order to avoid store/restore of large FPU state, it is assumed that 43 // this code and page fault handling doesn't use FPU. 44 // Instead this is called manually when handling IRQ or syscall. 45 extern "C" void _fp_save(aarch64_fpu_state *fpu); 46 extern "C" void _fp_restore(aarch64_fpu_state *fpu); 47 48 void 49 arch_int_enable_io_interrupt(int32 irq) 50 { 51 InterruptController *ic = InterruptController::Get(); 52 if (ic != NULL) 53 ic->EnableInterrupt(irq); 54 } 55 56 57 void 58 arch_int_disable_io_interrupt(int32 irq) 59 { 60 InterruptController *ic = InterruptController::Get(); 61 if (ic != NULL) 62 ic->DisableInterrupt(irq); 63 } 64 65 66 int32 67 arch_int_assign_to_cpu(int32 irq, int32 cpu) 68 { 69 // Not yet supported. 70 return 0; 71 } 72 73 74 static void 75 print_iframe(const char *event, struct iframe *frame) 76 { 77 if (event) 78 dprintf("Exception: %s\n", event); 79 80 dprintf("ELR=%016lx SPSR=%016lx\n", 81 frame->elr, frame->spsr); 82 dprintf("LR=%016lx SP =%016lx\n", 83 frame->lr, frame->sp); 84 } 85 86 87 status_t 88 arch_int_init(kernel_args *args) 89 { 90 return B_OK; 91 } 92 93 94 status_t 95 arch_int_init_post_vm(kernel_args *args) 96 { 97 InterruptController *ic = NULL; 98 if (strcmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2) == 0) { 99 ic = new(std::nothrow) GICv2InterruptController( 100 args->arch_args.interrupt_controller.regs1.start, 101 args->arch_args.interrupt_controller.regs2.start); 102 } 103 104 if (ic == NULL) 105 return B_ERROR; 106 107 return B_OK; 108 } 109 110 111 status_t 112 arch_int_init_io(kernel_args* args) 113 { 114 return B_OK; 115 } 116 117 118 status_t 119 arch_int_init_post_device_manager(struct kernel_args *args) 120 { 121 return B_ENTRY_NOT_FOUND; 122 } 123 124 125 // TODO: reuse things from VMSAv8TranslationMap 126 127 128 static int page_bits = 12; 129 130 static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); 131 static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); 132 static constexpr uint64_t kAttrSWDBM = (1UL << 55); 133 static constexpr uint64_t kAttrAF = (1UL << 10); 134 static constexpr uint64_t kAttrAP2 = (1UL << 7); 135 136 137 static uint64_t* 138 TableFromPa(phys_addr_t pa) 139 { 140 return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa); 141 } 142 143 144 static bool 145 fixup_entry(phys_addr_t ptPa, int level, addr_t va, bool wr) 146 { 147 int tableBits = page_bits - 3; 148 uint64_t tableMask = (1UL << tableBits) - 1; 149 150 int shift = tableBits * (3 - level) + page_bits; 151 uint64_t entrySize = 1UL << shift; 152 uint64_t entryMask = entrySize - 1; 153 154 int index = (va >> shift) & tableMask; 155 156 uint64_t *pte = &TableFromPa(ptPa)[index]; 157 158 int type = *pte & 0x3; 159 uint64_t addr = *pte & kPteAddrMask; 160 161 if ((level == 3 && type == 0x3) || (level < 3 && type == 0x1)) { 162 if (!wr && (*pte & kAttrAF) == 0) { 163 atomic_or64((int64*)pte, kAttrAF); 164 return true; 165 } 166 if (wr && (*pte & kAttrSWDBM) != 0 && (*pte & kAttrAP2) != 0) { 167 atomic_and64((int64*)pte, ~kAttrAP2); 168 asm("tlbi vaae1is, %0 \n dsb ish"::"r"(va >> page_bits)); 169 return true; 170 } 171 } else if (level < 3 && type == 0x3) { 172 return fixup_entry(addr, level + 1, va, wr); 173 } 174 175 return false; 176 } 177 178 179 void 180 after_exception() 181 { 182 Thread* thread = thread_get_current_thread(); 183 cpu_status state = disable_interrupts(); 184 if (thread->post_interrupt_callback != NULL) { 185 void (*callback)(void*) = thread->post_interrupt_callback; 186 void* data = thread->post_interrupt_data; 187 188 thread->post_interrupt_callback = NULL; 189 thread->post_interrupt_data = NULL; 190 191 restore_interrupts(state); 192 193 callback(data); 194 } else if (thread->cpu->invoke_scheduler) { 195 SpinLocker schedulerLocker(thread->scheduler_lock); 196 scheduler_reschedule(B_THREAD_READY); 197 schedulerLocker.Unlock(); 198 restore_interrupts(state); 199 } 200 } 201 202 203 // Little helper class for handling the 204 // iframe stack as used by KDL. 205 class IFrameScope { 206 public: 207 IFrameScope(struct iframe *iframe) { 208 fThread = thread_get_current_thread(); 209 if (fThread) 210 arm64_push_iframe(&fThread->arch_info.iframes, iframe); 211 else 212 arm64_push_iframe(&gBootFrameStack, iframe); 213 } 214 215 virtual ~IFrameScope() { 216 // pop iframe 217 if (fThread) 218 arm64_pop_iframe(&fThread->arch_info.iframes); 219 else 220 arm64_pop_iframe(&gBootFrameStack); 221 } 222 private: 223 Thread* fThread; 224 }; 225 226 227 extern "C" void 228 do_sync_handler(iframe * frame) 229 { 230 #ifdef TRACE_ARCH_INT_IFRAMES 231 print_iframe("Sync abort", frame); 232 #endif 233 234 IFrameScope scope(frame); 235 236 bool isExec = false; 237 switch (ESR_ELx_EXCEPTION(frame->esr)) { 238 case EXCP_INSN_ABORT_L: 239 case EXCP_INSN_ABORT: 240 isExec = true; 241 case EXCP_DATA_ABORT_L: 242 case EXCP_DATA_ABORT: 243 { 244 bool write = (frame->esr & ISS_DATA_WnR) != 0; 245 bool known = false; 246 247 int initialLevel = VMSAv8TranslationMap::CalcStartLevel(48, 12); 248 phys_addr_t ptPa; 249 bool addrType = (frame->far & (1UL << 63)) != 0; 250 if (addrType) 251 ptPa = READ_SPECIALREG(TTBR1_EL1); 252 else 253 ptPa = READ_SPECIALREG(TTBR0_EL1); 254 255 switch (frame->esr & ISS_DATA_DFSC_MASK) { 256 case ISS_DATA_DFSC_TF_L0: 257 case ISS_DATA_DFSC_TF_L1: 258 case ISS_DATA_DFSC_TF_L2: 259 case ISS_DATA_DFSC_TF_L3: 260 known = true; 261 break; 262 263 case ISS_DATA_DFSC_AFF_L1: 264 case ISS_DATA_DFSC_AFF_L2: 265 case ISS_DATA_DFSC_AFF_L3: 266 known = true; 267 if (fixup_entry(ptPa, initialLevel, frame->far, false)) 268 return; 269 break; 270 271 case ISS_DATA_DFSC_PF_L1: 272 case ISS_DATA_DFSC_PF_L2: 273 case ISS_DATA_DFSC_PF_L3: 274 known = true; 275 if (write && fixup_entry(ptPa, initialLevel, frame->far, true)) 276 return; 277 break; 278 } 279 280 if (!known) 281 break; 282 283 if (debug_debugger_running()) { 284 Thread* thread = thread_get_current_thread(); 285 if (thread != NULL) { 286 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 287 if (cpu->fault_handler != 0) { 288 debug_set_page_fault_info(frame->far, frame->elr, 289 write ? DEBUG_PAGE_FAULT_WRITE : 0); 290 frame->elr = cpu->fault_handler; 291 frame->sp = cpu->fault_handler_stack_pointer; 292 return; 293 } 294 } 295 } 296 297 Thread *thread = thread_get_current_thread(); 298 ASSERT(thread); 299 300 bool isUser = (frame->spsr & PSR_M_MASK) == PSR_M_EL0t; 301 302 if ((frame->spsr & PSR_I) != 0) { 303 // interrupts disabled 304 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler); 305 if (thread->fault_handler != 0) { 306 frame->elr = handler; 307 return; 308 } 309 } else if (thread->page_faults_allowed != 0) { 310 dprintf("PF: %lx\n", frame->far); 311 enable_interrupts(); 312 addr_t ret = 0; 313 vm_page_fault(frame->far, frame->elr, write, isExec, isUser, &ret); 314 if (ret != 0) 315 frame->elr = ret; 316 return; 317 } 318 319 panic("unhandled pagefault! FAR=%lx ELR=%lx ESR=%lx", 320 frame->far, frame->elr, frame->esr); 321 break; 322 } 323 324 case EXCP_SVC64: 325 { 326 uint32 imm = (frame->esr & 0xffff); 327 328 uint32 count = imm & 0x1f; 329 uint32 syscall = imm >> 5; 330 331 uint64_t args[20]; 332 if (count > 20) { 333 frame->x[0] = B_ERROR; 334 return; 335 } 336 337 memset(args, 0, sizeof(args)); 338 memcpy(args, frame->x, (count < 8 ? count : 8) * 8); 339 340 if (count > 8) { 341 if (!IS_USER_ADDRESS(frame->sp) 342 || user_memcpy(&args[8], (void*)frame->sp, (count - 8) * 8) != B_OK) { 343 frame->x[0] = B_BAD_ADDRESS; 344 return; 345 } 346 } 347 348 _fp_save(&frame->fpu); 349 350 thread_at_kernel_entry(system_time()); 351 352 enable_interrupts(); 353 syscall_dispatcher(syscall, (void*)args, &frame->x[0]); 354 355 { 356 disable_interrupts(); 357 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 358 if ((thread_get_current_thread()->flags 359 & (THREAD_FLAGS_SIGNALS_PENDING 360 | THREAD_FLAGS_DEBUG_THREAD 361 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 362 enable_interrupts(); 363 thread_at_kernel_exit(); 364 } else { 365 thread_at_kernel_exit_no_signals(); 366 } 367 if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { 368 panic("syscall restart"); 369 } 370 } 371 372 _fp_restore(&frame->fpu); 373 374 return; 375 } 376 } 377 378 panic("unhandled exception! FAR=%lx ELR=%lx ESR=%lx (EC=%lx)", 379 frame->far, frame->elr, frame->esr, (frame->esr >> 26) & 0x3f); 380 } 381 382 383 extern "C" void 384 do_error_handler(iframe * frame) 385 { 386 #ifdef TRACE_ARCH_INT_IFRAMES 387 print_iframe("Error", frame); 388 #endif 389 390 IFrameScope scope(frame); 391 392 panic("unhandled error! FAR=%lx ELR=%lx ESR=%lx", frame->far, frame->elr, frame->esr); 393 } 394 395 396 extern "C" void 397 do_irq_handler(iframe * frame) 398 { 399 #ifdef TRACE_ARCH_INT_IFRAMES 400 print_iframe("IRQ", frame); 401 #endif 402 403 IFrameScope scope(frame); 404 405 _fp_save(&frame->fpu); 406 407 InterruptController *ic = InterruptController::Get(); 408 if (ic != NULL) 409 ic->HandleInterrupt(); 410 411 after_exception(); 412 413 _fp_restore(&frame->fpu); 414 } 415 416 417 extern "C" void 418 do_fiq_handler(iframe * frame) 419 { 420 #ifdef TRACE_ARCH_INT_IFRAMES 421 print_iframe("FIQ", frame); 422 #endif 423 424 IFrameScope scope(frame); 425 426 panic("do_fiq_handler"); 427 } 428