1 /* 2 * Copyright 2019-2022 Haiku, Inc. All Rights Reserved. 3 * Distributed under the terms of the MIT License. 4 */ 5 #include <int.h> 6 7 #include <arch/smp.h> 8 #include <boot/kernel_args.h> 9 #include <device_manager.h> 10 #include <kscheduler.h> 11 #include <ksyscalls.h> 12 #include <interrupt_controller.h> 13 #include <smp.h> 14 #include <thread.h> 15 #include <timer.h> 16 #include <util/AutoLock.h> 17 #include <util/DoublyLinkedList.h> 18 #include <util/kernel_cpp.h> 19 #include <vm/vm.h> 20 #include <vm/vm_priv.h> 21 #include <vm/VMAddressSpace.h> 22 #include "syscall_numbers.h" 23 #include "VMSAv8TranslationMap.h" 24 #include <string.h> 25 26 #include "soc.h" 27 #include "arch_int_gicv2.h" 28 29 #define TRACE_ARCH_INT 30 #ifdef TRACE_ARCH_INT 31 # define TRACE(x) dprintf x 32 #else 33 # define TRACE(x) ; 34 #endif 35 36 //#define TRACE_ARCH_INT_IFRAMES 37 38 // An iframe stack used in the early boot process when we don't have 39 // threads yet. 40 struct iframe_stack gBootFrameStack; 41 42 // In order to avoid store/restore of large FPU state, it is assumed that 43 // this code and page fault handling doesn't use FPU. 44 // Instead this is called manually when handling IRQ or syscall. 45 extern "C" void _fp_save(aarch64_fpu_state *fpu); 46 extern "C" void _fp_restore(aarch64_fpu_state *fpu); 47 48 void 49 arch_int_enable_io_interrupt(int32 irq) 50 { 51 InterruptController *ic = InterruptController::Get(); 52 if (ic != NULL) 53 ic->EnableInterrupt(irq); 54 } 55 56 57 void 58 arch_int_disable_io_interrupt(int32 irq) 59 { 60 InterruptController *ic = InterruptController::Get(); 61 if (ic != NULL) 62 ic->DisableInterrupt(irq); 63 } 64 65 66 int32 67 arch_int_assign_to_cpu(int32 irq, int32 cpu) 68 { 69 // Not yet supported. 70 return 0; 71 } 72 73 74 static void 75 print_iframe(const char *event, struct iframe *frame) 76 { 77 if (event) 78 dprintf("Exception: %s\n", event); 79 80 dprintf("ELR=%016lx SPSR=%016lx\n", 81 frame->elr, frame->spsr); 82 dprintf("LR=%016lx SP =%016lx\n", 83 frame->lr, frame->sp); 84 } 85 86 87 status_t 88 arch_int_init(kernel_args *args) 89 { 90 return B_OK; 91 } 92 93 94 status_t 95 arch_int_init_post_vm(kernel_args *args) 96 { 97 InterruptController *ic = NULL; 98 if (strcmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2) == 0) { 99 ic = new(std::nothrow) GICv2InterruptController( 100 args->arch_args.interrupt_controller.regs1.start, 101 args->arch_args.interrupt_controller.regs2.start); 102 } 103 104 if (ic == NULL) 105 return B_ERROR; 106 107 return B_OK; 108 } 109 110 111 status_t 112 arch_int_init_io(kernel_args* args) 113 { 114 return B_OK; 115 } 116 117 118 status_t 119 arch_int_init_post_device_manager(struct kernel_args *args) 120 { 121 return B_ENTRY_NOT_FOUND; 122 } 123 124 125 // TODO: reuse things from VMSAv8TranslationMap 126 127 128 static int page_bits = 12; 129 130 static uint64_t* 131 TableFromPa(phys_addr_t pa) 132 { 133 return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa); 134 } 135 136 137 static bool 138 fixup_entry(phys_addr_t ptPa, int level, addr_t va, bool wr) 139 { 140 int tableBits = page_bits - 3; 141 uint64_t tableMask = (1UL << tableBits) - 1; 142 143 int shift = tableBits * (3 - level) + page_bits; 144 uint64_t entrySize = 1UL << shift; 145 uint64_t entryMask = entrySize - 1; 146 147 int index = (va >> shift) & tableMask; 148 149 uint64_t *pte = &TableFromPa(ptPa)[index]; 150 uint64_t oldPte = atomic_get64((int64*)pte); 151 152 int type = oldPte & kPteTypeMask; 153 uint64_t addr = oldPte & kPteAddrMask; 154 155 if ((level == 3 && type == kPteTypeL3Page) || (level < 3 && type == kPteTypeL12Block)) { 156 if (!wr && (oldPte & kAttrAF) == 0) { 157 uint64_t newPte = oldPte | kAttrAF; 158 if ((uint64_t)atomic_test_and_set64((int64*)pte, newPte, oldPte) != oldPte) 159 return true; // If something changed, handle it by taking another fault 160 asm("dsb ishst"); 161 asm("isb"); 162 return true; 163 } 164 if (wr && (oldPte & kAttrSWDBM) != 0 && (oldPte & kAttrAPReadOnly) != 0) { 165 uint64_t newPte = oldPte & ~kAttrAPReadOnly; 166 if ((uint64_t)atomic_test_and_set64((int64*)pte, newPte, oldPte) != oldPte) 167 return true; 168 asm("dsb ishst"); 169 uint64_t ttbr0 = READ_SPECIALREG(TTBR0_EL1); 170 asm("tlbi vae1is, %0" ::"r"(((va >> 12) & kTLBIMask) | (ttbr0 & kASIDMask))); 171 asm("dsb ish"); 172 asm("isb"); 173 return true; 174 } 175 } else if (level < 3 && type == kPteTypeL012Table) { 176 return fixup_entry(addr, level + 1, va, wr); 177 } 178 179 return false; 180 } 181 182 183 void 184 after_exception() 185 { 186 Thread* thread = thread_get_current_thread(); 187 cpu_status state = disable_interrupts(); 188 if (thread->post_interrupt_callback != NULL) { 189 void (*callback)(void*) = thread->post_interrupt_callback; 190 void* data = thread->post_interrupt_data; 191 192 thread->post_interrupt_callback = NULL; 193 thread->post_interrupt_data = NULL; 194 195 restore_interrupts(state); 196 197 callback(data); 198 } else if (thread->cpu->invoke_scheduler) { 199 SpinLocker schedulerLocker(thread->scheduler_lock); 200 scheduler_reschedule(B_THREAD_READY); 201 schedulerLocker.Unlock(); 202 restore_interrupts(state); 203 } 204 } 205 206 207 // Little helper class for handling the 208 // iframe stack as used by KDL. 209 class IFrameScope { 210 public: 211 IFrameScope(struct iframe *iframe) { 212 fThread = thread_get_current_thread(); 213 if (fThread) 214 arm64_push_iframe(&fThread->arch_info.iframes, iframe); 215 else 216 arm64_push_iframe(&gBootFrameStack, iframe); 217 } 218 219 virtual ~IFrameScope() { 220 // pop iframe 221 if (fThread) 222 arm64_pop_iframe(&fThread->arch_info.iframes); 223 else 224 arm64_pop_iframe(&gBootFrameStack); 225 } 226 private: 227 Thread* fThread; 228 }; 229 230 231 extern "C" void 232 do_sync_handler(iframe * frame) 233 { 234 #ifdef TRACE_ARCH_INT_IFRAMES 235 print_iframe("Sync abort", frame); 236 #endif 237 238 IFrameScope scope(frame); 239 240 bool isExec = false; 241 switch (ESR_ELx_EXCEPTION(frame->esr)) { 242 case EXCP_INSN_ABORT_L: 243 case EXCP_INSN_ABORT: 244 isExec = true; 245 case EXCP_DATA_ABORT_L: 246 case EXCP_DATA_ABORT: 247 { 248 bool write = (frame->esr & ISS_DATA_WnR) != 0; 249 bool known = false; 250 251 int initialLevel = VMSAv8TranslationMap::CalcStartLevel(48, 12); 252 phys_addr_t ptPa; 253 bool addrType = (frame->far & (1UL << 63)) != 0; 254 if (addrType) 255 ptPa = READ_SPECIALREG(TTBR1_EL1); 256 else 257 ptPa = READ_SPECIALREG(TTBR0_EL1); 258 ptPa &= kTtbrBasePhysAddrMask; 259 260 switch (frame->esr & ISS_DATA_DFSC_MASK) { 261 case ISS_DATA_DFSC_TF_L0: 262 case ISS_DATA_DFSC_TF_L1: 263 case ISS_DATA_DFSC_TF_L2: 264 case ISS_DATA_DFSC_TF_L3: 265 known = true; 266 break; 267 268 case ISS_DATA_DFSC_AFF_L1: 269 case ISS_DATA_DFSC_AFF_L2: 270 case ISS_DATA_DFSC_AFF_L3: 271 known = true; 272 if (fixup_entry(ptPa, initialLevel, frame->far, false)) 273 return; 274 break; 275 276 case ISS_DATA_DFSC_PF_L1: 277 case ISS_DATA_DFSC_PF_L2: 278 case ISS_DATA_DFSC_PF_L3: 279 known = true; 280 if (write && fixup_entry(ptPa, initialLevel, frame->far, true)) 281 return; 282 break; 283 } 284 285 if (!known) 286 break; 287 288 if (debug_debugger_running()) { 289 Thread* thread = thread_get_current_thread(); 290 if (thread != NULL) { 291 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 292 if (cpu->fault_handler != 0) { 293 debug_set_page_fault_info(frame->far, frame->elr, 294 write ? DEBUG_PAGE_FAULT_WRITE : 0); 295 frame->elr = cpu->fault_handler; 296 frame->sp = cpu->fault_handler_stack_pointer; 297 return; 298 } 299 } 300 } 301 302 Thread *thread = thread_get_current_thread(); 303 ASSERT(thread); 304 305 bool isUser = (frame->spsr & PSR_M_MASK) == PSR_M_EL0t; 306 307 if ((frame->spsr & PSR_I) != 0) { 308 // interrupts disabled 309 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler); 310 if (thread->fault_handler != 0) { 311 frame->elr = handler; 312 return; 313 } 314 } else if (thread->page_faults_allowed != 0) { 315 dprintf("PF: %lx\n", frame->far); 316 enable_interrupts(); 317 addr_t ret = 0; 318 vm_page_fault(frame->far, frame->elr, write, isExec, isUser, &ret); 319 if (ret != 0) 320 frame->elr = ret; 321 return; 322 } 323 324 panic("unhandled pagefault! FAR=%lx ELR=%lx ESR=%lx", 325 frame->far, frame->elr, frame->esr); 326 break; 327 } 328 329 case EXCP_SVC64: 330 { 331 uint32 imm = (frame->esr & 0xffff); 332 333 uint32 count = imm & 0x1f; 334 uint32 syscall = imm >> 5; 335 336 uint64_t args[20]; 337 if (count > 20) { 338 frame->x[0] = B_ERROR; 339 return; 340 } 341 342 memset(args, 0, sizeof(args)); 343 memcpy(args, frame->x, (count < 8 ? count : 8) * 8); 344 345 if (count > 8) { 346 if (!IS_USER_ADDRESS(frame->sp) 347 || user_memcpy(&args[8], (void*)frame->sp, (count - 8) * 8) != B_OK) { 348 frame->x[0] = B_BAD_ADDRESS; 349 return; 350 } 351 } 352 353 _fp_save(&frame->fpu); 354 355 thread_at_kernel_entry(system_time()); 356 357 enable_interrupts(); 358 syscall_dispatcher(syscall, (void*)args, &frame->x[0]); 359 360 { 361 disable_interrupts(); 362 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 363 if ((thread_get_current_thread()->flags 364 & (THREAD_FLAGS_SIGNALS_PENDING 365 | THREAD_FLAGS_DEBUG_THREAD 366 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 367 enable_interrupts(); 368 thread_at_kernel_exit(); 369 } else { 370 thread_at_kernel_exit_no_signals(); 371 } 372 if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { 373 panic("syscall restart"); 374 } 375 } 376 377 _fp_restore(&frame->fpu); 378 379 return; 380 } 381 } 382 383 panic("unhandled exception! FAR=%lx ELR=%lx ESR=%lx (EC=%lx)", 384 frame->far, frame->elr, frame->esr, (frame->esr >> 26) & 0x3f); 385 } 386 387 388 extern "C" void 389 do_error_handler(iframe * frame) 390 { 391 #ifdef TRACE_ARCH_INT_IFRAMES 392 print_iframe("Error", frame); 393 #endif 394 395 IFrameScope scope(frame); 396 397 panic("unhandled error! FAR=%lx ELR=%lx ESR=%lx", frame->far, frame->elr, frame->esr); 398 } 399 400 401 extern "C" void 402 do_irq_handler(iframe * frame) 403 { 404 #ifdef TRACE_ARCH_INT_IFRAMES 405 print_iframe("IRQ", frame); 406 #endif 407 408 IFrameScope scope(frame); 409 410 _fp_save(&frame->fpu); 411 412 InterruptController *ic = InterruptController::Get(); 413 if (ic != NULL) 414 ic->HandleInterrupt(); 415 416 after_exception(); 417 418 _fp_restore(&frame->fpu); 419 } 420 421 422 extern "C" void 423 do_fiq_handler(iframe * frame) 424 { 425 #ifdef TRACE_ARCH_INT_IFRAMES 426 print_iframe("FIQ", frame); 427 #endif 428 429 IFrameScope scope(frame); 430 431 panic("do_fiq_handler"); 432 } 433