1 /* 2 * Copyright 2019 Haiku, Inc. All Rights Reserved. 3 * Distributed under the terms of the MIT License. 4 */ 5 #include <int.h> 6 7 #include <arch/smp.h> 8 #include <boot/kernel_args.h> 9 #include <device_manager.h> 10 #include <kscheduler.h> 11 #include <ksyscalls.h> 12 #include <interrupt_controller.h> 13 #include <smp.h> 14 #include <thread.h> 15 #include <timer.h> 16 #include <util/AutoLock.h> 17 #include <util/DoublyLinkedList.h> 18 #include <util/kernel_cpp.h> 19 #include <vm/vm.h> 20 #include <vm/vm_priv.h> 21 #include <vm/VMAddressSpace.h> 22 #include "syscall_numbers.h" 23 #include "VMSAv8TranslationMap.h" 24 #include <string.h> 25 26 #define TRACE_ARCH_INT 27 #ifdef TRACE_ARCH_INT 28 # define TRACE(x) dprintf x 29 #else 30 # define TRACE(x) ; 31 #endif 32 33 34 void 35 arch_int_enable_io_interrupt(int irq) 36 { 37 } 38 39 40 void 41 arch_int_disable_io_interrupt(int irq) 42 { 43 } 44 45 46 int32 47 arch_int_assign_to_cpu(int32 irq, int32 cpu) 48 { 49 // Not yet supported. 50 return 0; 51 } 52 53 54 status_t 55 arch_int_init(kernel_args *args) 56 { 57 return B_OK; 58 } 59 60 61 status_t 62 arch_int_init_post_vm(kernel_args *args) 63 { 64 return B_OK; 65 } 66 67 68 status_t 69 arch_int_init_io(kernel_args* args) 70 { 71 return B_OK; 72 } 73 74 75 status_t 76 arch_int_init_post_device_manager(struct kernel_args *args) 77 { 78 return B_ENTRY_NOT_FOUND; 79 } 80 81 82 // TODO: reuse things from VMSAv8TranslationMap 83 84 85 static int page_bits = 12; 86 87 static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); 88 static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); 89 static constexpr uint64_t kAttrSWDBM = (1UL << 55); 90 static constexpr uint64_t kAttrAF = (1UL << 10); 91 static constexpr uint64_t kAttrAP2 = (1UL << 7); 92 93 94 static uint64_t* 95 TableFromPa(phys_addr_t pa) 96 { 97 return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa); 98 } 99 100 101 static bool 102 fixup_entry(phys_addr_t ptPa, int level, addr_t va, bool wr) 103 { 104 int tableBits = page_bits - 3; 105 uint64_t tableMask = (1UL << tableBits) - 1; 106 107 int shift = tableBits * (3 - level) + page_bits; 108 uint64_t entrySize = 1UL << shift; 109 uint64_t entryMask = entrySize - 1; 110 111 int index = (va >> shift) & tableMask; 112 113 uint64_t *pte = &TableFromPa(ptPa)[index]; 114 115 int type = *pte & 0x3; 116 uint64_t addr = *pte & kPteAddrMask; 117 118 if ((level == 3 && type == 0x3) || (level < 3 && type == 0x1)) { 119 if (!wr && (*pte & kAttrAF) == 0) { 120 atomic_or64((int64*)pte, kAttrAF); 121 return true; 122 } 123 if (wr && (*pte & kAttrSWDBM) != 0 && (*pte & kAttrAP2) != 0) { 124 atomic_and64((int64*)pte, ~kAttrAP2); 125 asm("tlbi vaae1is, %0 \n dsb ish"::"r"(va >> page_bits)); 126 return true; 127 } 128 } else if (level < 3 && type == 0x3) { 129 return fixup_entry(addr, level + 1, va, wr); 130 } 131 132 return false; 133 } 134 135 136 void 137 after_exception() 138 { 139 Thread* thread = thread_get_current_thread(); 140 if (thread->cpu->invoke_scheduler) { 141 disable_interrupts(); 142 SpinLocker schedulerLocker(thread->scheduler_lock); 143 scheduler_reschedule(B_THREAD_READY); 144 } 145 } 146 147 148 extern "C" void 149 do_sync_handler(iframe * frame) 150 { 151 bool isExec = false; 152 switch (ESR_ELx_EXCEPTION(frame->esr)) { 153 case EXCP_INSN_ABORT_L: 154 case EXCP_INSN_ABORT: 155 isExec = true; 156 case EXCP_DATA_ABORT_L: 157 case EXCP_DATA_ABORT: 158 { 159 bool write = (frame->esr & ISS_DATA_WnR) != 0; 160 bool known = false; 161 162 int initialLevel = VMSAv8TranslationMap::CalcStartLevel(48, 12); 163 phys_addr_t ptPa; 164 bool addrType = (frame->far & (1UL << 63)) != 0; 165 if (addrType) 166 ptPa = READ_SPECIALREG(TTBR1_EL1); 167 else 168 ptPa = READ_SPECIALREG(TTBR0_EL1); 169 170 switch (frame->esr & ISS_DATA_DFSC_MASK) { 171 case ISS_DATA_DFSC_TF_L0: 172 case ISS_DATA_DFSC_TF_L1: 173 case ISS_DATA_DFSC_TF_L2: 174 case ISS_DATA_DFSC_TF_L3: 175 known = true; 176 break; 177 178 case ISS_DATA_DFSC_AFF_L1: 179 case ISS_DATA_DFSC_AFF_L2: 180 case ISS_DATA_DFSC_AFF_L3: 181 known = true; 182 if (fixup_entry(ptPa, initialLevel, frame->far, false)) 183 return; 184 break; 185 186 case ISS_DATA_DFSC_PF_L1: 187 case ISS_DATA_DFSC_PF_L2: 188 case ISS_DATA_DFSC_PF_L3: 189 known = true; 190 if (write && fixup_entry(ptPa, initialLevel, frame->far, true)) 191 return; 192 break; 193 } 194 195 if (!known) 196 break; 197 198 if (debug_debugger_running()) { 199 Thread* thread = thread_get_current_thread(); 200 if (thread != NULL) { 201 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 202 if (cpu->fault_handler != 0) { 203 debug_set_page_fault_info(frame->far, frame->elr, 204 write ? DEBUG_PAGE_FAULT_WRITE : 0); 205 frame->elr = cpu->fault_handler; 206 frame->sp = cpu->fault_handler_stack_pointer; 207 return; 208 } 209 } 210 } 211 212 Thread *thread = thread_get_current_thread(); 213 ASSERT(thread); 214 215 bool isUser = (frame->spsr & PSR_M_MASK) == PSR_M_EL0t; 216 217 if ((frame->spsr & PSR_I) != 0) { 218 // interrupts disabled 219 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler); 220 if (thread->fault_handler != 0) { 221 frame->elr = handler; 222 return; 223 } 224 } else if (thread->page_faults_allowed != 0) { 225 dprintf("PF: %lx\n", frame->far); 226 enable_interrupts(); 227 addr_t ret = 0; 228 vm_page_fault(frame->far, frame->elr, write, isExec, isUser, &ret); 229 if (ret != 0) 230 frame->elr = ret; 231 return; 232 } 233 234 panic("unhandled pagefault! FAR=%lx ELR=%lx ESR=%lx", 235 frame->far, frame->elr, frame->esr); 236 break; 237 } 238 239 case EXCP_SVC64: 240 { 241 uint32 imm = (frame->esr & 0xffff); 242 243 uint32 count = imm & 0x1f; 244 uint32 syscall = imm >> 5; 245 246 uint64_t args[20]; 247 if (count > 20) { 248 frame->x[0] = B_ERROR; 249 return; 250 } 251 252 memset(args, 0, sizeof(args)); 253 memcpy(args, frame->x, (count < 8 ? count : 8) * 8); 254 255 if (count > 8) { 256 if (!IS_USER_ADDRESS(frame->sp) 257 || user_memcpy(&args[8], (void*)frame->sp, (count - 8) * 8) != B_OK) { 258 frame->x[0] = B_BAD_ADDRESS; 259 return; 260 } 261 } 262 263 thread_at_kernel_entry(system_time()); 264 265 enable_interrupts(); 266 syscall_dispatcher(syscall, (void*)args, &frame->x[0]); 267 268 { 269 disable_interrupts(); 270 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 271 if ((thread_get_current_thread()->flags 272 & (THREAD_FLAGS_SIGNALS_PENDING 273 | THREAD_FLAGS_DEBUG_THREAD 274 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 275 enable_interrupts(); 276 thread_at_kernel_exit(); 277 } else { 278 thread_at_kernel_exit_no_signals(); 279 } 280 if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { 281 panic("syscall restart"); 282 } 283 } 284 285 return; 286 } 287 } 288 289 panic("unhandled exception! FAR=%lx ELR=%lx ESR=%lx (EC=%lx)", 290 frame->far, frame->elr, frame->esr, (frame->esr >> 26) & 0x3f); 291 } 292 293 294 extern "C" void 295 do_error_handler(iframe * frame) 296 { 297 panic("unhandled error! FAR=%lx ELR=%lx ESR=%lx", frame->far, frame->elr, frame->esr); 298 } 299 300 301 extern "C" void 302 do_irq_handler(iframe * frame) 303 { 304 InterruptController *ic = InterruptController::Get(); 305 if (ic != NULL) 306 ic->HandleInterrupt(); 307 308 after_exception(); 309 } 310 311 312 extern "C" void 313 do_fiq_handler(iframe * frame) 314 { 315 panic("do_fiq_handler"); 316 } 317