1 /* 2 * Copyright 2003-2011, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Adrien Destugues, pulkomandy@pulkomandy.tk 7 */ 8 9 10 #include <int.h> 11 #include <cpu.h> 12 #include <thread.h> 13 #include <vm/vm_priv.h> 14 #include <ksyscalls.h> 15 #include <syscall_numbers.h> 16 #include <arch_cpu_defs.h> 17 #include <arch_thread_types.h> 18 #include <arch/debug.h> 19 #include <util/AutoLock.h> 20 #include <Htif.h> 21 #include <Plic.h> 22 #include <Clint.h> 23 #include <AutoDeleterDrivers.h> 24 #include <ScopeExit.h> 25 #include "RISCV64VMTranslationMap.h" 26 27 #include <algorithm> 28 29 30 static uint32 sPlicContexts[SMP_MAX_CPUS]; 31 32 33 //#pragma mark debug output 34 35 static void 36 WriteMode(int mode) 37 { 38 switch (mode) { 39 case modeU: dprintf("u"); break; 40 case modeS: dprintf("s"); break; 41 case modeM: dprintf("m"); break; 42 default: dprintf("%d", mode); 43 } 44 } 45 46 47 static void 48 WriteModeSet(uint32_t val) 49 { 50 bool first = true; 51 dprintf("{"); 52 for (int i = 0; i < 32; i++) { 53 if (((1LL << i) & val) != 0) { 54 if (first) first = false; else dprintf(", "); 55 WriteMode(i); 56 } 57 } 58 dprintf("}"); 59 } 60 61 62 static void 63 WriteExt(uint64_t val) 64 { 65 switch (val) { 66 case 0: dprintf("off"); break; 67 case 1: dprintf("initial"); break; 68 case 2: dprintf("clean"); break; 69 case 3: dprintf("dirty"); break; 70 default: dprintf("%" B_PRId64, val); 71 } 72 } 73 74 75 static void 76 WriteSstatus(uint64_t val) 77 { 78 SstatusReg status{.val = val}; 79 dprintf("("); 80 dprintf("ie: "); WriteModeSet(status.ie); 81 dprintf(", pie: "); WriteModeSet(status.pie); 82 dprintf(", spp: "); WriteMode(status.spp); 83 dprintf(", fs: "); WriteExt(status.fs); 84 dprintf(", xs: "); WriteExt(status.xs); 85 dprintf(", sum: %d", (int)status.sum); 86 dprintf(", mxr: %d", (int)status.mxr); 87 dprintf(", uxl: %d", (int)status.uxl); 88 dprintf(", sd: %d", (int)status.sd); 89 dprintf(")"); 90 } 91 92 93 static void 94 WriteInterrupt(uint64_t val) 95 { 96 switch (val) { 97 case 0 + modeU: dprintf("uSoft"); break; 98 case 0 + modeS: dprintf("sSoft"); break; 99 case 0 + modeM: dprintf("mSoft"); break; 100 case 4 + modeU: dprintf("uTimer"); break; 101 case 4 + modeS: dprintf("sTimer"); break; 102 case 4 + modeM: dprintf("mTimer"); break; 103 case 8 + modeU: dprintf("uExtern"); break; 104 case 8 + modeS: dprintf("sExtern"); break; 105 case 8 + modeM: dprintf("mExtern"); break; 106 default: dprintf("%" B_PRId64, val); 107 } 108 } 109 110 111 static void 112 WriteInterruptSet(uint64_t val) 113 { 114 bool first = true; 115 dprintf("{"); 116 for (int i = 0; i < 64; i++) { 117 if (((1LL << i) & val) != 0) { 118 if (first) first = false; else dprintf(", "); 119 WriteInterrupt(i); 120 } 121 } 122 dprintf("}"); 123 } 124 125 126 static void 127 WriteCause(uint64_t cause) 128 { 129 if ((cause & causeInterrupt) == 0) { 130 dprintf("exception "); 131 switch (cause) { 132 case causeExecMisalign: dprintf("execMisalign"); break; 133 case causeExecAccessFault: dprintf("execAccessFault"); break; 134 case causeIllegalInst: dprintf("illegalInst"); break; 135 case causeBreakpoint: dprintf("breakpoint"); break; 136 case causeLoadMisalign: dprintf("loadMisalign"); break; 137 case causeLoadAccessFault: dprintf("loadAccessFault"); break; 138 case causeStoreMisalign: dprintf("storeMisalign"); break; 139 case causeStoreAccessFault: dprintf("storeAccessFault"); break; 140 case causeUEcall: dprintf("uEcall"); break; 141 case causeSEcall: dprintf("sEcall"); break; 142 case causeMEcall: dprintf("mEcall"); break; 143 case causeExecPageFault: dprintf("execPageFault"); break; 144 case causeLoadPageFault: dprintf("loadPageFault"); break; 145 case causeStorePageFault: dprintf("storePageFault"); break; 146 default: dprintf("%" B_PRId64, cause); 147 } 148 } else { 149 dprintf("interrupt "); WriteInterrupt(cause & ~causeInterrupt); 150 } 151 } 152 153 154 const static char* registerNames[] = { 155 " ra", " t6", " sp", " gp", 156 " tp", " t0", " t1", " t2", 157 " t5", " s1", " a0", " a1", 158 " a2", " a3", " a4", " a5", 159 " a6", " a7", " s2", " s3", 160 " s4", " s5", " s6", " s7", 161 " s8", " s9", "s10", "s11", 162 " t3", " t4", " fp", "epc" 163 }; 164 165 166 static void WriteRegisters(iframe* frame) 167 { 168 uint64* regs = &frame->ra; 169 for (int i = 0; i < 32; i += 4) { 170 dprintf( 171 " %s: 0x%016" B_PRIx64 172 " %s: 0x%016" B_PRIx64 173 " %s: 0x%016" B_PRIx64 174 " %s: 0x%016" B_PRIx64 "\n", 175 registerNames[i + 0], regs[i + 0], 176 registerNames[i + 1], regs[i + 1], 177 registerNames[i + 2], regs[i + 2], 178 registerNames[i + 3], regs[i + 3] 179 ); 180 } 181 } 182 183 184 static void 185 DumpMemory(uint64* adr, size_t len) 186 { 187 while (len > 0) { 188 if ((addr_t)adr % 0x10 == 0) 189 dprintf("%08" B_PRIxADDR " ", (addr_t)adr); 190 uint64 val; 191 if (user_memcpy(&val, adr++, sizeof(val)) < B_OK) { 192 dprintf(" ????????????????"); 193 } else { 194 dprintf(" %016" B_PRIx64, val); 195 } 196 if ((addr_t)adr % 0x10 == 0) 197 dprintf("\n"); 198 len -= 8; 199 } 200 if ((addr_t)adr % 0x10 != 0) 201 dprintf("\n"); 202 203 dprintf("%08" B_PRIxADDR "\n\n", (addr_t)adr); 204 } 205 206 207 void 208 WriteTrapInfo(iframe* frame) 209 { 210 InterruptsLocker locker; 211 dprintf("STrap("); WriteCause(frame->cause); dprintf(")\n"); 212 dprintf(" sstatus: "); WriteSstatus(frame->status); dprintf("\n"); 213 // dprintf(" sie: "); WriteInterruptSet(Sie()); dprintf("\n"); 214 // dprintf(" sip: "); WriteInterruptSet(Sip()); dprintf("\n"); 215 //dprintf(" stval: "); WritePC(Stval()); dprintf("\n"); 216 dprintf(" stval: 0x%" B_PRIx64 "\n", frame->tval); 217 // dprintf(" tp: 0x%" B_PRIxADDR "(%s)\n", Tp(), 218 // thread_get_current_thread()->name); 219 220 WriteRegisters(frame); 221 #if 0 222 dprintf(" kernel stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n", 223 thread_get_current_thread()->kernel_stack_base, 224 thread_get_current_thread()->kernel_stack_top - 1 225 ); 226 dprintf(" user stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n", 227 thread_get_current_thread()->user_stack_base, 228 thread_get_current_thread()->user_stack_base + 229 thread_get_current_thread()->user_stack_size - 1 230 ); 231 if (thread_get_current_thread()->arch_info.userFrame != NULL) { 232 WriteRegisters(thread_get_current_thread()->arch_info.userFrame); 233 234 dprintf("Stack memory dump:\n"); 235 DumpMemory( 236 (uint64*)thread_get_current_thread()->arch_info.userFrame->sp, 237 thread_get_current_thread()->user_stack_base + 238 thread_get_current_thread()->user_stack_size - 239 thread_get_current_thread()->arch_info.userFrame->sp 240 ); 241 // if (true) { 242 // } else { 243 // DumpMemory((uint64*)frame->sp, thread_get_current_thread()->kernel_stack_top - frame->sp); 244 // } 245 } 246 #endif 247 } 248 249 250 //#pragma mark - 251 252 static void 253 SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode, 254 addr_t signalAddress = 0, int32 signalError = B_ERROR) 255 { 256 if (SstatusReg{.val = Sstatus()}.spp == modeU) { 257 struct sigaction action; 258 Thread* thread = thread_get_current_thread(); 259 260 //DoStackTrace(Fp(), 0); 261 262 enable_interrupts(); 263 264 // If the thread has a signal handler for the signal, we simply send it 265 // the signal. Otherwise we notify the user debugger first. 266 if ((sigaction(signalNumber, NULL, &action) == 0 267 && action.sa_handler != SIG_DFL 268 && action.sa_handler != SIG_IGN) 269 || user_debug_exception_occurred(type, signalNumber)) { 270 Signal signal(signalNumber, signalCode, signalError, 271 thread->team->id); 272 signal.SetAddress((void*)signalAddress); 273 send_signal_to_thread(thread, signal, 0); 274 } 275 } else { 276 panic("Unexpected exception occurred in kernel mode!"); 277 } 278 } 279 280 281 static void 282 AfterInterrupt() 283 { 284 if (debug_debugger_running()) 285 return; 286 287 Thread* thread = thread_get_current_thread(); 288 cpu_status state = disable_interrupts(); 289 if (thread->cpu->invoke_scheduler) { 290 SpinLocker schedulerLocker(thread->scheduler_lock); 291 scheduler_reschedule(B_THREAD_READY); 292 schedulerLocker.Unlock(); 293 restore_interrupts(state); 294 } else if (thread->post_interrupt_callback != NULL) { 295 void (*callback)(void*) = thread->post_interrupt_callback; 296 void* data = thread->post_interrupt_data; 297 298 thread->post_interrupt_callback = NULL; 299 thread->post_interrupt_data = NULL; 300 301 restore_interrupts(state); 302 303 callback(data); 304 } 305 } 306 307 308 static bool 309 SetAccessedFlags(addr_t addr, bool isWrite) 310 { 311 VMAddressSpacePutter addressSpace; 312 if (IS_KERNEL_ADDRESS(addr)) 313 addressSpace.SetTo(VMAddressSpace::GetKernel()); 314 else if (IS_USER_ADDRESS(addr)) 315 addressSpace.SetTo(VMAddressSpace::GetCurrent()); 316 317 if(!addressSpace.IsSet()) 318 return false; 319 320 RISCV64VMTranslationMap* map 321 = (RISCV64VMTranslationMap*)addressSpace->TranslationMap(); 322 323 phys_addr_t physAdr; 324 uint32 pageFlags; 325 map->QueryInterrupt(addr, &physAdr, &pageFlags); 326 327 if ((PAGE_PRESENT & pageFlags) == 0) 328 return false; 329 330 if (isWrite) { 331 if ( 332 ((B_WRITE_AREA | B_KERNEL_WRITE_AREA) & pageFlags) != 0 333 && ((PAGE_ACCESSED | PAGE_MODIFIED) & pageFlags) 334 != (PAGE_ACCESSED | PAGE_MODIFIED) 335 ) { 336 map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED); 337 return true; 338 } 339 } else { 340 if ( 341 ((B_READ_AREA | B_KERNEL_READ_AREA) & pageFlags) != 0 342 && (PAGE_ACCESSED & pageFlags) == 0 343 ) { 344 map->SetFlags(addr, PAGE_ACCESSED); 345 return true; 346 } 347 } 348 return false; 349 } 350 351 352 extern "C" void 353 STrap(iframe* frame) 354 { 355 // dprintf("STrap("); WriteCause(Scause()); dprintf(")\n"); 356 357 /* 358 iframe oldFrame = *frame; 359 const auto& frameChangeChecker = MakeScopeExit([&]() { 360 InterruptsLocker locker; 361 bool first = true; 362 for (int i = 0; i < 32; i++) { 363 uint64 oldVal = ((int64*)&oldFrame)[i]; 364 uint64 newVal = ((int64*)frame)[i]; 365 if (oldVal != newVal) { 366 if (first) { 367 dprintf("FrameChangeChecker, thread: %" B_PRId32 "(%s)\n", thread_get_current_thread()->id, thread_get_current_thread()->name); 368 first = false; 369 } 370 dprintf(" %s: %#" B_PRIxADDR " -> %#" B_PRIxADDR "\n", registerNames[i], oldVal, newVal); 371 } 372 } 373 374 if (frame->epc == 0) 375 panic("FrameChangeChecker: EPC = 0"); 376 }); 377 */ 378 switch (frame->cause) { 379 case causeExecPageFault: 380 case causeLoadPageFault: 381 case causeStorePageFault: { 382 if (SetAccessedFlags(Stval(), frame->cause == causeStorePageFault)) 383 return; 384 } 385 } 386 387 if (SstatusReg{.val = frame->status}.spp == modeU) { 388 thread_get_current_thread()->arch_info.userFrame = frame; 389 thread_get_current_thread()->arch_info.oldA0 = frame->a0; 390 thread_at_kernel_entry(system_time()); 391 } 392 const auto& kernelExit = ScopeExit([&]() { 393 if (SstatusReg{.val = frame->status}.spp == modeU) { 394 disable_interrupts(); 395 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 396 if ((thread_get_current_thread()->flags 397 & (THREAD_FLAGS_SIGNALS_PENDING 398 | THREAD_FLAGS_DEBUG_THREAD 399 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 400 enable_interrupts(); 401 thread_at_kernel_exit(); 402 } else { 403 thread_at_kernel_exit_no_signals(); 404 } 405 if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { 406 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL); 407 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED); 408 409 frame->a0 = thread_get_current_thread()->arch_info.oldA0; 410 frame->epc -= 4; 411 } 412 thread_get_current_thread()->arch_info.userFrame = NULL; 413 } 414 }); 415 416 switch (frame->cause) { 417 case causeIllegalInst: { 418 return SendSignal(B_INVALID_OPCODE_EXCEPTION, SIGILL, ILL_ILLOPC, 419 frame->epc); 420 } 421 case causeExecMisalign: 422 case causeLoadMisalign: 423 case causeStoreMisalign: { 424 return SendSignal(B_ALIGNMENT_EXCEPTION, SIGBUS, BUS_ADRALN, 425 Stval()); 426 } 427 case causeBreakpoint: { 428 if (SstatusReg{.val = frame->status}.spp == modeU) { 429 user_debug_breakpoint_hit(false); 430 } else { 431 panic("hit kernel breakpoint"); 432 } 433 return; 434 } 435 case causeExecAccessFault: 436 case causeLoadAccessFault: 437 case causeStoreAccessFault: { 438 return SendSignal(B_SEGMENT_VIOLATION, SIGBUS, BUS_ADRERR, 439 Stval()); 440 } 441 case causeExecPageFault: 442 case causeLoadPageFault: 443 case causeStorePageFault: { 444 uint64 stval = Stval(); 445 446 if (debug_debugger_running()) { 447 Thread* thread = thread_get_current_thread(); 448 if (thread != NULL) { 449 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 450 if (cpu->fault_handler != 0) { 451 debug_set_page_fault_info(stval, frame->epc, 452 (frame->cause == causeStorePageFault) 453 ? DEBUG_PAGE_FAULT_WRITE : 0); 454 frame->epc = cpu->fault_handler; 455 frame->sp = cpu->fault_handler_stack_pointer; 456 return; 457 } 458 459 if (thread->fault_handler != 0) { 460 kprintf("ERROR: thread::fault_handler used in kernel " 461 "debugger!\n"); 462 debug_set_page_fault_info(stval, frame->epc, 463 frame->cause == causeStorePageFault 464 ? DEBUG_PAGE_FAULT_WRITE : 0); 465 frame->epc = (addr_t)thread->fault_handler; 466 return; 467 } 468 } 469 470 panic("page fault in debugger without fault handler! Touching " 471 "address %p from ip %p\n", (void*)stval, (void*)frame->epc); 472 return; 473 } 474 475 if (SstatusReg{.val = frame->status}.pie == 0) { 476 // user_memcpy() failure 477 Thread* thread = thread_get_current_thread(); 478 if (thread != NULL && thread->fault_handler != 0) { 479 addr_t handler = (addr_t)(thread->fault_handler); 480 if (frame->epc != handler) { 481 frame->epc = handler; 482 return; 483 } 484 } 485 panic("page fault with interrupts disabled@!dump_virt_page %#" B_PRIx64, stval); 486 } 487 488 addr_t newIP = 0; 489 enable_interrupts(); 490 491 vm_page_fault(stval, frame->epc, frame->cause == causeStorePageFault, 492 frame->cause == causeExecPageFault, 493 SstatusReg{.val = frame->status}.spp == modeU, &newIP); 494 495 if (newIP != 0) 496 frame->epc = newIP; 497 498 return; 499 } 500 case causeInterrupt + sSoftInt: { 501 ClearBitsSip(1 << sSoftInt); 502 // dprintf("sSoftInt(%" B_PRId32 ")\n", smp_get_current_cpu()); 503 smp_intercpu_int_handler(smp_get_current_cpu()); 504 AfterInterrupt(); 505 return; 506 } 507 case causeInterrupt + sTimerInt: { 508 ClearBitsSie(1 << sTimerInt); 509 // dprintf("sTimerInt(%" B_PRId32 ")\n", smp_get_current_cpu()); 510 timer_interrupt(); 511 AfterInterrupt(); 512 return; 513 } 514 case causeInterrupt + sExternInt: { 515 uint64 irq = gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete; 516 int_io_interrupt_handler(irq, true); 517 gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete = irq; 518 AfterInterrupt(); 519 return; 520 } 521 case causeUEcall: { 522 frame->epc += 4; // skip ecall 523 uint64 syscall = frame->t0; 524 uint64 args[20]; 525 if (syscall < (uint64)kSyscallCount) { 526 uint32 argCnt = kExtendedSyscallInfos[syscall].parameter_count; 527 memcpy(&args[0], &frame->a0, 528 sizeof(uint64)*std::min<uint32>(argCnt, 8)); 529 if (argCnt > 8) { 530 if (status_t res = user_memcpy(&args[8], (void*)frame->sp, 531 sizeof(uint64)*(argCnt - 8)) < B_OK) { 532 dprintf("can't read syscall arguments on user " 533 "stack\n"); 534 frame->a0 = res; 535 return; 536 } 537 } 538 } 539 /* 540 switch (syscall) { 541 case SYSCALL_READ_PORT_ETC: 542 case SYSCALL_WRITE_PORT_ETC: 543 DoStackTrace(Fp(), 0); 544 break; 545 } 546 */ 547 // dprintf("syscall: %s\n", kExtendedSyscallInfos[syscall].name); 548 549 enable_interrupts(); 550 uint64 returnValue = 0; 551 syscall_dispatcher(syscall, (void*)args, &returnValue); 552 frame->a0 = returnValue; 553 return; 554 } 555 } 556 panic("unhandled STrap"); 557 } 558 559 560 //#pragma mark - 561 562 status_t 563 arch_int_init(kernel_args* args) 564 { 565 dprintf("arch_int_init()\n"); 566 567 for (uint32 i = 0; i < args->num_cpus; i++) { 568 dprintf(" CPU %" B_PRIu32 ":\n", i); 569 dprintf(" hartId: %" B_PRIu32 "\n", args->arch_args.hartIds[i]); 570 dprintf(" plicContext: %" B_PRIu32 "\n", args->arch_args.plicContexts[i]); 571 } 572 573 for (uint32 i = 0; i < args->num_cpus; i++) 574 sPlicContexts[i] = args->arch_args.plicContexts[i]; 575 576 // TODO: read from FDT 577 reserve_io_interrupt_vectors(128, 0, INTERRUPT_TYPE_IRQ); 578 579 for (uint32 i = 0; i < args->num_cpus; i++) 580 gPlicRegs->contexts[sPlicContexts[i]].priorityThreshold = 0; 581 582 return B_OK; 583 } 584 585 586 status_t 587 arch_int_init_post_vm(kernel_args* args) 588 { 589 return B_OK; 590 } 591 592 593 status_t 594 arch_int_init_post_device_manager(struct kernel_args* args) 595 { 596 return B_OK; 597 } 598 599 600 status_t 601 arch_int_init_io(kernel_args* args) 602 { 603 return B_OK; 604 } 605 606 607 void 608 arch_int_enable_io_interrupt(int32 irq) 609 { 610 dprintf("arch_int_enable_io_interrupt(%" B_PRId32 ")\n", irq); 611 gPlicRegs->priority[irq] = 1; 612 gPlicRegs->enable[sPlicContexts[0]][irq / 32] |= 1 << (irq % 32); 613 } 614 615 616 void 617 arch_int_disable_io_interrupt(int32 irq) 618 { 619 dprintf("arch_int_disable_io_interrupt(%" B_PRId32 ")\n", irq); 620 gPlicRegs->priority[irq] = 0; 621 gPlicRegs->enable[sPlicContexts[0]][irq / 32] &= ~(1 << (irq % 32)); 622 } 623 624 625 int32 626 arch_int_assign_to_cpu(int32 irq, int32 cpu) 627 { 628 // Not yet supported. 629 return 0; 630 } 631 632 633 #undef arch_int_enable_interrupts 634 #undef arch_int_disable_interrupts 635 #undef arch_int_restore_interrupts 636 #undef arch_int_are_interrupts_enabled 637 638 639 extern "C" void 640 arch_int_enable_interrupts() 641 { 642 arch_int_enable_interrupts_inline(); 643 } 644 645 646 extern "C" int 647 arch_int_disable_interrupts() 648 { 649 return arch_int_disable_interrupts_inline(); 650 } 651 652 653 extern "C" void 654 arch_int_restore_interrupts(int oldState) 655 { 656 arch_int_restore_interrupts_inline(oldState); 657 } 658 659 660 extern "C" bool 661 arch_int_are_interrupts_enabled() 662 { 663 return arch_int_are_interrupts_enabled_inline(); 664 } 665