1 /* 2 * Copyright 2003-2011, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Adrien Destugues, pulkomandy@pulkomandy.tk 7 */ 8 9 10 #include <int.h> 11 #include <cpu.h> 12 #include <thread.h> 13 #include <vm/vm_priv.h> 14 #include <ksyscalls.h> 15 #include <syscall_numbers.h> 16 #include <arch_cpu_defs.h> 17 #include <arch_thread_types.h> 18 #include <arch/debug.h> 19 #include <util/AutoLock.h> 20 #include <Htif.h> 21 #include <Plic.h> 22 #include <Clint.h> 23 #include <AutoDeleterDrivers.h> 24 #include "RISCV64VMTranslationMap.h" 25 26 #include <algorithm> 27 28 29 static uint32 sPlicContexts[SMP_MAX_CPUS]; 30 31 32 //#pragma mark debug output 33 34 static void 35 WriteMode(int mode) 36 { 37 switch (mode) { 38 case modeU: dprintf("u"); break; 39 case modeS: dprintf("s"); break; 40 case modeM: dprintf("m"); break; 41 default: dprintf("%d", mode); 42 } 43 } 44 45 46 static void 47 WriteModeSet(uint32_t val) 48 { 49 bool first = true; 50 dprintf("{"); 51 for (int i = 0; i < 32; i++) { 52 if (((1LL << i) & val) != 0) { 53 if (first) first = false; else dprintf(", "); 54 WriteMode(i); 55 } 56 } 57 dprintf("}"); 58 } 59 60 61 static void 62 WriteExt(uint64_t val) 63 { 64 switch (val) { 65 case 0: dprintf("off"); break; 66 case 1: dprintf("initial"); break; 67 case 2: dprintf("clean"); break; 68 case 3: dprintf("dirty"); break; 69 default: dprintf("%" B_PRId64, val); 70 } 71 } 72 73 74 static void 75 WriteSstatus(uint64_t val) 76 { 77 SstatusReg status(val); 78 dprintf("("); 79 dprintf("ie: "); WriteModeSet(status.ie); 80 dprintf(", pie: "); WriteModeSet(status.pie); 81 dprintf(", spp: "); WriteMode(status.spp); 82 dprintf(", fs: "); WriteExt(status.fs); 83 dprintf(", xs: "); WriteExt(status.xs); 84 dprintf(", sum: %d", (int)status.sum); 85 dprintf(", mxr: %d", (int)status.mxr); 86 dprintf(", uxl: %d", (int)status.uxl); 87 dprintf(", sd: %d", (int)status.sd); 88 dprintf(")"); 89 } 90 91 92 static void 93 WriteInterrupt(uint64_t val) 94 { 95 switch (val) { 96 case 0 + modeU: dprintf("uSoft"); break; 97 case 0 + modeS: dprintf("sSoft"); break; 98 case 0 + modeM: dprintf("mSoft"); break; 99 case 4 + modeU: dprintf("uTimer"); break; 100 case 4 + modeS: dprintf("sTimer"); break; 101 case 4 + modeM: dprintf("mTimer"); break; 102 case 8 + modeU: dprintf("uExtern"); break; 103 case 8 + modeS: dprintf("sExtern"); break; 104 case 8 + modeM: dprintf("mExtern"); break; 105 default: dprintf("%" B_PRId64, val); 106 } 107 } 108 109 110 static void 111 WriteInterruptSet(uint64_t val) 112 { 113 bool first = true; 114 dprintf("{"); 115 for (int i = 0; i < 64; i++) { 116 if (((1LL << i) & val) != 0) { 117 if (first) first = false; else dprintf(", "); 118 WriteInterrupt(i); 119 } 120 } 121 dprintf("}"); 122 } 123 124 125 static void 126 WriteCause(uint64_t cause) 127 { 128 if ((cause & causeInterrupt) == 0) { 129 dprintf("exception "); 130 switch (cause) { 131 case causeExecMisalign: dprintf("execMisalign"); break; 132 case causeExecAccessFault: dprintf("execAccessFault"); break; 133 case causeIllegalInst: dprintf("illegalInst"); break; 134 case causeBreakpoint: dprintf("breakpoint"); break; 135 case causeLoadMisalign: dprintf("loadMisalign"); break; 136 case causeLoadAccessFault: dprintf("loadAccessFault"); break; 137 case causeStoreMisalign: dprintf("storeMisalign"); break; 138 case causeStoreAccessFault: dprintf("storeAccessFault"); break; 139 case causeUEcall: dprintf("uEcall"); break; 140 case causeSEcall: dprintf("sEcall"); break; 141 case causeMEcall: dprintf("mEcall"); break; 142 case causeExecPageFault: dprintf("execPageFault"); break; 143 case causeLoadPageFault: dprintf("loadPageFault"); break; 144 case causeStorePageFault: dprintf("storePageFault"); break; 145 default: dprintf("%" B_PRId64, cause); 146 } 147 } else { 148 dprintf("interrupt "); WriteInterrupt(cause & ~causeInterrupt); 149 } 150 } 151 152 153 const static char* registerNames[] = { 154 " ra", " t6", " sp", " gp", 155 " tp", " t0", " t1", " t2", 156 " t5", " s1", " a0", " a1", 157 " a2", " a3", " a4", " a5", 158 " a6", " a7", " s2", " s3", 159 " s4", " s5", " s6", " s7", 160 " s8", " s9", "s10", "s11", 161 " t3", " t4", " fp", "epc" 162 }; 163 164 165 static void WriteRegisters(iframe* frame) 166 { 167 uint64* regs = &frame->ra; 168 for (int i = 0; i < 32; i += 4) { 169 dprintf( 170 " %s: 0x%016" B_PRIx64 171 " %s: 0x%016" B_PRIx64 172 " %s: 0x%016" B_PRIx64 173 " %s: 0x%016" B_PRIx64 "\n", 174 registerNames[i + 0], regs[i + 0], 175 registerNames[i + 1], regs[i + 1], 176 registerNames[i + 2], regs[i + 2], 177 registerNames[i + 3], regs[i + 3] 178 ); 179 } 180 } 181 182 183 static void 184 DumpMemory(uint64* adr, size_t len) 185 { 186 while (len > 0) { 187 if ((addr_t)adr % 0x10 == 0) 188 dprintf("%08" B_PRIxADDR " ", (addr_t)adr); 189 uint64 val; 190 if (user_memcpy(&val, adr++, sizeof(val)) < B_OK) { 191 dprintf(" ????????????????"); 192 } else { 193 dprintf(" %016" B_PRIx64, val); 194 } 195 if ((addr_t)adr % 0x10 == 0) 196 dprintf("\n"); 197 len -= 8; 198 } 199 if ((addr_t)adr % 0x10 != 0) 200 dprintf("\n"); 201 202 dprintf("%08" B_PRIxADDR "\n\n", (addr_t)adr); 203 } 204 205 206 void 207 WriteTrapInfo(iframe* frame) 208 { 209 InterruptsLocker locker; 210 dprintf("STrap("); WriteCause(frame->cause); dprintf(")\n"); 211 dprintf(" sstatus: "); WriteSstatus(frame->status); dprintf("\n"); 212 // dprintf(" sie: "); WriteInterruptSet(Sie()); dprintf("\n"); 213 // dprintf(" sip: "); WriteInterruptSet(Sip()); dprintf("\n"); 214 //dprintf(" stval: "); WritePC(Stval()); dprintf("\n"); 215 dprintf(" stval: 0x%" B_PRIx64 "\n", frame->tval); 216 // dprintf(" tp: 0x%" B_PRIxADDR "(%s)\n", Tp(), 217 // thread_get_current_thread()->name); 218 219 WriteRegisters(frame); 220 #if 0 221 dprintf(" kernel stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n", 222 thread_get_current_thread()->kernel_stack_base, 223 thread_get_current_thread()->kernel_stack_top - 1 224 ); 225 dprintf(" user stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n", 226 thread_get_current_thread()->user_stack_base, 227 thread_get_current_thread()->user_stack_base + 228 thread_get_current_thread()->user_stack_size - 1 229 ); 230 if (thread_get_current_thread()->arch_info.userFrame != NULL) { 231 WriteRegisters(thread_get_current_thread()->arch_info.userFrame); 232 233 dprintf("Stack memory dump:\n"); 234 DumpMemory( 235 (uint64*)thread_get_current_thread()->arch_info.userFrame->sp, 236 thread_get_current_thread()->user_stack_base + 237 thread_get_current_thread()->user_stack_size - 238 thread_get_current_thread()->arch_info.userFrame->sp 239 ); 240 // if (true) { 241 // } else { 242 // DumpMemory((uint64*)frame->sp, thread_get_current_thread()->kernel_stack_top - frame->sp); 243 // } 244 } 245 #endif 246 } 247 248 249 //#pragma mark - 250 251 static void 252 SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode, 253 addr_t signalAddress = 0, int32 signalError = B_ERROR) 254 { 255 if (SstatusReg(Sstatus()).spp == modeU) { 256 struct sigaction action; 257 Thread* thread = thread_get_current_thread(); 258 259 DoStackTrace(Fp(), 0); 260 261 enable_interrupts(); 262 263 // If the thread has a signal handler for the signal, we simply send it 264 // the signal. Otherwise we notify the user debugger first. 265 if ((sigaction(signalNumber, NULL, &action) == 0 266 && action.sa_handler != SIG_DFL 267 && action.sa_handler != SIG_IGN) 268 || user_debug_exception_occurred(type, signalNumber)) { 269 Signal signal(signalNumber, signalCode, signalError, 270 thread->team->id); 271 signal.SetAddress((void*)signalAddress); 272 send_signal_to_thread(thread, signal, 0); 273 } 274 } else { 275 panic("Unexpected exception occurred in kernel mode!"); 276 } 277 } 278 279 280 static void 281 AfterInterrupt() 282 { 283 if (debug_debugger_running()) 284 return; 285 286 Thread* thread = thread_get_current_thread(); 287 cpu_status state = disable_interrupts(); 288 if (thread->cpu->invoke_scheduler) { 289 SpinLocker schedulerLocker(thread->scheduler_lock); 290 scheduler_reschedule(B_THREAD_READY); 291 schedulerLocker.Unlock(); 292 restore_interrupts(state); 293 } else if (thread->post_interrupt_callback != NULL) { 294 void (*callback)(void*) = thread->post_interrupt_callback; 295 void* data = thread->post_interrupt_data; 296 297 thread->post_interrupt_callback = NULL; 298 thread->post_interrupt_data = NULL; 299 300 restore_interrupts(state); 301 302 callback(data); 303 } 304 } 305 306 307 static bool 308 SetAccessedFlags(addr_t addr, bool isWrite) 309 { 310 VMAddressSpacePutter addressSpace; 311 if (IS_KERNEL_ADDRESS(addr)) 312 addressSpace.SetTo(VMAddressSpace::GetKernel()); 313 else if (IS_USER_ADDRESS(addr)) 314 addressSpace.SetTo(VMAddressSpace::GetCurrent()); 315 316 if(!addressSpace.IsSet()) 317 return false; 318 319 RISCV64VMTranslationMap* map 320 = (RISCV64VMTranslationMap*)addressSpace->TranslationMap(); 321 322 phys_addr_t physAdr; 323 uint32 pageFlags; 324 map->QueryInterrupt(addr, &physAdr, &pageFlags); 325 326 if ((PAGE_PRESENT & pageFlags) == 0) 327 return false; 328 329 if (isWrite) { 330 if ( 331 ((B_WRITE_AREA | B_KERNEL_WRITE_AREA) & pageFlags) != 0 332 && ((PAGE_ACCESSED | PAGE_MODIFIED) & pageFlags) 333 != (PAGE_ACCESSED | PAGE_MODIFIED) 334 ) { 335 map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED); 336 /* 337 dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); 338 */ 339 return true; 340 } 341 } else { 342 if ( 343 ((B_READ_AREA | B_KERNEL_READ_AREA) & pageFlags) != 0 344 && (PAGE_ACCESSED & pageFlags) == 0 345 ) { 346 map->SetFlags(addr, PAGE_ACCESSED); 347 /* 348 dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); 349 */ 350 return true; 351 } 352 } 353 return false; 354 } 355 356 357 // TODO: needs moved into an arch-agnostic location? 358 359 template<typename F> 360 class ScopeExit 361 { 362 public: 363 explicit ScopeExit(F&& fn) : fFn(fn) 364 { 365 } 366 367 ~ScopeExit() 368 { 369 fFn(); 370 } 371 372 ScopeExit(ScopeExit&& other) : fFn(std::move(other.fFn)) 373 { 374 } 375 376 private: 377 ScopeExit(const ScopeExit&); 378 ScopeExit& operator=(const ScopeExit&); 379 380 private: 381 F fFn; 382 }; 383 384 template<typename F> 385 ScopeExit<F> MakeScopeExit(F&& fn) 386 { 387 return ScopeExit<F>(std::move(fn)); 388 } 389 390 391 extern "C" void 392 STrap(iframe* frame) 393 { 394 // dprintf("STrap("); WriteCause(Scause()); dprintf(")\n"); 395 396 /* 397 iframe oldFrame = *frame; 398 const auto& frameChangeChecker = MakeScopeExit([&]() { 399 InterruptsLocker locker; 400 bool first = true; 401 for (int i = 0; i < 32; i++) { 402 uint64 oldVal = ((int64*)&oldFrame)[i]; 403 uint64 newVal = ((int64*)frame)[i]; 404 if (oldVal != newVal) { 405 if (first) { 406 dprintf("FrameChangeChecker, thread: %" B_PRId32 "(%s)\n", thread_get_current_thread()->id, thread_get_current_thread()->name); 407 first = false; 408 } 409 dprintf(" %s: %#" B_PRIxADDR " -> %#" B_PRIxADDR "\n", registerNames[i], oldVal, newVal); 410 } 411 } 412 413 if (frame->epc == 0) 414 panic("FrameChangeChecker: EPC = 0"); 415 }); 416 */ 417 switch (frame->cause) { 418 case causeExecPageFault: 419 case causeLoadPageFault: 420 case causeStorePageFault: { 421 if (SetAccessedFlags(Stval(), frame->cause == causeStorePageFault)) 422 return; 423 } 424 } 425 426 if (SstatusReg(frame->status).spp == modeU) { 427 thread_get_current_thread()->arch_info.userFrame = frame; 428 thread_get_current_thread()->arch_info.oldA0 = frame->a0; 429 thread_at_kernel_entry(system_time()); 430 } 431 const auto& kernelExit = MakeScopeExit([&]() { 432 if (SstatusReg(frame->status).spp == modeU) { 433 disable_interrupts(); 434 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 435 if ((thread_get_current_thread()->flags 436 & (THREAD_FLAGS_SIGNALS_PENDING 437 | THREAD_FLAGS_DEBUG_THREAD 438 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 439 enable_interrupts(); 440 thread_at_kernel_exit(); 441 } else { 442 thread_at_kernel_exit_no_signals(); 443 } 444 if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { 445 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL); 446 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED); 447 448 frame->a0 = thread_get_current_thread()->arch_info.oldA0; 449 frame->epc -= 4; 450 } 451 thread_get_current_thread()->arch_info.userFrame = NULL; 452 } 453 }); 454 455 switch (frame->cause) { 456 case causeIllegalInst: { 457 return SendSignal(B_INVALID_OPCODE_EXCEPTION, SIGILL, ILL_ILLOPC, 458 frame->epc); 459 } 460 case causeExecMisalign: 461 case causeLoadMisalign: 462 case causeStoreMisalign: { 463 return SendSignal(B_ALIGNMENT_EXCEPTION, SIGBUS, BUS_ADRALN, 464 Stval()); 465 } 466 // case causeBreakpoint: 467 case causeExecAccessFault: 468 case causeLoadAccessFault: 469 case causeStoreAccessFault: { 470 return SendSignal(B_SEGMENT_VIOLATION, SIGBUS, BUS_ADRERR, 471 Stval()); 472 } 473 case causeExecPageFault: 474 case causeLoadPageFault: 475 case causeStorePageFault: { 476 uint64 stval = Stval(); 477 478 if (debug_debugger_running()) { 479 Thread* thread = thread_get_current_thread(); 480 if (thread != NULL) { 481 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 482 if (cpu->fault_handler != 0) { 483 debug_set_page_fault_info(stval, frame->epc, 484 (frame->cause == causeStorePageFault) 485 ? DEBUG_PAGE_FAULT_WRITE : 0); 486 frame->epc = cpu->fault_handler; 487 frame->sp = cpu->fault_handler_stack_pointer; 488 return; 489 } 490 491 if (thread->fault_handler != 0) { 492 kprintf("ERROR: thread::fault_handler used in kernel " 493 "debugger!\n"); 494 debug_set_page_fault_info(stval, frame->epc, 495 frame->cause == causeStorePageFault 496 ? DEBUG_PAGE_FAULT_WRITE : 0); 497 frame->epc = (addr_t)thread->fault_handler; 498 return; 499 } 500 } 501 502 panic("page fault in debugger without fault handler! Touching " 503 "address %p from ip %p\n", (void*)stval, (void*)frame->epc); 504 return; 505 } 506 507 if (SstatusReg(frame->status).pie == 0) { 508 // user_memcpy() failure 509 Thread* thread = thread_get_current_thread(); 510 if (thread != NULL && thread->fault_handler != 0) { 511 addr_t handler = (addr_t)(thread->fault_handler); 512 if (frame->epc != handler) { 513 frame->epc = handler; 514 return; 515 } 516 } 517 panic("page fault with interrupts disabled@!dump_virt_page %#" B_PRIx64, stval); 518 } 519 520 addr_t newIP = 0; 521 enable_interrupts(); 522 523 vm_page_fault(stval, frame->epc, frame->cause == causeStorePageFault, 524 frame->cause == causeExecPageFault, 525 SstatusReg(frame->status).spp == modeU, &newIP); 526 527 if (newIP != 0) 528 frame->epc = newIP; 529 530 return; 531 } 532 case causeInterrupt + sSoftInt: { 533 SetSip(Sip() & ~(1 << sSoftInt)); 534 // dprintf("sSoftInt(%" B_PRId32 ")\n", smp_get_current_cpu()); 535 smp_intercpu_int_handler(smp_get_current_cpu()); 536 AfterInterrupt(); 537 return; 538 } 539 case causeInterrupt + sTimerInt: { 540 // SetSie(Sie() & ~(1 << sTimerInt)); 541 // dprintf("sTimerInt(%" B_PRId32 ")\n", smp_get_current_cpu()); 542 timer_interrupt(); 543 AfterInterrupt(); 544 return; 545 } 546 case causeInterrupt + sExternInt: { 547 uint64 irq = gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete; 548 int_io_interrupt_handler(irq, true); 549 gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete = irq; 550 AfterInterrupt(); 551 return; 552 } 553 case causeUEcall: { 554 frame->epc += 4; // skip ecall 555 uint64 syscall = frame->t0; 556 uint64 args[20]; 557 if (syscall < (uint64)kSyscallCount) { 558 uint32 argCnt = kExtendedSyscallInfos[syscall].parameter_count; 559 memcpy(&args[0], &frame->a0, 560 sizeof(uint64)*std::min<uint32>(argCnt, 8)); 561 if (argCnt > 8) { 562 if (status_t res = user_memcpy(&args[8], (void*)frame->sp, 563 sizeof(uint64)*(argCnt - 8)) < B_OK) { 564 dprintf("can't read syscall arguments on user " 565 "stack\n"); 566 frame->a0 = res; 567 return; 568 } 569 } 570 } 571 /* 572 switch (syscall) { 573 case SYSCALL_READ_PORT_ETC: 574 case SYSCALL_WRITE_PORT_ETC: 575 DoStackTrace(Fp(), 0); 576 break; 577 } 578 */ 579 // dprintf("syscall: %s\n", kExtendedSyscallInfos[syscall].name); 580 581 enable_interrupts(); 582 uint64 returnValue = 0; 583 syscall_dispatcher(syscall, (void*)args, &returnValue); 584 frame->a0 = returnValue; 585 return; 586 } 587 } 588 panic("unhandled STrap"); 589 } 590 591 592 //#pragma mark - 593 594 status_t 595 arch_int_init(kernel_args* args) 596 { 597 dprintf("arch_int_init()\n"); 598 599 for (uint32 i = 0; i < args->num_cpus; i++) { 600 dprintf(" CPU %" B_PRIu32 ":\n", i); 601 dprintf(" hartId: %" B_PRIu32 "\n", args->arch_args.hartIds[i]); 602 dprintf(" plicContext: %" B_PRIu32 "\n", args->arch_args.plicContexts[i]); 603 } 604 605 for (uint32 i = 0; i < args->num_cpus; i++) 606 sPlicContexts[i] = args->arch_args.plicContexts[i]; 607 608 // TODO: read from FDT 609 reserve_io_interrupt_vectors(128, 0, INTERRUPT_TYPE_IRQ); 610 611 for (uint32 i = 0; i < args->num_cpus; i++) 612 gPlicRegs->contexts[sPlicContexts[i]].priorityThreshold = 0; 613 614 return B_OK; 615 } 616 617 618 status_t 619 arch_int_init_post_vm(kernel_args* args) 620 { 621 return B_OK; 622 } 623 624 625 status_t 626 arch_int_init_post_device_manager(struct kernel_args* args) 627 { 628 return B_OK; 629 } 630 631 632 status_t 633 arch_int_init_io(kernel_args* args) 634 { 635 return B_OK; 636 } 637 638 639 void 640 arch_int_enable_io_interrupt(int irq) 641 { 642 dprintf("arch_int_enable_io_interrupt(%d)\n", irq); 643 gPlicRegs->priority[irq] = 1; 644 gPlicRegs->enable[sPlicContexts[0]][irq / 32] |= 1 << (irq % 32); 645 } 646 647 648 void 649 arch_int_disable_io_interrupt(int irq) 650 { 651 dprintf("arch_int_disable_io_interrupt(%d)\n", irq); 652 gPlicRegs->priority[irq] = 0; 653 gPlicRegs->enable[sPlicContexts[0]][irq / 32] &= ~(1 << (irq % 32)); 654 } 655 656 657 int32 658 arch_int_assign_to_cpu(int32 irq, int32 cpu) 659 { 660 // Not yet supported. 661 return 0; 662 } 663 664 665 #undef arch_int_enable_interrupts 666 #undef arch_int_disable_interrupts 667 #undef arch_int_restore_interrupts 668 #undef arch_int_are_interrupts_enabled 669 670 671 extern "C" void 672 arch_int_enable_interrupts() 673 { 674 arch_int_enable_interrupts_inline(); 675 } 676 677 678 extern "C" int 679 arch_int_disable_interrupts() 680 { 681 return arch_int_disable_interrupts_inline(); 682 } 683 684 685 extern "C" void 686 arch_int_restore_interrupts(int oldState) 687 { 688 arch_int_restore_interrupts_inline(oldState); 689 } 690 691 692 extern "C" bool 693 arch_int_are_interrupts_enabled() 694 { 695 return arch_int_are_interrupts_enabled_inline(); 696 } 697