1 /* 2 * Copyright 2003-2011, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Adrien Destugues, pulkomandy@pulkomandy.tk 7 */ 8 9 10 #include <int.h> 11 #include <cpu.h> 12 #include <thread.h> 13 #include <vm/vm_priv.h> 14 #include <ksyscalls.h> 15 #include <syscall_numbers.h> 16 #include <arch_cpu_defs.h> 17 #include <arch_thread_types.h> 18 #include <arch/debug.h> 19 #include <util/AutoLock.h> 20 #include <Htif.h> 21 #include <Plic.h> 22 #include <Clint.h> 23 #include <AutoDeleterDrivers.h> 24 #include <ScopeExit.h> 25 #include "RISCV64VMTranslationMap.h" 26 27 #include <algorithm> 28 29 30 static uint32 sPlicContexts[SMP_MAX_CPUS]; 31 32 33 //#pragma mark debug output 34 35 static void 36 WriteMode(int mode) 37 { 38 switch (mode) { 39 case modeU: dprintf("u"); break; 40 case modeS: dprintf("s"); break; 41 case modeM: dprintf("m"); break; 42 default: dprintf("%d", mode); 43 } 44 } 45 46 47 static void 48 WriteModeSet(uint32_t val) 49 { 50 bool first = true; 51 dprintf("{"); 52 for (int i = 0; i < 32; i++) { 53 if (((1LL << i) & val) != 0) { 54 if (first) first = false; else dprintf(", "); 55 WriteMode(i); 56 } 57 } 58 dprintf("}"); 59 } 60 61 62 static void 63 WriteExt(uint64_t val) 64 { 65 switch (val) { 66 case 0: dprintf("off"); break; 67 case 1: dprintf("initial"); break; 68 case 2: dprintf("clean"); break; 69 case 3: dprintf("dirty"); break; 70 default: dprintf("%" B_PRId64, val); 71 } 72 } 73 74 75 static void 76 WriteSstatus(uint64_t val) 77 { 78 SstatusReg status{.val = val}; 79 dprintf("("); 80 dprintf("ie: "); WriteModeSet(status.ie); 81 dprintf(", pie: "); WriteModeSet(status.pie); 82 dprintf(", spp: "); WriteMode(status.spp); 83 dprintf(", fs: "); WriteExt(status.fs); 84 dprintf(", xs: "); WriteExt(status.xs); 85 dprintf(", sum: %d", (int)status.sum); 86 dprintf(", mxr: %d", (int)status.mxr); 87 dprintf(", uxl: %d", (int)status.uxl); 88 dprintf(", sd: %d", (int)status.sd); 89 dprintf(")"); 90 } 91 92 93 static void 94 WriteInterrupt(uint64_t val) 95 { 96 switch (val) { 97 case 0 + modeU: dprintf("uSoft"); break; 98 case 0 + modeS: dprintf("sSoft"); break; 99 case 0 + modeM: dprintf("mSoft"); break; 100 case 4 + modeU: dprintf("uTimer"); break; 101 case 4 + modeS: dprintf("sTimer"); break; 102 case 4 + modeM: dprintf("mTimer"); break; 103 case 8 + modeU: dprintf("uExtern"); break; 104 case 8 + modeS: dprintf("sExtern"); break; 105 case 8 + modeM: dprintf("mExtern"); break; 106 default: dprintf("%" B_PRId64, val); 107 } 108 } 109 110 111 static void 112 WriteInterruptSet(uint64_t val) 113 { 114 bool first = true; 115 dprintf("{"); 116 for (int i = 0; i < 64; i++) { 117 if (((1LL << i) & val) != 0) { 118 if (first) first = false; else dprintf(", "); 119 WriteInterrupt(i); 120 } 121 } 122 dprintf("}"); 123 } 124 125 126 static void 127 WriteCause(uint64_t cause) 128 { 129 if ((cause & causeInterrupt) == 0) { 130 dprintf("exception "); 131 switch (cause) { 132 case causeExecMisalign: dprintf("execMisalign"); break; 133 case causeExecAccessFault: dprintf("execAccessFault"); break; 134 case causeIllegalInst: dprintf("illegalInst"); break; 135 case causeBreakpoint: dprintf("breakpoint"); break; 136 case causeLoadMisalign: dprintf("loadMisalign"); break; 137 case causeLoadAccessFault: dprintf("loadAccessFault"); break; 138 case causeStoreMisalign: dprintf("storeMisalign"); break; 139 case causeStoreAccessFault: dprintf("storeAccessFault"); break; 140 case causeUEcall: dprintf("uEcall"); break; 141 case causeSEcall: dprintf("sEcall"); break; 142 case causeMEcall: dprintf("mEcall"); break; 143 case causeExecPageFault: dprintf("execPageFault"); break; 144 case causeLoadPageFault: dprintf("loadPageFault"); break; 145 case causeStorePageFault: dprintf("storePageFault"); break; 146 default: dprintf("%" B_PRId64, cause); 147 } 148 } else { 149 dprintf("interrupt "); WriteInterrupt(cause & ~causeInterrupt); 150 } 151 } 152 153 154 const static char* registerNames[] = { 155 " ra", " t6", " sp", " gp", 156 " tp", " t0", " t1", " t2", 157 " t5", " s1", " a0", " a1", 158 " a2", " a3", " a4", " a5", 159 " a6", " a7", " s2", " s3", 160 " s4", " s5", " s6", " s7", 161 " s8", " s9", "s10", "s11", 162 " t3", " t4", " fp", "epc" 163 }; 164 165 166 static void WriteRegisters(iframe* frame) 167 { 168 uint64* regs = &frame->ra; 169 for (int i = 0; i < 32; i += 4) { 170 dprintf( 171 " %s: 0x%016" B_PRIx64 172 " %s: 0x%016" B_PRIx64 173 " %s: 0x%016" B_PRIx64 174 " %s: 0x%016" B_PRIx64 "\n", 175 registerNames[i + 0], regs[i + 0], 176 registerNames[i + 1], regs[i + 1], 177 registerNames[i + 2], regs[i + 2], 178 registerNames[i + 3], regs[i + 3] 179 ); 180 } 181 } 182 183 184 static void 185 DumpMemory(uint64* adr, size_t len) 186 { 187 while (len > 0) { 188 if ((addr_t)adr % 0x10 == 0) 189 dprintf("%08" B_PRIxADDR " ", (addr_t)adr); 190 uint64 val; 191 if (user_memcpy(&val, adr++, sizeof(val)) < B_OK) { 192 dprintf(" ????????????????"); 193 } else { 194 dprintf(" %016" B_PRIx64, val); 195 } 196 if ((addr_t)adr % 0x10 == 0) 197 dprintf("\n"); 198 len -= 8; 199 } 200 if ((addr_t)adr % 0x10 != 0) 201 dprintf("\n"); 202 203 dprintf("%08" B_PRIxADDR "\n\n", (addr_t)adr); 204 } 205 206 207 void 208 WriteTrapInfo(iframe* frame) 209 { 210 InterruptsLocker locker; 211 dprintf("STrap("); WriteCause(frame->cause); dprintf(")\n"); 212 dprintf(" sstatus: "); WriteSstatus(frame->status); dprintf("\n"); 213 // dprintf(" sie: "); WriteInterruptSet(Sie()); dprintf("\n"); 214 // dprintf(" sip: "); WriteInterruptSet(Sip()); dprintf("\n"); 215 //dprintf(" stval: "); WritePC(Stval()); dprintf("\n"); 216 dprintf(" stval: 0x%" B_PRIx64 "\n", frame->tval); 217 // dprintf(" tp: 0x%" B_PRIxADDR "(%s)\n", Tp(), 218 // thread_get_current_thread()->name); 219 220 WriteRegisters(frame); 221 #if 0 222 dprintf(" kernel stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n", 223 thread_get_current_thread()->kernel_stack_base, 224 thread_get_current_thread()->kernel_stack_top - 1 225 ); 226 dprintf(" user stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n", 227 thread_get_current_thread()->user_stack_base, 228 thread_get_current_thread()->user_stack_base + 229 thread_get_current_thread()->user_stack_size - 1 230 ); 231 if (thread_get_current_thread()->arch_info.userFrame != NULL) { 232 WriteRegisters(thread_get_current_thread()->arch_info.userFrame); 233 234 dprintf("Stack memory dump:\n"); 235 DumpMemory( 236 (uint64*)thread_get_current_thread()->arch_info.userFrame->sp, 237 thread_get_current_thread()->user_stack_base + 238 thread_get_current_thread()->user_stack_size - 239 thread_get_current_thread()->arch_info.userFrame->sp 240 ); 241 // if (true) { 242 // } else { 243 // DumpMemory((uint64*)frame->sp, thread_get_current_thread()->kernel_stack_top - frame->sp); 244 // } 245 } 246 #endif 247 } 248 249 250 //#pragma mark - 251 252 static void 253 SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode, 254 addr_t signalAddress = 0, int32 signalError = B_ERROR) 255 { 256 if (SstatusReg{.val = Sstatus()}.spp == modeU) { 257 struct sigaction action; 258 Thread* thread = thread_get_current_thread(); 259 260 DoStackTrace(Fp(), 0); 261 262 enable_interrupts(); 263 264 // If the thread has a signal handler for the signal, we simply send it 265 // the signal. Otherwise we notify the user debugger first. 266 if ((sigaction(signalNumber, NULL, &action) == 0 267 && action.sa_handler != SIG_DFL 268 && action.sa_handler != SIG_IGN) 269 || user_debug_exception_occurred(type, signalNumber)) { 270 Signal signal(signalNumber, signalCode, signalError, 271 thread->team->id); 272 signal.SetAddress((void*)signalAddress); 273 send_signal_to_thread(thread, signal, 0); 274 } 275 } else { 276 panic("Unexpected exception occurred in kernel mode!"); 277 } 278 } 279 280 281 static void 282 AfterInterrupt() 283 { 284 if (debug_debugger_running()) 285 return; 286 287 Thread* thread = thread_get_current_thread(); 288 cpu_status state = disable_interrupts(); 289 if (thread->cpu->invoke_scheduler) { 290 SpinLocker schedulerLocker(thread->scheduler_lock); 291 scheduler_reschedule(B_THREAD_READY); 292 schedulerLocker.Unlock(); 293 restore_interrupts(state); 294 } else if (thread->post_interrupt_callback != NULL) { 295 void (*callback)(void*) = thread->post_interrupt_callback; 296 void* data = thread->post_interrupt_data; 297 298 thread->post_interrupt_callback = NULL; 299 thread->post_interrupt_data = NULL; 300 301 restore_interrupts(state); 302 303 callback(data); 304 } 305 } 306 307 308 static bool 309 SetAccessedFlags(addr_t addr, bool isWrite) 310 { 311 VMAddressSpacePutter addressSpace; 312 if (IS_KERNEL_ADDRESS(addr)) 313 addressSpace.SetTo(VMAddressSpace::GetKernel()); 314 else if (IS_USER_ADDRESS(addr)) 315 addressSpace.SetTo(VMAddressSpace::GetCurrent()); 316 317 if(!addressSpace.IsSet()) 318 return false; 319 320 RISCV64VMTranslationMap* map 321 = (RISCV64VMTranslationMap*)addressSpace->TranslationMap(); 322 323 phys_addr_t physAdr; 324 uint32 pageFlags; 325 map->QueryInterrupt(addr, &physAdr, &pageFlags); 326 327 if ((PAGE_PRESENT & pageFlags) == 0) 328 return false; 329 330 if (isWrite) { 331 if ( 332 ((B_WRITE_AREA | B_KERNEL_WRITE_AREA) & pageFlags) != 0 333 && ((PAGE_ACCESSED | PAGE_MODIFIED) & pageFlags) 334 != (PAGE_ACCESSED | PAGE_MODIFIED) 335 ) { 336 map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED); 337 /* 338 dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); 339 */ 340 return true; 341 } 342 } else { 343 if ( 344 ((B_READ_AREA | B_KERNEL_READ_AREA) & pageFlags) != 0 345 && (PAGE_ACCESSED & pageFlags) == 0 346 ) { 347 map->SetFlags(addr, PAGE_ACCESSED); 348 /* 349 dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); 350 */ 351 return true; 352 } 353 } 354 return false; 355 } 356 357 358 extern "C" void 359 STrap(iframe* frame) 360 { 361 // dprintf("STrap("); WriteCause(Scause()); dprintf(")\n"); 362 363 /* 364 iframe oldFrame = *frame; 365 const auto& frameChangeChecker = MakeScopeExit([&]() { 366 InterruptsLocker locker; 367 bool first = true; 368 for (int i = 0; i < 32; i++) { 369 uint64 oldVal = ((int64*)&oldFrame)[i]; 370 uint64 newVal = ((int64*)frame)[i]; 371 if (oldVal != newVal) { 372 if (first) { 373 dprintf("FrameChangeChecker, thread: %" B_PRId32 "(%s)\n", thread_get_current_thread()->id, thread_get_current_thread()->name); 374 first = false; 375 } 376 dprintf(" %s: %#" B_PRIxADDR " -> %#" B_PRIxADDR "\n", registerNames[i], oldVal, newVal); 377 } 378 } 379 380 if (frame->epc == 0) 381 panic("FrameChangeChecker: EPC = 0"); 382 }); 383 */ 384 switch (frame->cause) { 385 case causeExecPageFault: 386 case causeLoadPageFault: 387 case causeStorePageFault: { 388 if (SetAccessedFlags(Stval(), frame->cause == causeStorePageFault)) 389 return; 390 } 391 } 392 393 if (SstatusReg{.val = frame->status}.spp == modeU) { 394 thread_get_current_thread()->arch_info.userFrame = frame; 395 thread_get_current_thread()->arch_info.oldA0 = frame->a0; 396 thread_at_kernel_entry(system_time()); 397 } 398 const auto& kernelExit = ScopeExit([&]() { 399 if (SstatusReg{.val = frame->status}.spp == modeU) { 400 disable_interrupts(); 401 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 402 if ((thread_get_current_thread()->flags 403 & (THREAD_FLAGS_SIGNALS_PENDING 404 | THREAD_FLAGS_DEBUG_THREAD 405 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 406 enable_interrupts(); 407 thread_at_kernel_exit(); 408 } else { 409 thread_at_kernel_exit_no_signals(); 410 } 411 if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { 412 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL); 413 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED); 414 415 frame->a0 = thread_get_current_thread()->arch_info.oldA0; 416 frame->epc -= 4; 417 } 418 thread_get_current_thread()->arch_info.userFrame = NULL; 419 } 420 }); 421 422 switch (frame->cause) { 423 case causeIllegalInst: { 424 return SendSignal(B_INVALID_OPCODE_EXCEPTION, SIGILL, ILL_ILLOPC, 425 frame->epc); 426 } 427 case causeExecMisalign: 428 case causeLoadMisalign: 429 case causeStoreMisalign: { 430 return SendSignal(B_ALIGNMENT_EXCEPTION, SIGBUS, BUS_ADRALN, 431 Stval()); 432 } 433 case causeBreakpoint: { 434 if (SstatusReg{.val = frame->status}.spp == modeU) { 435 user_debug_breakpoint_hit(false); 436 } else { 437 panic("hit kernel breakpoint"); 438 } 439 return; 440 } 441 case causeExecAccessFault: 442 case causeLoadAccessFault: 443 case causeStoreAccessFault: { 444 return SendSignal(B_SEGMENT_VIOLATION, SIGBUS, BUS_ADRERR, 445 Stval()); 446 } 447 case causeExecPageFault: 448 case causeLoadPageFault: 449 case causeStorePageFault: { 450 uint64 stval = Stval(); 451 452 if (debug_debugger_running()) { 453 Thread* thread = thread_get_current_thread(); 454 if (thread != NULL) { 455 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 456 if (cpu->fault_handler != 0) { 457 debug_set_page_fault_info(stval, frame->epc, 458 (frame->cause == causeStorePageFault) 459 ? DEBUG_PAGE_FAULT_WRITE : 0); 460 frame->epc = cpu->fault_handler; 461 frame->sp = cpu->fault_handler_stack_pointer; 462 return; 463 } 464 465 if (thread->fault_handler != 0) { 466 kprintf("ERROR: thread::fault_handler used in kernel " 467 "debugger!\n"); 468 debug_set_page_fault_info(stval, frame->epc, 469 frame->cause == causeStorePageFault 470 ? DEBUG_PAGE_FAULT_WRITE : 0); 471 frame->epc = (addr_t)thread->fault_handler; 472 return; 473 } 474 } 475 476 panic("page fault in debugger without fault handler! Touching " 477 "address %p from ip %p\n", (void*)stval, (void*)frame->epc); 478 return; 479 } 480 481 if (SstatusReg{.val = frame->status}.pie == 0) { 482 // user_memcpy() failure 483 Thread* thread = thread_get_current_thread(); 484 if (thread != NULL && thread->fault_handler != 0) { 485 addr_t handler = (addr_t)(thread->fault_handler); 486 if (frame->epc != handler) { 487 frame->epc = handler; 488 return; 489 } 490 } 491 panic("page fault with interrupts disabled@!dump_virt_page %#" B_PRIx64, stval); 492 } 493 494 addr_t newIP = 0; 495 enable_interrupts(); 496 497 vm_page_fault(stval, frame->epc, frame->cause == causeStorePageFault, 498 frame->cause == causeExecPageFault, 499 SstatusReg{.val = frame->status}.spp == modeU, &newIP); 500 501 if (newIP != 0) 502 frame->epc = newIP; 503 504 return; 505 } 506 case causeInterrupt + sSoftInt: { 507 ClearBitsSip(1 << sSoftInt); 508 // dprintf("sSoftInt(%" B_PRId32 ")\n", smp_get_current_cpu()); 509 smp_intercpu_int_handler(smp_get_current_cpu()); 510 AfterInterrupt(); 511 return; 512 } 513 case causeInterrupt + sTimerInt: { 514 ClearBitsSie(1 << sTimerInt); 515 // dprintf("sTimerInt(%" B_PRId32 ")\n", smp_get_current_cpu()); 516 timer_interrupt(); 517 AfterInterrupt(); 518 return; 519 } 520 case causeInterrupt + sExternInt: { 521 uint64 irq = gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete; 522 int_io_interrupt_handler(irq, true); 523 gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete = irq; 524 AfterInterrupt(); 525 return; 526 } 527 case causeUEcall: { 528 frame->epc += 4; // skip ecall 529 uint64 syscall = frame->t0; 530 uint64 args[20]; 531 if (syscall < (uint64)kSyscallCount) { 532 uint32 argCnt = kExtendedSyscallInfos[syscall].parameter_count; 533 memcpy(&args[0], &frame->a0, 534 sizeof(uint64)*std::min<uint32>(argCnt, 8)); 535 if (argCnt > 8) { 536 if (status_t res = user_memcpy(&args[8], (void*)frame->sp, 537 sizeof(uint64)*(argCnt - 8)) < B_OK) { 538 dprintf("can't read syscall arguments on user " 539 "stack\n"); 540 frame->a0 = res; 541 return; 542 } 543 } 544 } 545 /* 546 switch (syscall) { 547 case SYSCALL_READ_PORT_ETC: 548 case SYSCALL_WRITE_PORT_ETC: 549 DoStackTrace(Fp(), 0); 550 break; 551 } 552 */ 553 // dprintf("syscall: %s\n", kExtendedSyscallInfos[syscall].name); 554 555 enable_interrupts(); 556 uint64 returnValue = 0; 557 syscall_dispatcher(syscall, (void*)args, &returnValue); 558 frame->a0 = returnValue; 559 return; 560 } 561 } 562 panic("unhandled STrap"); 563 } 564 565 566 //#pragma mark - 567 568 status_t 569 arch_int_init(kernel_args* args) 570 { 571 dprintf("arch_int_init()\n"); 572 573 for (uint32 i = 0; i < args->num_cpus; i++) { 574 dprintf(" CPU %" B_PRIu32 ":\n", i); 575 dprintf(" hartId: %" B_PRIu32 "\n", args->arch_args.hartIds[i]); 576 dprintf(" plicContext: %" B_PRIu32 "\n", args->arch_args.plicContexts[i]); 577 } 578 579 for (uint32 i = 0; i < args->num_cpus; i++) 580 sPlicContexts[i] = args->arch_args.plicContexts[i]; 581 582 // TODO: read from FDT 583 reserve_io_interrupt_vectors(128, 0, INTERRUPT_TYPE_IRQ); 584 585 for (uint32 i = 0; i < args->num_cpus; i++) 586 gPlicRegs->contexts[sPlicContexts[i]].priorityThreshold = 0; 587 588 return B_OK; 589 } 590 591 592 status_t 593 arch_int_init_post_vm(kernel_args* args) 594 { 595 return B_OK; 596 } 597 598 599 status_t 600 arch_int_init_post_device_manager(struct kernel_args* args) 601 { 602 return B_OK; 603 } 604 605 606 status_t 607 arch_int_init_io(kernel_args* args) 608 { 609 return B_OK; 610 } 611 612 613 void 614 arch_int_enable_io_interrupt(int irq) 615 { 616 dprintf("arch_int_enable_io_interrupt(%d)\n", irq); 617 gPlicRegs->priority[irq] = 1; 618 gPlicRegs->enable[sPlicContexts[0]][irq / 32] |= 1 << (irq % 32); 619 } 620 621 622 void 623 arch_int_disable_io_interrupt(int irq) 624 { 625 dprintf("arch_int_disable_io_interrupt(%d)\n", irq); 626 gPlicRegs->priority[irq] = 0; 627 gPlicRegs->enable[sPlicContexts[0]][irq / 32] &= ~(1 << (irq % 32)); 628 } 629 630 631 int32 632 arch_int_assign_to_cpu(int32 irq, int32 cpu) 633 { 634 // Not yet supported. 635 return 0; 636 } 637 638 639 #undef arch_int_enable_interrupts 640 #undef arch_int_disable_interrupts 641 #undef arch_int_restore_interrupts 642 #undef arch_int_are_interrupts_enabled 643 644 645 extern "C" void 646 arch_int_enable_interrupts() 647 { 648 arch_int_enable_interrupts_inline(); 649 } 650 651 652 extern "C" int 653 arch_int_disable_interrupts() 654 { 655 return arch_int_disable_interrupts_inline(); 656 } 657 658 659 extern "C" void 660 arch_int_restore_interrupts(int oldState) 661 { 662 arch_int_restore_interrupts_inline(oldState); 663 } 664 665 666 extern "C" bool 667 arch_int_are_interrupts_enabled() 668 { 669 return arch_int_are_interrupts_enabled_inline(); 670 } 671