1/* 2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com. 3 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8#include <asm_defs.h> 9 10#include <thread_types.h> 11 12#include <arch/x86/descriptors.h> 13#include <arch/x86/arch_altcodepatch.h> 14#include <arch/x86/arch_cpu.h> 15#include <arch/x86/arch_kernel.h> 16 17#include "asm_offsets.h" 18#include "syscall_numbers.h" 19#include "syscall_table.h" 20 21 22// Push the remainder of the interrupt frame onto the stack. 23#define PUSH_IFRAME_BOTTOM(iframeType) \ 24 push %rax; /* orig_rax */ \ 25 push %rax; \ 26 push %rbx; \ 27 push %rcx; \ 28 push %rdx; \ 29 push %rdi; \ 30 push %rsi; \ 31 push %rbp; \ 32 push %r8; \ 33 push %r9; \ 34 push %r10; \ 35 push %r11; \ 36 push %r12; \ 37 push %r13; \ 38 push %r14; \ 39 push %r15; \ 40 pushq $0; \ 41 push $iframeType; 42 43 44// Restore the interrupt frame. 45#define RESTORE_IFRAME() \ 46 add $16, %rsp; \ 47 pop %r15; \ 48 pop %r14; \ 49 pop %r13; \ 50 pop %r12; \ 51 pop %r11; \ 52 pop %r10; \ 53 pop %r9; \ 54 pop %r8; \ 55 pop %rbp; \ 56 pop %rsi; \ 57 pop %rdi; \ 58 pop %rdx; \ 59 pop %rcx; \ 60 pop %rbx; \ 61 pop %rax; \ 62 addq $24, %rsp; 63 64 65// The macros below require R12 to contain the current thread pointer. R12 is 66// callee-save so will be preserved through all function calls and only needs 67// to be obtained once. R13 is used to store the system call start time, will 68// also be preserved. 69 70#define LOCK_THREAD_TIME() \ 71 leaq THREAD_time_lock(%r12), %rdi; \ 72 call acquire_spinlock; 73 74#define UNLOCK_THREAD_TIME() \ 75 leaq THREAD_time_lock(%r12), %rdi; \ 76 call release_spinlock; \ 77 78#define UPDATE_THREAD_USER_TIME() \ 79 LOCK_THREAD_TIME() \ 80 \ 81 call system_time; \ 82 \ 83 /* Preserve system_time for post syscall debug */ \ 84 movq %rax, %r13; \ 85 \ 86 /* thread->user_time += now - thread->last_time; */ \ 87 subq THREAD_last_time(%r12), %rax; \ 88 addq %rax, THREAD_user_time(%r12); \ 89 \ 90 /* thread->last_time = now; */ \ 91 movq %r13, THREAD_last_time(%r12); \ 92 \ 93 /* thread->in_kernel = true; */ \ 94 movb $1, THREAD_in_kernel(%r12); \ 95 \ 96 UNLOCK_THREAD_TIME() 97 98#define UPDATE_THREAD_KERNEL_TIME() \ 99 LOCK_THREAD_TIME() \ 100 \ 101 call system_time; \ 102 movq %rax, %r13; \ 103 \ 104 /* thread->kernel_time += now - thread->last_time; */ \ 105 subq THREAD_last_time(%r12), %rax; \ 106 addq %rax, THREAD_kernel_time(%r12); \ 107 \ 108 /* thread->last_time = now; */ \ 109 movq %r13, THREAD_last_time(%r12); \ 110 \ 111 /* thread->in_kernel = false; */ \ 112 movb $0, THREAD_in_kernel(%r12); \ 113 \ 114 UNLOCK_THREAD_TIME() 115 116#define STOP_USER_DEBUGGING() \ 117 testl $(THREAD_FLAGS_BREAKPOINTS_INSTALLED \ 118 | THREAD_FLAGS_SINGLE_STEP), THREAD_flags(%r12); \ 119 jz 1f; \ 120 call x86_exit_user_debug_at_kernel_entry; \ 121 1: 122 123#define CLEAR_FPU_STATE() \ 124 pxor %xmm0, %xmm0; \ 125 pxor %xmm1, %xmm1; \ 126 pxor %xmm2, %xmm2; \ 127 pxor %xmm3, %xmm3; \ 128 pxor %xmm4, %xmm4; \ 129 pxor %xmm5, %xmm5; \ 130 pxor %xmm6, %xmm6; \ 131 pxor %xmm7, %xmm7; \ 132 pxor %xmm8, %xmm8; \ 133 pxor %xmm9, %xmm9; \ 134 pxor %xmm10, %xmm10; \ 135 pxor %xmm11, %xmm11; \ 136 pxor %xmm12, %xmm12; \ 137 pxor %xmm13, %xmm13; \ 138 pxor %xmm14, %xmm14; \ 139 pxor %xmm15, %xmm15 140 141// The following code defines the interrupt service routines for all 256 142// interrupts. It creates a block of handlers, each 16 bytes, that the IDT 143// initialization code just loops through. 144 145// Interrupt with no error code, pushes a 0 error code. 146#define DEFINE_ISR(nr) \ 147 .align 16; \ 148 ASM_CLAC \ 149 push $0; \ 150 push $nr; \ 151 jmp int_bottom; 152 153// Interrupt with an error code. 154#define DEFINE_ISR_E(nr) \ 155 .align 16; \ 156 ASM_CLAC \ 157 push $nr; \ 158 jmp int_bottom; 159 160// Array of interrupt service routines. 161.align 16 162SYMBOL(isr_array): 163 // Exceptions (0-19) and reserved interrupts (20-31). 164 DEFINE_ISR(0) 165 DEFINE_ISR(1) 166 DEFINE_ISR(2) 167 DEFINE_ISR(3) 168 DEFINE_ISR(4) 169 DEFINE_ISR(5) 170 DEFINE_ISR(6) 171 DEFINE_ISR(7) 172 DEFINE_ISR_E(8) 173 DEFINE_ISR(9) 174 DEFINE_ISR_E(10) 175 DEFINE_ISR_E(11) 176 DEFINE_ISR_E(12) 177 DEFINE_ISR_E(13) 178 DEFINE_ISR_E(14) 179 DEFINE_ISR(15) 180 DEFINE_ISR(16) 181 DEFINE_ISR_E(17) 182 DEFINE_ISR(18) 183 DEFINE_ISR(19) 184 DEFINE_ISR(20) 185 DEFINE_ISR(21) 186 DEFINE_ISR(22) 187 DEFINE_ISR(23) 188 DEFINE_ISR(24) 189 DEFINE_ISR(25) 190 DEFINE_ISR(26) 191 DEFINE_ISR(27) 192 DEFINE_ISR(28) 193 DEFINE_ISR(29) 194 DEFINE_ISR(30) 195 DEFINE_ISR(31) 196 197 // User-defined ISRs (32-255) - none take an error code. 198 .Lintr = 32 199 .rept 224 200 DEFINE_ISR(.Lintr) 201 .Lintr = .Lintr+1 202 .endr 203 204 205// Common interrupt handling code. 206STATIC_FUNCTION(int_bottom): 207 // Coming from user-mode requires special handling. 208 testl $3, 24(%rsp) 209 jnz int_bottom_user 210 211 // Push the rest of the interrupt frame to the stack. 212 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) 213 214 cld 215 216 // Frame pointer is the iframe. 217 movq %rsp, %rbp 218 219 // Set the RF (resume flag) in RFLAGS. This prevents an instruction 220 // breakpoint on the instruction we're returning to to trigger a debug 221 // exception. 222 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) 223 224 subq $512, %rsp 225 andq $~15, %rsp 226 fxsaveq (%rsp) 227 228 // Call the interrupt handler. 229 movq %rbp, %rdi 230 movq IFRAME_vector(%rbp), %rax 231 call *gInterruptHandlerTable(, %rax, 8) 232 233 fxrstorq (%rsp) 234 movq %rbp, %rsp 235 236 // Restore the saved registers. 237 RESTORE_IFRAME() 238 239 iretq 240FUNCTION_END(int_bottom) 241 242 243// Handler for an interrupt that occurred in user-mode. 244STATIC_FUNCTION(int_bottom_user): 245 // Load the kernel GS segment base. 246 swapgs 247 248 // Push the rest of the interrupt frame to the stack. 249 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) 250 cld 251 252 // Frame pointer is the iframe. 253 movq %rsp, %rbp 254 255 subq $512, %rsp 256 andq $~15, %rsp 257 fxsaveq (%rsp) 258 movq %rsp, IFRAME_fpu(%rbp) 259 260 // Set the RF (resume flag) in RFLAGS. This prevents an instruction 261 // breakpoint on the instruction we're returning to to trigger a debug 262 // exception. 263 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) 264 265 // Get thread pointer. 266 movq %gs:0, %r12 267 268 STOP_USER_DEBUGGING() 269 UPDATE_THREAD_USER_TIME() 270 271 // Call the interrupt handler. 272 movq %rbp, %rdi 273 movq IFRAME_vector(%rbp), %rax 274 call *gInterruptHandlerTable(, %rax, 8) 275 276 // If there are no signals pending or we're not debugging, we can avoid 277 // most of the work here, just need to update the kernel time. 278 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 279 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 280 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 281 , THREAD_flags(%r12) 282 jnz .Lkernel_exit_work 283 284 cli 285 286 UPDATE_THREAD_KERNEL_TIME() 287 288 fxrstorq (%rsp) 289 movq %rbp, %rsp 290 291 // Restore the saved registers. 292 RESTORE_IFRAME() 293 294 // Restore the previous GS base and return. 295 swapgs 296 iretq 297 298.Lkernel_exit_work: 299 // Slow path for return to userland. 300 301 // Do we need to handle signals? 302 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \ 303 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 304 , THREAD_flags(%r12) 305 jnz .Lkernel_exit_handle_signals 306 cli 307 call thread_at_kernel_exit_no_signals 308 309.Lkernel_exit_work_done: 310 // Install breakpoints, if defined. 311 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 312 jz 1f 313 movq %rbp, %rdi 314 call x86_init_user_debug_at_kernel_exit 3151: 316 fxrstorq (%rsp) 317 movq %rbp, %rsp 318 319 // Restore the saved registers. 320 RESTORE_IFRAME() 321 322 // Restore the previous GS base and return. 323 swapgs 324 iretq 325 326.Lkernel_exit_handle_signals: 327 // thread_at_kernel_exit requires interrupts to be enabled, it will disable 328 // them after. 329 sti 330 call thread_at_kernel_exit 331 jmp .Lkernel_exit_work_done 332FUNCTION_END(int_bottom_user) 333 334 335// SYSCALL entry point. 336FUNCTION(x86_64_syscall_entry): 337 // Upon entry, RSP still points at the user stack. Load the kernel GS 338 // segment base address, which points at the current thread's arch_thread 339 // structure. This contains our kernel stack pointer and a temporary 340 // scratch space to store the user stack pointer in before we can push it 341 // to the stack. 342 swapgs 343 movq %rsp, %gs:ARCH_THREAD_user_rsp 344 movq %gs:ARCH_THREAD_syscall_rsp, %rsp 345 346 // The following pushes de-align the stack by 8 bytes, so account for that first. 347 sub $8, %rsp 348 349 // Set up an iframe on the stack (R11 = saved RFLAGS, RCX = saved RIP). 350 push $USER_DATA_SELECTOR // ss 351 push %gs:ARCH_THREAD_user_rsp // rsp 352 push %r11 // flags 353 push $USER_CODE_SELECTOR // cs 354 push %rcx // ip 355 push $0 // error_code 356 push $99 // vector 357 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_SYSCALL) 358 359 cld 360 361 // Frame pointer is the iframe. 362 movq %rsp, %rbp 363 364 // Preserve call number (R14 is callee-save), get thread pointer. 365 movq %rax, %r14 366 movq %gs:0, %r12 367 368 STOP_USER_DEBUGGING() 369 UPDATE_THREAD_USER_TIME() 370 371 // No longer need interrupts disabled. 372 sti 373 374 // Check whether the syscall number is valid. 375 cmpq $SYSCALL_COUNT, %r14 376 jae .Lsyscall_return 377 378 // Get the system call table entry. Note I'm hardcoding the shift because 379 // sizeof(syscall_info) is 16 and scale factors of 16 aren't supported, 380 // so can't just do leaq kSyscallInfos(, %rax, SYSCALL_INFO_sizeof). 381 movq %r14, %rax 382 shlq $4, %rax 383 leaq kSyscallInfos(, %rax, 1), %rax 384 385 // Check the number of call arguments, greater than 6 (6 * 8 = 48) requires 386 // a stack copy. 387 movq SYSCALL_INFO_parameter_size(%rax), %rcx 388 cmpq $48, %rcx 389 ja .Lsyscall_stack_args 390 391.Lperform_syscall: 392 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12) 393 jnz .Lpre_syscall_debug 394 395.Lpre_syscall_debug_done: 396 // Restore the arguments from the iframe. UPDATE_THREAD_USER_TIME() makes 397 // 2 function calls which means they may have been overwritten. Note that 398 // argument 4 is in R10 on the frame rather than RCX as RCX is used by 399 // SYSCALL. 400 movq IFRAME_di(%rbp), %rdi 401 movq IFRAME_si(%rbp), %rsi 402 movq IFRAME_dx(%rbp), %rdx 403 movq IFRAME_r10(%rbp), %rcx 404 movq IFRAME_r8(%rbp), %r8 405 movq IFRAME_r9(%rbp), %r9 406 407 // TODO: pre-syscall tracing 408 409 // Call the function and save its return value. 410 call *SYSCALL_INFO_function(%rax) 411 movq %rax, IFRAME_ax(%rbp) 412 413 // TODO: post-syscall tracing 414 415.Lsyscall_return: 416 // Restore the original stack pointer and return. 417 movq %rbp, %rsp 418 419 // Clear the restarted flag. 420 testl $THREAD_FLAGS_SYSCALL_RESTARTED, THREAD_flags(%r12) 421 jz 2f 4221: 423 movl THREAD_flags(%r12), %eax 424 movl %eax, %edx 425 andl $~THREAD_FLAGS_SYSCALL_RESTARTED, %edx 426 lock 427 cmpxchgl %edx, THREAD_flags(%r12) 428 jnz 1b 4292: 430 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 431 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 432 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP | THREAD_FLAGS_RESTART_SYSCALL) \ 433 , THREAD_flags(%r12) 434 jnz .Lpost_syscall_work 435 436 cli 437 438 UPDATE_THREAD_KERNEL_TIME() 439 440 // If we've just restored a signal frame, use the IRET path. 441 cmpq $SYSCALL_RESTORE_SIGNAL_FRAME, %r14 442 je .Lrestore_fpu 443 444 CLEAR_FPU_STATE() 445 446 // Restore the iframe and RCX/R11 for SYSRET. 447 RESTORE_IFRAME() 448 pop %rcx 449 addq $8, %rsp 450 pop %r11 451 pop %rsp 452 453 // Restore previous GS base and return. 454 swapgs 455 sysretq 456 457.Lpre_syscall_debug: 458 // user_debug_pre_syscall expects a pointer to a block of arguments, need 459 // to push the register arguments onto the stack. 460 push IFRAME_r9(%rbp) 461 push IFRAME_r8(%rbp) 462 push IFRAME_r10(%rbp) 463 push IFRAME_dx(%rbp) 464 push IFRAME_si(%rbp) 465 push IFRAME_di(%rbp) 466 movq %r14, %rdi // syscall number 467 movq %rsp, %rsi 468 push %rax 469 call user_debug_pre_syscall 470 pop %rax 471 addq $48, %rsp 472 jmp .Lpre_syscall_debug_done 473 474.Lpost_syscall_work: 475 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12) 476 jz 1f 477 478 // Post-syscall debugging. Same as above, need a block of arguments. 479 push IFRAME_r9(%rbp) 480 push IFRAME_r8(%rbp) 481 push IFRAME_r10(%rbp) 482 push IFRAME_dx(%rbp) 483 push IFRAME_si(%rbp) 484 push IFRAME_di(%rbp) 485 movq %r14, %rdi // syscall number 486 movq %rsp, %rsi 487 movq IFRAME_ax(%rbp), %rdx // return value 488 movq %r13, %rcx // start time, preserved earlier 489 call user_debug_post_syscall 490 addq $48, %rsp 4911: 492 // Do we need to handle signals? 493 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \ 494 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 495 , THREAD_flags(%r12) 496 jnz .Lpost_syscall_handle_signals 497 cli 498 call thread_at_kernel_exit_no_signals 499 500.Lpost_syscall_work_done: 501 // Handle syscall restarting. 502 testl $THREAD_FLAGS_RESTART_SYSCALL, THREAD_flags(%r12) 503 jz 1f 504 movq %rsp, %rdi 505 call x86_restart_syscall 5061: 507 // Install breakpoints, if defined. 508 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 509 jz 1f 510 movq %rbp, %rdi 511 call x86_init_user_debug_at_kernel_exit 5121: 513 // On this return path it is possible that the frame has been modified, 514 // for example to execute a signal handler. In this case it is safer to 515 // return via IRET. 516 CLEAR_FPU_STATE() 517 jmp .Liret 518 519.Lrestore_fpu: 520 movq IFRAME_fpu(%rbp), %rax 521 fxrstorq (%rax) 522.Liret: 523 // Restore the saved registers. 524 RESTORE_IFRAME() 525 526 // Restore the previous GS base and return. 527 swapgs 528 iretq 529 530.Lpost_syscall_handle_signals: 531 call thread_at_kernel_exit 532 jmp .Lpost_syscall_work_done 533 534.Lsyscall_stack_args: 535 // Some arguments are on the stack, work out what we need to copy. 6 536 // arguments (48 bytes) are already in registers. 537 // RAX = syscall table entry address, RCX = argument size. 538 subq $48, %rcx 539 540 // Get the address to copy from. 541 movq IFRAME_user_sp(%rbp), %rsi 542 addq $8, %rsi 543 movabs $(USER_BASE + USER_SIZE), %rdx 544 cmp %rdx, %rsi 545 jae .Lbad_syscall_args 546 547 // Make space on the stack. 548 subq %rcx, %rsp 549 andq $~15, %rsp 550 movq %rsp, %rdi 551 552 // Set a fault handler. 553 movq $.Lbad_syscall_args, THREAD_fault_handler(%r12) 554 555 ASM_STAC 556 557 // Copy them by quadwords. 558 shrq $3, %rcx 559 rep 560 movsq 561 ASM_CLAC 562 movq $0, THREAD_fault_handler(%r12) 563 564 // Perform the call. 565 jmp .Lperform_syscall 566 567.Lbad_syscall_args: 568 movq $0, THREAD_fault_handler(%r12) 569 movq %rbp, %rsp 570 jmp .Lsyscall_return 571FUNCTION_END(x86_64_syscall_entry) 572 573 574/*! \fn void x86_return_to_userland(iframe* frame) 575 \brief Returns to the userland environment given by \a frame. 576 577 Before returning to userland all potentially necessary kernel exit work is 578 done. 579 580 \a frame must point to a location somewhere on the caller's stack (e.g. a 581 local variable). 582 The function must be called with interrupts disabled. 583 584 \param frame The iframe defining the userland environment. 585*/ 586FUNCTION(x86_return_to_userland): 587 movq %rdi, %rbp 588 movq %rbp, %rsp 589 590 // Perform kernel exit work. 591 movq %gs:0, %r12 592 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 593 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 594 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 595 , THREAD_flags(%r12) 596 jnz .Luserland_return_work 597 598 // update the thread's kernel time and return 599 UPDATE_THREAD_KERNEL_TIME() 600 601 // Restore the frame and return. 602 RESTORE_IFRAME() 603 swapgs 604 iretq 605.Luserland_return_work: 606 // Slow path for return to userland. 607 608 // Do we need to handle signals? 609 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \ 610 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 611 , THREAD_flags(%r12) 612 jnz .Luserland_return_handle_signals 613 cli 614 call thread_at_kernel_exit_no_signals 615 616.Luserland_return_work_done: 617 // Install breakpoints, if defined. 618 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 619 jz 1f 620 movq %rbp, %rdi 621 call x86_init_user_debug_at_kernel_exit 6221: 623 // Restore the saved registers. 624 RESTORE_IFRAME() 625 626 // Restore the previous GS base and return. 627 swapgs 628 iretq 629.Luserland_return_handle_signals: 630 // thread_at_kernel_exit requires interrupts to be enabled, it will disable 631 // them after. 632 sti 633 call thread_at_kernel_exit 634 jmp .Luserland_return_work_done 635FUNCTION_END(x86_return_to_userland) 636