1/* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 7#include <asm_defs.h> 8 9#include <thread_types.h> 10 11#include <arch/x86/descriptors.h> 12#include <arch/x86/arch_cpu.h> 13#include <arch/x86/arch_kernel.h> 14 15#include "asm_offsets.h" 16#include "syscall_numbers.h" 17#include "syscall_table.h" 18 19 20// Push the remainder of the interrupt frame onto the stack. 21#define PUSH_IFRAME_BOTTOM(iframeType) \ 22 push %rax; /* orig_rax */ \ 23 push %rax; \ 24 push %rbx; \ 25 push %rcx; \ 26 push %rdx; \ 27 push %rdi; \ 28 push %rsi; \ 29 push %rbp; \ 30 push %r8; \ 31 push %r9; \ 32 push %r10; \ 33 push %r11; \ 34 push %r12; \ 35 push %r13; \ 36 push %r14; \ 37 push %r15; \ 38 pushq $0; \ 39 push $iframeType; 40 41 42// Restore the interrupt frame. 43#define RESTORE_IFRAME() \ 44 add $16, %rsp; \ 45 pop %r15; \ 46 pop %r14; \ 47 pop %r13; \ 48 pop %r12; \ 49 pop %r11; \ 50 pop %r10; \ 51 pop %r9; \ 52 pop %r8; \ 53 pop %rbp; \ 54 pop %rsi; \ 55 pop %rdi; \ 56 pop %rdx; \ 57 pop %rcx; \ 58 pop %rbx; \ 59 pop %rax; \ 60 addq $24, %rsp; 61 62 63// The macros below require R12 to contain the current thread pointer. R12 is 64// callee-save so will be preserved through all function calls and only needs 65// to be obtained once. R13 is used to store the system call start time, will 66// also be preserved. 67 68#define LOCK_THREAD_TIME() \ 69 leaq THREAD_time_lock(%r12), %rdi; \ 70 call acquire_spinlock; 71 72#define UNLOCK_THREAD_TIME() \ 73 leaq THREAD_time_lock(%r12), %rdi; \ 74 call release_spinlock; \ 75 76#define UPDATE_THREAD_USER_TIME() \ 77 LOCK_THREAD_TIME() \ 78 \ 79 call system_time; \ 80 \ 81 /* Preserve system_time for post syscall debug */ \ 82 movq %rax, %r13; \ 83 \ 84 /* thread->user_time += now - thread->last_time; */ \ 85 subq THREAD_last_time(%r12), %rax; \ 86 addq %rax, THREAD_user_time(%r12); \ 87 \ 88 /* thread->last_time = now; */ \ 89 movq %r13, THREAD_last_time(%r12); \ 90 \ 91 /* thread->in_kernel = true; */ \ 92 movb $1, THREAD_in_kernel(%r12); \ 93 \ 94 UNLOCK_THREAD_TIME() 95 96#define UPDATE_THREAD_KERNEL_TIME() \ 97 LOCK_THREAD_TIME() \ 98 \ 99 call system_time; \ 100 movq %rax, %r13; \ 101 \ 102 /* thread->kernel_time += now - thread->last_time; */ \ 103 subq THREAD_last_time(%r12), %rax; \ 104 addq %rax, THREAD_kernel_time(%r12); \ 105 \ 106 /* thread->last_time = now; */ \ 107 movq %r13, THREAD_last_time(%r12); \ 108 \ 109 /* thread->in_kernel = false; */ \ 110 movb $0, THREAD_in_kernel(%r12); \ 111 \ 112 UNLOCK_THREAD_TIME() 113 114#define STOP_USER_DEBUGGING() \ 115 testl $(THREAD_FLAGS_BREAKPOINTS_INSTALLED \ 116 | THREAD_FLAGS_SINGLE_STEP), THREAD_flags(%r12); \ 117 jz 1f; \ 118 call x86_exit_user_debug_at_kernel_entry; \ 119 1: 120 121#define CLEAR_FPU_STATE() \ 122 pxor %xmm0, %xmm0; \ 123 pxor %xmm1, %xmm1; \ 124 pxor %xmm2, %xmm2; \ 125 pxor %xmm3, %xmm3; \ 126 pxor %xmm4, %xmm4; \ 127 pxor %xmm5, %xmm5; \ 128 pxor %xmm6, %xmm6; \ 129 pxor %xmm7, %xmm7; \ 130 pxor %xmm8, %xmm8; \ 131 pxor %xmm9, %xmm9; \ 132 pxor %xmm10, %xmm10; \ 133 pxor %xmm11, %xmm11; \ 134 pxor %xmm12, %xmm12; \ 135 pxor %xmm13, %xmm13; \ 136 pxor %xmm14, %xmm14; \ 137 pxor %xmm15, %xmm15 138 139// The following code defines the interrupt service routines for all 256 140// interrupts. It creates a block of handlers, each 16 bytes, that the IDT 141// initialization code just loops through. 142 143// Interrupt with no error code, pushes a 0 error code. 144#define DEFINE_ISR(nr) \ 145 .align 16; \ 146 push $0; \ 147 push $nr; \ 148 jmp int_bottom; 149 150// Interrupt with an error code. 151#define DEFINE_ISR_E(nr) \ 152 .align 16; \ 153 push $nr; \ 154 jmp int_bottom; 155 156// Array of interrupt service routines. 157.align 16 158SYMBOL(isr_array): 159 // Exceptions (0-19) and reserved interrupts (20-31). 160 DEFINE_ISR(0) 161 DEFINE_ISR(1) 162 DEFINE_ISR(2) 163 DEFINE_ISR(3) 164 DEFINE_ISR(4) 165 DEFINE_ISR(5) 166 DEFINE_ISR(6) 167 DEFINE_ISR(7) 168 DEFINE_ISR_E(8) 169 DEFINE_ISR(9) 170 DEFINE_ISR_E(10) 171 DEFINE_ISR_E(11) 172 DEFINE_ISR_E(12) 173 DEFINE_ISR_E(13) 174 DEFINE_ISR_E(14) 175 DEFINE_ISR(15) 176 DEFINE_ISR(16) 177 DEFINE_ISR_E(17) 178 DEFINE_ISR(18) 179 DEFINE_ISR(19) 180 DEFINE_ISR(20) 181 DEFINE_ISR(21) 182 DEFINE_ISR(22) 183 DEFINE_ISR(23) 184 DEFINE_ISR(24) 185 DEFINE_ISR(25) 186 DEFINE_ISR(26) 187 DEFINE_ISR(27) 188 DEFINE_ISR(28) 189 DEFINE_ISR(29) 190 DEFINE_ISR(30) 191 DEFINE_ISR(31) 192 193 // User-defined ISRs (32-255) - none take an error code. 194 .Lintr = 32 195 .rept 224 196 DEFINE_ISR(.Lintr) 197 .Lintr = .Lintr+1 198 .endr 199 200 201// Common interrupt handling code. 202STATIC_FUNCTION(int_bottom): 203 // Coming from user-mode requires special handling. 204 testl $3, 24(%rsp) 205 jnz int_bottom_user 206 207 // Push the rest of the interrupt frame to the stack. 208 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) 209 210 cld 211 212 // Frame pointer is the iframe. 213 movq %rsp, %rbp 214 215 // Set the RF (resume flag) in RFLAGS. This prevents an instruction 216 // breakpoint on the instruction we're returning to to trigger a debug 217 // exception. 218 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) 219 220 subq $512, %rsp 221 andq $~15, %rsp 222 fxsaveq (%rsp) 223 224 // Call the interrupt handler. 225 movq %rbp, %rdi 226 movq IFRAME_vector(%rbp), %rax 227 call *gInterruptHandlerTable(, %rax, 8) 228 229 fxrstorq (%rsp) 230 movq %rbp, %rsp 231 232 // Restore the saved registers. 233 RESTORE_IFRAME() 234 235 iretq 236FUNCTION_END(int_bottom) 237 238 239// Handler for an interrupt that occurred in user-mode. 240STATIC_FUNCTION(int_bottom_user): 241 // Load the kerrnel GS segment base. 242 swapgs 243 244 // Push the rest of the interrupt frame to the stack. 245 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) 246 cld 247 248 // Frame pointer is the iframe. 249 movq %rsp, %rbp 250 251 subq $512, %rsp 252 andq $~15, %rsp 253 fxsaveq (%rsp) 254 movq %rsp, IFRAME_fpu(%rbp) 255 256 // Set the RF (resume flag) in RFLAGS. This prevents an instruction 257 // breakpoint on the instruction we're returning to to trigger a debug 258 // exception. 259 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) 260 261 // Get thread pointer. 262 movq %gs:0, %r12 263 264 STOP_USER_DEBUGGING() 265 UPDATE_THREAD_USER_TIME() 266 267 // Call the interrupt handler. 268 movq %rbp, %rdi 269 movq IFRAME_vector(%rbp), %rax 270 call *gInterruptHandlerTable(, %rax, 8) 271 272 // If there are no signals pending or we're not debugging, we can avoid 273 // most of the work here, just need to update the kernel time. 274 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 275 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 276 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 277 , THREAD_flags(%r12) 278 jnz .Lkernel_exit_work 279 280 cli 281 282 UPDATE_THREAD_KERNEL_TIME() 283 284 fxrstorq (%rsp) 285 movq %rbp, %rsp 286 287 // Restore the saved registers. 288 RESTORE_IFRAME() 289 290 // Restore the previous GS base and return. 291 swapgs 292 iretq 293 294.Lkernel_exit_work: 295 // Slow path for return to userland. 296 297 // Do we need to handle signals? 298 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \ 299 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 300 , THREAD_flags(%r12) 301 jnz .Lkernel_exit_handle_signals 302 cli 303 call thread_at_kernel_exit_no_signals 304 305.Lkernel_exit_work_done: 306 // Install breakpoints, if defined. 307 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 308 jz 1f 309 movq %rbp, %rdi 310 call x86_init_user_debug_at_kernel_exit 3111: 312 fxrstorq (%rsp) 313 movq %rbp, %rsp 314 315 // Restore the saved registers. 316 RESTORE_IFRAME() 317 318 // Restore the previous GS base and return. 319 swapgs 320 iretq 321 322.Lkernel_exit_handle_signals: 323 // thread_at_kernel_exit requires interrupts to be enabled, it will disable 324 // them after. 325 sti 326 call thread_at_kernel_exit 327 jmp .Lkernel_exit_work_done 328FUNCTION_END(int_bottom_user) 329 330 331// SYSCALL entry point. 332FUNCTION(x86_64_syscall_entry): 333 // Upon entry, RSP still points at the user stack. Load the kernel GS 334 // segment base address, which points at the current thread's arch_thread 335 // structure. This contains our kernel stack pointer and a temporary 336 // scratch space to store the user stack pointer in before we can push it 337 // to the stack. 338 swapgs 339 movq %rsp, %gs:ARCH_THREAD_user_rsp 340 movq %gs:ARCH_THREAD_syscall_rsp, %rsp 341 342 // Set up an iframe on the stack (R11 = saved RFLAGS, RCX = saved RIP). 343 push $USER_DATA_SELECTOR // ss 344 push %gs:ARCH_THREAD_user_rsp // rsp 345 push %r11 // flags 346 push $USER_CODE_SELECTOR // cs 347 push %rcx // ip 348 push $0 // error_code 349 push $99 // vector 350 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_SYSCALL) 351 352 cld 353 354 // Frame pointer is the iframe. 355 movq %rsp, %rbp 356 andq $~15, %rsp 357 358 // Preserve call number (R14 is callee-save), get thread pointer. 359 movq %rax, %r14 360 movq %gs:0, %r12 361 362 STOP_USER_DEBUGGING() 363 UPDATE_THREAD_USER_TIME() 364 365 // No longer need interrupts disabled. 366 sti 367 368 // Check whether the syscall number is valid. 369 cmpq $SYSCALL_COUNT, %r14 370 jae .Lsyscall_return 371 372 // Get the system call table entry. Note I'm hardcoding the shift because 373 // sizeof(syscall_info) is 16 and scale factors of 16 aren't supported, 374 // so can't just do leaq kSyscallInfos(, %rax, SYSCALL_INFO_sizeof). 375 movq %r14, %rax 376 shlq $4, %rax 377 leaq kSyscallInfos(, %rax, 1), %rax 378 379 // Check the number of call arguments, greater than 6 (6 * 8 = 48) requires 380 // a stack copy. 381 movq SYSCALL_INFO_parameter_size(%rax), %rcx 382 cmpq $48, %rcx 383 ja .Lsyscall_stack_args 384 385.Lperform_syscall: 386 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12) 387 jnz .Lpre_syscall_debug 388 389.Lpre_syscall_debug_done: 390 // Restore the arguments from the iframe. UPDATE_THREAD_USER_TIME() makes 391 // 2 function calls which means they may have been overwritten. Note that 392 // argument 4 is in R10 on the frame rather than RCX as RCX is used by 393 // SYSCALL. 394 movq IFRAME_di(%rbp), %rdi 395 movq IFRAME_si(%rbp), %rsi 396 movq IFRAME_dx(%rbp), %rdx 397 movq IFRAME_r10(%rbp), %rcx 398 movq IFRAME_r8(%rbp), %r8 399 movq IFRAME_r9(%rbp), %r9 400 401 // TODO: pre-syscall tracing 402 403 // Call the function and save its return value. 404 call *SYSCALL_INFO_function(%rax) 405 movq %rax, IFRAME_ax(%rbp) 406 407 // TODO: post-syscall tracing 408 409.Lsyscall_return: 410 // Restore the original stack pointer and return. 411 movq %rbp, %rsp 412 413 // Clear the restarted flag. 414 testl $THREAD_FLAGS_SYSCALL_RESTARTED, THREAD_flags(%r12) 415 jz 2f 4161: 417 movl THREAD_flags(%r12), %eax 418 movl %eax, %edx 419 andl $~THREAD_FLAGS_SYSCALL_RESTARTED, %edx 420 lock 421 cmpxchgl %edx, THREAD_flags(%r12) 422 jnz 1b 4232: 424 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 425 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 426 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP | THREAD_FLAGS_RESTART_SYSCALL) \ 427 , THREAD_flags(%r12) 428 jnz .Lpost_syscall_work 429 430 cli 431 432 UPDATE_THREAD_KERNEL_TIME() 433 434 // If we've just restored a signal frame, use the IRET path. 435 cmpq $SYSCALL_RESTORE_SIGNAL_FRAME, %r14 436 je .Lrestore_fpu 437 438 CLEAR_FPU_STATE() 439 440 // Restore the iframe and RCX/R11 for SYSRET. 441 RESTORE_IFRAME() 442 pop %rcx 443 addq $8, %rsp 444 pop %r11 445 pop %rsp 446 447 // Restore previous GS base and return. 448 swapgs 449 sysretq 450 451.Lpre_syscall_debug: 452 // user_debug_pre_syscall expects a pointer to a block of arguments, need 453 // to push the register arguments onto the stack. 454 push IFRAME_r9(%rbp) 455 push IFRAME_r8(%rbp) 456 push IFRAME_r10(%rbp) 457 push IFRAME_dx(%rbp) 458 push IFRAME_si(%rbp) 459 push IFRAME_di(%rbp) 460 movq %r14, %rdi // syscall number 461 movq %rsp, %rsi 462 push %rax 463 call user_debug_pre_syscall 464 pop %rax 465 addq $48, %rsp 466 jmp .Lpre_syscall_debug_done 467 468.Lpost_syscall_work: 469 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12) 470 jz 1f 471 472 // Post-syscall debugging. Same as above, need a block of arguments. 473 push IFRAME_r9(%rbp) 474 push IFRAME_r8(%rbp) 475 push IFRAME_r10(%rbp) 476 push IFRAME_dx(%rbp) 477 push IFRAME_si(%rbp) 478 push IFRAME_di(%rbp) 479 movq %r14, %rdi // syscall number 480 movq %rsp, %rsi 481 movq IFRAME_ax(%rbp), %rdx // return value 482 movq %r13, %rcx // start time, preserved earlier 483 call user_debug_post_syscall 484 addq $48, %rsp 4851: 486 // Do we need to handle signals? 487 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \ 488 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 489 , THREAD_flags(%r12) 490 jnz .Lpost_syscall_handle_signals 491 cli 492 call thread_at_kernel_exit_no_signals 493 494.Lpost_syscall_work_done: 495 // Handle syscall restarting. 496 testl $THREAD_FLAGS_RESTART_SYSCALL, THREAD_flags(%r12) 497 jz 1f 498 movq %rsp, %rdi 499 call x86_restart_syscall 5001: 501 // Install breakpoints, if defined. 502 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 503 jz 1f 504 movq %rbp, %rdi 505 call x86_init_user_debug_at_kernel_exit 5061: 507 // On this return path it is possible that the frame has been modified, 508 // for example to execute a signal handler. In this case it is safer to 509 // return via IRET. 510 CLEAR_FPU_STATE() 511 jmp .Liret 512 513.Lrestore_fpu: 514 movq IFRAME_fpu(%rbp), %rax 515 fxrstorq (%rax) 516.Liret: 517 // Restore the saved registers. 518 RESTORE_IFRAME() 519 520 // Restore the previous GS base and return. 521 swapgs 522 iretq 523 524.Lpost_syscall_handle_signals: 525 call thread_at_kernel_exit 526 jmp .Lpost_syscall_work_done 527 528.Lsyscall_stack_args: 529 // Some arguments are on the stack, work out what we need to copy. 6 530 // arguments (48 bytes) are already in registers. 531 // RAX = syscall table entry address, RCX = argument size. 532 subq $48, %rcx 533 534 // Get the address to copy from. 535 movq IFRAME_user_sp(%rbp), %rsi 536 addq $8, %rsi 537 movabs $(USER_BASE + USER_SIZE), %rdx 538 cmp %rdx, %rsi 539 jae .Lbad_syscall_args 540 541 // Make space on the stack. 542 subq %rcx, %rsp 543 andq $~15, %rsp 544 movq %rsp, %rdi 545 546 // Set a fault handler. 547 movq $.Lbad_syscall_args, THREAD_fault_handler(%r12) 548 549 // Copy them by quadwords. 550 shrq $3, %rcx 551 rep 552 movsq 553 movq $0, THREAD_fault_handler(%r12) 554 555 // Perform the call. 556 jmp .Lperform_syscall 557 558.Lbad_syscall_args: 559 movq $0, THREAD_fault_handler(%r12) 560 movq %rbp, %rsp 561 jmp .Lsyscall_return 562FUNCTION_END(x86_64_syscall_entry) 563 564 565/*! \fn void x86_return_to_userland(iframe* frame) 566 \brief Returns to the userland environment given by \a frame. 567 568 Before returning to userland all potentially necessary kernel exit work is 569 done. 570 571 \a frame must point to a location somewhere on the caller's stack (e.g. a 572 local variable). 573 The function must be called with interrupts disabled. 574 575 \param frame The iframe defining the userland environment. 576*/ 577FUNCTION(x86_return_to_userland): 578 movq %rdi, %rbp 579 movq %rbp, %rsp 580 581 // Perform kernel exit work. 582 movq %gs:0, %r12 583 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 584 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 585 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 586 , THREAD_flags(%r12) 587 jnz .Luserland_return_work 588 589 // update the thread's kernel time and return 590 UPDATE_THREAD_KERNEL_TIME() 591 592 // Restore the frame and return. 593 RESTORE_IFRAME() 594 swapgs 595 iretq 596.Luserland_return_work: 597 // Slow path for return to userland. 598 599 // Do we need to handle signals? 600 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \ 601 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \ 602 , THREAD_flags(%r12) 603 jnz .Luserland_return_handle_signals 604 cli 605 call thread_at_kernel_exit_no_signals 606 607.Luserland_return_work_done: 608 // Install breakpoints, if defined. 609 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 610 jz 1f 611 movq %rbp, %rdi 612 call x86_init_user_debug_at_kernel_exit 6131: 614 // Restore the saved registers. 615 RESTORE_IFRAME() 616 617 // Restore the previous GS base and return. 618 swapgs 619 iretq 620.Luserland_return_handle_signals: 621 // thread_at_kernel_exit requires interrupts to be enabled, it will disable 622 // them after. 623 sti 624 call thread_at_kernel_exit 625 jmp .Luserland_return_work_done 626FUNCTION_END(x86_return_to_userland) 627