1/* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 7#include <asm_defs.h> 8 9#include <thread_types.h> 10 11#include <arch/x86/descriptors.h> 12#include <arch/x86/arch_cpu.h> 13#include <arch/x86/arch_kernel.h> 14 15#include "asm_offsets.h" 16#include "syscall_numbers.h" 17#include "syscall_table.h" 18 19 20// Push the remainder of the interrupt frame onto the stack. 21#define PUSH_IFRAME_BOTTOM(iframeType) \ 22 push %rax; /* orig_rax */ \ 23 push %rax; \ 24 push %rbx; \ 25 push %rcx; \ 26 push %rdx; \ 27 push %rdi; \ 28 push %rsi; \ 29 push %rbp; \ 30 push %r8; \ 31 push %r9; \ 32 push %r10; \ 33 push %r11; \ 34 push %r12; \ 35 push %r13; \ 36 push %r14; \ 37 push %r15; \ 38 push $iframeType; 39 40// Restore the interrupt frame. 41#define RESTORE_IFRAME() \ 42 add $8, %rsp; \ 43 pop %r15; \ 44 pop %r14; \ 45 pop %r13; \ 46 pop %r12; \ 47 pop %r11; \ 48 pop %r10; \ 49 pop %r9; \ 50 pop %r8; \ 51 pop %rbp; \ 52 pop %rsi; \ 53 pop %rdi; \ 54 pop %rdx; \ 55 pop %rcx; \ 56 pop %rbx; \ 57 pop %rax; \ 58 addq $24, %rsp; 59 60 61// The macros below require R12 to contain the current thread pointer. R12 is 62// callee-save so will be preserved through all function calls and only needs 63// to be obtained once. R13 is used to store the system call start time, will 64// also be preserved. 65 66#define LOCK_THREAD_TIME() \ 67 leaq THREAD_time_lock(%r12), %rdi; \ 68 call acquire_spinlock; 69 70#define UNLOCK_THREAD_TIME() \ 71 leaq THREAD_time_lock(%r12), %rdi; \ 72 call release_spinlock; \ 73 74#define UPDATE_THREAD_USER_TIME() \ 75 LOCK_THREAD_TIME() \ 76 \ 77 call system_time; \ 78 \ 79 /* Preserve system_time for post syscall debug */ \ 80 movq %rax, %r13; \ 81 \ 82 /* thread->user_time += now - thread->last_time; */ \ 83 subq THREAD_last_time(%r12), %rax; \ 84 addq %rax, THREAD_user_time(%r12); \ 85 \ 86 /* thread->last_time = now; */ \ 87 movq %r13, THREAD_last_time(%r12); \ 88 \ 89 /* thread->in_kernel = true; */ \ 90 movb $1, THREAD_in_kernel(%r12); \ 91 \ 92 UNLOCK_THREAD_TIME() 93 94#define UPDATE_THREAD_KERNEL_TIME() \ 95 LOCK_THREAD_TIME() \ 96 \ 97 call system_time; \ 98 movq %rax, %r13; \ 99 \ 100 /* thread->kernel_time += now - thread->last_time; */ \ 101 subq THREAD_last_time(%r12), %rax; \ 102 addq %rax, THREAD_kernel_time(%r12); \ 103 \ 104 /* thread->last_time = now; */ \ 105 movq %r13, THREAD_last_time(%r12); \ 106 \ 107 /* thread->in_kernel = false; */ \ 108 movb $0, THREAD_in_kernel(%r12); \ 109 \ 110 UNLOCK_THREAD_TIME() 111 112#define STOP_USER_DEBUGGING() \ 113 testl $(THREAD_FLAGS_BREAKPOINTS_INSTALLED \ 114 | THREAD_FLAGS_SINGLE_STEP), THREAD_flags(%r12); \ 115 jz 1f; \ 116 call x86_exit_user_debug_at_kernel_entry; \ 117 1: 118 119 120// The following code defines the interrupt service routines for all 256 121// interrupts. It creates a block of handlers, each 16 bytes, that the IDT 122// initialization code just loops through. 123 124// Interrupt with no error code, pushes a 0 error code. 125#define DEFINE_ISR(nr) \ 126 .align 16; \ 127 push $0; \ 128 push $nr; \ 129 jmp int_bottom; 130 131// Interrupt with an error code. 132#define DEFINE_ISR_E(nr) \ 133 .align 16; \ 134 push $nr; \ 135 jmp int_bottom; 136 137// Array of interrupt service routines. 138.align 16 139SYMBOL(isr_array): 140 // Exceptions (0-19) and reserved interrupts (20-31). 141 DEFINE_ISR(0) 142 DEFINE_ISR(1) 143 DEFINE_ISR(2) 144 DEFINE_ISR(3) 145 DEFINE_ISR(4) 146 DEFINE_ISR(5) 147 DEFINE_ISR(6) 148 DEFINE_ISR(7) 149 DEFINE_ISR_E(8) 150 DEFINE_ISR(9) 151 DEFINE_ISR_E(10) 152 DEFINE_ISR_E(11) 153 DEFINE_ISR_E(12) 154 DEFINE_ISR_E(13) 155 DEFINE_ISR_E(14) 156 DEFINE_ISR(15) 157 DEFINE_ISR(16) 158 DEFINE_ISR_E(17) 159 DEFINE_ISR(18) 160 DEFINE_ISR(19) 161 DEFINE_ISR(20) 162 DEFINE_ISR(21) 163 DEFINE_ISR(22) 164 DEFINE_ISR(23) 165 DEFINE_ISR(24) 166 DEFINE_ISR(25) 167 DEFINE_ISR(26) 168 DEFINE_ISR(27) 169 DEFINE_ISR(28) 170 DEFINE_ISR(29) 171 DEFINE_ISR(30) 172 DEFINE_ISR(31) 173 174 // User-defined ISRs (32-255) - none take an error code. 175 .Lintr = 32 176 .rept 224 177 DEFINE_ISR(.Lintr) 178 .Lintr = .Lintr+1 179 .endr 180 181 182// Common interrupt handling code. 183STATIC_FUNCTION(int_bottom): 184 // Coming from user-mode requires special handling. 185 testl $3, 24(%rsp) 186 jnz int_bottom_user 187 188 // Push the rest of the interrupt frame to the stack. 189 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) 190 191 cld 192 193 // Frame pointer is the iframe. 194 movq %rsp, %rbp 195 196 // Set the RF (resume flag) in RFLAGS. This prevents an instruction 197 // breakpoint on the instruction we're returning to to trigger a debug 198 // exception. 199 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) 200 201 // Call the interrupt handler. 202 movq %rsp, %rdi 203 movq IFRAME_vector(%rsp), %rax 204 call *gInterruptHandlerTable(, %rax, 8) 205 206 // Restore the saved registers. 207 RESTORE_IFRAME() 208 209 iretq 210FUNCTION_END(int_bottom) 211 212 213// Handler for an interrupt that occurred in user-mode. 214STATIC_FUNCTION(int_bottom_user): 215 // Load the kerrnel GS segment base. 216 swapgs 217 218 // Push the rest of the interrupt frame to the stack. 219 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) 220 221 cld 222 223 // Frame pointer is the iframe. 224 movq %rsp, %rbp 225 226 // Set the RF (resume flag) in RFLAGS. This prevents an instruction 227 // breakpoint on the instruction we're returning to to trigger a debug 228 // exception. 229 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) 230 231 // Get thread pointer. 232 movq %gs:0, %r12 233 234 STOP_USER_DEBUGGING() 235 UPDATE_THREAD_USER_TIME() 236 237 // Call the interrupt handler. 238 movq %rsp, %rdi 239 movq IFRAME_vector(%rsp), %rax 240 call *gInterruptHandlerTable(, %rax, 8) 241 242 // If there are no signals pending or we're not debugging, we can avoid 243 // most of the work here, just need to update the kernel time. 244 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 245 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \ 246 , THREAD_flags(%r12) 247 jnz .Lkernel_exit_work 248 249 cli 250 251 UPDATE_THREAD_KERNEL_TIME() 252 253 // Restore the saved registers. 254 RESTORE_IFRAME() 255 256 // Restore the previous GS base and return. 257 swapgs 258 iretq 259 260.Lkernel_exit_work: 261 // Slow path for return to userland. 262 263 // Do we need to handle signals? 264 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \ 265 , THREAD_flags(%r12) 266 jnz .Lkernel_exit_handle_signals 267 cli 268 call thread_at_kernel_exit_no_signals 269 270.Lkernel_exit_work_done: 271 // Install breakpoints, if defined. 272 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 273 jz 1f 274 movq %rbp, %rdi 275 call x86_init_user_debug_at_kernel_exit 2761: 277 // Restore the saved registers. 278 RESTORE_IFRAME() 279 280 // Restore the previous GS base and return. 281 swapgs 282 iretq 283 284.Lkernel_exit_handle_signals: 285 // thread_at_kernel_exit requires interrupts to be enabled, it will disable 286 // them after. 287 sti 288 call thread_at_kernel_exit 289 jmp .Lkernel_exit_work_done 290FUNCTION_END(int_bottom_user) 291 292 293// SYSCALL entry point. 294FUNCTION(x86_64_syscall_entry): 295 // Upon entry, RSP still points at the user stack. Load the kernel GS 296 // segment base address, which points at the current thread's arch_thread 297 // structure. This contains our kernel stack pointer and a temporary 298 // scratch space to store the user stack pointer in before we can push it 299 // to the stack. 300 swapgs 301 movq %rsp, %gs:ARCH_THREAD_user_rsp 302 movq %gs:ARCH_THREAD_syscall_rsp, %rsp 303 304 // Set up an iframe on the stack (R11 = saved RFLAGS, RCX = saved RIP). 305 push $USER_DATA_SEG // ss 306 push %gs:ARCH_THREAD_user_rsp // rsp 307 push %r11 // flags 308 push $USER_CODE_SEG // cs 309 push %rcx // ip 310 push $0 // error_code 311 push $0 // vector 312 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_SYSCALL) 313 314 cld 315 316 // Frame pointer is the iframe. 317 movq %rsp, %rbp 318 319 // Preserve call number (R14 is callee-save), get thread pointer. 320 movq %rax, %r14 321 movq %gs:0, %r12 322 323 STOP_USER_DEBUGGING() 324 UPDATE_THREAD_USER_TIME() 325 326 // No longer need interrupts disabled. 327 sti 328 329 // Check whether the syscall number is valid. 330 cmpq $SYSCALL_COUNT, %r14 331 jae .Lsyscall_return 332 333 // Get the system call table entry. Note I'm hardcoding the shift because 334 // sizeof(syscall_info) is 16 and scale factors of 16 aren't supported, 335 // so can't just do leaq kSyscallInfos(, %rax, SYSCALL_INFO_sizeof). 336 movq %r14, %rax 337 shlq $4, %rax 338 leaq kSyscallInfos(, %rax, 1), %rax 339 340 // Check the number of call arguments, greater than 6 (6 * 8 = 48) requires 341 // a stack copy. 342 movq SYSCALL_INFO_parameter_size(%rax), %rcx 343 cmpq $48, %rcx 344 ja .Lsyscall_stack_args 345 346.Lperform_syscall: 347 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12) 348 jnz .Lpre_syscall_debug 349 350.Lpre_syscall_debug_done: 351 // Restore the arguments from the iframe. UPDATE_THREAD_USER_TIME() makes 352 // 2 function calls which means they may have been overwritten. Note that 353 // argument 4 is in R10 on the frame rather than RCX as RCX is used by 354 // SYSCALL. 355 movq IFRAME_di(%rbp), %rdi 356 movq IFRAME_si(%rbp), %rsi 357 movq IFRAME_dx(%rbp), %rdx 358 movq IFRAME_r10(%rbp), %rcx 359 movq IFRAME_r8(%rbp), %r8 360 movq IFRAME_r9(%rbp), %r9 361 362 // TODO: pre-syscall tracing 363 364 // Call the function and save its return value. 365 call *SYSCALL_INFO_function(%rax) 366 movq %rax, IFRAME_ax(%rbp) 367 368 // TODO: post-syscall tracing 369 370 // Restore the original stack pointer and return. 371 movq %rbp, %rsp 372 373.Lsyscall_return: 374 // Clear the restarted flag. 375 testl $THREAD_FLAGS_SYSCALL_RESTARTED, THREAD_flags(%r12) 376 jz 2f 3771: 378 movl THREAD_flags(%r12), %eax 379 movl %eax, %edx 380 andl $~THREAD_FLAGS_SYSCALL_RESTARTED, %edx 381 lock 382 cmpxchgl %edx, THREAD_flags(%r12) 383 jnz 1b 3842: 385 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 386 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \ 387 | THREAD_FLAGS_RESTART_SYSCALL) \ 388 , THREAD_flags(%r12) 389 jnz .Lpost_syscall_work 390 391 cli 392 393 UPDATE_THREAD_KERNEL_TIME() 394 395 // If we've just restored a signal frame, use the IRET path. 396 cmpq $SYSCALL_RESTORE_SIGNAL_FRAME, %r14 397 je .Liret 398 399 // Restore the iframe and RCX/R11 for SYSRET. 400 RESTORE_IFRAME() 401 pop %rcx 402 addq $8, %rsp 403 pop %r11 404 pop %rsp 405 406 // Restore previous GS base and return. 407 swapgs 408 sysretq 409 410.Lpre_syscall_debug: 411 // user_debug_pre_syscall expects a pointer to a block of arguments, need 412 // to push the register arguments onto the stack. 413 push IFRAME_r9(%rbp) 414 push IFRAME_r8(%rbp) 415 push IFRAME_r10(%rbp) 416 push IFRAME_dx(%rbp) 417 push IFRAME_si(%rbp) 418 push IFRAME_di(%rbp) 419 movq %r14, %rdi // syscall number 420 movq %rsp, %rsi 421 push %rax 422 call user_debug_pre_syscall 423 pop %rax 424 addq $48, %rsp 425 jmp .Lpre_syscall_debug_done 426 427.Lpost_syscall_work: 428 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12) 429 jz 1f 430 431 // Post-syscall debugging. Same as above, need a block of arguments. 432 push IFRAME_r9(%rbp) 433 push IFRAME_r8(%rbp) 434 push IFRAME_r10(%rbp) 435 push IFRAME_dx(%rbp) 436 push IFRAME_si(%rbp) 437 push IFRAME_di(%rbp) 438 movq %r14, %rdi // syscall number 439 movq %rsp, %rsi 440 movq IFRAME_ax(%rbp), %rdx // return value 441 movq %r13, %rcx // start time, preserved earlier 442 call user_debug_post_syscall 443 addq $48, %rsp 4441: 445 // Do we need to handle signals? 446 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \ 447 , THREAD_flags(%r12) 448 jnz .Lpost_syscall_handle_signals 449 cli 450 call thread_at_kernel_exit_no_signals 451 452.Lpost_syscall_work_done: 453 // Handle syscall restarting. 454 testl $THREAD_FLAGS_RESTART_SYSCALL, THREAD_flags(%r12) 455 jz 1f 456 movq %rsp, %rdi 457 call x86_restart_syscall 4581: 459 // Install breakpoints, if defined. 460 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) 461 jz .Liret 462 movq %rbp, %rdi 463 call x86_init_user_debug_at_kernel_exit 464 465 // On this return path it is possible that the frame has been modified, 466 // for example to execute a signal handler. In this case it is safer to 467 // return via IRET. 468 469.Liret: 470 // Restore the saved registers. 471 RESTORE_IFRAME() 472 473 // Restore the previous GS base and return. 474 swapgs 475 iretq 476 477.Lpost_syscall_handle_signals: 478 call thread_at_kernel_exit 479 jmp .Lpost_syscall_work_done 480 481.Lsyscall_stack_args: 482 // Some arguments are on the stack, work out what we need to copy. 6 483 // arguments (48 bytes) are already in registers. 484 // RAX = syscall table entry address, RCX = argument size. 485 subq $48, %rcx 486 487 // Get the address to copy from. 488 movq IFRAME_user_sp(%rbp), %rsi 489 addq $8, %rsi 490 movabs $(USER_BASE + USER_SIZE), %rdx 491 cmp %rdx, %rsi 492 jae .Lbad_syscall_args 493 494 // Make space on the stack. 495 subq %rcx, %rsp 496 movq %rsp, %rdi 497 498 // Set a fault handler. 499 movq $.Lbad_syscall_args, THREAD_fault_handler(%r12) 500 501 // Copy them by quadwords. 502 shrq $3, %rcx 503 rep 504 movsq 505 movq $0, THREAD_fault_handler(%r12) 506 507 // Perform the call. 508 jmp .Lperform_syscall 509 510.Lbad_syscall_args: 511 movq $0, THREAD_fault_handler(%r12) 512 movq %rbp, %rsp 513 jmp .Lsyscall_return 514FUNCTION_END(x86_64_syscall_entry) 515 516 517/*! \fn void x86_return_to_userland(iframe* frame) 518 \brief Returns to the userland environment given by \a frame. 519 520 Before returning to userland all potentially necessary kernel exit work is 521 done. 522 523 \a frame must point to a location somewhere on the caller's stack (e.g. a 524 local variable). 525 The function must be called with interrupts disabled. 526 527 \param frame The iframe defining the userland environment. 528*/ 529FUNCTION(x86_return_to_userland): 530 movq %rdi, %rbp 531 movq %rbp, %rsp 532 533 // Perform kernel exit work. 534 movq %gs:0, %r12 535 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ 536 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \ 537 , THREAD_flags(%r12) 538 jnz .Lkernel_exit_work 539 540 // update the thread's kernel time and return 541 UPDATE_THREAD_KERNEL_TIME() 542 543 // Restore the frame and return. 544 RESTORE_IFRAME() 545 swapgs 546 iretq 547FUNCTION_END(x86_return_to_userland) 548