1 /* 2 * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. 3 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include <arch/x86/descriptors.h> 9 10 #include <boot/kernel_args.h> 11 #include <cpu.h> 12 #include <tls.h> 13 #include <vm/vm.h> 14 #include <vm/vm_priv.h> 15 16 #include <arch/int.h> 17 #include <arch/user_debugger.h> 18 19 20 template<typename T, T (*Function)(unsigned), unsigned N, unsigned ...Index> 21 struct GenerateTable : GenerateTable<T, Function, N - 1, N - 1, Index...> { 22 }; 23 24 template<typename T, T (*Function)(unsigned), unsigned ...Index> 25 struct GenerateTable<T, Function, 0, Index...> { 26 GenerateTable() 27 : 28 fTable { Function(Index)... } 29 { 30 } 31 32 T fTable[sizeof...(Index)]; 33 }; 34 35 enum class DescriptorType : unsigned { 36 DataWritable = 0x2, 37 CodeExecuteOnly = 0x8, 38 TSS = 0x9, 39 }; 40 41 class Descriptor { 42 public: 43 constexpr Descriptor(); 44 inline Descriptor(uint32_t first, uint32_t second); 45 constexpr Descriptor(DescriptorType type, bool kernelOnly, 46 bool compatMode = false); 47 48 protected: 49 union { 50 struct [[gnu::packed]] { 51 uint16_t fLimit0; 52 unsigned fBase0 :24; 53 unsigned fType :4; 54 unsigned fSystem :1; 55 unsigned fDPL :2; 56 unsigned fPresent :1; 57 unsigned fLimit1 :4; 58 unsigned fUnused :1; 59 unsigned fLong :1; 60 unsigned fDB :1; 61 unsigned fGranularity :1; 62 uint8_t fBase1; 63 }; 64 65 uint32_t fDescriptor[2]; 66 }; 67 }; 68 69 class TSSDescriptor : public Descriptor { 70 public: 71 inline TSSDescriptor(uintptr_t base, size_t limit); 72 73 const Descriptor& GetLower() const { return *this; } 74 const Descriptor& GetUpper() const { return fSecond; } 75 76 static void LoadTSS(unsigned index); 77 78 private: 79 Descriptor fSecond; 80 }; 81 82 83 class UserTLSDescriptor : public Descriptor { 84 public: 85 inline UserTLSDescriptor(uintptr_t base, size_t limit); 86 87 const Descriptor& GetDescriptor() const { return *this; } 88 }; 89 90 91 class GlobalDescriptorTable { 92 public: 93 constexpr GlobalDescriptorTable(); 94 95 inline void Load() const; 96 97 unsigned SetTSS(unsigned cpu, 98 const TSSDescriptor& tss); 99 unsigned SetUserTLS(unsigned cpu, 100 addr_t base, size_t limit); 101 private: 102 static constexpr unsigned kFirstTSS = 6; 103 static constexpr unsigned kDescriptorCount 104 = kFirstTSS + SMP_MAX_CPUS * 3; 105 106 alignas(uint64_t) Descriptor fTable[kDescriptorCount]; 107 }; 108 109 enum class InterruptDescriptorType : unsigned { 110 Interrupt = 14, 111 Trap, 112 }; 113 114 class [[gnu::packed]] InterruptDescriptor { 115 public: 116 constexpr InterruptDescriptor(uintptr_t isr, 117 unsigned ist, bool kernelOnly); 118 constexpr InterruptDescriptor(uintptr_t isr); 119 120 static InterruptDescriptor Generate(unsigned index); 121 122 private: 123 uint16_t fBase0; 124 uint16_t fSelector; 125 unsigned fIST :3; 126 unsigned fReserved0 :5; 127 unsigned fType :4; 128 unsigned fReserved1 :1; 129 unsigned fDPL :2; 130 unsigned fPresent :1; 131 uint16_t fBase1; 132 uint32_t fBase2; 133 uint32_t fReserved2; 134 }; 135 136 class InterruptDescriptorTable { 137 public: 138 inline void Load() const; 139 140 static constexpr unsigned kDescriptorCount = 256; 141 142 private: 143 typedef GenerateTable<InterruptDescriptor, InterruptDescriptor::Generate, 144 kDescriptorCount> TableType; 145 alignas(uint64_t) TableType fTable; 146 }; 147 148 class InterruptServiceRoutine { 149 alignas(16) uint8_t fDummy[16]; 150 }; 151 152 extern const InterruptServiceRoutine 153 isr_array[InterruptDescriptorTable::kDescriptorCount]; 154 155 static GlobalDescriptorTable sGDT; 156 static InterruptDescriptorTable sIDT; 157 158 typedef void interrupt_handler_function(iframe* frame); 159 interrupt_handler_function* 160 gInterruptHandlerTable[InterruptDescriptorTable::kDescriptorCount]; 161 162 163 constexpr bool 164 is_code_segment(DescriptorType type) 165 { 166 return type == DescriptorType::CodeExecuteOnly; 167 }; 168 169 170 constexpr 171 Descriptor::Descriptor() 172 : 173 fDescriptor { 0, 0 } 174 { 175 static_assert(sizeof(Descriptor) == sizeof(uint64_t), 176 "Invalid Descriptor size."); 177 } 178 179 180 Descriptor::Descriptor(uint32_t first, uint32_t second) 181 : 182 fDescriptor { first, second } 183 { 184 } 185 186 187 constexpr 188 Descriptor::Descriptor(DescriptorType type, bool kernelOnly, bool compatMode) 189 : 190 fLimit0(-1), 191 fBase0(0), 192 fType(static_cast<unsigned>(type)), 193 fSystem(1), 194 fDPL(kernelOnly ? 0 : 3), 195 fPresent(1), 196 fLimit1(0xf), 197 fUnused(0), 198 fLong(is_code_segment(type) && !compatMode ? 1 : 0), 199 fDB(is_code_segment(type) && !compatMode ? 0 : 1), 200 fGranularity(1), 201 fBase1(0) 202 { 203 } 204 205 206 TSSDescriptor::TSSDescriptor(uintptr_t base, size_t limit) 207 : 208 fSecond(base >> 32, 0) 209 { 210 fLimit0 = static_cast<uint16_t>(limit); 211 fBase0 = base & 0xffffff; 212 fType = static_cast<unsigned>(DescriptorType::TSS); 213 fPresent = 1; 214 fLimit1 = (limit >> 16) & 0xf; 215 fBase1 = static_cast<uint8_t>(base >> 24); 216 } 217 218 219 void 220 TSSDescriptor::LoadTSS(unsigned index) 221 { 222 asm volatile("ltr %w0" : : "r" (index << 3)); 223 } 224 225 226 UserTLSDescriptor::UserTLSDescriptor(uintptr_t base, size_t limit) 227 : Descriptor(DescriptorType::DataWritable, false) 228 { 229 fLimit0 = static_cast<uint16_t>(limit); 230 fBase0 = base & 0xffffff; 231 fLimit1 = (limit >> 16) & 0xf; 232 fBase1 = static_cast<uint8_t>(base >> 24); 233 } 234 235 236 constexpr 237 GlobalDescriptorTable::GlobalDescriptorTable() 238 : 239 fTable { 240 Descriptor(), 241 Descriptor(DescriptorType::CodeExecuteOnly, true), 242 Descriptor(DescriptorType::DataWritable, true), 243 Descriptor(DescriptorType::CodeExecuteOnly, false, true), 244 Descriptor(DescriptorType::DataWritable, false), 245 Descriptor(DescriptorType::CodeExecuteOnly, false), 246 } 247 { 248 static_assert(kDescriptorCount <= 8192, 249 "GDT cannot contain more than 8192 descriptors"); 250 } 251 252 253 void 254 GlobalDescriptorTable::Load() const 255 { 256 struct [[gnu::packed]] { 257 uint16_t fLimit; 258 const void* fAddress; 259 } gdtDescriptor = { 260 sizeof(fTable) - 1, 261 static_cast<const void*>(fTable), 262 }; 263 264 asm volatile("lgdt %0" : : "m" (gdtDescriptor)); 265 } 266 267 268 unsigned 269 GlobalDescriptorTable::SetTSS(unsigned cpu, const TSSDescriptor& tss) 270 { 271 auto index = kFirstTSS + cpu * 3; 272 ASSERT(index + 1 < kDescriptorCount); 273 fTable[index] = tss.GetLower(); 274 fTable[index + 1] = tss.GetUpper(); 275 return index; 276 } 277 278 279 unsigned 280 GlobalDescriptorTable::SetUserTLS(unsigned cpu, uintptr_t base, size_t limit) 281 { 282 auto index = kFirstTSS + cpu * 3 + 2; 283 ASSERT(index + 1 < kDescriptorCount); 284 UserTLSDescriptor desc(base, limit); 285 fTable[index] = desc.GetDescriptor(); 286 return index; 287 } 288 289 290 constexpr 291 InterruptDescriptor::InterruptDescriptor(uintptr_t isr, unsigned ist, 292 bool kernelOnly) 293 : 294 fBase0(isr), 295 fSelector(KERNEL_CODE_SELECTOR), 296 fIST(ist), 297 fReserved0(0), 298 fType(static_cast<unsigned>(InterruptDescriptorType::Interrupt)), 299 fReserved1(0), 300 fDPL(kernelOnly ? 0 : 3), 301 fPresent(1), 302 fBase1(isr >> 16), 303 fBase2(isr >> 32), 304 fReserved2(0) 305 { 306 static_assert(sizeof(InterruptDescriptor) == sizeof(uint64_t) * 2, 307 "Invalid InterruptDescriptor size."); 308 } 309 310 311 constexpr 312 InterruptDescriptor::InterruptDescriptor(uintptr_t isr) 313 : 314 InterruptDescriptor(isr, 0, true) 315 { 316 } 317 318 319 void 320 InterruptDescriptorTable::Load() const 321 { 322 struct [[gnu::packed]] { 323 uint16_t fLimit; 324 const void* fAddress; 325 } gdtDescriptor = { 326 sizeof(fTable) - 1, 327 static_cast<const void*>(fTable.fTable), 328 }; 329 330 asm volatile("lidt %0" : : "m" (gdtDescriptor)); 331 } 332 333 334 InterruptDescriptor 335 InterruptDescriptor::Generate(unsigned index) 336 { 337 return index == 3 338 ? InterruptDescriptor(uintptr_t(isr_array + index), 0, false) 339 : (index == 8 340 ? InterruptDescriptor(uintptr_t(isr_array + index), 1, true) 341 : InterruptDescriptor(uintptr_t(isr_array + index))); 342 } 343 344 345 // #pragma mark - Exception handlers 346 347 348 static void 349 x86_64_general_protection_fault(iframe* frame) 350 { 351 if (debug_debugger_running()) { 352 // Handle GPFs if there is a debugger fault handler installed, for 353 // non-canonical address accesses. 354 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 355 if (cpu->fault_handler != 0) { 356 debug_set_page_fault_info(0, frame->ip, DEBUG_PAGE_FAULT_NO_INFO); 357 frame->ip = cpu->fault_handler; 358 frame->bp = cpu->fault_handler_stack_pointer; 359 return; 360 } 361 } 362 363 x86_unexpected_exception(frame); 364 } 365 366 367 static void 368 x86_64_stack_fault_exception(iframe* frame) 369 { 370 // Non-canonical address accesses which reference the stack cause a stack 371 // fault exception instead of GPF. However, we can treat it like a GPF. 372 x86_64_general_protection_fault(frame); 373 } 374 375 376 // #pragma mark - 377 378 379 void 380 x86_descriptors_preboot_init_percpu(kernel_args* args, int cpu) 381 { 382 new(&sGDT) GlobalDescriptorTable; 383 sGDT.Load(); 384 385 memset(&gCPU[cpu].arch.tss, 0, sizeof(struct tss)); 386 gCPU[cpu].arch.tss.io_map_base = sizeof(struct tss); 387 388 // Set up the double fault IST entry (see x86_descriptors_init()). 389 struct tss* tss = &gCPU[cpu].arch.tss; 390 size_t stackSize; 391 tss->ist1 = (addr_t)x86_get_double_fault_stack(cpu, &stackSize); 392 tss->ist1 += stackSize; 393 394 // Set up the descriptor for this TSS. 395 auto tssIndex = sGDT.SetTSS(cpu, 396 TSSDescriptor(uintptr_t(&gCPU[cpu].arch.tss), sizeof(struct tss))); 397 TSSDescriptor::LoadTSS(tssIndex); 398 399 sGDT.SetUserTLS(cpu, 0, TLS_COMPAT_SIZE); 400 401 new(&sIDT) InterruptDescriptorTable; 402 sIDT.Load(); 403 } 404 405 406 void 407 x86_descriptors_init(kernel_args* args) 408 { 409 // Initialize the interrupt handler table. 410 interrupt_handler_function** table = gInterruptHandlerTable; 411 for (uint32 i = 0; i < ARCH_INTERRUPT_BASE; i++) 412 table[i] = x86_invalid_exception; 413 for (uint32 i = ARCH_INTERRUPT_BASE; 414 i < InterruptDescriptorTable::kDescriptorCount; i++) { 415 table[i] = x86_hardware_interrupt; 416 } 417 418 table[0] = x86_unexpected_exception; // Divide Error Exception (#DE) 419 table[1] = x86_handle_debug_exception; // Debug Exception (#DB) 420 table[2] = x86_fatal_exception; // NMI Interrupt 421 table[3] = x86_handle_breakpoint_exception; // Breakpoint Exception (#BP) 422 table[4] = x86_unexpected_exception; // Overflow Exception (#OF) 423 table[5] = x86_unexpected_exception; // BOUND Range Exceeded Exception (#BR) 424 table[6] = x86_unexpected_exception; // Invalid Opcode Exception (#UD) 425 table[7] = x86_fatal_exception; // Device Not Available Exception (#NM) 426 table[8] = x86_fatal_exception; // Double Fault Exception (#DF) 427 table[9] = x86_fatal_exception; // Coprocessor Segment Overrun 428 table[10] = x86_fatal_exception; // Invalid TSS Exception (#TS) 429 table[11] = x86_fatal_exception; // Segment Not Present (#NP) 430 table[12] = x86_64_stack_fault_exception; // Stack Fault Exception (#SS) 431 table[13] = x86_64_general_protection_fault; // General Protection Exception (#GP) 432 table[14] = x86_page_fault_exception; // Page-Fault Exception (#PF) 433 table[16] = x86_unexpected_exception; // x87 FPU Floating-Point Error (#MF) 434 table[17] = x86_unexpected_exception; // Alignment Check Exception (#AC) 435 table[18] = x86_fatal_exception; // Machine-Check Exception (#MC) 436 table[19] = x86_unexpected_exception; // SIMD Floating-Point Exception (#XF) 437 } 438 439 440 unsigned 441 x86_64_set_user_tls_segment_base(int cpu, addr_t base) 442 { 443 return sGDT.SetUserTLS(cpu, base, TLS_COMPAT_SIZE); 444 } 445