1 /* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "long.h" 8 9 #include <algorithm> 10 11 #include <KernelExport.h> 12 13 // Include the x86_64 version of descriptors.h 14 #define __x86_64__ 15 #include <arch/x86/descriptors.h> 16 #undef __x86_64__ 17 18 #include <arch_system_info.h> 19 #include <boot/platform.h> 20 #include <boot/heap.h> 21 #include <boot/stage2.h> 22 #include <boot/stdio.h> 23 #include <kernel.h> 24 25 #include "debug.h" 26 #include "smp.h" 27 #include "mmu.h" 28 29 30 static const uint64 kTableMappingFlags = 0x7; 31 static const uint64 kLargePageMappingFlags = 0x183; 32 static const uint64 kPageMappingFlags = 0x103; 33 // Global, R/W, Present 34 35 extern "C" void long_enter_kernel(int currentCPU, uint64 stackTop); 36 37 extern uint32 gLongPhysicalGDT; 38 extern uint64 gLongVirtualGDT; 39 extern uint32 gLongPhysicalPML4; 40 extern uint64 gLongKernelEntry; 41 42 43 /*! Convert a 32-bit address to a 64-bit address. */ 44 static inline uint64 45 fix_address(uint64 address) 46 { 47 if(address >= KERNEL_LOAD_BASE) 48 return address - KERNEL_LOAD_BASE + KERNEL_LOAD_BASE_64_BIT; 49 else 50 return address; 51 } 52 53 54 template<typename Type> 55 inline void 56 fix_address(FixedWidthPointer<Type>& p) 57 { 58 if (p != NULL) 59 p.SetTo(fix_address(p.Get())); 60 } 61 62 63 static void 64 long_gdt_init() 65 { 66 // Allocate memory for the GDT. 67 segment_descriptor* gdt = (segment_descriptor*) 68 mmu_allocate_page(&gKernelArgs.arch_args.phys_gdt); 69 gKernelArgs.arch_args.vir_gdt = fix_address((addr_t)gdt); 70 71 dprintf("GDT at phys 0x%lx, virt 0x%llx\n", gKernelArgs.arch_args.phys_gdt, 72 gKernelArgs.arch_args.vir_gdt); 73 74 clear_segment_descriptor(&gdt[0]); 75 76 // Set up code/data segments (TSS segments set up later in the kernel). 77 set_segment_descriptor(&gdt[KERNEL_CODE_SEG / 8], DT_CODE_EXECUTE_ONLY, 78 DPL_KERNEL); 79 set_segment_descriptor(&gdt[KERNEL_DATA_SEG / 8], DT_DATA_WRITEABLE, 80 DPL_KERNEL); 81 set_segment_descriptor(&gdt[USER_CODE_SEG / 8], DT_CODE_EXECUTE_ONLY, 82 DPL_USER); 83 set_segment_descriptor(&gdt[USER_DATA_SEG / 8], DT_DATA_WRITEABLE, 84 DPL_USER); 85 86 // Used by long_enter_kernel(). 87 gLongPhysicalGDT = gKernelArgs.arch_args.phys_gdt; 88 gLongVirtualGDT = gKernelArgs.arch_args.vir_gdt; 89 } 90 91 92 static void 93 long_idt_init() 94 { 95 interrupt_descriptor* idt = (interrupt_descriptor*) 96 mmu_allocate_page(&gKernelArgs.arch_args.phys_idt); 97 gKernelArgs.arch_args.vir_idt = fix_address((addr_t)idt); 98 99 dprintf("IDT at phys %#lx, virt %#llx\n", gKernelArgs.arch_args.phys_idt, 100 gKernelArgs.arch_args.vir_idt); 101 102 // The 32-bit kernel gets an IDT with the loader's exception handlers until 103 // it can set up its own. Can't do that here because they won't work after 104 // switching to long mode. Therefore, just clear the IDT and leave the 105 // kernel to set it up. 106 memset(idt, 0, B_PAGE_SIZE); 107 } 108 109 110 static void 111 long_mmu_init() 112 { 113 uint64* pml4; 114 uint64* pdpt; 115 uint64* pageDir; 116 uint64* pageTable; 117 addr_t physicalAddress; 118 119 // Allocate the top level PML4. 120 pml4 = (uint64*)mmu_allocate_page(&gKernelArgs.arch_args.phys_pgdir); 121 memset(pml4, 0, B_PAGE_SIZE); 122 gKernelArgs.arch_args.vir_pgdir = fix_address((uint64)(addr_t)pml4); 123 124 // Store the virtual memory usage information. 125 gKernelArgs.virtual_allocated_range[0].start = KERNEL_LOAD_BASE_64_BIT; 126 gKernelArgs.virtual_allocated_range[0].size = mmu_get_virtual_usage(); 127 gKernelArgs.num_virtual_allocated_ranges = 1; 128 gKernelArgs.arch_args.virtual_end = ROUNDUP(KERNEL_LOAD_BASE_64_BIT 129 + gKernelArgs.virtual_allocated_range[0].size, 0x200000); 130 131 // Find the highest physical memory address. We map all physical memory 132 // into the kernel address space, so we want to make sure we map everything 133 // we have available. 134 uint64 maxAddress = 0; 135 for (uint32 i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 136 maxAddress = std::max(maxAddress, 137 gKernelArgs.physical_memory_range[i].start 138 + gKernelArgs.physical_memory_range[i].size); 139 } 140 141 // Want to map at least 4GB, there may be stuff other than usable RAM that 142 // could be in the first 4GB of physical address space. 143 maxAddress = std::max(maxAddress, (uint64)0x100000000ll); 144 maxAddress = ROUNDUP(maxAddress, 0x40000000); 145 146 // Currently only use 1 PDPT (512GB). This will need to change if someone 147 // wants to use Haiku on a box with more than 512GB of RAM but that's 148 // probably not going to happen any time soon. 149 if (maxAddress / 0x40000000 > 512) 150 panic("Can't currently support more than 512GB of RAM!"); 151 152 // Create page tables for the physical map area. Also map this PDPT 153 // temporarily at the bottom of the address space so that we are identity 154 // mapped. 155 156 pdpt = (uint64*)mmu_allocate_page(&physicalAddress); 157 memset(pdpt, 0, B_PAGE_SIZE); 158 pml4[510] = physicalAddress | kTableMappingFlags; 159 pml4[0] = physicalAddress | kTableMappingFlags; 160 161 for (uint64 i = 0; i < maxAddress; i += 0x40000000) { 162 pageDir = (uint64*)mmu_allocate_page(&physicalAddress); 163 memset(pageDir, 0, B_PAGE_SIZE); 164 pdpt[i / 0x40000000] = physicalAddress | kTableMappingFlags; 165 166 for (uint64 j = 0; j < 0x40000000; j += 0x200000) { 167 pageDir[j / 0x200000] = (i + j) | kLargePageMappingFlags; 168 } 169 170 mmu_free(pageDir, B_PAGE_SIZE); 171 } 172 173 mmu_free(pdpt, B_PAGE_SIZE); 174 175 // Allocate tables for the kernel mappings. 176 177 pdpt = (uint64*)mmu_allocate_page(&physicalAddress); 178 memset(pdpt, 0, B_PAGE_SIZE); 179 pml4[511] = physicalAddress | kTableMappingFlags; 180 181 pageDir = (uint64*)mmu_allocate_page(&physicalAddress); 182 memset(pageDir, 0, B_PAGE_SIZE); 183 pdpt[510] = physicalAddress | kTableMappingFlags; 184 185 // We can now allocate page tables and duplicate the mappings across from 186 // the 32-bit address space to them. 187 pageTable = NULL; 188 for (uint32 i = 0; i < gKernelArgs.virtual_allocated_range[0].size 189 / B_PAGE_SIZE; i++) { 190 if ((i % 512) == 0) { 191 if (pageTable) 192 mmu_free(pageTable, B_PAGE_SIZE); 193 194 pageTable = (uint64*)mmu_allocate_page(&physicalAddress); 195 memset(pageTable, 0, B_PAGE_SIZE); 196 pageDir[i / 512] = physicalAddress | kTableMappingFlags; 197 } 198 199 // Get the physical address to map. 200 if (!mmu_get_virtual_mapping(KERNEL_LOAD_BASE + (i * B_PAGE_SIZE), 201 &physicalAddress)) 202 continue; 203 204 pageTable[i % 512] = physicalAddress | kPageMappingFlags; 205 } 206 207 if (pageTable) 208 mmu_free(pageTable, B_PAGE_SIZE); 209 mmu_free(pageDir, B_PAGE_SIZE); 210 mmu_free(pdpt, B_PAGE_SIZE); 211 212 // Sort the address ranges. 213 sort_address_ranges(gKernelArgs.physical_memory_range, 214 gKernelArgs.num_physical_memory_ranges); 215 sort_address_ranges(gKernelArgs.physical_allocated_range, 216 gKernelArgs.num_physical_allocated_ranges); 217 sort_address_ranges(gKernelArgs.virtual_allocated_range, 218 gKernelArgs.num_virtual_allocated_ranges); 219 220 dprintf("phys memory ranges:\n"); 221 for (uint32 i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 222 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 223 gKernelArgs.physical_memory_range[i].start, 224 gKernelArgs.physical_memory_range[i].size); 225 } 226 227 dprintf("allocated phys memory ranges:\n"); 228 for (uint32 i = 0; i < gKernelArgs.num_physical_allocated_ranges; i++) { 229 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 230 gKernelArgs.physical_allocated_range[i].start, 231 gKernelArgs.physical_allocated_range[i].size); 232 } 233 234 dprintf("allocated virt memory ranges:\n"); 235 for (uint32 i = 0; i < gKernelArgs.num_virtual_allocated_ranges; i++) { 236 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 237 gKernelArgs.virtual_allocated_range[i].start, 238 gKernelArgs.virtual_allocated_range[i].size); 239 } 240 241 gLongPhysicalPML4 = gKernelArgs.arch_args.phys_pgdir; 242 } 243 244 245 static void 246 convert_preloaded_image(preloaded_elf64_image* image) 247 { 248 fix_address(image->next); 249 fix_address(image->name); 250 fix_address(image->debug_string_table); 251 fix_address(image->syms); 252 fix_address(image->rel); 253 fix_address(image->rela); 254 fix_address(image->pltrel); 255 fix_address(image->debug_symbols); 256 } 257 258 259 /*! Convert all addresses in kernel_args to 64-bit addresses. */ 260 static void 261 convert_kernel_args() 262 { 263 fix_address(gKernelArgs.boot_volume); 264 fix_address(gKernelArgs.vesa_modes); 265 fix_address(gKernelArgs.edid_info); 266 fix_address(gKernelArgs.debug_output); 267 fix_address(gKernelArgs.boot_splash); 268 fix_address(gKernelArgs.arch_args.apic); 269 fix_address(gKernelArgs.arch_args.hpet); 270 271 convert_preloaded_image(static_cast<preloaded_elf64_image*>( 272 gKernelArgs.kernel_image.Pointer())); 273 fix_address(gKernelArgs.kernel_image); 274 275 // Iterate over the preloaded images. Must save the next address before 276 // converting, as the next pointer will be converted. 277 preloaded_image* image = gKernelArgs.preloaded_images; 278 fix_address(gKernelArgs.preloaded_images); 279 while (image != NULL) { 280 preloaded_image* next = image->next; 281 convert_preloaded_image(static_cast<preloaded_elf64_image*>(image)); 282 image = next; 283 } 284 285 // Set correct kernel args range addresses. 286 dprintf("kernel args ranges:\n"); 287 for (uint32 i = 0; i < gKernelArgs.num_kernel_args_ranges; i++) { 288 gKernelArgs.kernel_args_range[i].start = fix_address( 289 gKernelArgs.kernel_args_range[i].start); 290 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 291 gKernelArgs.kernel_args_range[i].start, 292 gKernelArgs.kernel_args_range[i].size); 293 } 294 295 // Fix driver settings files. 296 driver_settings_file* file = gKernelArgs.driver_settings; 297 fix_address(gKernelArgs.driver_settings); 298 while (file != NULL) { 299 driver_settings_file* next = file->next; 300 fix_address(file->next); 301 fix_address(file->buffer); 302 file = next; 303 } 304 } 305 306 307 static void 308 long_smp_start_kernel(void) 309 { 310 uint32 cpu = smp_get_current_cpu(); 311 312 // Important. Make sure supervisor threads can fault on read only pages... 313 asm("movl %%eax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1)); 314 asm("cld"); 315 asm("fninit"); 316 317 // Fix our kernel stack address. 318 gKernelArgs.cpu_kstack[cpu].start 319 = fix_address(gKernelArgs.cpu_kstack[cpu].start); 320 321 long_enter_kernel(cpu, gKernelArgs.cpu_kstack[cpu].start 322 + gKernelArgs.cpu_kstack[cpu].size); 323 324 panic("Shouldn't get here"); 325 } 326 327 328 void 329 long_start_kernel() 330 { 331 // Check whether long mode is supported. 332 cpuid_info info; 333 get_current_cpuid(&info, 0x80000001); 334 if ((info.regs.edx & (1 << 29)) == 0) 335 panic("64-bit kernel requires a 64-bit CPU"); 336 337 preloaded_elf64_image *image = static_cast<preloaded_elf64_image *>( 338 gKernelArgs.kernel_image.Pointer()); 339 340 smp_init_other_cpus(); 341 342 long_gdt_init(); 343 long_idt_init(); 344 long_mmu_init(); 345 debug_cleanup(); 346 convert_kernel_args(); 347 348 // Save the kernel entry point address. 349 gLongKernelEntry = image->elf_header.e_entry; 350 dprintf("kernel entry at %#llx\n", gLongKernelEntry); 351 352 // Fix our kernel stack address. 353 gKernelArgs.cpu_kstack[0].start 354 = fix_address(gKernelArgs.cpu_kstack[0].start); 355 356 // We're about to enter the kernel -- disable console output. 357 stdout = NULL; 358 359 smp_boot_other_cpus(long_smp_start_kernel); 360 361 // Enter the kernel! 362 long_enter_kernel(0, gKernelArgs.cpu_kstack[0].start 363 + gKernelArgs.cpu_kstack[0].size); 364 365 panic("Shouldn't get here"); 366 } 367