1 /* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "long.h" 8 9 #include <algorithm> 10 11 #include <KernelExport.h> 12 13 // Include the x86_64 version of descriptors.h 14 #define __x86_64__ 15 #include <arch/x86/descriptors.h> 16 #undef __x86_64__ 17 18 #include <arch_system_info.h> 19 #include <boot/platform.h> 20 #include <boot/heap.h> 21 #include <boot/stage2.h> 22 #include <boot/stdio.h> 23 #include <kernel.h> 24 25 #include "debug.h" 26 #include "mmu.h" 27 #include "smp.h" 28 29 30 static const uint64 kTableMappingFlags = 0x7; 31 static const uint64 kLargePageMappingFlags = 0x183; 32 static const uint64 kPageMappingFlags = 0x103; 33 // Global, R/W, Present 34 35 extern "C" void long_enter_kernel(int currentCPU, uint64 stackTop); 36 37 extern uint64 gLongGDT; 38 extern uint32 gLongPhysicalPML4; 39 extern uint64 gLongKernelEntry; 40 41 42 /*! Convert a 32-bit address to a 64-bit address. */ 43 static inline uint64 44 fix_address(uint64 address) 45 { 46 if(address >= KERNEL_LOAD_BASE) 47 return address - KERNEL_LOAD_BASE + KERNEL_LOAD_BASE_64_BIT; 48 else 49 return address; 50 } 51 52 53 template<typename Type> 54 inline void 55 fix_address(FixedWidthPointer<Type>& p) 56 { 57 if (p != NULL) 58 p.SetTo(fix_address(p.Get())); 59 } 60 61 62 static void 63 long_gdt_init() 64 { 65 STATIC_ASSERT(BOOT_GDT_SEGMENT_COUNT > KERNEL_CODE_SEGMENT 66 && BOOT_GDT_SEGMENT_COUNT > KERNEL_DATA_SEGMENT 67 && BOOT_GDT_SEGMENT_COUNT > USER_CODE_SEGMENT 68 && BOOT_GDT_SEGMENT_COUNT > USER_DATA_SEGMENT); 69 70 clear_segment_descriptor(&gBootGDT[0]); 71 72 // Set up code/data segments (TSS segments set up later in the kernel). 73 set_segment_descriptor(&gBootGDT[KERNEL_CODE_SEGMENT], DT_CODE_EXECUTE_ONLY, 74 DPL_KERNEL); 75 set_segment_descriptor(&gBootGDT[KERNEL_DATA_SEGMENT], DT_DATA_WRITEABLE, 76 DPL_KERNEL); 77 set_segment_descriptor(&gBootGDT[USER_CODE_SEGMENT], DT_CODE_EXECUTE_ONLY, 78 DPL_USER); 79 set_segment_descriptor(&gBootGDT[USER_DATA_SEGMENT], DT_DATA_WRITEABLE, 80 DPL_USER); 81 82 // Used by long_enter_kernel(). 83 gLongGDT = fix_address((addr_t)gBootGDT); 84 dprintf("GDT at 0x%llx\n", gLongGDT); 85 } 86 87 88 static void 89 long_mmu_init() 90 { 91 uint64* pml4; 92 uint64* pdpt; 93 uint64* pageDir; 94 uint64* pageTable; 95 addr_t physicalAddress; 96 97 // Allocate the top level PML4. 98 pml4 = (uint64*)mmu_allocate_page(&gKernelArgs.arch_args.phys_pgdir); 99 memset(pml4, 0, B_PAGE_SIZE); 100 gKernelArgs.arch_args.vir_pgdir = fix_address((uint64)(addr_t)pml4); 101 102 // Store the virtual memory usage information. 103 gKernelArgs.virtual_allocated_range[0].start = KERNEL_LOAD_BASE_64_BIT; 104 gKernelArgs.virtual_allocated_range[0].size = mmu_get_virtual_usage(); 105 gKernelArgs.num_virtual_allocated_ranges = 1; 106 gKernelArgs.arch_args.virtual_end = ROUNDUP(KERNEL_LOAD_BASE_64_BIT 107 + gKernelArgs.virtual_allocated_range[0].size, 0x200000); 108 109 // Find the highest physical memory address. We map all physical memory 110 // into the kernel address space, so we want to make sure we map everything 111 // we have available. 112 uint64 maxAddress = 0; 113 for (uint32 i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 114 maxAddress = std::max(maxAddress, 115 gKernelArgs.physical_memory_range[i].start 116 + gKernelArgs.physical_memory_range[i].size); 117 } 118 119 // Want to map at least 4GB, there may be stuff other than usable RAM that 120 // could be in the first 4GB of physical address space. 121 maxAddress = std::max(maxAddress, (uint64)0x100000000ll); 122 maxAddress = ROUNDUP(maxAddress, 0x40000000); 123 124 // Currently only use 1 PDPT (512GB). This will need to change if someone 125 // wants to use Haiku on a box with more than 512GB of RAM but that's 126 // probably not going to happen any time soon. 127 if (maxAddress / 0x40000000 > 512) 128 panic("Can't currently support more than 512GB of RAM!"); 129 130 // Create page tables for the physical map area. Also map this PDPT 131 // temporarily at the bottom of the address space so that we are identity 132 // mapped. 133 134 pdpt = (uint64*)mmu_allocate_page(&physicalAddress); 135 memset(pdpt, 0, B_PAGE_SIZE); 136 pml4[510] = physicalAddress | kTableMappingFlags; 137 pml4[0] = physicalAddress | kTableMappingFlags; 138 139 for (uint64 i = 0; i < maxAddress; i += 0x40000000) { 140 pageDir = (uint64*)mmu_allocate_page(&physicalAddress); 141 memset(pageDir, 0, B_PAGE_SIZE); 142 pdpt[i / 0x40000000] = physicalAddress | kTableMappingFlags; 143 144 for (uint64 j = 0; j < 0x40000000; j += 0x200000) { 145 pageDir[j / 0x200000] = (i + j) | kLargePageMappingFlags; 146 } 147 148 mmu_free(pageDir, B_PAGE_SIZE); 149 } 150 151 mmu_free(pdpt, B_PAGE_SIZE); 152 153 // Allocate tables for the kernel mappings. 154 155 pdpt = (uint64*)mmu_allocate_page(&physicalAddress); 156 memset(pdpt, 0, B_PAGE_SIZE); 157 pml4[511] = physicalAddress | kTableMappingFlags; 158 159 pageDir = (uint64*)mmu_allocate_page(&physicalAddress); 160 memset(pageDir, 0, B_PAGE_SIZE); 161 pdpt[510] = physicalAddress | kTableMappingFlags; 162 163 // We can now allocate page tables and duplicate the mappings across from 164 // the 32-bit address space to them. 165 pageTable = NULL; 166 for (uint32 i = 0; i < gKernelArgs.virtual_allocated_range[0].size 167 / B_PAGE_SIZE; i++) { 168 if ((i % 512) == 0) { 169 if (pageTable) 170 mmu_free(pageTable, B_PAGE_SIZE); 171 172 pageTable = (uint64*)mmu_allocate_page(&physicalAddress); 173 memset(pageTable, 0, B_PAGE_SIZE); 174 pageDir[i / 512] = physicalAddress | kTableMappingFlags; 175 } 176 177 // Get the physical address to map. 178 if (!mmu_get_virtual_mapping(KERNEL_LOAD_BASE + (i * B_PAGE_SIZE), 179 &physicalAddress)) 180 continue; 181 182 pageTable[i % 512] = physicalAddress | kPageMappingFlags; 183 } 184 185 if (pageTable) 186 mmu_free(pageTable, B_PAGE_SIZE); 187 mmu_free(pageDir, B_PAGE_SIZE); 188 mmu_free(pdpt, B_PAGE_SIZE); 189 190 // Sort the address ranges. 191 sort_address_ranges(gKernelArgs.physical_memory_range, 192 gKernelArgs.num_physical_memory_ranges); 193 sort_address_ranges(gKernelArgs.physical_allocated_range, 194 gKernelArgs.num_physical_allocated_ranges); 195 sort_address_ranges(gKernelArgs.virtual_allocated_range, 196 gKernelArgs.num_virtual_allocated_ranges); 197 198 dprintf("phys memory ranges:\n"); 199 for (uint32 i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 200 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 201 gKernelArgs.physical_memory_range[i].start, 202 gKernelArgs.physical_memory_range[i].size); 203 } 204 205 dprintf("allocated phys memory ranges:\n"); 206 for (uint32 i = 0; i < gKernelArgs.num_physical_allocated_ranges; i++) { 207 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 208 gKernelArgs.physical_allocated_range[i].start, 209 gKernelArgs.physical_allocated_range[i].size); 210 } 211 212 dprintf("allocated virt memory ranges:\n"); 213 for (uint32 i = 0; i < gKernelArgs.num_virtual_allocated_ranges; i++) { 214 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 215 gKernelArgs.virtual_allocated_range[i].start, 216 gKernelArgs.virtual_allocated_range[i].size); 217 } 218 219 gLongPhysicalPML4 = gKernelArgs.arch_args.phys_pgdir; 220 } 221 222 223 static void 224 convert_preloaded_image(preloaded_elf64_image* image) 225 { 226 fix_address(image->next); 227 fix_address(image->name); 228 fix_address(image->debug_string_table); 229 fix_address(image->syms); 230 fix_address(image->rel); 231 fix_address(image->rela); 232 fix_address(image->pltrel); 233 fix_address(image->debug_symbols); 234 } 235 236 237 /*! Convert all addresses in kernel_args to 64-bit addresses. */ 238 static void 239 convert_kernel_args() 240 { 241 fix_address(gKernelArgs.boot_volume); 242 fix_address(gKernelArgs.vesa_modes); 243 fix_address(gKernelArgs.edid_info); 244 fix_address(gKernelArgs.debug_output); 245 fix_address(gKernelArgs.previous_debug_output); 246 fix_address(gKernelArgs.boot_splash); 247 fix_address(gKernelArgs.arch_args.apic); 248 fix_address(gKernelArgs.arch_args.hpet); 249 250 convert_preloaded_image(static_cast<preloaded_elf64_image*>( 251 gKernelArgs.kernel_image.Pointer())); 252 fix_address(gKernelArgs.kernel_image); 253 254 // Iterate over the preloaded images. Must save the next address before 255 // converting, as the next pointer will be converted. 256 preloaded_image* image = gKernelArgs.preloaded_images; 257 fix_address(gKernelArgs.preloaded_images); 258 while (image != NULL) { 259 preloaded_image* next = image->next; 260 convert_preloaded_image(static_cast<preloaded_elf64_image*>(image)); 261 image = next; 262 } 263 264 // Set correct kernel args range addresses. 265 dprintf("kernel args ranges:\n"); 266 for (uint32 i = 0; i < gKernelArgs.num_kernel_args_ranges; i++) { 267 gKernelArgs.kernel_args_range[i].start = fix_address( 268 gKernelArgs.kernel_args_range[i].start); 269 dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 270 gKernelArgs.kernel_args_range[i].start, 271 gKernelArgs.kernel_args_range[i].size); 272 } 273 274 // Fix driver settings files. 275 driver_settings_file* file = gKernelArgs.driver_settings; 276 fix_address(gKernelArgs.driver_settings); 277 while (file != NULL) { 278 driver_settings_file* next = file->next; 279 fix_address(file->next); 280 fix_address(file->buffer); 281 file = next; 282 } 283 } 284 285 286 static void 287 enable_sse() 288 { 289 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 290 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 291 } 292 293 294 static void 295 long_smp_start_kernel(void) 296 { 297 uint32 cpu = smp_get_current_cpu(); 298 299 // Important. Make sure supervisor threads can fault on read only pages... 300 asm("movl %%eax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1)); 301 asm("cld"); 302 asm("fninit"); 303 enable_sse(); 304 305 // Fix our kernel stack address. 306 gKernelArgs.cpu_kstack[cpu].start 307 = fix_address(gKernelArgs.cpu_kstack[cpu].start); 308 309 long_enter_kernel(cpu, gKernelArgs.cpu_kstack[cpu].start 310 + gKernelArgs.cpu_kstack[cpu].size); 311 312 panic("Shouldn't get here"); 313 } 314 315 316 void 317 long_start_kernel() 318 { 319 // Check whether long mode is supported. 320 cpuid_info info; 321 get_current_cpuid(&info, 0x80000001, 0); 322 if ((info.regs.edx & (1 << 29)) == 0) 323 panic("64-bit kernel requires a 64-bit CPU"); 324 325 enable_sse(); 326 327 preloaded_elf64_image *image = static_cast<preloaded_elf64_image *>( 328 gKernelArgs.kernel_image.Pointer()); 329 330 smp_init_other_cpus(); 331 332 long_gdt_init(); 333 debug_cleanup(); 334 long_mmu_init(); 335 convert_kernel_args(); 336 337 // Save the kernel entry point address. 338 gLongKernelEntry = image->elf_header.e_entry; 339 dprintf("kernel entry at %#llx\n", gLongKernelEntry); 340 341 // Fix our kernel stack address. 342 gKernelArgs.cpu_kstack[0].start 343 = fix_address(gKernelArgs.cpu_kstack[0].start); 344 345 // We're about to enter the kernel -- disable console output. 346 stdout = NULL; 347 348 smp_boot_other_cpus(long_smp_start_kernel); 349 350 // Enter the kernel! 351 long_enter_kernel(0, gKernelArgs.cpu_kstack[0].start 352 + gKernelArgs.cpu_kstack[0].size); 353 354 panic("Shouldn't get here"); 355 } 356