1 /* 2 * Copyright 2016 Haiku, Inc. All rights reserved. 3 * Copyright 2014, Jessica Hamilton, jessica.l.hamilton@gmail.com. 4 * Copyright 2014, Henry Harrington, henry.harrington@gmail.com. 5 * Distributed under the terms of the MIT License. 6 */ 7 8 9 #include <algorithm> 10 11 #include <boot/addr_range.h> 12 #include <boot/platform.h> 13 #include <boot/stage2.h> 14 #include <kernel/arch/x86/arch_kernel.h> 15 #include <kernel/kernel.h> 16 17 #include "efi_platform.h" 18 #include "mmu.h" 19 20 21 struct allocated_memory_region { 22 allocated_memory_region *next; 23 uint64_t vaddr; 24 uint64_t paddr; 25 size_t size; 26 bool released; 27 }; 28 29 30 static uint64_t next_virtual_address = KERNEL_LOAD_BASE_64_BIT + 32 * 1024 * 1024; 31 static allocated_memory_region *allocated_memory_regions = NULL; 32 33 34 static uint64_t mmu_allocate_page() 35 { 36 EFI_PHYSICAL_ADDRESS addr; 37 EFI_STATUS s = kBootServices->AllocatePages(AllocateAnyPages, EfiLoaderData, 1, &addr); 38 if (s != EFI_SUCCESS) 39 panic("Unabled to allocate memory: %li", s); 40 41 return addr; 42 } 43 44 45 uint64_t 46 mmu_generate_post_efi_page_tables(UINTN memory_map_size, 47 EFI_MEMORY_DESCRIPTOR *memory_map, UINTN descriptor_size, 48 UINTN descriptor_version) 49 { 50 // Generate page tables, matching bios_ia32/long.cpp. 51 uint64_t *pml4; 52 uint64_t *pdpt; 53 uint64_t *pageDir; 54 uint64_t *pageTable; 55 56 // Allocate the top level PML4. 57 pml4 = NULL; 58 if (platform_allocate_region((void**)&pml4, B_PAGE_SIZE, 0, false) != B_OK) 59 panic("Failed to allocate PML4."); 60 gKernelArgs.arch_args.phys_pgdir = (uint32_t)(addr_t)pml4; 61 memset(pml4, 0, B_PAGE_SIZE); 62 platform_bootloader_address_to_kernel_address(pml4, &gKernelArgs.arch_args.vir_pgdir); 63 64 // Store the virtual memory usage information. 65 gKernelArgs.virtual_allocated_range[0].start = KERNEL_LOAD_BASE_64_BIT; 66 gKernelArgs.virtual_allocated_range[0].size = next_virtual_address - KERNEL_LOAD_BASE_64_BIT; 67 gKernelArgs.num_virtual_allocated_ranges = 1; 68 gKernelArgs.arch_args.virtual_end = ROUNDUP(KERNEL_LOAD_BASE_64_BIT 69 + gKernelArgs.virtual_allocated_range[0].size, 0x200000); 70 71 // Find the highest physical memory address. We map all physical memory 72 // into the kernel address space, so we want to make sure we map everything 73 // we have available. 74 uint64 maxAddress = 0; 75 for (UINTN i = 0; i < memory_map_size / descriptor_size; ++i) { 76 EFI_MEMORY_DESCRIPTOR *entry = (EFI_MEMORY_DESCRIPTOR *)((addr_t)memory_map + i * descriptor_size); 77 maxAddress = std::max(maxAddress, 78 entry->PhysicalStart + entry->NumberOfPages * 4096); 79 } 80 81 // Want to map at least 4GB, there may be stuff other than usable RAM that 82 // could be in the first 4GB of physical address space. 83 maxAddress = std::max(maxAddress, (uint64)0x100000000ll); 84 maxAddress = ROUNDUP(maxAddress, 0x40000000); 85 86 // Currently only use 1 PDPT (512GB). This will need to change if someone 87 // wants to use Haiku on a box with more than 512GB of RAM but that's 88 // probably not going to happen any time soon. 89 if (maxAddress / 0x40000000 > 512) 90 panic("Can't currently support more than 512GB of RAM!"); 91 92 // Create page tables for the physical map area. Also map this PDPT 93 // temporarily at the bottom of the address space so that we are identity 94 // mapped. 95 96 pdpt = (uint64*)mmu_allocate_page(); 97 memset(pdpt, 0, B_PAGE_SIZE); 98 pml4[510] = (addr_t)pdpt | kTableMappingFlags; 99 pml4[0] = (addr_t)pdpt | kTableMappingFlags; 100 101 for (uint64 i = 0; i < maxAddress; i += 0x40000000) { 102 pageDir = (uint64*)mmu_allocate_page(); 103 memset(pageDir, 0, B_PAGE_SIZE); 104 pdpt[i / 0x40000000] = (addr_t)pageDir | kTableMappingFlags; 105 106 for (uint64 j = 0; j < 0x40000000; j += 0x200000) { 107 pageDir[j / 0x200000] = (i + j) | kLargePageMappingFlags; 108 } 109 } 110 111 // Allocate tables for the kernel mappings. 112 113 pdpt = (uint64*)mmu_allocate_page(); 114 memset(pdpt, 0, B_PAGE_SIZE); 115 pml4[511] = (addr_t)pdpt | kTableMappingFlags; 116 117 pageDir = (uint64*)mmu_allocate_page(); 118 memset(pageDir, 0, B_PAGE_SIZE); 119 pdpt[510] = (addr_t)pageDir | kTableMappingFlags; 120 121 // We can now allocate page tables and duplicate the mappings across from 122 // the 32-bit address space to them. 123 pageTable = NULL; // shush, compiler. 124 for (uint32 i = 0; i < gKernelArgs.virtual_allocated_range[0].size 125 / B_PAGE_SIZE; i++) { 126 if ((i % 512) == 0) { 127 pageTable = (uint64*)mmu_allocate_page(); 128 memset(pageTable, 0, B_PAGE_SIZE); 129 pageDir[i / 512] = (addr_t)pageTable | kTableMappingFlags; 130 } 131 132 // Get the physical address to map. 133 void *phys; 134 if (platform_kernel_address_to_bootloader_address(KERNEL_LOAD_BASE_64_BIT + (i * B_PAGE_SIZE), 135 &phys) != B_OK) 136 continue; 137 138 pageTable[i % 512] = (addr_t)phys | kPageMappingFlags; 139 } 140 141 return (uint64)pml4; 142 } 143 144 145 // Called after EFI boot services exit. 146 // Currently assumes that the memory map is sane... Sorted and no overlapping 147 // regions. 148 void 149 mmu_post_efi_setup(UINTN memory_map_size, EFI_MEMORY_DESCRIPTOR *memory_map, UINTN descriptor_size, UINTN descriptor_version) 150 { 151 // Add physical memory to the kernel args and update virtual addresses for EFI regions.. 152 addr_t addr = (addr_t)memory_map; 153 gKernelArgs.num_physical_memory_ranges = 0; 154 for (UINTN i = 0; i < memory_map_size / descriptor_size; ++i) { 155 EFI_MEMORY_DESCRIPTOR *entry = (EFI_MEMORY_DESCRIPTOR *)(addr + i * descriptor_size); 156 switch (entry->Type) { 157 case EfiLoaderCode: 158 case EfiLoaderData: 159 case EfiBootServicesCode: 160 case EfiBootServicesData: 161 case EfiConventionalMemory: { 162 // Usable memory. 163 // Ignore memory below 1MB and above 512GB. 164 uint64_t base = entry->PhysicalStart; 165 uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096; 166 if (base < 0x100000) 167 base = 0x100000; 168 if (end > (512ull * 1024 * 1024 * 1024)) 169 end = 512ull * 1024 * 1024 * 1024; 170 if (base >= end) 171 break; 172 uint64_t size = end - base; 173 174 insert_physical_memory_range(base, size); 175 // LoaderData memory is bootloader allocated memory, possibly 176 // containing the kernel or loaded drivers. 177 if (entry->Type == EfiLoaderData) 178 insert_physical_allocated_range(base, size); 179 break; 180 } 181 case EfiACPIReclaimMemory: 182 // ACPI reclaim -- physical memory we could actually use later 183 gKernelArgs.ignored_physical_memory += entry->NumberOfPages * 4096; 184 break; 185 case EfiRuntimeServicesCode: 186 case EfiRuntimeServicesData: 187 entry->VirtualStart = entry->PhysicalStart; 188 break; 189 } 190 } 191 192 // Sort the address ranges. 193 sort_address_ranges(gKernelArgs.physical_memory_range, 194 gKernelArgs.num_physical_memory_ranges); 195 sort_address_ranges(gKernelArgs.physical_allocated_range, 196 gKernelArgs.num_physical_allocated_ranges); 197 sort_address_ranges(gKernelArgs.virtual_allocated_range, 198 gKernelArgs.num_virtual_allocated_ranges); 199 200 // Switch EFI to virtual mode, using the kernel pmap. 201 // Something involving ConvertPointer might need to be done after this? 202 // http://wiki.phoenix.com/wiki/index.php/EFI_RUNTIME_SERVICES#SetVirtualAddressMap.28.29 203 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, descriptor_version, memory_map); 204 205 // Important. Make sure supervisor threads can fault on read only pages... 206 asm("mov %%rax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1)); 207 } 208 209 210 // Platform allocator. 211 // The bootloader assumes that bootloader address space == kernel address space. 212 // This is not true until just before the kernel is booted, so an ugly hack is 213 // used to cover the difference. platform_allocate_region allocates addresses 214 // in bootloader space, but can convert them to kernel space. The ELF loader 215 // accesses kernel memory via Mao(), and much later in the boot process, 216 // addresses in the kernel argument struct are converted from bootloader 217 // addresses to kernel addresses. 218 219 extern "C" status_t 220 platform_allocate_region(void **_address, size_t size, uint8 /* protection */, bool exactAddress) 221 { 222 // We don't have any control over the page tables, give up right away if an 223 // exactAddress is wanted. 224 if (exactAddress) 225 return B_NO_MEMORY; 226 227 EFI_PHYSICAL_ADDRESS addr; 228 size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE); 229 allocated_memory_region *region = new(std::nothrow) allocated_memory_region; 230 231 if (region == NULL) 232 return B_NO_MEMORY; 233 234 EFI_STATUS status = kBootServices->AllocatePages(AllocateAnyPages, 235 EfiLoaderData, aligned_size / B_PAGE_SIZE, &addr); 236 if (status != EFI_SUCCESS) { 237 delete region; 238 return B_NO_MEMORY; 239 } 240 241 // Addresses above 512GB not supported. 242 // Memory map regions above 512GB can be ignored, but if EFI returns pages 243 // above that there's nothing that can be done to fix it. 244 if (addr + size > (512ull * 1024 * 1024 * 1024)) 245 panic("Can't currently support more than 512GB of RAM!"); 246 247 region->next = allocated_memory_regions; 248 allocated_memory_regions = region; 249 region->vaddr = 0; 250 region->paddr = addr; 251 region->size = size; 252 region->released = false; 253 254 if (*_address != NULL) { 255 region->vaddr = (uint64_t)*_address; 256 } 257 258 //dprintf("Allocated region %#lx (requested %p) %#lx %lu\n", region->vaddr, *_address, region->paddr, region->size); 259 260 *_address = (void *)region->paddr; 261 262 return B_OK; 263 } 264 265 266 /*! 267 Neither \a virtualAddress nor \a size need to be aligned, but the function 268 will map all pages the range intersects with. 269 If physicalAddress is not page-aligned, the returned virtual address will 270 have the same "misalignment". 271 */ 272 extern "C" addr_t 273 mmu_map_physical_memory(addr_t physicalAddress, size_t size, uint32 flags) 274 { 275 addr_t pageOffset = physicalAddress & (B_PAGE_SIZE - 1); 276 277 physicalAddress -= pageOffset; 278 size += pageOffset; 279 280 if (insert_physical_allocated_range(physicalAddress, ROUNDUP(size, B_PAGE_SIZE)) != B_OK) 281 return B_NO_MEMORY; 282 283 return physicalAddress + pageOffset; 284 } 285 286 287 extern "C" void 288 mmu_free(void *virtualAddress, size_t size) 289 { 290 addr_t physicalAddress = (addr_t)virtualAddress; 291 addr_t pageOffset = physicalAddress & (B_PAGE_SIZE - 1); 292 293 physicalAddress -= pageOffset; 294 size += pageOffset; 295 296 size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE); 297 298 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 299 if (region->paddr == physicalAddress && region->size == aligned_size) { 300 region->released = true; 301 return; 302 } 303 } 304 } 305 306 307 static allocated_memory_region * 308 get_region(void *address, size_t size) 309 { 310 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 311 if (region->paddr == (uint64_t)address && region->size == size) { 312 return region; 313 } 314 } 315 return 0; 316 } 317 318 319 static void 320 convert_physical_ranges() { 321 addr_range *range = gKernelArgs.physical_allocated_range; 322 uint32 num_ranges = gKernelArgs.num_physical_allocated_ranges; 323 324 for (uint32 i = 0; i < num_ranges; ++i) { 325 allocated_memory_region *region = new(std::nothrow) allocated_memory_region; 326 327 if (!region) 328 panic("Couldn't add allocated region"); 329 330 // Addresses above 512GB not supported. 331 // Memory map regions above 512GB can be ignored, but if EFI returns pages above 332 // that there's nothing that can be done to fix it. 333 if (range[i].start + range[i].size > (512ull * 1024 * 1024 * 1024)) 334 panic("Can't currently support more than 512GB of RAM!"); 335 336 region->next = allocated_memory_regions; 337 allocated_memory_regions = region; 338 region->vaddr = 0; 339 region->paddr = range[i].start; 340 region->size = range[i].size; 341 region->released = false; 342 343 // Clear out the allocated range 344 range[i].start = 0; 345 range[i].size = 0; 346 gKernelArgs.num_physical_allocated_ranges--; 347 } 348 } 349 350 351 extern "C" status_t 352 platform_bootloader_address_to_kernel_address(void *address, uint64_t *_result) 353 { 354 // Convert any physical ranges prior to looking up address 355 convert_physical_ranges(); 356 357 uint64_t addr = (uint64_t)address; 358 359 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 360 if (region->paddr <= addr && addr < region->paddr + region->size) { 361 // Lazily allocate virtual memory. 362 if (region->vaddr == 0) { 363 region->vaddr = next_virtual_address; 364 next_virtual_address += ROUNDUP(region->size, B_PAGE_SIZE); 365 } 366 *_result = region->vaddr + (addr - region->paddr); 367 //dprintf("Converted bootloader address %p in region %#lx-%#lx to %#lx\n", 368 // address, region->paddr, region->paddr + region->size, *_result); 369 return B_OK; 370 } 371 } 372 373 return B_ERROR; 374 } 375 376 377 extern "C" status_t 378 platform_kernel_address_to_bootloader_address(uint64_t address, void **_result) 379 { 380 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 381 if (region->vaddr != 0 && region->vaddr <= address && address < region->vaddr + region->size) { 382 *_result = (void *)(region->paddr + (address - region->vaddr)); 383 //dprintf("Converted kernel address %#lx in region %#lx-%#lx to %p\n", 384 // address, region->vaddr, region->vaddr + region->size, *_result); 385 return B_OK; 386 } 387 } 388 389 return B_ERROR; 390 } 391 392 393 extern "C" status_t 394 platform_free_region(void *address, size_t size) 395 { 396 //dprintf("Release region %p %lu\n", address, size); 397 allocated_memory_region *region = get_region(address, size); 398 if (!region) 399 panic("Unknown region??"); 400 401 kBootServices->FreePages((EFI_PHYSICAL_ADDRESS)address, ROUNDUP(size, B_PAGE_SIZE) / B_PAGE_SIZE); 402 403 return B_OK; 404 } 405