1 /* 2 * Copyright 2016 Haiku, Inc. All rights reserved. 3 * Copyright 2014, Jessica Hamilton, jessica.l.hamilton@gmail.com. 4 * Copyright 2014, Henry Harrington, henry.harrington@gmail.com. 5 * Distributed under the terms of the MIT License. 6 */ 7 8 9 #include <algorithm> 10 11 #include <boot/platform.h> 12 #include <boot/stage2.h> 13 #include <kernel/arch/x86/arch_kernel.h> 14 #include <kernel/kernel.h> 15 16 #include "efi_platform.h" 17 #include "mmu.h" 18 19 20 struct allocated_memory_region { 21 allocated_memory_region *next; 22 uint64_t vaddr; 23 uint64_t paddr; 24 size_t size; 25 bool released; 26 }; 27 28 29 static uint64_t next_virtual_address = KERNEL_LOAD_BASE_64_BIT + 32 * 1024 * 1024; 30 static allocated_memory_region *allocated_memory_regions = NULL; 31 32 33 static uint64_t mmu_allocate_page() 34 { 35 EFI_PHYSICAL_ADDRESS addr; 36 EFI_STATUS s = kBootServices->AllocatePages(AllocateAnyPages, EfiLoaderData, 1, &addr); 37 if (s != EFI_SUCCESS) 38 panic("Unabled to allocate memory: %li", s); 39 40 return addr; 41 } 42 43 44 uint64_t 45 mmu_generate_post_efi_page_tables(UINTN memory_map_size, 46 EFI_MEMORY_DESCRIPTOR *memory_map, UINTN descriptor_size, 47 UINTN descriptor_version) 48 { 49 // Generate page tables, matching bios_ia32/long.cpp. 50 uint64_t *pml4; 51 uint64_t *pdpt; 52 uint64_t *pageDir; 53 uint64_t *pageTable; 54 55 // Allocate the top level PML4. 56 pml4 = NULL; 57 if (platform_allocate_region((void**)&pml4, B_PAGE_SIZE, 0, false) != B_OK) 58 panic("Failed to allocate PML4."); 59 gKernelArgs.arch_args.phys_pgdir = (uint32_t)(addr_t)pml4; 60 memset(pml4, 0, B_PAGE_SIZE); 61 platform_bootloader_address_to_kernel_address(pml4, &gKernelArgs.arch_args.vir_pgdir); 62 63 // Store the virtual memory usage information. 64 gKernelArgs.virtual_allocated_range[0].start = KERNEL_LOAD_BASE_64_BIT; 65 gKernelArgs.virtual_allocated_range[0].size = next_virtual_address - KERNEL_LOAD_BASE_64_BIT; 66 gKernelArgs.num_virtual_allocated_ranges = 1; 67 gKernelArgs.arch_args.virtual_end = ROUNDUP(KERNEL_LOAD_BASE_64_BIT 68 + gKernelArgs.virtual_allocated_range[0].size, 0x200000); 69 70 // Find the highest physical memory address. We map all physical memory 71 // into the kernel address space, so we want to make sure we map everything 72 // we have available. 73 uint64 maxAddress = 0; 74 for (UINTN i = 0; i < memory_map_size / descriptor_size; ++i) { 75 EFI_MEMORY_DESCRIPTOR *entry = (EFI_MEMORY_DESCRIPTOR *)((addr_t)memory_map + i * descriptor_size); 76 maxAddress = std::max(maxAddress, 77 entry->PhysicalStart + entry->NumberOfPages * 4096); 78 } 79 80 // Want to map at least 4GB, there may be stuff other than usable RAM that 81 // could be in the first 4GB of physical address space. 82 maxAddress = std::max(maxAddress, (uint64)0x100000000ll); 83 maxAddress = ROUNDUP(maxAddress, 0x40000000); 84 85 // Currently only use 1 PDPT (512GB). This will need to change if someone 86 // wants to use Haiku on a box with more than 512GB of RAM but that's 87 // probably not going to happen any time soon. 88 if (maxAddress / 0x40000000 > 512) 89 panic("Can't currently support more than 512GB of RAM!"); 90 91 // Create page tables for the physical map area. Also map this PDPT 92 // temporarily at the bottom of the address space so that we are identity 93 // mapped. 94 95 pdpt = (uint64*)mmu_allocate_page(); 96 memset(pdpt, 0, B_PAGE_SIZE); 97 pml4[510] = (addr_t)pdpt | kTableMappingFlags; 98 pml4[0] = (addr_t)pdpt | kTableMappingFlags; 99 100 for (uint64 i = 0; i < maxAddress; i += 0x40000000) { 101 pageDir = (uint64*)mmu_allocate_page(); 102 memset(pageDir, 0, B_PAGE_SIZE); 103 pdpt[i / 0x40000000] = (addr_t)pageDir | kTableMappingFlags; 104 105 for (uint64 j = 0; j < 0x40000000; j += 0x200000) { 106 pageDir[j / 0x200000] = (i + j) | kLargePageMappingFlags; 107 } 108 } 109 110 // Allocate tables for the kernel mappings. 111 112 pdpt = (uint64*)mmu_allocate_page(); 113 memset(pdpt, 0, B_PAGE_SIZE); 114 pml4[511] = (addr_t)pdpt | kTableMappingFlags; 115 116 pageDir = (uint64*)mmu_allocate_page(); 117 memset(pageDir, 0, B_PAGE_SIZE); 118 pdpt[510] = (addr_t)pageDir | kTableMappingFlags; 119 120 // We can now allocate page tables and duplicate the mappings across from 121 // the 32-bit address space to them. 122 pageTable = NULL; // shush, compiler. 123 for (uint32 i = 0; i < gKernelArgs.virtual_allocated_range[0].size 124 / B_PAGE_SIZE; i++) { 125 if ((i % 512) == 0) { 126 pageTable = (uint64*)mmu_allocate_page(); 127 memset(pageTable, 0, B_PAGE_SIZE); 128 pageDir[i / 512] = (addr_t)pageTable | kTableMappingFlags; 129 } 130 131 // Get the physical address to map. 132 void *phys; 133 if (platform_kernel_address_to_bootloader_address(KERNEL_LOAD_BASE_64_BIT + (i * B_PAGE_SIZE), 134 &phys) != B_OK) 135 continue; 136 137 pageTable[i % 512] = (addr_t)phys | kPageMappingFlags; 138 } 139 140 return (uint64)pml4; 141 } 142 143 144 // Called after EFI boot services exit. 145 // Currently assumes that the memory map is sane... Sorted and no overlapping 146 // regions. 147 void 148 mmu_post_efi_setup(UINTN memory_map_size, EFI_MEMORY_DESCRIPTOR *memory_map, UINTN descriptor_size, UINTN descriptor_version) 149 { 150 // Add physical memory to the kernel args and update virtual addresses for EFI regions.. 151 addr_t addr = (addr_t)memory_map; 152 gKernelArgs.num_physical_memory_ranges = 0; 153 for (UINTN i = 0; i < memory_map_size / descriptor_size; ++i) { 154 EFI_MEMORY_DESCRIPTOR *entry = (EFI_MEMORY_DESCRIPTOR *)(addr + i * descriptor_size); 155 switch (entry->Type) { 156 case EfiLoaderCode: 157 case EfiLoaderData: 158 case EfiBootServicesCode: 159 case EfiBootServicesData: 160 case EfiConventionalMemory: { 161 // Usable memory. 162 // Ignore memory below 1MB and above 512GB. 163 uint64_t base = entry->PhysicalStart; 164 uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096; 165 if (base < 0x100000) 166 base = 0x100000; 167 if (end > (512ull * 1024 * 1024 * 1024)) 168 end = 512ull * 1024 * 1024 * 1024; 169 if (base >= end) 170 break; 171 uint64_t size = end - base; 172 173 insert_physical_memory_range(base, size); 174 // LoaderData memory is bootloader allocated memory, possibly 175 // containing the kernel or loaded drivers. 176 if (entry->Type == EfiLoaderData) 177 insert_physical_allocated_range(base, size); 178 break; 179 } 180 case EfiACPIReclaimMemory: 181 // ACPI reclaim -- physical memory we could actually use later 182 gKernelArgs.ignored_physical_memory += entry->NumberOfPages * 4096; 183 break; 184 case EfiRuntimeServicesCode: 185 case EfiRuntimeServicesData: 186 entry->VirtualStart = entry->PhysicalStart; 187 break; 188 } 189 } 190 191 // Sort the address ranges. 192 sort_address_ranges(gKernelArgs.physical_memory_range, 193 gKernelArgs.num_physical_memory_ranges); 194 sort_address_ranges(gKernelArgs.physical_allocated_range, 195 gKernelArgs.num_physical_allocated_ranges); 196 sort_address_ranges(gKernelArgs.virtual_allocated_range, 197 gKernelArgs.num_virtual_allocated_ranges); 198 199 // Switch EFI to virtual mode, using the kernel pmap. 200 // Something involving ConvertPointer might need to be done after this? 201 // http://wiki.phoenix.com/wiki/index.php/EFI_RUNTIME_SERVICES#SetVirtualAddressMap.28.29 202 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, descriptor_version, memory_map); 203 204 // Important. Make sure supervisor threads can fault on read only pages... 205 asm("mov %%rax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1)); 206 } 207 208 209 // Platform allocator. 210 // The bootloader assumes that bootloader address space == kernel address space. 211 // This is not true until just before the kernel is booted, so an ugly hack is 212 // used to cover the difference. platform_allocate_region allocates addresses 213 // in bootloader space, but can convert them to kernel space. The ELF loader 214 // accesses kernel memory via Mao(), and much later in the boot process, 215 // addresses in the kernel argument struct are converted from bootloader 216 // addresses to kernel addresses. 217 218 extern "C" status_t 219 platform_allocate_region(void **_address, size_t size, uint8 /* protection */, bool exactAddress) 220 { 221 // We don't have any control over the page tables, give up right away if an 222 // exactAddress is wanted. 223 if (exactAddress) 224 return B_NO_MEMORY; 225 226 EFI_PHYSICAL_ADDRESS addr; 227 size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE); 228 allocated_memory_region *region = new(std::nothrow) allocated_memory_region; 229 230 if (region == NULL) 231 return B_NO_MEMORY; 232 233 EFI_STATUS status = kBootServices->AllocatePages(AllocateAnyPages, 234 EfiLoaderData, aligned_size / B_PAGE_SIZE, &addr); 235 if (status != EFI_SUCCESS) { 236 delete region; 237 return B_NO_MEMORY; 238 } 239 240 // Addresses above 512GB not supported. 241 // Memory map regions above 512GB can be ignored, but if EFI returns pages 242 // above that there's nothing that can be done to fix it. 243 if (addr + size > (512ull * 1024 * 1024 * 1024)) 244 panic("Can't currently support more than 512GB of RAM!"); 245 246 region->next = allocated_memory_regions; 247 allocated_memory_regions = region; 248 region->vaddr = 0; 249 region->paddr = addr; 250 region->size = size; 251 region->released = false; 252 253 if (*_address != NULL) { 254 region->vaddr = (uint64_t)*_address; 255 } 256 257 //dprintf("Allocated region %#lx (requested %p) %#lx %lu\n", region->vaddr, *_address, region->paddr, region->size); 258 259 *_address = (void *)region->paddr; 260 261 return B_OK; 262 } 263 264 265 /*! 266 Neither \a virtualAddress nor \a size need to be aligned, but the function 267 will map all pages the range intersects with. 268 If physicalAddress is not page-aligned, the returned virtual address will 269 have the same "misalignment". 270 */ 271 extern "C" addr_t 272 mmu_map_physical_memory(addr_t physicalAddress, size_t size, uint32 flags) 273 { 274 addr_t pageOffset = physicalAddress & (B_PAGE_SIZE - 1); 275 276 physicalAddress -= pageOffset; 277 size += pageOffset; 278 279 size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE); 280 allocated_memory_region *region = new(std::nothrow) allocated_memory_region; 281 282 if (!region) 283 return B_NO_MEMORY; 284 285 // Addresses above 512GB not supported. 286 // Memory map regions above 512GB can be ignored, but if EFI returns pages above 287 // that there's nothing that can be done to fix it. 288 if (physicalAddress + size > (512ull * 1024 * 1024 * 1024)) 289 panic("Can't currently support more than 512GB of RAM!"); 290 291 region->next = allocated_memory_regions; 292 allocated_memory_regions = region; 293 region->vaddr = 0; 294 region->paddr = physicalAddress; 295 region->size = aligned_size; 296 region->released = false; 297 298 return physicalAddress + pageOffset; 299 } 300 301 302 extern "C" void 303 mmu_free(void *virtualAddress, size_t size) 304 { 305 addr_t physicalAddress = (addr_t)virtualAddress; 306 addr_t pageOffset = physicalAddress & (B_PAGE_SIZE - 1); 307 308 physicalAddress -= pageOffset; 309 size += pageOffset; 310 311 size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE); 312 313 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 314 if (region->paddr == physicalAddress && region->size == aligned_size) { 315 region->released = true; 316 return; 317 } 318 } 319 } 320 321 322 static allocated_memory_region * 323 get_region(void *address, size_t size) 324 { 325 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 326 if (region->paddr == (uint64_t)address && region->size == size) { 327 return region; 328 } 329 } 330 return 0; 331 } 332 333 334 extern "C" status_t 335 platform_bootloader_address_to_kernel_address(void *address, uint64_t *_result) 336 { 337 uint64_t addr = (uint64_t)address; 338 339 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 340 if (region->paddr <= addr && addr < region->paddr + region->size) { 341 // Lazily allocate virtual memory. 342 if (region->vaddr == 0) { 343 region->vaddr = next_virtual_address; 344 next_virtual_address += ROUNDUP(region->size, B_PAGE_SIZE); 345 } 346 *_result = region->vaddr + (addr - region->paddr); 347 //dprintf("Converted bootloader address %p in region %#lx-%#lx to %#lx\n", 348 // address, region->paddr, region->paddr + region->size, *_result); 349 return B_OK; 350 } 351 } 352 353 return B_ERROR; 354 } 355 356 357 extern "C" status_t 358 platform_kernel_address_to_bootloader_address(uint64_t address, void **_result) 359 { 360 for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) { 361 if (region->vaddr != 0 && region->vaddr <= address && address < region->vaddr + region->size) { 362 *_result = (void *)(region->paddr + (address - region->vaddr)); 363 //dprintf("Converted kernel address %#lx in region %#lx-%#lx to %p\n", 364 // address, region->vaddr, region->vaddr + region->size, *_result); 365 return B_OK; 366 } 367 } 368 369 return B_ERROR; 370 } 371 372 373 extern "C" status_t 374 platform_free_region(void *address, size_t size) 375 { 376 //dprintf("Release region %p %lu\n", address, size); 377 allocated_memory_region *region = get_region(address, size); 378 if (!region) 379 panic("Unknown region??"); 380 381 kBootServices->FreePages((EFI_PHYSICAL_ADDRESS)address, ROUNDUP(size, B_PAGE_SIZE) / B_PAGE_SIZE); 382 383 return B_OK; 384 } 385