1 /* 2 * Copyright 2019-2022 Haiku, Inc. All rights reserved. 3 * Released under the terms of the MIT License. 4 */ 5 6 7 #include <algorithm> 8 9 #include <kernel.h> 10 #include <arch_kernel.h> 11 #include <boot/platform.h> 12 #include <boot/stage2.h> 13 #include <efi/types.h> 14 #include <efi/boot-services.h> 15 #include <string.h> 16 17 #include "efi_platform.h" 18 #include "generic_mmu.h" 19 #include "mmu.h" 20 21 22 //#define TRACE_MMU 23 #ifdef TRACE_MMU 24 # define TRACE(x...) dprintf(x) 25 #else 26 # define TRACE(x...) ; 27 #endif 28 29 30 //#define TRACE_MEMORY_MAP 31 32 // Ignore memory above 512GB 33 #define PHYSICAL_MEMORY_LOW 0x00000000 34 #define PHYSICAL_MEMORY_HIGH 0x8000000000ull 35 36 #define RESERVED_MEMORY_BASE 0x80000000 37 38 phys_addr_t sPageTable = 0; 39 40 41 static inline 42 void *VirtFromPhys(uint64_t physAdr) 43 { 44 return (void*)physAdr; 45 } 46 47 48 static uint64_t 49 SignExtendVirtAdr(uint64_t virtAdr) 50 { 51 if (((uint64_t)1 << 38) & virtAdr) 52 return virtAdr | 0xFFFFFF8000000000; 53 return virtAdr; 54 } 55 56 57 static void 58 WritePteFlags(uint32 flags) 59 { 60 bool first = true; 61 dprintf("{"); 62 for (uint32 i = 0; i < 32; i++) { 63 if ((1 << i) & flags) { 64 if (first) first = false; else dprintf(", "); 65 switch (i) { 66 case pteValid: dprintf("valid"); break; 67 case pteRead: dprintf("read"); break; 68 case pteWrite: dprintf("write"); break; 69 case pteExec: dprintf("exec"); break; 70 case pteUser: dprintf("user"); break; 71 case pteGlobal: dprintf("global"); break; 72 case pteAccessed: dprintf("accessed"); break; 73 case pteDirty: dprintf("dirty"); break; 74 default: dprintf("%" B_PRIu32, i); 75 } 76 } 77 } 78 dprintf("}"); 79 } 80 81 82 static void 83 DumpPageWrite(uint64_t virtAdr, uint64_t physAdr, size_t size, uint64 flags, uint64& firstVirt, 84 uint64& firstPhys, uint64& firstFlags, uint64& len) 85 { 86 if (virtAdr == firstVirt + len && physAdr == firstPhys + len && flags == firstFlags) { 87 len += size; 88 } else { 89 if (len != 0) { 90 dprintf(" 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR, 91 firstVirt, firstVirt + (len - 1)); 92 dprintf(": 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR ", %#" B_PRIxADDR ", ", 93 firstPhys, firstPhys + (len - 1), len); 94 WritePteFlags(firstFlags); dprintf("\n"); 95 } 96 firstVirt = virtAdr; 97 firstPhys = physAdr; 98 firstFlags = flags; 99 len = size; 100 } 101 } 102 103 104 static void 105 DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, uint64& firstVirt, uint64& firstPhys, 106 uint64& firstFlags, uint64& len) 107 { 108 for (uint32 i = 0; i < pteCount; i++) { 109 if (((1 << pteValid) & pte[i].flags) != 0) { 110 if ((((1 << pteRead) | (1 << pteWrite) | (1 << pteExec)) & pte[i].flags) == 0) { 111 if (level == 0) 112 panic("internal page table on level 0"); 113 114 DumpPageTableInt((Pte*)VirtFromPhys(pageSize*pte[i].ppn), 115 virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level)), 116 level - 1, firstVirt, firstPhys, firstFlags, len); 117 } else { 118 DumpPageWrite( 119 SignExtendVirtAdr(virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level))), 120 pte[i].ppn * B_PAGE_SIZE, 121 1 << (pageBits + pteIdxBits*level), 122 pte[i].flags, 123 firstVirt, firstPhys, firstFlags, len); 124 } 125 } 126 } 127 } 128 129 130 static int 131 DumpPageTable(uint64 satp) 132 { 133 SatpReg satpReg(satp); 134 Pte* root = (Pte*)VirtFromPhys(satpReg.ppn * B_PAGE_SIZE); 135 136 dprintf("PageTable:\n"); 137 uint64 firstVirt = 0; 138 uint64 firstPhys = 0; 139 uint64 firstFlags = 0; 140 uint64 len = 0; 141 DumpPageTableInt(root, 0, 2, firstVirt, firstPhys, firstFlags, len); 142 DumpPageWrite(0, 0, 0, 0, firstVirt, firstPhys, firstFlags, len); 143 144 return 0; 145 } 146 147 148 static Pte* 149 LookupPte(addr_t virtAdr, bool alloc) 150 { 151 Pte *pte = (Pte*)VirtFromPhys(sPageTable); 152 for (int level = 2; level > 0; level --) { 153 pte += VirtAdrPte(virtAdr, level); 154 if (((1 << pteValid) & pte->flags) == 0) { 155 if (!alloc) 156 return NULL; 157 pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; 158 if (pte->ppn == 0) 159 return NULL; 160 memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); 161 pte->flags |= (1 << pteValid); 162 } 163 pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); 164 } 165 pte += VirtAdrPte(virtAdr, 0); 166 return pte; 167 } 168 169 170 static void 171 Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags) 172 { 173 // TRACE("Map(%#" B_PRIxADDR ", %#" B_PRIxADDR ")\n", virtAdr, physAdr); 174 Pte* pte = LookupPte(virtAdr, true); 175 if (pte == NULL) panic("can't allocate page table"); 176 177 pte->ppn = physAdr / B_PAGE_SIZE; 178 pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags; 179 } 180 181 182 static void 183 MapRange(addr_t virtAdr, phys_addr_t physAdr, size_t size, uint64 flags) 184 { 185 TRACE("MapRange(%#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" 186 B_PRIxADDR ")\n", virtAdr, virtAdr + (size - 1), physAdr, physAdr + (size - 1), size); 187 for (size_t i = 0; i < size; i += B_PAGE_SIZE) 188 Map(virtAdr + i, physAdr + i, flags); 189 190 ASSERT_ALWAYS(insert_virtual_allocated_range(virtAdr, size) >= B_OK); 191 } 192 193 194 static void 195 insert_virtual_range_to_keep(uint64 start, uint64 size) 196 { 197 status_t status = insert_address_range( 198 gKernelArgs.arch_args.virtual_ranges_to_keep, 199 &gKernelArgs.arch_args.num_virtual_ranges_to_keep, 200 MAX_VIRTUAL_RANGES_TO_KEEP, start, size); 201 202 if (status == B_ENTRY_NOT_FOUND) 203 panic("too many virtual ranges to keep"); 204 else if (status != B_OK) 205 panic("failed to add virtual range to keep"); 206 } 207 208 209 static void 210 MapAddrRange(addr_range& range, uint64 flags) 211 { 212 if (range.size == 0) { 213 range.start = 0; 214 return; 215 } 216 217 phys_addr_t physAdr = range.start; 218 range.start = get_next_virtual_address(range.size); 219 220 MapRange(range.start, physAdr, range.size, flags); 221 insert_virtual_range_to_keep(range.start, range.size); 222 } 223 224 225 static void 226 PreallocKernelRange() 227 { 228 Pte* root = (Pte*)VirtFromPhys(sPageTable); 229 for (uint64 i = VirtAdrPte(KERNEL_BASE, 2); i <= VirtAdrPte(KERNEL_TOP, 2); 230 i++) { 231 Pte *pte = &root[i]; 232 pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; 233 if (pte->ppn == 0) panic("can't alloc early physical page"); 234 memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); 235 pte->flags |= (1 << pteValid); 236 } 237 } 238 239 240 uint64 241 GetSatp() 242 { 243 SatpReg satp; 244 satp.ppn = sPageTable / B_PAGE_SIZE; 245 satp.asid = 0; 246 satp.mode = satpModeSv39; 247 return satp.val; 248 } 249 250 251 static void 252 GetPhysMemRange(addr_range& range) 253 { 254 phys_addr_t beg = (phys_addr_t)(-1), end = 0; 255 if (gKernelArgs.num_physical_memory_ranges <= 0) 256 beg = 0; 257 else { 258 for (size_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 259 beg = std::min(beg, gKernelArgs.physical_memory_range[i].start); 260 end = std::max(end, gKernelArgs.physical_memory_range[i].start + gKernelArgs.physical_memory_range[i].size); 261 } 262 } 263 range.start = beg; 264 range.size = end - beg; 265 } 266 267 268 //#pragma mark - 269 270 271 void 272 arch_mmu_init() 273 { 274 } 275 276 277 void 278 arch_mmu_post_efi_setup(size_t memory_map_size, 279 efi_memory_descriptor *memory_map, size_t descriptor_size, 280 uint32_t descriptor_version) 281 { 282 build_physical_allocated_list(memory_map_size, memory_map, 283 descriptor_size, descriptor_version); 284 285 // Switch EFI to virtual mode, using the kernel pmap. 286 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, 287 descriptor_version, memory_map); 288 289 #ifdef TRACE_MEMORY_MAP 290 dprintf("phys memory ranges:\n"); 291 for (uint32_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 292 uint64 start = gKernelArgs.physical_memory_range[i].start; 293 uint64 size = gKernelArgs.physical_memory_range[i].size; 294 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 295 start, start + size, size); 296 } 297 298 dprintf("allocated phys memory ranges:\n"); 299 for (uint32_t i = 0; i < gKernelArgs.num_physical_allocated_ranges; i++) { 300 uint64 start = gKernelArgs.physical_allocated_range[i].start; 301 uint64 size = gKernelArgs.physical_allocated_range[i].size; 302 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 303 start, start + size, size); 304 } 305 306 dprintf("allocated virt memory ranges:\n"); 307 for (uint32_t i = 0; i < gKernelArgs.num_virtual_allocated_ranges; i++) { 308 uint64 start = gKernelArgs.virtual_allocated_range[i].start; 309 uint64 size = gKernelArgs.virtual_allocated_range[i].size; 310 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 311 start, start + size, size); 312 } 313 314 dprintf("virt memory ranges to keep:\n"); 315 for (uint32_t i = 0; i < gKernelArgs.arch_args.num_virtual_ranges_to_keep; i++) { 316 uint64 start = gKernelArgs.arch_args.virtual_ranges_to_keep[i].start; 317 uint64 size = gKernelArgs.arch_args.virtual_ranges_to_keep[i].size; 318 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 319 start, start + size, size); 320 } 321 #endif 322 } 323 324 325 static void 326 fix_memory_map_for_m_mode(size_t memoryMapSize, efi_memory_descriptor* memoryMap, 327 size_t descriptorSize, uint32_t descriptorVersion) 328 { 329 addr_t addr = (addr_t)memoryMap; 330 331 for (size_t i = 0; i < memoryMapSize / descriptorSize; ++i) { 332 efi_memory_descriptor* entry = (efi_memory_descriptor *)(addr + i * descriptorSize); 333 if (entry->PhysicalStart == RESERVED_MEMORY_BASE) { 334 entry->Type = EfiReservedMemoryType; 335 } 336 } 337 } 338 339 340 uint64 341 arch_mmu_generate_post_efi_page_tables(size_t memoryMapSize, efi_memory_descriptor* memoryMap, 342 size_t descriptorSize, uint32_t descriptorVersion) 343 { 344 sPageTable = mmu_allocate_page(); 345 memset(VirtFromPhys(sPageTable), 0, B_PAGE_SIZE); 346 TRACE("sPageTable: %#" B_PRIxADDR "\n", sPageTable); 347 348 PreallocKernelRange(); 349 350 gKernelArgs.num_virtual_allocated_ranges = 0; 351 gKernelArgs.arch_args.num_virtual_ranges_to_keep = 0; 352 353 fix_memory_map_for_m_mode(memoryMapSize, memoryMap, descriptorSize, descriptorVersion); 354 355 build_physical_memory_list(memoryMapSize, memoryMap, descriptorSize, descriptorVersion, 356 PHYSICAL_MEMORY_LOW, PHYSICAL_MEMORY_HIGH); 357 358 addr_range physMemRange; 359 GetPhysMemRange(physMemRange); 360 TRACE("physMemRange: %#" B_PRIxADDR ", %#" B_PRIxSIZE "\n", 361 physMemRange.start, physMemRange.size); 362 363 // Physical memory mapping 364 gKernelArgs.arch_args.physMap.start = KERNEL_TOP + 1 - physMemRange.size; 365 gKernelArgs.arch_args.physMap.size = physMemRange.size; 366 MapRange(gKernelArgs.arch_args.physMap.start, physMemRange.start, physMemRange.size, 367 (1 << pteRead) | (1 << pteWrite)); 368 369 // Boot loader 370 TRACE("Boot loader:\n"); 371 for (size_t i = 0; i < memoryMapSize / descriptorSize; ++i) { 372 efi_memory_descriptor* entry = &memoryMap[i]; 373 switch (entry->Type) { 374 case EfiLoaderCode: 375 case EfiLoaderData: 376 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, 377 (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 378 break; 379 default: 380 ; 381 } 382 } 383 TRACE("Boot loader stack\n"); 384 addr_t sp = Sp(); 385 addr_t stackTop = ROUNDDOWN(sp - 1024*64, B_PAGE_SIZE); 386 TRACE(" SP: %#" B_PRIxADDR "\n", sp); 387 388 // EFI runtime services 389 TRACE("EFI runtime services:\n"); 390 for (size_t i = 0; i < memoryMapSize / descriptorSize; ++i) { 391 efi_memory_descriptor* entry = &memoryMap[i]; 392 if ((entry->Attribute & EFI_MEMORY_RUNTIME) != 0) 393 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, 394 (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 395 } 396 397 // Memory regions 398 TRACE("Regions:\n"); 399 void* cookie = NULL; 400 addr_t virtAdr; 401 phys_addr_t physAdr; 402 size_t size; 403 while (mmu_next_region(&cookie, &virtAdr, &physAdr, &size)) { 404 MapRange(virtAdr, physAdr, size, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 405 } 406 407 // Devices 408 TRACE("Devices:\n"); 409 MapAddrRange(gKernelArgs.arch_args.clint, (1 << pteRead) | (1 << pteWrite)); 410 MapAddrRange(gKernelArgs.arch_args.htif, (1 << pteRead) | (1 << pteWrite)); 411 MapAddrRange(gKernelArgs.arch_args.plic, (1 << pteRead) | (1 << pteWrite)); 412 413 if (strcmp(gKernelArgs.arch_args.uart.kind, "") != 0) { 414 MapRange(gKernelArgs.arch_args.uart.regs.start, 415 gKernelArgs.arch_args.uart.regs.start, 416 gKernelArgs.arch_args.uart.regs.size, 417 (1 << pteRead) | (1 << pteWrite)); 418 MapAddrRange(gKernelArgs.arch_args.uart.regs, 419 (1 << pteRead) | (1 << pteWrite)); 420 } 421 422 sort_address_ranges(gKernelArgs.virtual_allocated_range, 423 gKernelArgs.num_virtual_allocated_ranges); 424 425 DumpPageTable(GetSatp()); 426 427 return GetSatp(); 428 } 429