1 /* 2 * Copyright 2019-2022 Haiku, Inc. All rights reserved. 3 * Released under the terms of the MIT License. 4 */ 5 6 7 #include <algorithm> 8 9 #include <kernel.h> 10 #include <arch_kernel.h> 11 #include <boot/platform.h> 12 #include <boot/stage2.h> 13 #include <efi/types.h> 14 #include <efi/boot-services.h> 15 #include <string.h> 16 17 #include "efi_platform.h" 18 #include "generic_mmu.h" 19 #include "mmu.h" 20 21 22 //#define TRACE_MMU 23 #ifdef TRACE_MMU 24 # define TRACE(x...) dprintf(x) 25 #else 26 # define TRACE(x...) ; 27 #endif 28 29 30 //#define TRACE_MEMORY_MAP 31 32 // Ignore memory above 512GB 33 #define PHYSICAL_MEMORY_LOW 0x00000000 34 #define PHYSICAL_MEMORY_HIGH 0x8000000000ull 35 36 #define RESERVED_MEMORY_BASE 0x80000000 37 38 phys_addr_t sPageTable = 0; 39 40 41 static inline 42 void *VirtFromPhys(uint64_t physAdr) 43 { 44 return (void*)physAdr; 45 } 46 47 48 static uint64_t 49 SignExtendVirtAdr(uint64_t virtAdr) 50 { 51 if (((uint64_t)1 << 38) & virtAdr) 52 return virtAdr | 0xFFFFFF8000000000; 53 return virtAdr; 54 } 55 56 57 static void 58 WritePteFlags(uint32 flags) 59 { 60 bool first = true; 61 dprintf("{"); 62 for (uint32 i = 0; i < 32; i++) { 63 if ((1 << i) & flags) { 64 if (first) first = false; else dprintf(", "); 65 switch (i) { 66 case 0: dprintf("valid"); break; 67 case 1: dprintf("read"); break; 68 case 2: dprintf("write"); break; 69 case 3: dprintf("exec"); break; 70 case 4: dprintf("user"); break; 71 case 5: dprintf("global"); break; 72 case 6: dprintf("accessed"); break; 73 case 7: dprintf("dirty"); break; 74 default: dprintf("%" B_PRIu32, i); 75 } 76 } 77 } 78 dprintf("}"); 79 } 80 81 82 static void 83 DumpPageWrite(uint64_t virtAdr, uint64_t physAdr, size_t size, uint64 flags, uint64& firstVirt, 84 uint64& firstPhys, uint64& firstFlags, uint64& len) 85 { 86 if (virtAdr == firstVirt + len && physAdr == firstPhys + len && flags == firstFlags) { 87 len += size; 88 } else { 89 if (len != 0) { 90 dprintf(" 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR, 91 firstVirt, firstVirt + (len - 1)); 92 dprintf(": 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR ", %#" B_PRIxADDR ", ", 93 firstPhys, firstPhys + (len - 1), len); 94 WritePteFlags(firstFlags); dprintf("\n"); 95 } 96 firstVirt = virtAdr; 97 firstPhys = physAdr; 98 firstFlags = flags; 99 len = size; 100 } 101 } 102 103 104 static void 105 DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, uint64& firstVirt, uint64& firstPhys, 106 uint64& firstFlags, uint64& len) 107 { 108 for (uint32 i = 0; i < pteCount; i++) { 109 if (pte[i].isValid) { 110 if (!pte[i].isRead && !pte[i].isWrite && !pte[i].isExec) { 111 if (level == 0) 112 panic("internal page table on level 0"); 113 114 DumpPageTableInt((Pte*)VirtFromPhys(B_PAGE_SIZE*pte[i].ppn), 115 virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level)), 116 level - 1, firstVirt, firstPhys, firstFlags, len); 117 } else { 118 DumpPageWrite( 119 SignExtendVirtAdr(virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level))), 120 pte[i].ppn * B_PAGE_SIZE, 121 1 << (pageBits + pteIdxBits*level), 122 pte[i].val & 0xff, 123 firstVirt, firstPhys, firstFlags, len); 124 } 125 } 126 } 127 } 128 129 130 static int 131 DumpPageTable(uint64 satp) 132 { 133 SatpReg satpReg{.val = satp}; 134 Pte* root = (Pte*)VirtFromPhys(satpReg.ppn * B_PAGE_SIZE); 135 136 dprintf("PageTable:\n"); 137 uint64 firstVirt = 0; 138 uint64 firstPhys = 0; 139 uint64 firstFlags = 0; 140 uint64 len = 0; 141 DumpPageTableInt(root, 0, 2, firstVirt, firstPhys, firstFlags, len); 142 DumpPageWrite(0, 0, 0, 0, firstVirt, firstPhys, firstFlags, len); 143 144 return 0; 145 } 146 147 148 static Pte* 149 LookupPte(addr_t virtAdr, bool alloc) 150 { 151 Pte *pte = (Pte*)VirtFromPhys(sPageTable); 152 for (int level = 2; level > 0; level --) { 153 pte += VirtAdrPte(virtAdr, level); 154 if (!pte->isValid) { 155 if (!alloc) 156 return NULL; 157 uint64 ppn = mmu_allocate_page() / B_PAGE_SIZE; 158 if (ppn == 0) 159 return NULL; 160 memset((Pte*)VirtFromPhys(B_PAGE_SIZE * ppn), 0, B_PAGE_SIZE); 161 Pte newPte { 162 .isValid = true, 163 .isGlobal = IS_KERNEL_ADDRESS(virtAdr), 164 .ppn = ppn 165 }; 166 pte->val = newPte.val; 167 } 168 pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); 169 } 170 pte += VirtAdrPte(virtAdr, 0); 171 return pte; 172 } 173 174 175 static void 176 Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags) 177 { 178 // TRACE("Map(%#" B_PRIxADDR ", %#" B_PRIxADDR ")\n", virtAdr, physAdr); 179 Pte* pte = LookupPte(virtAdr, true); 180 if (pte == NULL) panic("can't allocate page table"); 181 182 Pte newPte { 183 .isValid = true, 184 .isGlobal = IS_KERNEL_ADDRESS(virtAdr), 185 .isAccessed = true, 186 .isDirty = true, 187 }; 188 newPte.val |= flags; 189 190 pte->val = newPte.val; 191 } 192 193 194 static void 195 MapRange(addr_t virtAdr, phys_addr_t physAdr, size_t size, uint64 flags) 196 { 197 TRACE("MapRange(%#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" 198 B_PRIxADDR ")\n", virtAdr, virtAdr + (size - 1), physAdr, physAdr + (size - 1), size); 199 for (size_t i = 0; i < size; i += B_PAGE_SIZE) 200 Map(virtAdr + i, physAdr + i, flags); 201 202 ASSERT_ALWAYS(insert_virtual_allocated_range(virtAdr, size) >= B_OK); 203 } 204 205 206 static void 207 insert_virtual_range_to_keep(uint64 start, uint64 size) 208 { 209 status_t status = insert_address_range( 210 gKernelArgs.arch_args.virtual_ranges_to_keep, 211 &gKernelArgs.arch_args.num_virtual_ranges_to_keep, 212 MAX_VIRTUAL_RANGES_TO_KEEP, start, size); 213 214 if (status == B_ENTRY_NOT_FOUND) 215 panic("too many virtual ranges to keep"); 216 else if (status != B_OK) 217 panic("failed to add virtual range to keep"); 218 } 219 220 221 static void 222 MapAddrRange(addr_range& range, uint64 flags) 223 { 224 if (range.size == 0) { 225 range.start = 0; 226 return; 227 } 228 229 phys_addr_t physAdr = range.start; 230 range.start = get_next_virtual_address(range.size); 231 232 MapRange(range.start, physAdr, range.size, flags); 233 insert_virtual_range_to_keep(range.start, range.size); 234 } 235 236 237 static void 238 PreallocKernelRange() 239 { 240 Pte* root = (Pte*)VirtFromPhys(sPageTable); 241 for (uint64 i = VirtAdrPte(KERNEL_BASE, 2); i <= VirtAdrPte(KERNEL_TOP, 2); 242 i++) { 243 Pte* pte = &root[i]; 244 uint64 ppn = mmu_allocate_page() / B_PAGE_SIZE; 245 if (ppn == 0) panic("can't alloc early physical page"); 246 memset(VirtFromPhys(B_PAGE_SIZE * ppn), 0, B_PAGE_SIZE); 247 Pte newPte { 248 .isValid = true, 249 .isGlobal = true, 250 .ppn = ppn 251 }; 252 pte->val = newPte.val; 253 } 254 } 255 256 257 static uint64 258 GetSatp() 259 { 260 return SatpReg{ 261 .ppn = sPageTable / B_PAGE_SIZE, 262 .asid = 0, 263 .mode = satpModeSv39 264 }.val; 265 } 266 267 268 static void 269 GetPhysMemRange(addr_range& range) 270 { 271 phys_addr_t beg = (phys_addr_t)(-1), end = 0; 272 if (gKernelArgs.num_physical_memory_ranges <= 0) 273 beg = 0; 274 else { 275 for (size_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 276 beg = std::min(beg, gKernelArgs.physical_memory_range[i].start); 277 end = std::max(end, gKernelArgs.physical_memory_range[i].start + gKernelArgs.physical_memory_range[i].size); 278 } 279 } 280 range.start = beg; 281 range.size = end - beg; 282 } 283 284 285 //#pragma mark - 286 287 288 void 289 arch_mmu_init() 290 { 291 } 292 293 294 void 295 arch_mmu_post_efi_setup(size_t memory_map_size, 296 efi_memory_descriptor *memory_map, size_t descriptor_size, 297 uint32_t descriptor_version) 298 { 299 build_physical_allocated_list(memory_map_size, memory_map, 300 descriptor_size, descriptor_version); 301 302 // Switch EFI to virtual mode, using the kernel pmap. 303 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, 304 descriptor_version, memory_map); 305 306 #ifdef TRACE_MEMORY_MAP 307 dprintf("phys memory ranges:\n"); 308 for (uint32_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 309 uint64 start = gKernelArgs.physical_memory_range[i].start; 310 uint64 size = gKernelArgs.physical_memory_range[i].size; 311 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 312 start, start + size, size); 313 } 314 315 dprintf("allocated phys memory ranges:\n"); 316 for (uint32_t i = 0; i < gKernelArgs.num_physical_allocated_ranges; i++) { 317 uint64 start = gKernelArgs.physical_allocated_range[i].start; 318 uint64 size = gKernelArgs.physical_allocated_range[i].size; 319 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 320 start, start + size, size); 321 } 322 323 dprintf("allocated virt memory ranges:\n"); 324 for (uint32_t i = 0; i < gKernelArgs.num_virtual_allocated_ranges; i++) { 325 uint64 start = gKernelArgs.virtual_allocated_range[i].start; 326 uint64 size = gKernelArgs.virtual_allocated_range[i].size; 327 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 328 start, start + size, size); 329 } 330 331 dprintf("virt memory ranges to keep:\n"); 332 for (uint32_t i = 0; i < gKernelArgs.arch_args.num_virtual_ranges_to_keep; i++) { 333 uint64 start = gKernelArgs.arch_args.virtual_ranges_to_keep[i].start; 334 uint64 size = gKernelArgs.arch_args.virtual_ranges_to_keep[i].size; 335 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 336 start, start + size, size); 337 } 338 #endif 339 } 340 341 342 static void 343 fix_memory_map_for_m_mode(size_t memoryMapSize, efi_memory_descriptor* memoryMap, 344 size_t descriptorSize, uint32_t descriptorVersion) 345 { 346 addr_t addr = (addr_t)memoryMap; 347 348 for (size_t i = 0; i < memoryMapSize / descriptorSize; ++i) { 349 efi_memory_descriptor* entry = (efi_memory_descriptor *)(addr + i * descriptorSize); 350 if (entry->PhysicalStart == RESERVED_MEMORY_BASE) { 351 entry->Type = EfiReservedMemoryType; 352 } 353 } 354 } 355 356 357 uint64 358 arch_mmu_generate_post_efi_page_tables(size_t memoryMapSize, efi_memory_descriptor* memoryMap, 359 size_t descriptorSize, uint32_t descriptorVersion) 360 { 361 sPageTable = mmu_allocate_page(); 362 memset(VirtFromPhys(sPageTable), 0, B_PAGE_SIZE); 363 TRACE("sPageTable: %#" B_PRIxADDR "\n", sPageTable); 364 365 PreallocKernelRange(); 366 367 gKernelArgs.num_virtual_allocated_ranges = 0; 368 gKernelArgs.arch_args.num_virtual_ranges_to_keep = 0; 369 370 fix_memory_map_for_m_mode(memoryMapSize, memoryMap, descriptorSize, descriptorVersion); 371 372 build_physical_memory_list(memoryMapSize, memoryMap, descriptorSize, descriptorVersion, 373 PHYSICAL_MEMORY_LOW, PHYSICAL_MEMORY_HIGH); 374 375 addr_range physMemRange; 376 GetPhysMemRange(physMemRange); 377 TRACE("physMemRange: %#" B_PRIxADDR ", %#" B_PRIxSIZE "\n", 378 physMemRange.start, physMemRange.size); 379 380 // Physical memory mapping 381 gKernelArgs.arch_args.physMap.start = KERNEL_TOP + 1 - physMemRange.size; 382 gKernelArgs.arch_args.physMap.size = physMemRange.size; 383 MapRange(gKernelArgs.arch_args.physMap.start, physMemRange.start, physMemRange.size, 384 Pte {.isRead = true, .isWrite = true}.val); 385 386 // Boot loader 387 TRACE("Boot loader:\n"); 388 for (size_t i = 0; i < memoryMapSize / descriptorSize; ++i) { 389 efi_memory_descriptor* entry = &memoryMap[i]; 390 switch (entry->Type) { 391 case EfiLoaderCode: 392 case EfiLoaderData: 393 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, 394 Pte {.isRead = true, .isWrite = true, .isExec = true}.val); 395 break; 396 default: 397 ; 398 } 399 } 400 TRACE("Boot loader stack\n"); 401 addr_t sp = Sp(); 402 addr_t stackTop = ROUNDDOWN(sp - 1024*64, B_PAGE_SIZE); 403 TRACE(" SP: %#" B_PRIxADDR "\n", sp); 404 405 // EFI runtime services 406 TRACE("EFI runtime services:\n"); 407 for (size_t i = 0; i < memoryMapSize / descriptorSize; ++i) { 408 efi_memory_descriptor* entry = &memoryMap[i]; 409 if ((entry->Attribute & EFI_MEMORY_RUNTIME) != 0) 410 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, 411 Pte {.isRead = true, .isWrite = true, .isExec = true}.val); 412 } 413 414 // Memory regions 415 TRACE("Regions:\n"); 416 void* cookie = NULL; 417 addr_t virtAdr; 418 phys_addr_t physAdr; 419 size_t size; 420 while (mmu_next_region(&cookie, &virtAdr, &physAdr, &size)) { 421 MapRange(virtAdr, physAdr, size, Pte {.isRead = true, .isWrite = true, .isExec = true}.val); 422 } 423 424 // Devices 425 TRACE("Devices:\n"); 426 MapAddrRange(gKernelArgs.arch_args.clint, Pte {.isRead = true, .isWrite = true}.val); 427 MapAddrRange(gKernelArgs.arch_args.htif, Pte {.isRead = true, .isWrite = true}.val); 428 MapAddrRange(gKernelArgs.arch_args.plic, Pte {.isRead = true, .isWrite = true}.val); 429 430 if (strcmp(gKernelArgs.arch_args.uart.kind, "") != 0) { 431 MapRange(gKernelArgs.arch_args.uart.regs.start, 432 gKernelArgs.arch_args.uart.regs.start, 433 gKernelArgs.arch_args.uart.regs.size, 434 Pte {.isRead = true, .isWrite = true}.val); 435 MapAddrRange(gKernelArgs.arch_args.uart.regs, 436 Pte {.isRead = true, .isWrite = true}.val); 437 } 438 439 sort_address_ranges(gKernelArgs.virtual_allocated_range, 440 gKernelArgs.num_virtual_allocated_ranges); 441 442 DumpPageTable(GetSatp()); 443 444 return GetSatp(); 445 } 446