1 /* 2 * Copyright 2019-2020 Haiku, Inc. All rights reserved. 3 * Released under the terms of the MIT License. 4 */ 5 6 7 #include <algorithm> 8 9 #include <kernel.h> 10 #include <arch_kernel.h> 11 #include <boot/platform.h> 12 #include <boot/stage2.h> 13 #include <efi/types.h> 14 #include <efi/boot-services.h> 15 #include <string.h> 16 17 #include "mmu.h" 18 #include "efi_platform.h" 19 20 21 phys_addr_t sPageTable = 0; 22 23 24 static inline 25 void *VirtFromPhys(uint64_t physAdr) 26 { 27 return (void*)physAdr; 28 } 29 30 31 static uint64_t 32 SignExtendVirtAdr(uint64_t virtAdr) 33 { 34 if (((uint64_t)1 << 38) & virtAdr) 35 return virtAdr | 0xFFFFFF8000000000; 36 return virtAdr; 37 } 38 39 40 static void 41 WritePteFlags(uint32 flags) 42 { 43 bool first = true; 44 dprintf("{"); 45 for (uint32 i = 0; i < 32; i++) { 46 if ((1 << i) & flags) { 47 if (first) first = false; else dprintf(", "); 48 switch (i) { 49 case pteValid: dprintf("valid"); break; 50 case pteRead: dprintf("read"); break; 51 case pteWrite: dprintf("write"); break; 52 case pteExec: dprintf("exec"); break; 53 case pteUser: dprintf("user"); break; 54 case pteGlobal: dprintf("global"); break; 55 case pteAccessed: dprintf("accessed"); break; 56 case pteDirty: dprintf("dirty"); break; 57 default: dprintf("%" B_PRIu32, i); 58 } 59 } 60 } 61 dprintf("}"); 62 } 63 64 65 static void 66 DumpPageWrite(uint64_t virtAdr, uint64_t physAdr, size_t size, uint64 flags, uint64& firstVirt, 67 uint64& firstPhys, uint64& firstFlags, uint64& len) 68 { 69 if (virtAdr == firstVirt + len && physAdr == firstPhys + len && flags == firstFlags) { 70 len += size; 71 } else { 72 if (len != 0) { 73 dprintf(" 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR, 74 firstVirt, firstVirt + (len - 1)); 75 dprintf(": 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR ", %#" B_PRIxADDR ", ", 76 firstPhys, firstPhys + (len - 1), len); 77 WritePteFlags(firstFlags); dprintf("\n"); 78 } 79 firstVirt = virtAdr; 80 firstPhys = physAdr; 81 firstFlags = flags; 82 len = size; 83 } 84 } 85 86 87 static void 88 DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, uint64& firstVirt, uint64& firstPhys, 89 uint64& firstFlags, uint64& len) 90 { 91 for (uint32 i = 0; i < pteCount; i++) { 92 if (((1 << pteValid) & pte[i].flags) != 0) { 93 if ((((1 << pteRead) | (1 << pteWrite) | (1 << pteExec)) & pte[i].flags) == 0) { 94 if (level == 0) 95 panic("internal page table on level 0"); 96 97 DumpPageTableInt((Pte*)VirtFromPhys(pageSize*pte[i].ppn), 98 virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level)), 99 level - 1, firstVirt, firstPhys, firstFlags, len); 100 } else { 101 DumpPageWrite( 102 SignExtendVirtAdr(virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level))), 103 pte[i].ppn * B_PAGE_SIZE, 104 1 << (pageBits + pteIdxBits*level), 105 pte[i].flags, 106 firstVirt, firstPhys, firstFlags, len); 107 } 108 } 109 } 110 } 111 112 113 static int 114 DumpPageTable(uint64 satp) 115 { 116 SatpReg satpReg(satp); 117 Pte* root = (Pte*)VirtFromPhys(satpReg.ppn * B_PAGE_SIZE); 118 119 dprintf("PageTable:\n"); 120 uint64 firstVirt = 0; 121 uint64 firstPhys = 0; 122 uint64 firstFlags = 0; 123 uint64 len = 0; 124 DumpPageTableInt(root, 0, 2, firstVirt, firstPhys, firstFlags, len); 125 DumpPageWrite(0, 0, 0, 0, firstVirt, firstPhys, firstFlags, len); 126 127 return 0; 128 } 129 130 131 static Pte* 132 LookupPte(addr_t virtAdr, bool alloc) 133 { 134 Pte *pte = (Pte*)VirtFromPhys(sPageTable); 135 for (int level = 2; level > 0; level --) { 136 pte += VirtAdrPte(virtAdr, level); 137 if (((1 << pteValid) & pte->flags) == 0) { 138 if (!alloc) 139 return NULL; 140 pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; 141 if (pte->ppn == 0) 142 return NULL; 143 memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); 144 pte->flags |= (1 << pteValid); 145 } 146 pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); 147 } 148 pte += VirtAdrPte(virtAdr, 0); 149 return pte; 150 } 151 152 153 static void 154 Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags) 155 { 156 // dprintf("Map(%#" B_PRIxADDR ", %#" B_PRIxADDR ")\n", virtAdr, physAdr); 157 Pte* pte = LookupPte(virtAdr, true); 158 if (pte == NULL) panic("can't allocate page table"); 159 160 pte->ppn = physAdr / B_PAGE_SIZE; 161 pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags; 162 } 163 164 165 static void 166 MapRange(addr_t virtAdr, phys_addr_t physAdr, size_t size, uint64 flags) 167 { 168 dprintf("MapRange(%#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" 169 B_PRIxADDR ")\n", virtAdr, virtAdr + (size - 1), physAdr, physAdr + (size - 1), size); 170 for (size_t i = 0; i < size; i += B_PAGE_SIZE) 171 Map(virtAdr + i, physAdr + i, flags); 172 173 ASSERT_ALWAYS(insert_virtual_allocated_range(virtAdr, size) >= B_OK); 174 } 175 176 177 static void 178 MapAddrRange(addr_range& range, uint64 flags) 179 { 180 if (range.size == 0) { 181 range.start = 0; 182 return; 183 } 184 185 phys_addr_t physAdr = range.start; 186 range.start = get_next_virtual_address(range.size); 187 188 MapRange(range.start, physAdr, range.size, flags); 189 190 if (gKernelArgs.arch_args.num_virtual_ranges_to_keep 191 >= MAX_VIRTUAL_RANGES_TO_KEEP) 192 panic("too many virtual ranges to keep"); 193 194 gKernelArgs.arch_args.virtual_ranges_to_keep[ 195 gKernelArgs.arch_args.num_virtual_ranges_to_keep++] = range; 196 } 197 198 199 static void 200 PreallocKernelRange() 201 { 202 Pte* root = (Pte*)VirtFromPhys(sPageTable); 203 for (uint64 i = VirtAdrPte(KERNEL_BASE, 2); i <= VirtAdrPte(KERNEL_TOP, 2); 204 i++) { 205 Pte *pte = &root[i]; 206 pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; 207 if (pte->ppn == 0) panic("can't alloc early physical page"); 208 memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); 209 pte->flags |= (1 << pteValid); 210 } 211 } 212 213 214 uint64 215 GetSatp() 216 { 217 SatpReg satp; 218 satp.ppn = sPageTable / B_PAGE_SIZE; 219 satp.asid = 0; 220 satp.mode = satpModeSv39; 221 return satp.val; 222 } 223 224 225 static void 226 GetPhysMemRange(addr_range& range) 227 { 228 phys_addr_t beg = (phys_addr_t)(-1), end = 0; 229 if (gKernelArgs.num_physical_memory_ranges <= 0) 230 beg = 0; 231 else { 232 for (size_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 233 beg = std::min(beg, gKernelArgs.physical_memory_range[i].start); 234 end = std::max(end, gKernelArgs.physical_memory_range[i].start + gKernelArgs.physical_memory_range[i].size); 235 } 236 } 237 range.start = beg; 238 range.size = end - beg; 239 } 240 241 242 static void 243 FillPhysicalMemoryMap(size_t memory_map_size, 244 efi_memory_descriptor *memory_map, size_t descriptor_size, 245 uint32_t descriptor_version) 246 { 247 // Add physical memory to the kernel args and update virtual addresses for 248 // EFI regions. 249 gKernelArgs.num_physical_memory_ranges = 0; 250 251 // First scan: Add all usable ranges 252 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 253 efi_memory_descriptor* entry = &memory_map[i]; 254 switch (entry->Type) { 255 case EfiLoaderCode: 256 case EfiLoaderData: 257 case EfiBootServicesCode: 258 case EfiBootServicesData: 259 case EfiConventionalMemory: { 260 // Usable memory. 261 uint64_t base = entry->PhysicalStart; 262 uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096; 263 uint64_t originalSize = end - base; 264 265 // PMP protected memory, unusable 266 if (base == 0x80000000) 267 break; 268 269 gKernelArgs.ignored_physical_memory 270 += originalSize - (std::max(end, base) - base); 271 272 if (base >= end) 273 break; 274 uint64_t size = end - base; 275 276 insert_physical_memory_range(base, size); 277 break; 278 } 279 case EfiACPIReclaimMemory: 280 // ACPI reclaim -- physical memory we could actually use later 281 break; 282 case EfiRuntimeServicesCode: 283 case EfiRuntimeServicesData: 284 entry->VirtualStart = entry->PhysicalStart; 285 break; 286 } 287 } 288 289 uint64_t initialPhysicalMemory = total_physical_memory(); 290 291 // Second scan: Remove everything reserved that may overlap 292 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 293 efi_memory_descriptor* entry = &memory_map[i]; 294 switch (entry->Type) { 295 case EfiLoaderCode: 296 case EfiLoaderData: 297 case EfiBootServicesCode: 298 case EfiBootServicesData: 299 case EfiConventionalMemory: 300 break; 301 default: 302 uint64_t base = entry->PhysicalStart; 303 uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096; 304 remove_physical_memory_range(base, end - base); 305 } 306 } 307 308 gKernelArgs.ignored_physical_memory 309 += initialPhysicalMemory - total_physical_memory(); 310 311 sort_address_ranges(gKernelArgs.physical_memory_range, 312 gKernelArgs.num_physical_memory_ranges); 313 } 314 315 316 static void 317 FillPhysicalAllocatedMemoryMap(size_t memory_map_size, 318 efi_memory_descriptor *memory_map, size_t descriptor_size, 319 uint32_t descriptor_version) 320 { 321 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 322 efi_memory_descriptor* entry = &memory_map[i]; 323 switch (entry->Type) { 324 case EfiLoaderData: 325 insert_physical_allocated_range(entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE); 326 break; 327 default: 328 ; 329 } 330 } 331 sort_address_ranges(gKernelArgs.physical_allocated_range, 332 gKernelArgs.num_physical_allocated_ranges); 333 } 334 335 336 //#pragma mark - 337 338 339 void 340 arch_mmu_init() 341 { 342 } 343 344 345 void 346 arch_mmu_post_efi_setup(size_t memory_map_size, 347 efi_memory_descriptor *memory_map, size_t descriptor_size, 348 uint32_t descriptor_version) 349 { 350 FillPhysicalAllocatedMemoryMap(memory_map_size, memory_map, descriptor_size, descriptor_version); 351 352 // Switch EFI to virtual mode, using the kernel pmap. 353 // Something involving ConvertPointer might need to be done after this? 354 // http://wiki.phoenix.com/wiki/index.php/EFI_RUNTIME_SERVICES 355 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, 356 descriptor_version, memory_map); 357 } 358 359 360 uint64 361 arch_mmu_generate_post_efi_page_tables(size_t memory_map_size, 362 efi_memory_descriptor *memory_map, size_t descriptor_size, 363 uint32_t descriptor_version) 364 { 365 sPageTable = mmu_allocate_page(); 366 memset(VirtFromPhys(sPageTable), 0, B_PAGE_SIZE); 367 dprintf("sPageTable: %#" B_PRIxADDR "\n", sPageTable); 368 369 PreallocKernelRange(); 370 371 gKernelArgs.num_virtual_allocated_ranges = 0; 372 gKernelArgs.arch_args.num_virtual_ranges_to_keep = 0; 373 FillPhysicalMemoryMap(memory_map_size, memory_map, descriptor_size, descriptor_version); 374 375 addr_range physMemRange; 376 GetPhysMemRange(physMemRange); 377 dprintf("physMemRange: %#" B_PRIxADDR ", %#" B_PRIxSIZE "\n", physMemRange.start, physMemRange.size); 378 379 // Physical memory mapping 380 gKernelArgs.arch_args.physMap.start = KERNEL_TOP + 1 - physMemRange.size; 381 gKernelArgs.arch_args.physMap.size = physMemRange.size; 382 MapRange(gKernelArgs.arch_args.physMap.start, physMemRange.start, physMemRange.size, (1 << pteRead) | (1 << pteWrite)); 383 384 // Boot loader 385 dprintf("Boot loader:\n"); 386 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 387 efi_memory_descriptor* entry = &memory_map[i]; 388 switch (entry->Type) { 389 case EfiLoaderCode: 390 case EfiLoaderData: 391 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 392 break; 393 default: 394 ; 395 } 396 } 397 dprintf("Boot loader stack\n"); 398 addr_t sp = Sp(); 399 addr_t stackTop = ROUNDDOWN(sp - 1024*64, B_PAGE_SIZE); 400 dprintf(" SP: %#" B_PRIxADDR "\n", sp); 401 402 // EFI runtime services 403 dprintf("EFI runtime services:\n"); 404 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 405 efi_memory_descriptor* entry = &memory_map[i]; 406 if ((entry->Attribute & EFI_MEMORY_RUNTIME) != 0) 407 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 408 } 409 410 // Memory regions 411 dprintf("Regions:\n"); 412 void* cookie = NULL; 413 addr_t virtAdr; 414 phys_addr_t physAdr; 415 size_t size; 416 while (mmu_next_region(&cookie, &virtAdr, &physAdr, &size)) { 417 MapRange(virtAdr, physAdr, size, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 418 } 419 420 // Devices 421 dprintf("Devices:\n"); 422 MapAddrRange(gKernelArgs.arch_args.clint, (1 << pteRead) | (1 << pteWrite)); 423 MapAddrRange(gKernelArgs.arch_args.htif, (1 << pteRead) | (1 << pteWrite)); 424 MapAddrRange(gKernelArgs.arch_args.plic, (1 << pteRead) | (1 << pteWrite)); 425 426 if (strcmp(gKernelArgs.arch_args.uart.kind, "") != 0) { 427 MapRange(gKernelArgs.arch_args.uart.regs.start, 428 gKernelArgs.arch_args.uart.regs.start, 429 gKernelArgs.arch_args.uart.regs.size, 430 (1 << pteRead) | (1 << pteWrite)); 431 MapAddrRange(gKernelArgs.arch_args.uart.regs, 432 (1 << pteRead) | (1 << pteWrite)); 433 } 434 435 sort_address_ranges(gKernelArgs.virtual_allocated_range, gKernelArgs.num_virtual_allocated_ranges); 436 437 DumpPageTable(GetSatp()); 438 439 return GetSatp(); 440 } 441