1 /* 2 * Copyright 2019-2020 Haiku, Inc. All rights reserved. 3 * Released under the terms of the MIT License. 4 */ 5 6 7 #include <algorithm> 8 9 #include <kernel.h> 10 #include <arch_kernel.h> 11 #include <boot/platform.h> 12 #include <boot/stage2.h> 13 #include <efi/types.h> 14 #include <efi/boot-services.h> 15 #include <string.h> 16 17 #include "mmu.h" 18 #include "efi_platform.h" 19 20 21 phys_addr_t sPageTable = 0; 22 23 24 static inline 25 void *VirtFromPhys(uint64_t physAdr) 26 { 27 return (void*)physAdr; 28 } 29 30 31 static uint64_t 32 SignExtendVirtAdr(uint64_t virtAdr) 33 { 34 if (((uint64_t)1 << 38) & virtAdr) 35 return virtAdr | 0xFFFFFF8000000000; 36 return virtAdr; 37 } 38 39 40 static void 41 WritePteFlags(uint32 flags) 42 { 43 bool first = true; 44 dprintf("{"); 45 for (uint32 i = 0; i < 32; i++) { 46 if ((1 << i) & flags) { 47 if (first) first = false; else dprintf(", "); 48 switch (i) { 49 case pteValid: dprintf("valid"); break; 50 case pteRead: dprintf("read"); break; 51 case pteWrite: dprintf("write"); break; 52 case pteExec: dprintf("exec"); break; 53 case pteUser: dprintf("user"); break; 54 case pteGlobal: dprintf("global"); break; 55 case pteAccessed: dprintf("accessed"); break; 56 case pteDirty: dprintf("dirty"); break; 57 default: dprintf("%" B_PRIu32, i); 58 } 59 } 60 } 61 dprintf("}"); 62 } 63 64 65 static void 66 DumpPageWrite(uint64_t virtAdr, uint64_t physAdr, size_t size, uint64 flags, uint64& firstVirt, uint64& firstPhys, uint64& firstFlags, uint64& len) 67 { 68 if (virtAdr == firstVirt + len && physAdr == firstPhys + len && flags == firstFlags) { 69 len += size; 70 } else { 71 if (len != 0) { 72 dprintf(" 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR, 73 firstVirt, firstVirt + (len - 1)); 74 dprintf(": 0x%08" B_PRIxADDR " - 0x%08" B_PRIxADDR ", %#" B_PRIxADDR ", ", firstPhys, firstPhys + (len - 1), len); 75 WritePteFlags(firstFlags); dprintf("\n"); 76 } 77 firstVirt = virtAdr; 78 firstPhys = physAdr; 79 firstFlags = flags; 80 len = size; 81 } 82 } 83 84 85 static void 86 DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, uint64& firstVirt, uint64& firstPhys, uint64& firstFlags, uint64& len) 87 { 88 for (uint32 i = 0; i < pteCount; i++) { 89 if (((1 << pteValid) & pte[i].flags) != 0) { 90 if ((((1 << pteRead) | (1 << pteWrite) | (1 << pteExec)) & pte[i].flags) == 0) { 91 if (level == 0) 92 panic("internal page table on level 0"); 93 94 DumpPageTableInt((Pte*)VirtFromPhys(pageSize*pte[i].ppn), 95 virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level)), 96 level - 1, firstVirt, firstPhys, firstFlags, len); 97 } else { 98 DumpPageWrite( 99 SignExtendVirtAdr(virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level))), 100 pte[i].ppn * B_PAGE_SIZE, 101 1 << (pageBits + pteIdxBits*level), 102 pte[i].flags, 103 firstVirt, firstPhys, firstFlags, len); 104 } 105 } 106 } 107 } 108 109 110 static int 111 DumpPageTable(uint64 satp) 112 { 113 SatpReg satpReg(satp); 114 Pte* root = (Pte*)VirtFromPhys(satpReg.ppn * B_PAGE_SIZE); 115 116 dprintf("PageTable:\n"); 117 uint64 firstVirt = 0; 118 uint64 firstPhys = 0; 119 uint64 firstFlags = 0; 120 uint64 len = 0; 121 DumpPageTableInt(root, 0, 2, firstVirt, firstPhys, firstFlags, len); 122 DumpPageWrite(0, 0, 0, 0, firstVirt, firstPhys, firstFlags, len); 123 124 return 0; 125 } 126 127 128 static Pte* 129 LookupPte(addr_t virtAdr, bool alloc) 130 { 131 Pte *pte = (Pte*)VirtFromPhys(sPageTable); 132 for (int level = 2; level > 0; level --) { 133 pte += VirtAdrPte(virtAdr, level); 134 if (((1 << pteValid) & pte->flags) == 0) { 135 if (!alloc) 136 return NULL; 137 pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; 138 if (pte->ppn == 0) 139 return NULL; 140 memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); 141 pte->flags |= (1 << pteValid); 142 } 143 pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); 144 } 145 pte += VirtAdrPte(virtAdr, 0); 146 return pte; 147 } 148 149 150 static void 151 Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags) 152 { 153 // dprintf("Map(%#" B_PRIxADDR ", %#" B_PRIxADDR ")\n", virtAdr, physAdr); 154 Pte* pte = LookupPte(virtAdr, true); 155 if (pte == NULL) panic("can't allocate page table"); 156 157 pte->ppn = physAdr / B_PAGE_SIZE; 158 pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags; 159 } 160 161 162 static void 163 MapRange(addr_t virtAdr, phys_addr_t physAdr, size_t size, uint64 flags) 164 { 165 dprintf("MapRange(%#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" B_PRIxADDR " - %#" B_PRIxADDR ", %#" 166 B_PRIxADDR ")\n", virtAdr, virtAdr + (size - 1), physAdr, physAdr + (size - 1), size); 167 for (size_t i = 0; i < size; i += B_PAGE_SIZE) 168 Map(virtAdr + i, physAdr + i, flags); 169 170 ASSERT_ALWAYS(insert_virtual_allocated_range(virtAdr, size) >= B_OK); 171 } 172 173 174 static void 175 MapAddrRange(addr_range& range, uint64 flags) 176 { 177 if (range.size == 0) { 178 range.start = 0; 179 return; 180 } 181 182 phys_addr_t physAdr = range.start; 183 range.start = get_next_virtual_address(range.size); 184 185 MapRange(range.start, physAdr, range.size, flags); 186 187 if (gKernelArgs.arch_args.num_virtual_ranges_to_keep 188 >= MAX_VIRTUAL_RANGES_TO_KEEP) 189 panic("too many virtual ranges to keep"); 190 191 gKernelArgs.arch_args.virtual_ranges_to_keep[ 192 gKernelArgs.arch_args.num_virtual_ranges_to_keep++] = range; 193 } 194 195 196 static void 197 PreallocKernelRange() 198 { 199 Pte* root = (Pte*)VirtFromPhys(sPageTable); 200 for (uint64 i = VirtAdrPte(KERNEL_BASE, 2); i <= VirtAdrPte(KERNEL_TOP, 2); 201 i++) { 202 Pte *pte = &root[i]; 203 pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; 204 if (pte->ppn == 0) panic("can't alloc early physical page"); 205 memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); 206 pte->flags |= (1 << pteValid); 207 } 208 } 209 210 211 uint64 212 GetSatp() 213 { 214 SatpReg satp; 215 satp.ppn = sPageTable / B_PAGE_SIZE; 216 satp.asid = 0; 217 satp.mode = satpModeSv39; 218 return satp.val; 219 } 220 221 222 static void 223 GetPhysMemRange(addr_range& range) 224 { 225 phys_addr_t beg = (phys_addr_t)(-1), end = 0; 226 if (gKernelArgs.num_physical_memory_ranges <= 0) 227 beg = 0; 228 else { 229 for (size_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 230 beg = std::min(beg, gKernelArgs.physical_memory_range[i].start); 231 end = std::max(end, gKernelArgs.physical_memory_range[i].start + gKernelArgs.physical_memory_range[i].size); 232 } 233 } 234 range.start = beg; 235 range.size = end - beg; 236 } 237 238 239 static void 240 FillPhysicalMemoryMap(size_t memory_map_size, 241 efi_memory_descriptor *memory_map, size_t descriptor_size, 242 uint32_t descriptor_version) 243 { 244 // Add physical memory to the kernel args and update virtual addresses for 245 // EFI regions. 246 gKernelArgs.num_physical_memory_ranges = 0; 247 248 // First scan: Add all usable ranges 249 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 250 efi_memory_descriptor* entry = &memory_map[i]; 251 switch (entry->Type) { 252 case EfiLoaderCode: 253 case EfiLoaderData: 254 case EfiBootServicesCode: 255 case EfiBootServicesData: 256 case EfiConventionalMemory: { 257 // Usable memory. 258 uint64_t base = entry->PhysicalStart; 259 uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096; 260 uint64_t originalSize = end - base; 261 262 // PMP protected memory, unusable 263 if (base == 0x80000000) 264 break; 265 266 gKernelArgs.ignored_physical_memory 267 += originalSize - (std::max(end, base) - base); 268 269 if (base >= end) 270 break; 271 uint64_t size = end - base; 272 273 insert_physical_memory_range(base, size); 274 break; 275 } 276 case EfiACPIReclaimMemory: 277 // ACPI reclaim -- physical memory we could actually use later 278 break; 279 case EfiRuntimeServicesCode: 280 case EfiRuntimeServicesData: 281 entry->VirtualStart = entry->PhysicalStart; 282 break; 283 } 284 } 285 286 uint64_t initialPhysicalMemory = total_physical_memory(); 287 288 // Second scan: Remove everything reserved that may overlap 289 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 290 efi_memory_descriptor* entry = &memory_map[i]; 291 switch (entry->Type) { 292 case EfiLoaderCode: 293 case EfiLoaderData: 294 case EfiBootServicesCode: 295 case EfiBootServicesData: 296 case EfiConventionalMemory: 297 break; 298 default: 299 uint64_t base = entry->PhysicalStart; 300 uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096; 301 remove_physical_memory_range(base, end - base); 302 } 303 } 304 305 gKernelArgs.ignored_physical_memory 306 += initialPhysicalMemory - total_physical_memory(); 307 308 sort_address_ranges(gKernelArgs.physical_memory_range, 309 gKernelArgs.num_physical_memory_ranges); 310 } 311 312 313 static void 314 FillPhysicalAllocatedMemoryMap(size_t memory_map_size, 315 efi_memory_descriptor *memory_map, size_t descriptor_size, 316 uint32_t descriptor_version) 317 { 318 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 319 efi_memory_descriptor* entry = &memory_map[i]; 320 switch (entry->Type) { 321 case EfiLoaderData: 322 insert_physical_allocated_range(entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE); 323 break; 324 default: 325 ; 326 } 327 } 328 sort_address_ranges(gKernelArgs.physical_allocated_range, 329 gKernelArgs.num_physical_allocated_ranges); 330 } 331 332 333 //#pragma mark - 334 335 336 void 337 arch_mmu_init() 338 { 339 } 340 341 342 void 343 arch_mmu_post_efi_setup(size_t memory_map_size, 344 efi_memory_descriptor *memory_map, size_t descriptor_size, 345 uint32_t descriptor_version) 346 { 347 FillPhysicalAllocatedMemoryMap(memory_map_size, memory_map, descriptor_size, descriptor_version); 348 349 // Switch EFI to virtual mode, using the kernel pmap. 350 // Something involving ConvertPointer might need to be done after this? 351 // http://wiki.phoenix.com/wiki/index.php/EFI_RUNTIME_SERVICES 352 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, 353 descriptor_version, memory_map); 354 } 355 356 357 uint64 358 arch_mmu_generate_post_efi_page_tables(size_t memory_map_size, 359 efi_memory_descriptor *memory_map, size_t descriptor_size, 360 uint32_t descriptor_version) 361 { 362 sPageTable = mmu_allocate_page(); 363 memset(VirtFromPhys(sPageTable), 0, B_PAGE_SIZE); 364 dprintf("sPageTable: %#" B_PRIxADDR "\n", sPageTable); 365 366 PreallocKernelRange(); 367 368 gKernelArgs.num_virtual_allocated_ranges = 0; 369 gKernelArgs.arch_args.num_virtual_ranges_to_keep = 0; 370 FillPhysicalMemoryMap(memory_map_size, memory_map, descriptor_size, descriptor_version); 371 372 addr_range physMemRange; 373 GetPhysMemRange(physMemRange); 374 dprintf("physMemRange: %#" B_PRIxADDR ", %#" B_PRIxSIZE "\n", physMemRange.start, physMemRange.size); 375 376 // Physical memory mapping 377 gKernelArgs.arch_args.physMap.start = KERNEL_TOP + 1 - physMemRange.size; 378 gKernelArgs.arch_args.physMap.size = physMemRange.size; 379 MapRange(gKernelArgs.arch_args.physMap.start, physMemRange.start, physMemRange.size, (1 << pteRead) | (1 << pteWrite)); 380 381 // Boot loader 382 dprintf("Boot loader:\n"); 383 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 384 efi_memory_descriptor* entry = &memory_map[i]; 385 switch (entry->Type) { 386 case EfiLoaderCode: 387 case EfiLoaderData: 388 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 389 break; 390 default: 391 ; 392 } 393 } 394 dprintf("Boot loader stack\n"); 395 addr_t sp = Sp(); 396 addr_t stackTop = ROUNDDOWN(sp - 1024*64, B_PAGE_SIZE); 397 dprintf(" SP: %#" B_PRIxADDR "\n", sp); 398 399 // EFI runtime services 400 dprintf("EFI runtime services:\n"); 401 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 402 efi_memory_descriptor* entry = &memory_map[i]; 403 if ((entry->Attribute & EFI_MEMORY_RUNTIME) != 0) 404 MapRange(entry->VirtualStart, entry->PhysicalStart, entry->NumberOfPages * B_PAGE_SIZE, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 405 } 406 407 // Memory regions 408 dprintf("Regions:\n"); 409 void* cookie = NULL; 410 addr_t virtAdr; 411 phys_addr_t physAdr; 412 size_t size; 413 while (mmu_next_region(&cookie, &virtAdr, &physAdr, &size)) { 414 MapRange(virtAdr, physAdr, size, (1 << pteRead) | (1 << pteWrite) | (1 << pteExec)); 415 } 416 417 // Devices 418 dprintf("Devices:\n"); 419 MapAddrRange(gKernelArgs.arch_args.clint, (1 << pteRead) | (1 << pteWrite)); 420 MapAddrRange(gKernelArgs.arch_args.htif, (1 << pteRead) | (1 << pteWrite)); 421 MapAddrRange(gKernelArgs.arch_args.plic, (1 << pteRead) | (1 << pteWrite)); 422 423 if (strcmp(gKernelArgs.arch_args.uart.kind, "") != 0) { 424 MapRange(gKernelArgs.arch_args.uart.regs.start, 425 gKernelArgs.arch_args.uart.regs.start, 426 gKernelArgs.arch_args.uart.regs.size, 427 (1 << pteRead) | (1 << pteWrite)); 428 MapAddrRange(gKernelArgs.arch_args.uart.regs, 429 (1 << pteRead) | (1 << pteWrite)); 430 } 431 432 sort_address_ranges(gKernelArgs.virtual_allocated_range, gKernelArgs.num_virtual_allocated_ranges); 433 434 DumpPageTable(GetSatp()); 435 436 return GetSatp(); 437 } 438