1 /* 2 * Copyright 2019-2022 Haiku, Inc. All rights reserved. 3 * Released under the terms of the MIT License. 4 */ 5 6 #include <boot/platform.h> 7 #include <boot/stage2.h> 8 9 #include "efi_platform.h" 10 #include "generic_mmu.h" 11 #include "mmu.h" 12 13 #include "aarch64.h" 14 #include "arch_mmu.h" 15 16 // #define TRACE_MMU 17 #ifdef TRACE_MMU 18 # define TRACE(x...) dprintf(x) 19 #else 20 # define TRACE(x...) ; 21 #endif 22 23 24 //#define TRACE_MEMORY_MAP 25 //#define TRACE_PAGE_DIRECTORY 26 27 // Ignore memory above 512GB 28 #define PHYSICAL_MEMORY_LOW 0x00000000 29 #define PHYSICAL_MEMORY_HIGH 0x8000000000ull 30 31 ARMv8TranslationRegime::TranslationDescriptor translation4Kb48bits = { 32 {L0_SHIFT, L0_ADDR_MASK, false, true, false }, 33 {L1_SHIFT, Ln_ADDR_MASK, true, true, false }, 34 {L2_SHIFT, Ln_ADDR_MASK, true, true, false }, 35 {L3_SHIFT, Ln_ADDR_MASK, false, false, true } 36 }; 37 38 39 ARMv8TranslationRegime CurrentRegime(translation4Kb48bits); 40 /* ARM port */ 41 static uint64_t* sPageDirectory = NULL; 42 // static uint64_t* sFirstPageTable = NULL; 43 static uint64_t* sNextPageTable = NULL; 44 // static uint64_t* sLastPageTable = NULL; 45 46 47 const char* 48 granule_type_str(int tg) 49 { 50 switch (tg) { 51 case TG_4KB: 52 return "4KB"; 53 case TG_16KB: 54 return "16KB"; 55 case TG_64KB: 56 return "64KB"; 57 default: 58 return "Invalid Granule"; 59 } 60 } 61 62 63 void 64 arch_mmu_dump_table(uint64* table, uint8 currentLevel) 65 { 66 ARMv8TranslationTableDescriptor ttd(table); 67 68 if (currentLevel >= CurrentRegime.MaxLevels()) { 69 // This should not happen 70 panic("Too many levels ..."); 71 return; 72 } 73 74 uint64 EntriesPerLevel = arch_mmu_entries_per_granularity(CurrentRegime.Granularity()); 75 for (uint i = 0 ; i < EntriesPerLevel; i++) { 76 if (!ttd.IsInvalid()) { 77 TRACE("Level %d, @%0lx: TTD %016lx\t", currentLevel, ttd.Location(), ttd.Value()); 78 if (ttd.IsTable() && currentLevel < 3) { 79 TRACE("Table! Next Level:\n"); 80 arch_mmu_dump_table(ttd.Dereference(), currentLevel + 1); 81 } 82 if (ttd.IsBlock() || (ttd.IsPage() && currentLevel == 3)) { 83 TRACE("Block/Page"); 84 85 if (i & 1) { // 2 entries per row 86 TRACE("\n"); 87 } else { 88 TRACE("\t"); 89 } 90 } 91 } 92 ttd.Next(); 93 } 94 } 95 96 97 #ifdef TRACE_PAGE_DIRECTORY 98 void 99 arch_mmu_dump_present_tables() 100 { 101 uint64 address = arch_mmu_base_register(); 102 dprintf("Under TTBR0: %lx\n", address); 103 104 arch_mmu_dump_table(reinterpret_cast<uint64*>(address), 0); 105 106 /* We are willing to transition, but still in EL2, present MMU configuration 107 * for user is present in EL2 by TTBR0_EL2. Kernel side is not active, but 108 * allocated under sPageDirectory, defined under TTBR1_EL1. 109 */ 110 dprintf("Under allocated TTBR1_EL1:\n"); 111 arch_mmu_dump_table(sPageDirectory, 0); 112 } 113 #endif 114 115 116 void arch_mmu_setup_EL1(uint64 tcr) { 117 118 // Enable TTBR1 119 tcr &= ~TCR_EPD1_DISABLE; 120 121 // Set space for kernel space 122 tcr &= ~T1SZ_MASK; // Clear 123 // TODO: Compiler dependency? 124 tcr |= TCR_T1SZ(__builtin_popcountl(KERNEL_BASE)); 125 126 WRITE_SPECIALREG(TCR_EL1, tcr); 127 } 128 129 130 uint64 131 map_region(addr_t virt_addr, addr_t phys_addr, size_t size, 132 uint32_t level, uint64_t flags, uint64* descriptor) 133 { 134 ARMv8TranslationTableDescriptor ttd(descriptor); 135 136 if (level >= CurrentRegime.MaxLevels()) { 137 panic("Too many levels at mapping\n"); 138 } 139 140 uint64 currentLevelSize = CurrentRegime.EntrySize(level); 141 142 ttd.JumpTo(CurrentRegime.DescriptorIndex(virt_addr, level)); 143 144 uint64 remainingSizeInTable = CurrentRegime.TableSize(level) 145 - currentLevelSize * CurrentRegime.DescriptorIndex(virt_addr, level); 146 147 TRACE("Level %x, Processing desc %lx indexing %lx\n", 148 level, reinterpret_cast<uint64>(descriptor), ttd.Location()); 149 150 if (ttd.IsInvalid()) { 151 // If the physical has the same alignment we could make a block here 152 // instead of using a complete next level table 153 if (size >= currentLevelSize && CurrentRegime.Aligned(phys_addr, level)) { 154 // Set it as block or page 155 if (CurrentRegime.BlocksAllowed(level)) { 156 ttd.SetAsBlock(reinterpret_cast<uint64*>(phys_addr), flags); 157 } else { 158 // Most likely in Level 3... 159 ttd.SetAsPage(reinterpret_cast<uint64*>(phys_addr), flags); 160 } 161 162 // Expand! 163 int64 expandedSize = (size > remainingSizeInTable)?remainingSizeInTable:size; 164 165 do { 166 phys_addr += currentLevelSize; 167 expandedSize -= currentLevelSize; 168 if (expandedSize > 0) { 169 ttd.Next(); 170 if (CurrentRegime.BlocksAllowed(level)) { 171 ttd.SetAsBlock(reinterpret_cast<uint64*>(phys_addr), flags); 172 } else { 173 // Most likely in Level 3... 174 ttd.SetAsPage(reinterpret_cast<uint64*>(phys_addr), flags); 175 } 176 } 177 } while (expandedSize > 0); 178 179 return (size > remainingSizeInTable)?(size - remainingSizeInTable):0; 180 181 } else { 182 // Set it to next level 183 uint64 offset = 0; 184 uint64 remainingSize = size; 185 do { 186 uint64* page = NULL; 187 if (ttd.IsInvalid()) { 188 // our region is too small would need to create a level below 189 page = CurrentRegime.AllocatePage(); 190 ttd.SetToTable(page, flags); 191 } else if (ttd.IsTable()) { 192 // Next table is allocated, follow it 193 page = ttd.Dereference(); 194 } else { 195 panic("Required contiguous descriptor in use by Block/Page for %lx\n", ttd.Location()); 196 } 197 198 uint64 unprocessedSize = map_region(virt_addr + offset, 199 phys_addr + offset, remainingSize, level + 1, flags, page); 200 201 offset = remainingSize - unprocessedSize; 202 203 remainingSize = unprocessedSize; 204 205 ttd.Next(); 206 207 } while (remainingSize > 0); 208 209 return 0; 210 } 211 212 } else { 213 214 if ((ttd.IsBlock() && CurrentRegime.BlocksAllowed(level)) 215 || (ttd.IsPage() && CurrentRegime.PagesAllowed(level)) 216 ) { 217 // TODO: Review, overlap? expand? 218 panic("Re-setting a Block/Page descriptor for %lx\n", ttd.Location()); 219 return 0; 220 } else if (ttd.IsTable() && CurrentRegime.TablesAllowed(level)) { 221 // Next Level 222 map_region(virt_addr, phys_addr, size, level + 1, flags, ttd.Dereference()); 223 return 0; 224 } else { 225 panic("All descriptor types processed for %lx\n", ttd.Location()); 226 return 0; 227 } 228 } 229 } 230 231 232 static void 233 map_range(addr_t virt_addr, phys_addr_t phys_addr, size_t size, uint64_t flags) 234 { 235 TRACE("map 0x%0lx --> 0x%0lx, len=0x%0lx, flags=0x%0lx\n", 236 (uint64_t)virt_addr, (uint64_t)phys_addr, (uint64_t)size, flags); 237 238 // TODO: Review why we get ranges with 0 size ... 239 if (size == 0) { 240 TRACE("Requesing 0 size map\n"); 241 return; 242 } 243 244 // TODO: Review this case 245 if (phys_addr == READ_SPECIALREG(TTBR1_EL1)) { 246 TRACE("Trying to map the TTBR itself?!\n"); 247 return; 248 } 249 250 if (arch_mmu_read_access(virt_addr) && arch_mmu_read_access(virt_addr + size)) { 251 TRACE("Range already covered in current MMU\n"); 252 return; 253 } 254 255 uint64 address; 256 257 if (arch_mmu_is_kernel_address(virt_addr)) { 258 // Use TTBR1 259 address = READ_SPECIALREG(TTBR1_EL1); 260 } else { 261 // ok, but USE instead TTBR0 262 address = READ_SPECIALREG(TTBR0_EL1); 263 } 264 265 map_region(virt_addr, phys_addr, size, 0, flags, reinterpret_cast<uint64*>(address)); 266 267 // for (addr_t offset = 0; offset < size; offset += B_PAGE_SIZE) { 268 // map_page(virt_addr + offset, phys_addr + offset, flags); 269 // } 270 271 ASSERT_ALWAYS(insert_virtual_allocated_range(virt_addr, size) >= B_OK); 272 } 273 274 275 void 276 arch_mmu_init() 277 { 278 // Stub 279 } 280 281 282 void 283 arch_mmu_post_efi_setup(size_t memory_map_size, 284 efi_memory_descriptor* memory_map, size_t descriptor_size, 285 uint32_t descriptor_version) 286 { 287 build_physical_allocated_list(memory_map_size, memory_map, 288 descriptor_size, descriptor_version); 289 290 // Switch EFI to virtual mode, using the kernel pmap. 291 kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, 292 descriptor_version, memory_map); 293 294 #ifdef TRACE_MEMORY_MAP 295 dprintf("phys memory ranges:\n"); 296 for (uint32_t i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 297 uint64 start = gKernelArgs.physical_memory_range[i].start; 298 uint64 size = gKernelArgs.physical_memory_range[i].size; 299 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 300 start, start + size, size); 301 } 302 303 dprintf("allocated phys memory ranges:\n"); 304 for (uint32_t i = 0; i < gKernelArgs.num_physical_allocated_ranges; i++) { 305 uint64 start = gKernelArgs.physical_allocated_range[i].start; 306 uint64 size = gKernelArgs.physical_allocated_range[i].size; 307 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 308 start, start + size, size); 309 } 310 311 dprintf("allocated virt memory ranges:\n"); 312 for (uint32_t i = 0; i < gKernelArgs.num_virtual_allocated_ranges; i++) { 313 uint64 start = gKernelArgs.virtual_allocated_range[i].start; 314 uint64 size = gKernelArgs.virtual_allocated_range[i].size; 315 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 316 start, start + size, size); 317 } 318 319 dprintf("virt memory ranges to keep:\n"); 320 for (uint32_t i = 0; i < gKernelArgs.arch_args.num_virtual_ranges_to_keep; i++) { 321 uint64 start = gKernelArgs.arch_args.virtual_ranges_to_keep[i].start; 322 uint64 size = gKernelArgs.arch_args.virtual_ranges_to_keep[i].size; 323 dprintf(" 0x%08" B_PRIx64 "-0x%08" B_PRIx64 ", length 0x%08" B_PRIx64 "\n", 324 start, start + size, size); 325 } 326 #endif 327 } 328 329 330 void 331 arch_mmu_allocate_kernel_page_tables(void) 332 { 333 uint64* page = NULL; 334 uint64 ttbr1 = READ_SPECIALREG(TTBR1_EL1); 335 336 // Trust possible previous allocations of TTBR1 337 // only if we come from a preset EL1 context 338 if (ttbr1 != 0ll) { 339 if (arch_exception_level() == 1) { 340 page = reinterpret_cast<uint64*>(ttbr1); 341 TRACE("Reusing TTBR1_EL1 present : %" B_PRIx64 "\n", ttbr1); 342 } else if (arch_exception_level() == 2) { 343 TRACE("Ignoring EL1 TTBR1(%" B_PRIx64") tables\n", ttbr1); 344 } 345 } 346 347 // NOTE: On devices supporting multiple translation base registers, TTBR0 must 348 // be used solely. 349 if (page == NULL) { 350 page = CurrentRegime.AllocatePage(); 351 if (page != NULL) { 352 WRITE_SPECIALREG(TTBR1_EL1, page); 353 } else { 354 panic("Not enough memory for kernel initial page\n"); 355 } 356 } 357 358 sPageDirectory = page; 359 } 360 361 362 uint32_t 363 arch_mmu_generate_post_efi_page_tables(size_t memory_map_size, 364 efi_memory_descriptor* memory_map, size_t descriptor_size, 365 uint32_t descriptor_version) 366 { 367 addr_t memory_map_addr = (addr_t)memory_map; 368 369 MemoryAttributeIndirection currentMair; 370 371 // arch_mmu_allocate_page_tables(); 372 arch_mmu_allocate_kernel_page_tables(); 373 374 build_physical_memory_list(memory_map_size, memory_map, 375 descriptor_size, descriptor_version, 376 PHYSICAL_MEMORY_LOW, PHYSICAL_MEMORY_HIGH); 377 378 TRACE("Mapping EFI_MEMORY_RUNTIME\n"); 379 for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { 380 efi_memory_descriptor* entry = (efi_memory_descriptor*)(memory_map_addr + i * descriptor_size); 381 if ((entry->Attribute & EFI_MEMORY_RUNTIME) != 0) 382 map_range(entry->VirtualStart, entry->PhysicalStart, 383 entry->NumberOfPages * B_PAGE_SIZE, 384 ARMv8TranslationTableDescriptor::DefaultCodeAttribute | currentMair.MaskOf(MAIR_NORMAL_WB)); 385 } 386 387 TRACE("Mapping \"next\" regions\n"); 388 void* cookie = NULL; 389 addr_t vaddr; 390 phys_addr_t paddr; 391 size_t size; 392 while (mmu_next_region(&cookie, &vaddr, &paddr, &size)) { 393 map_range(vaddr, paddr, size, 394 ARMv8TranslationTableDescriptor::DefaultCodeAttribute 395 | currentMair.MaskOf(MAIR_NORMAL_WB)); 396 } 397 398 // TODO: We actually can only map physical RAM, mapping everything 399 // could cause unwanted MMIO or bus errors on real hardware. 400 map_range(KERNEL_PMAP_BASE, 0, KERNEL_PMAP_SIZE - 1, 401 ARMv8TranslationTableDescriptor::DefaultCodeAttribute 402 | currentMair.MaskOf(MAIR_NORMAL_WB)); 403 404 if (gKernelArgs.arch_args.uart.kind[0] != 0) { 405 // Map uart because we want to use it during early boot. 406 uint64 regs_start = gKernelArgs.arch_args.uart.regs.start; 407 uint64 regs_size = ROUNDUP(gKernelArgs.arch_args.uart.regs.size, B_PAGE_SIZE); 408 uint64 base = get_next_virtual_address(regs_size); 409 410 map_range(base, regs_start, regs_size, 411 ARMv8TranslationTableDescriptor::DefaultPeripheralAttribute | 412 currentMair.MaskOf(MAIR_DEVICE_nGnRnE)); 413 414 gKernelArgs.arch_args.uart.regs.start = base; 415 } 416 417 sort_address_ranges(gKernelArgs.virtual_allocated_range, 418 gKernelArgs.num_virtual_allocated_ranges); 419 420 addr_t vir_pgdir; 421 platform_bootloader_address_to_kernel_address((void*)sPageDirectory, &vir_pgdir); 422 423 gKernelArgs.arch_args.phys_pgdir = (uint64)sPageDirectory; 424 gKernelArgs.arch_args.vir_pgdir = (uint32)vir_pgdir; 425 gKernelArgs.arch_args.next_pagetable = (uint64)(sNextPageTable) - (uint64)sPageDirectory; 426 427 TRACE("gKernelArgs.arch_args.phys_pgdir = 0x%08x\n", 428 (uint32_t)gKernelArgs.arch_args.phys_pgdir); 429 TRACE("gKernelArgs.arch_args.vir_pgdir = 0x%08x\n", 430 (uint32_t)gKernelArgs.arch_args.vir_pgdir); 431 TRACE("gKernelArgs.arch_args.next_pagetable = 0x%08x\n", 432 (uint32_t)gKernelArgs.arch_args.next_pagetable); 433 434 #ifdef TRACE_PAGE_DIRECTORY 435 arch_mmu_dump_present_tables(); 436 #endif 437 438 return (uint64_t)sPageDirectory; 439 } 440