1a25542e7Smilek7 /* 2a25542e7Smilek7 * Copyright 2022 Haiku, Inc. All Rights Reserved. 3a25542e7Smilek7 * Distributed under the terms of the MIT License. 4a25542e7Smilek7 */ 5a25542e7Smilek7 #include "VMSAv8TranslationMap.h" 6a25542e7Smilek7 7a25542e7Smilek7 #include <util/AutoLock.h> 8a25542e7Smilek7 #include <util/ThreadAutoLock.h> 9a25542e7Smilek7 #include <vm/vm_page.h> 10a25542e7Smilek7 #include <vm/vm_priv.h> 11a25542e7Smilek7 12a25542e7Smilek7 13a25542e7Smilek7 static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); 14a25542e7Smilek7 static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); 15a25542e7Smilek7 1618a27fe0SOwen Anderson static constexpr uint64_t kPteTypeMask = 0x3; 1718a27fe0SOwen Anderson static constexpr uint64_t kPteTypeL012Table = 0x3; 1818a27fe0SOwen Anderson static constexpr uint64_t kPteTypeL012Block = 0x1; 1918a27fe0SOwen Anderson static constexpr uint64_t kPteTypeL3Page = 0x3; 2018a27fe0SOwen Anderson 21a25542e7Smilek7 static constexpr uint64_t kAttrSWDBM = (1UL << 55); 22a25542e7Smilek7 static constexpr uint64_t kAttrUXN = (1UL << 54); 23a25542e7Smilek7 static constexpr uint64_t kAttrPXN = (1UL << 53); 24a25542e7Smilek7 static constexpr uint64_t kAttrDBM = (1UL << 51); 25a25542e7Smilek7 static constexpr uint64_t kAttrNG = (1UL << 11); 26a25542e7Smilek7 static constexpr uint64_t kAttrAF = (1UL << 10); 27*108f6fdcSOwen Anderson static constexpr uint64_t kAttrSHInnerShareable = (3UL << 8); 28*108f6fdcSOwen Anderson static constexpr uint64_t kAttrAPReadOnly = (1UL << 7); 29*108f6fdcSOwen Anderson static constexpr uint64_t kAttrAPUserAccess = (1UL << 6); 30a25542e7Smilek7 317908993dSOwen Anderson static constexpr uint64_t kTLBIMask = ((1UL << 44) - 1); 327908993dSOwen Anderson 33a25542e7Smilek7 uint32_t VMSAv8TranslationMap::fHwFeature; 34a25542e7Smilek7 uint64_t VMSAv8TranslationMap::fMair; 35a25542e7Smilek7 369fad0a5cSOwen Anderson // ASID Management 379fad0a5cSOwen Anderson static constexpr size_t kAsidBits = 8; 389fad0a5cSOwen Anderson static constexpr size_t kNumAsids = (1 << kAsidBits); 397908993dSOwen Anderson static spinlock sAsidLock = B_SPINLOCK_INITIALIZER; 409fad0a5cSOwen Anderson // A bitmap to track which ASIDs are in use. 419fad0a5cSOwen Anderson static uint64 sAsidBitMap[kNumAsids / 64] = {}; 429fad0a5cSOwen Anderson // A mapping from ASID to translation map. 439fad0a5cSOwen Anderson static VMSAv8TranslationMap* sAsidMapping[kNumAsids] = {}; 449fad0a5cSOwen Anderson 459fad0a5cSOwen Anderson 469fad0a5cSOwen Anderson static void 479fad0a5cSOwen Anderson free_asid(size_t asid) 489fad0a5cSOwen Anderson { 499fad0a5cSOwen Anderson for (size_t i = 0; i < B_COUNT_OF(sAsidBitMap); ++i) { 509fad0a5cSOwen Anderson if (asid < 64) { 519fad0a5cSOwen Anderson sAsidBitMap[i] &= ~(uint64_t{1} << asid); 529fad0a5cSOwen Anderson return; 539fad0a5cSOwen Anderson } 549fad0a5cSOwen Anderson asid -= 64; 559fad0a5cSOwen Anderson } 569fad0a5cSOwen Anderson 579fad0a5cSOwen Anderson panic("Could not free ASID!"); 589fad0a5cSOwen Anderson } 599fad0a5cSOwen Anderson 609fad0a5cSOwen Anderson 619fad0a5cSOwen Anderson static size_t 629fad0a5cSOwen Anderson alloc_first_free_asid(void) 639fad0a5cSOwen Anderson { 649fad0a5cSOwen Anderson int asid = 0; 659fad0a5cSOwen Anderson for (size_t i = 0; i < B_COUNT_OF(sAsidBitMap); ++i) { 669fad0a5cSOwen Anderson int avail = __builtin_ffsll(~sAsidBitMap[i]); 679fad0a5cSOwen Anderson if (avail != 0) { 689fad0a5cSOwen Anderson sAsidBitMap[i] |= (uint64_t{1} << (avail-1)); 699fad0a5cSOwen Anderson asid += (avail - 1); 709fad0a5cSOwen Anderson return asid; 719fad0a5cSOwen Anderson } 729fad0a5cSOwen Anderson asid += 64; 739fad0a5cSOwen Anderson } 749fad0a5cSOwen Anderson 759fad0a5cSOwen Anderson return kNumAsids; 769fad0a5cSOwen Anderson } 777908993dSOwen Anderson 78a25542e7Smilek7 79a25542e7Smilek7 VMSAv8TranslationMap::VMSAv8TranslationMap( 80a25542e7Smilek7 bool kernel, phys_addr_t pageTable, int pageBits, int vaBits, int minBlockLevel) 81a25542e7Smilek7 : 82a25542e7Smilek7 fIsKernel(kernel), 83a25542e7Smilek7 fPageTable(pageTable), 84a25542e7Smilek7 fPageBits(pageBits), 85a25542e7Smilek7 fVaBits(vaBits), 867908993dSOwen Anderson fMinBlockLevel(minBlockLevel), 879fad0a5cSOwen Anderson fASID(-1), 889fad0a5cSOwen Anderson fRefcount(0) 89a25542e7Smilek7 { 90a25542e7Smilek7 dprintf("VMSAv8TranslationMap\n"); 91a25542e7Smilek7 92a25542e7Smilek7 fInitialLevel = CalcStartLevel(fVaBits, fPageBits); 93a25542e7Smilek7 } 94a25542e7Smilek7 95a25542e7Smilek7 96a25542e7Smilek7 VMSAv8TranslationMap::~VMSAv8TranslationMap() 97a25542e7Smilek7 { 987908993dSOwen Anderson ASSERT(!fIsKernel); 999fad0a5cSOwen Anderson ASSERT(fRefcount == 0); 1007908993dSOwen Anderson { 1017908993dSOwen Anderson ThreadCPUPinner pinner(thread_get_current_thread()); 1027908993dSOwen Anderson FreeTable(fPageTable, 0, fInitialLevel, [](int level, uint64_t oldPte) {}); 1037908993dSOwen Anderson } 104a25542e7Smilek7 1057908993dSOwen Anderson { 1067908993dSOwen Anderson InterruptsSpinLocker locker(sAsidLock); 1077908993dSOwen Anderson 1089fad0a5cSOwen Anderson if (fASID != -1) { 1097908993dSOwen Anderson sAsidMapping[fASID] = NULL; 1109fad0a5cSOwen Anderson free_asid(fASID); 1117908993dSOwen Anderson } 112a25542e7Smilek7 } 1139fad0a5cSOwen Anderson } 1149fad0a5cSOwen Anderson 1159fad0a5cSOwen Anderson 1169fad0a5cSOwen Anderson // Switch user map into TTBR0. 1179fad0a5cSOwen Anderson // Passing kernel map here configures empty page table. 1189fad0a5cSOwen Anderson void 1199fad0a5cSOwen Anderson VMSAv8TranslationMap::SwitchUserMap(VMSAv8TranslationMap *from, VMSAv8TranslationMap *to) 1209fad0a5cSOwen Anderson { 1219fad0a5cSOwen Anderson SpinLocker locker(sAsidLock); 1229fad0a5cSOwen Anderson 1239fad0a5cSOwen Anderson if (!from->fIsKernel) { 1249fad0a5cSOwen Anderson from->fRefcount--; 1259fad0a5cSOwen Anderson } 1269fad0a5cSOwen Anderson 1279fad0a5cSOwen Anderson if (!to->fIsKernel) { 1289fad0a5cSOwen Anderson to->fRefcount++; 1299fad0a5cSOwen Anderson } else { 1309fad0a5cSOwen Anderson arch_vm_install_empty_table_ttbr0(); 1319fad0a5cSOwen Anderson return; 1329fad0a5cSOwen Anderson } 1339fad0a5cSOwen Anderson 1349fad0a5cSOwen Anderson ASSERT(to->fPageTable != 0); 1359fad0a5cSOwen Anderson uint64_t ttbr = to->fPageTable | ((fHwFeature & HW_COMMON_NOT_PRIVATE) != 0 ? 1 : 0); 1369fad0a5cSOwen Anderson 1379fad0a5cSOwen Anderson if (to->fASID != -1) { 1389fad0a5cSOwen Anderson WRITE_SPECIALREG(TTBR0_EL1, ((uint64_t)to->fASID << 48) | ttbr); 1399fad0a5cSOwen Anderson asm("isb"); 1409fad0a5cSOwen Anderson return; 1419fad0a5cSOwen Anderson } 1429fad0a5cSOwen Anderson 1439fad0a5cSOwen Anderson size_t allocatedAsid = alloc_first_free_asid(); 1449fad0a5cSOwen Anderson if (allocatedAsid != kNumAsids) { 1459fad0a5cSOwen Anderson to->fASID = allocatedAsid; 1469fad0a5cSOwen Anderson sAsidMapping[allocatedAsid] = to; 1479fad0a5cSOwen Anderson 1489fad0a5cSOwen Anderson WRITE_SPECIALREG(TTBR0_EL1, (allocatedAsid << 48) | ttbr); 1499fad0a5cSOwen Anderson asm("isb"); 1509fad0a5cSOwen Anderson return; 1519fad0a5cSOwen Anderson } 1529fad0a5cSOwen Anderson 1539fad0a5cSOwen Anderson for (size_t i = 0; i < kNumAsids; ++i) { 1549fad0a5cSOwen Anderson if (sAsidMapping[i]->fRefcount == 0) { 1559fad0a5cSOwen Anderson sAsidMapping[i]->fASID = -1; 1569fad0a5cSOwen Anderson to->fASID = i; 1579fad0a5cSOwen Anderson sAsidMapping[i] = to; 1589fad0a5cSOwen Anderson 1599fad0a5cSOwen Anderson WRITE_SPECIALREG(TTBR0_EL1, (i << 48) | ttbr); 1609fad0a5cSOwen Anderson asm("dsb ishst"); 1619fad0a5cSOwen Anderson asm("tlbi aside1is, %0" :: "r" (i << 48)); 1629fad0a5cSOwen Anderson asm("dsb ish"); 1639fad0a5cSOwen Anderson asm("isb"); 1649fad0a5cSOwen Anderson return; 1659fad0a5cSOwen Anderson } 1669fad0a5cSOwen Anderson } 1679fad0a5cSOwen Anderson 1689fad0a5cSOwen Anderson panic("cannot assign ASID"); 1699fad0a5cSOwen Anderson } 170a25542e7Smilek7 171a25542e7Smilek7 172a25542e7Smilek7 int 173a25542e7Smilek7 VMSAv8TranslationMap::CalcStartLevel(int vaBits, int pageBits) 174a25542e7Smilek7 { 175a25542e7Smilek7 int level = 4; 176a25542e7Smilek7 177a25542e7Smilek7 int bitsLeft = vaBits - pageBits; 178a25542e7Smilek7 while (bitsLeft > 0) { 179a25542e7Smilek7 int tableBits = pageBits - 3; 180a25542e7Smilek7 bitsLeft -= tableBits; 181a25542e7Smilek7 level--; 182a25542e7Smilek7 } 183a25542e7Smilek7 184a25542e7Smilek7 ASSERT(level >= 0); 185a25542e7Smilek7 186a25542e7Smilek7 return level; 187a25542e7Smilek7 } 188a25542e7Smilek7 189a25542e7Smilek7 190a25542e7Smilek7 bool 191a25542e7Smilek7 VMSAv8TranslationMap::Lock() 192a25542e7Smilek7 { 193a25542e7Smilek7 recursive_lock_lock(&fLock); 194a25542e7Smilek7 return true; 195a25542e7Smilek7 } 196a25542e7Smilek7 197a25542e7Smilek7 198a25542e7Smilek7 void 199a25542e7Smilek7 VMSAv8TranslationMap::Unlock() 200a25542e7Smilek7 { 201a25542e7Smilek7 if (recursive_lock_get_recursion(&fLock) == 1) { 202a25542e7Smilek7 // we're about to release it for the last time 203a25542e7Smilek7 Flush(); 204a25542e7Smilek7 } 205a25542e7Smilek7 recursive_lock_unlock(&fLock); 206a25542e7Smilek7 } 207a25542e7Smilek7 208a25542e7Smilek7 209a25542e7Smilek7 addr_t 210a25542e7Smilek7 VMSAv8TranslationMap::MappedSize() const 211a25542e7Smilek7 { 212a25542e7Smilek7 panic("VMSAv8TranslationMap::MappedSize not implemented"); 213a25542e7Smilek7 return 0; 214a25542e7Smilek7 } 215a25542e7Smilek7 216a25542e7Smilek7 217a25542e7Smilek7 size_t 218a25542e7Smilek7 VMSAv8TranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const 219a25542e7Smilek7 { 220a25542e7Smilek7 size_t result = 0; 221a25542e7Smilek7 size_t size = end - start + 1; 222a25542e7Smilek7 223a25542e7Smilek7 for (int i = fInitialLevel; i < 3; i++) { 224a25542e7Smilek7 int tableBits = fPageBits - 3; 225a25542e7Smilek7 int shift = tableBits * (3 - i) + fPageBits; 226a25542e7Smilek7 uint64_t entrySize = 1UL << shift; 227a25542e7Smilek7 228a25542e7Smilek7 result += size / entrySize + 2; 229a25542e7Smilek7 } 230a25542e7Smilek7 231a25542e7Smilek7 return result; 232a25542e7Smilek7 } 233a25542e7Smilek7 234a25542e7Smilek7 235a25542e7Smilek7 uint64_t* 236a25542e7Smilek7 VMSAv8TranslationMap::TableFromPa(phys_addr_t pa) 237a25542e7Smilek7 { 238a25542e7Smilek7 return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa); 239a25542e7Smilek7 } 240a25542e7Smilek7 241a25542e7Smilek7 242a25542e7Smilek7 uint64_t 243a25542e7Smilek7 VMSAv8TranslationMap::MakeBlock(phys_addr_t pa, int level, uint64_t attr) 244a25542e7Smilek7 { 245a25542e7Smilek7 ASSERT(level >= fMinBlockLevel && level < 4); 246a25542e7Smilek7 247a25542e7Smilek7 return pa | attr | (level == 3 ? 0x3 : 0x1); 248a25542e7Smilek7 } 249a25542e7Smilek7 250a25542e7Smilek7 2517908993dSOwen Anderson template<typename EntryRemoved> 252a25542e7Smilek7 void 2537908993dSOwen Anderson VMSAv8TranslationMap::FreeTable(phys_addr_t ptPa, uint64_t va, int level, 2547908993dSOwen Anderson EntryRemoved &&entryRemoved) 255a25542e7Smilek7 { 2567908993dSOwen Anderson ASSERT(level < 4); 257a25542e7Smilek7 258a25542e7Smilek7 int tableBits = fPageBits - 3; 259a25542e7Smilek7 uint64_t tableSize = 1UL << tableBits; 2607908993dSOwen Anderson uint64_t vaMask = (1UL << fVaBits) - 1; 261a25542e7Smilek7 2627908993dSOwen Anderson int shift = tableBits * (3 - level) + fPageBits; 2637908993dSOwen Anderson uint64_t entrySize = 1UL << shift; 2647908993dSOwen Anderson 2657908993dSOwen Anderson uint64_t nextVa = va; 266a25542e7Smilek7 uint64_t* pt = TableFromPa(ptPa); 267a25542e7Smilek7 for (uint64_t i = 0; i < tableSize; i++) { 2687908993dSOwen Anderson uint64_t oldPte = (uint64_t) atomic_get_and_set64((int64*) &pt[i], 0); 2697908993dSOwen Anderson 27018a27fe0SOwen Anderson if (level < 3 && (oldPte & kPteTypeMask) == kPteTypeL012Table) { 2717908993dSOwen Anderson FreeTable(oldPte & kPteAddrMask, nextVa, level + 1, entryRemoved); 27218a27fe0SOwen Anderson } else if ((oldPte & kPteTypeMask) != 0) { 2737908993dSOwen Anderson uint64_t fullVa = (fIsKernel ? ~vaMask : 0) | nextVa; 2747908993dSOwen Anderson asm("dsb ishst"); 2757908993dSOwen Anderson asm("tlbi vaae1is, %0" :: "r" ((fullVa >> 12) & kTLBIMask)); 2767908993dSOwen Anderson // Does it correctly flush block entries at level < 3? We don't use them anyway though. 2777908993dSOwen Anderson // TODO: Flush only currently used ASID (using vae1is) 2787908993dSOwen Anderson entryRemoved(level, oldPte); 279a25542e7Smilek7 } 280a25542e7Smilek7 2817908993dSOwen Anderson nextVa += entrySize; 2827908993dSOwen Anderson } 2837908993dSOwen Anderson 2847908993dSOwen Anderson asm("dsb ish"); 2857908993dSOwen Anderson 286a25542e7Smilek7 vm_page* page = vm_lookup_page(ptPa >> fPageBits); 2877908993dSOwen Anderson DEBUG_PAGE_ACCESS_START(page); 288a25542e7Smilek7 vm_page_set_state(page, PAGE_STATE_FREE); 289a25542e7Smilek7 } 290a25542e7Smilek7 291a25542e7Smilek7 29218a27fe0SOwen Anderson // Make a new page sub-table. 29318a27fe0SOwen Anderson // The parent table is `ptPa`, and the new sub-table's PTE will be at `index` 29418a27fe0SOwen Anderson // in it. 29518a27fe0SOwen Anderson // Returns the physical address of the new table, or the address of the existing 29618a27fe0SOwen Anderson // one if the PTE is already filled. 297a25542e7Smilek7 phys_addr_t 298a25542e7Smilek7 VMSAv8TranslationMap::MakeTable( 299a25542e7Smilek7 phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation) 300a25542e7Smilek7 { 30118a27fe0SOwen Anderson ASSERT(level < 3); 302a25542e7Smilek7 30318a27fe0SOwen Anderson uint64_t* ptePtr = TableFromPa(ptPa) + index; 30418a27fe0SOwen Anderson uint64_t oldPte = atomic_get64((int64*) ptePtr); 305a25542e7Smilek7 30618a27fe0SOwen Anderson int type = oldPte & kPteTypeMask; 30718a27fe0SOwen Anderson if (type == kPteTypeL012Table) { 30818a27fe0SOwen Anderson // This is table entry already, just return it 309a25542e7Smilek7 return oldPte & kPteAddrMask; 31018a27fe0SOwen Anderson } else if (reservation != nullptr) { 31118a27fe0SOwen Anderson // Create new table there 31218a27fe0SOwen Anderson vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); 313a25542e7Smilek7 phys_addr_t newTablePa = page->physical_page_number << fPageBits; 31418a27fe0SOwen Anderson DEBUG_PAGE_ACCESS_END(page); 315a25542e7Smilek7 31618a27fe0SOwen Anderson // We only create mappings at the final level so we don't need to handle 31718a27fe0SOwen Anderson // splitting block mappings 31818a27fe0SOwen Anderson ASSERT(type != kPteTypeL012Block); 319a25542e7Smilek7 32018a27fe0SOwen Anderson // Ensure that writes to page being attached have completed 32118a27fe0SOwen Anderson asm("dsb ishst"); 322a25542e7Smilek7 32318a27fe0SOwen Anderson uint64_t oldPteRefetch = (uint64_t)atomic_test_and_set64((int64*) ptePtr, 32418a27fe0SOwen Anderson newTablePa | kPteTypeL012Table, oldPte); 32518a27fe0SOwen Anderson if (oldPteRefetch != oldPte) { 32618a27fe0SOwen Anderson // If the old PTE has mutated, it must be because another thread has allocated the 32718a27fe0SOwen Anderson // sub-table at the same time as us. If that has happened, deallocate the page we 32818a27fe0SOwen Anderson // setup and use the one they installed instead. 32918a27fe0SOwen Anderson ASSERT((oldPteRefetch & kPteTypeMask) == kPteTypeL012Table); 33018a27fe0SOwen Anderson DEBUG_PAGE_ACCESS_START(page); 33118a27fe0SOwen Anderson vm_page_set_state(page, PAGE_STATE_FREE); 33218a27fe0SOwen Anderson return oldPteRefetch & kPteAddrMask; 333a25542e7Smilek7 } 334a25542e7Smilek7 335a25542e7Smilek7 return newTablePa; 336a25542e7Smilek7 } 337a25542e7Smilek7 33818a27fe0SOwen Anderson // There's no existing table and we have no reservation 339a25542e7Smilek7 return 0; 340a25542e7Smilek7 } 341a25542e7Smilek7 342a25542e7Smilek7 343a25542e7Smilek7 void 344a25542e7Smilek7 VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size, 345a25542e7Smilek7 VMSAv8TranslationMap::VMAction action, uint64_t attr, vm_page_reservation* reservation) 346a25542e7Smilek7 { 347a25542e7Smilek7 ASSERT(level < 4); 348a25542e7Smilek7 ASSERT(ptPa != 0); 349a25542e7Smilek7 ASSERT(reservation != NULL || action != VMAction::MAP); 350a25542e7Smilek7 351a25542e7Smilek7 int tableBits = fPageBits - 3; 352a25542e7Smilek7 uint64_t tableMask = (1UL << tableBits) - 1; 353a25542e7Smilek7 354a25542e7Smilek7 int shift = tableBits * (3 - level) + fPageBits; 355a25542e7Smilek7 uint64_t entrySize = 1UL << shift; 356a25542e7Smilek7 357a25542e7Smilek7 uint64_t entryMask = entrySize - 1; 358a25542e7Smilek7 uint64_t nextVa = va; 359a25542e7Smilek7 uint64_t end = va + size; 360a25542e7Smilek7 int index; 361a25542e7Smilek7 362a25542e7Smilek7 // Handle misaligned header that straddles entry boundary in next-level table 363a25542e7Smilek7 if ((va & entryMask) != 0) { 364a25542e7Smilek7 uint64_t aligned = (va & ~entryMask) + entrySize; 365a25542e7Smilek7 if (end > aligned) { 366a25542e7Smilek7 index = (va >> shift) & tableMask; 367a25542e7Smilek7 phys_addr_t table = MakeTable(ptPa, level, index, reservation); 368a25542e7Smilek7 MapRange(table, level + 1, va, pa, aligned - va, action, attr, reservation); 369a25542e7Smilek7 nextVa = aligned; 370a25542e7Smilek7 } 371a25542e7Smilek7 } 372a25542e7Smilek7 373a25542e7Smilek7 // Handle fully aligned and appropriately sized chunks 374a25542e7Smilek7 while (nextVa + entrySize <= end) { 375a25542e7Smilek7 phys_addr_t targetPa = pa + (nextVa - va); 376a25542e7Smilek7 index = (nextVa >> shift) & tableMask; 377a25542e7Smilek7 378a25542e7Smilek7 bool blockAllowed = false; 379a25542e7Smilek7 if (action == VMAction::MAP) 380a25542e7Smilek7 blockAllowed = (level >= fMinBlockLevel && (targetPa & entryMask) == 0); 381a25542e7Smilek7 if (action == VMAction::SET_ATTR || action == VMAction::CLEAR_FLAGS) 382a25542e7Smilek7 blockAllowed = (MakeTable(ptPa, level, index, NULL) == 0); 383a25542e7Smilek7 if (action == VMAction::UNMAP) 384a25542e7Smilek7 blockAllowed = true; 385a25542e7Smilek7 386a25542e7Smilek7 if (blockAllowed) { 387a25542e7Smilek7 // Everything is aligned, we can make block mapping there 38818a27fe0SOwen Anderson uint64_t* pte = TableFromPa(ptPa) + index; 389a25542e7Smilek7 390a25542e7Smilek7 retry: 391a25542e7Smilek7 uint64_t oldPte = atomic_get64((int64*) pte); 392a25542e7Smilek7 393a25542e7Smilek7 if (action == VMAction::MAP || (oldPte & 0x1) != 0) { 394a25542e7Smilek7 uint64_t newPte = 0; 395a25542e7Smilek7 if (action == VMAction::MAP) { 396a25542e7Smilek7 newPte = MakeBlock(targetPa, level, attr); 397a25542e7Smilek7 } else if (action == VMAction::SET_ATTR) { 398a25542e7Smilek7 newPte = MakeBlock(oldPte & kPteAddrMask, level, MoveAttrFlags(attr, oldPte)); 399a25542e7Smilek7 } else if (action == VMAction::CLEAR_FLAGS) { 400a25542e7Smilek7 newPte = MakeBlock(oldPte & kPteAddrMask, level, ClearAttrFlags(oldPte, attr)); 401a25542e7Smilek7 } else if (action == VMAction::UNMAP) { 402a25542e7Smilek7 newPte = 0; 403a25542e7Smilek7 tmp_pte = oldPte; 404a25542e7Smilek7 } 405a25542e7Smilek7 406a25542e7Smilek7 // FIXME: this might not be enough on real hardware with SMP for some cases 407a25542e7Smilek7 if ((uint64_t) atomic_test_and_set64((int64*) pte, newPte, oldPte) != oldPte) 408a25542e7Smilek7 goto retry; 409a25542e7Smilek7 410a25542e7Smilek7 if (level < 3 && (oldPte & 0x3) == 0x3) { 411a25542e7Smilek7 // If we're replacing existing pagetable clean it up 4127908993dSOwen Anderson FreeTable(oldPte & kPteAddrMask, nextVa, level + 1, 4137908993dSOwen Anderson [](int level, uint64_t oldPte) {}); 414a25542e7Smilek7 } 415a25542e7Smilek7 } 416a25542e7Smilek7 } else { 417a25542e7Smilek7 // Otherwise handle mapping in next-level table 418a25542e7Smilek7 phys_addr_t table = MakeTable(ptPa, level, index, reservation); 419a25542e7Smilek7 MapRange(table, level + 1, nextVa, targetPa, entrySize, action, attr, reservation); 420a25542e7Smilek7 } 421a25542e7Smilek7 nextVa += entrySize; 422a25542e7Smilek7 } 423a25542e7Smilek7 424a25542e7Smilek7 // Handle misaligned tail area (or entirety of small area) in next-level table 425a25542e7Smilek7 if (nextVa < end) { 426a25542e7Smilek7 index = (nextVa >> shift) & tableMask; 427a25542e7Smilek7 phys_addr_t table = MakeTable(ptPa, level, index, reservation); 428a25542e7Smilek7 MapRange( 429a25542e7Smilek7 table, level + 1, nextVa, pa + (nextVa - va), end - nextVa, action, attr, reservation); 430a25542e7Smilek7 } 431a25542e7Smilek7 } 432a25542e7Smilek7 433a25542e7Smilek7 434a25542e7Smilek7 uint8_t 435a25542e7Smilek7 VMSAv8TranslationMap::MairIndex(uint8_t type) 436a25542e7Smilek7 { 437a25542e7Smilek7 for (int i = 0; i < 8; i++) 438a25542e7Smilek7 if (((fMair >> (i * 8)) & 0xff) == type) 439a25542e7Smilek7 return i; 440a25542e7Smilek7 441a25542e7Smilek7 panic("MAIR entry not found"); 442a25542e7Smilek7 return 0; 443a25542e7Smilek7 } 444a25542e7Smilek7 445a25542e7Smilek7 446a25542e7Smilek7 uint64_t 447a25542e7Smilek7 VMSAv8TranslationMap::ClearAttrFlags(uint64_t attr, uint32 flags) 448a25542e7Smilek7 { 449a25542e7Smilek7 attr &= kPteAttrMask; 450a25542e7Smilek7 451a25542e7Smilek7 if ((flags & PAGE_ACCESSED) != 0) 452a25542e7Smilek7 attr &= ~kAttrAF; 453a25542e7Smilek7 454a25542e7Smilek7 if ((flags & PAGE_MODIFIED) != 0 && (attr & kAttrSWDBM) != 0) 455*108f6fdcSOwen Anderson attr |= kAttrAPReadOnly; 456a25542e7Smilek7 457a25542e7Smilek7 return attr; 458a25542e7Smilek7 } 459a25542e7Smilek7 460a25542e7Smilek7 461a25542e7Smilek7 uint64_t 462a25542e7Smilek7 VMSAv8TranslationMap::MoveAttrFlags(uint64_t newAttr, uint64_t oldAttr) 463a25542e7Smilek7 { 464a25542e7Smilek7 if ((oldAttr & kAttrAF) != 0) 465a25542e7Smilek7 newAttr |= kAttrAF; 466*108f6fdcSOwen Anderson if (((newAttr & oldAttr) & kAttrSWDBM) != 0 && (oldAttr & kAttrAPReadOnly) == 0) 467*108f6fdcSOwen Anderson newAttr &= ~kAttrAPReadOnly; 468a25542e7Smilek7 469a25542e7Smilek7 return newAttr; 470a25542e7Smilek7 } 471a25542e7Smilek7 472a25542e7Smilek7 473a25542e7Smilek7 uint64_t 474a25542e7Smilek7 VMSAv8TranslationMap::GetMemoryAttr(uint32 attributes, uint32 memoryType, bool isKernel) 475a25542e7Smilek7 { 476a25542e7Smilek7 uint64_t attr = 0; 477a25542e7Smilek7 478a25542e7Smilek7 if (!isKernel) 479a25542e7Smilek7 attr |= kAttrNG; 480a25542e7Smilek7 481a25542e7Smilek7 if ((attributes & B_EXECUTE_AREA) == 0) 482a25542e7Smilek7 attr |= kAttrUXN; 483a25542e7Smilek7 if ((attributes & B_KERNEL_EXECUTE_AREA) == 0) 484a25542e7Smilek7 attr |= kAttrPXN; 485a25542e7Smilek7 486*108f6fdcSOwen Anderson // SWDBM is software reserved bit that we use to mark that 487*108f6fdcSOwen Anderson // writes are allowed, and fault handler should clear kAttrAPReadOnly. 488*108f6fdcSOwen Anderson // In that case kAttrAPReadOnly doubles as not-dirty bit. 489*108f6fdcSOwen Anderson // Additionally dirty state can be stored in SWDIRTY, in order not to lose 490*108f6fdcSOwen Anderson // dirty state when changing protection from RW to RO. 491a25542e7Smilek7 492*108f6fdcSOwen Anderson // All page permissions begin life in RO state. 493*108f6fdcSOwen Anderson attr |= kAttrAPReadOnly; 494*108f6fdcSOwen Anderson 495*108f6fdcSOwen Anderson // User-Execute implies User-Read, because it would break PAN otherwise 496*108f6fdcSOwen Anderson if ((attributes & B_READ_AREA) != 0 || (attributes & B_EXECUTE_AREA) != 0) 497*108f6fdcSOwen Anderson attr |= kAttrAPUserAccess; // Allow user reads 498*108f6fdcSOwen Anderson 499*108f6fdcSOwen Anderson if ((attributes & B_WRITE_AREA) != 0 || (attributes & B_KERNEL_WRITE_AREA) != 0) 500*108f6fdcSOwen Anderson attr |= kAttrSWDBM; // Mark as writeable 501*108f6fdcSOwen Anderson 502*108f6fdcSOwen Anderson // When supported by hardware copy our SWDBM bit into DBM, 503*108f6fdcSOwen Anderson // so that kAttrAPReadOnly is cleared on write attempt automatically 504*108f6fdcSOwen Anderson // without going through fault handler. 505*108f6fdcSOwen Anderson if ((fHwFeature & HW_DIRTY) != 0 && (attr & kAttrSWDBM) != 0) 506a25542e7Smilek7 attr |= kAttrDBM; 507a25542e7Smilek7 508*108f6fdcSOwen Anderson attr |= kAttrSHInnerShareable; // Inner Shareable 509a25542e7Smilek7 510*108f6fdcSOwen Anderson uint8_t type = MAIR_NORMAL_WB; 511*108f6fdcSOwen Anderson 512*108f6fdcSOwen Anderson if (memoryType & B_MTR_UC) 513*108f6fdcSOwen Anderson type = MAIR_DEVICE_nGnRnE; // TODO: This probably should be nGnRE for PCI 514*108f6fdcSOwen Anderson else if (memoryType & B_MTR_WC) 515*108f6fdcSOwen Anderson type = MAIR_NORMAL_NC; 516*108f6fdcSOwen Anderson else if (memoryType & B_MTR_WT) 517*108f6fdcSOwen Anderson type = MAIR_NORMAL_WT; 518*108f6fdcSOwen Anderson else if (memoryType & B_MTR_WP) 519*108f6fdcSOwen Anderson type = MAIR_NORMAL_WT; 520*108f6fdcSOwen Anderson else if (memoryType & B_MTR_WB) 521*108f6fdcSOwen Anderson type = MAIR_NORMAL_WB; 522*108f6fdcSOwen Anderson 523*108f6fdcSOwen Anderson attr |= MairIndex(type) << 2; 524a25542e7Smilek7 525a25542e7Smilek7 return attr; 526a25542e7Smilek7 } 527a25542e7Smilek7 528a25542e7Smilek7 529a25542e7Smilek7 status_t 530a25542e7Smilek7 VMSAv8TranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, uint32 memoryType, 531a25542e7Smilek7 vm_page_reservation* reservation) 532a25542e7Smilek7 { 533a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 534a25542e7Smilek7 535a25542e7Smilek7 uint64_t pageMask = (1UL << fPageBits) - 1; 536a25542e7Smilek7 uint64_t vaMask = (1UL << fVaBits) - 1; 537a25542e7Smilek7 538a25542e7Smilek7 ASSERT((va & pageMask) == 0); 539a25542e7Smilek7 ASSERT((pa & pageMask) == 0); 540a25542e7Smilek7 ASSERT(ValidateVa(va)); 541a25542e7Smilek7 542a25542e7Smilek7 uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel); 543a25542e7Smilek7 544a25542e7Smilek7 if (!fPageTable) { 545a25542e7Smilek7 vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); 546a25542e7Smilek7 fPageTable = page->physical_page_number << fPageBits; 547a25542e7Smilek7 } 548a25542e7Smilek7 549a25542e7Smilek7 MapRange( 550a25542e7Smilek7 fPageTable, fInitialLevel, va & vaMask, pa, B_PAGE_SIZE, VMAction::MAP, attr, reservation); 551a25542e7Smilek7 552a25542e7Smilek7 return B_OK; 553a25542e7Smilek7 } 554a25542e7Smilek7 555a25542e7Smilek7 556a25542e7Smilek7 status_t 557a25542e7Smilek7 VMSAv8TranslationMap::Unmap(addr_t start, addr_t end) 558a25542e7Smilek7 { 559a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 560a25542e7Smilek7 561a25542e7Smilek7 size_t size = end - start + 1; 562a25542e7Smilek7 563a25542e7Smilek7 uint64_t pageMask = (1UL << fPageBits) - 1; 564a25542e7Smilek7 uint64_t vaMask = (1UL << fVaBits) - 1; 565a25542e7Smilek7 566a25542e7Smilek7 ASSERT((start & pageMask) == 0); 567a25542e7Smilek7 ASSERT((size & pageMask) == 0); 568a25542e7Smilek7 ASSERT(ValidateVa(start)); 569a25542e7Smilek7 570a25542e7Smilek7 MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::UNMAP, 0, NULL); 571a25542e7Smilek7 572a25542e7Smilek7 return B_OK; 573a25542e7Smilek7 } 574a25542e7Smilek7 575a25542e7Smilek7 576a25542e7Smilek7 status_t 577a25542e7Smilek7 VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, bool updatePageQueue) 578a25542e7Smilek7 { 579a25542e7Smilek7 580a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 581a25542e7Smilek7 RecursiveLocker locker(fLock); 582a25542e7Smilek7 583a25542e7Smilek7 // TODO: replace this kludge 584a25542e7Smilek7 585a25542e7Smilek7 phys_addr_t pa; 586a25542e7Smilek7 uint64_t pte; 587a25542e7Smilek7 if (!WalkTable(fPageTable, fInitialLevel, address, &pa, &pte)) 588a25542e7Smilek7 return B_ENTRY_NOT_FOUND; 589a25542e7Smilek7 590a25542e7Smilek7 uint64_t vaMask = (1UL << fVaBits) - 1; 591a25542e7Smilek7 MapRange(fPageTable, fInitialLevel, address & vaMask, 0, B_PAGE_SIZE, VMAction::UNMAP, 0, NULL); 592a25542e7Smilek7 593a25542e7Smilek7 pinner.Unlock(); 594a25542e7Smilek7 locker.Detach(); 595*108f6fdcSOwen Anderson PageUnmapped(area, pa >> fPageBits, (tmp_pte & kAttrAF) != 0, (tmp_pte & kAttrAPReadOnly) == 0, 596a25542e7Smilek7 updatePageQueue); 597a25542e7Smilek7 598a25542e7Smilek7 return B_OK; 599a25542e7Smilek7 } 600a25542e7Smilek7 601a25542e7Smilek7 602a25542e7Smilek7 bool 603a25542e7Smilek7 VMSAv8TranslationMap::WalkTable( 604a25542e7Smilek7 phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* rpte) 605a25542e7Smilek7 { 606a25542e7Smilek7 int tableBits = fPageBits - 3; 607a25542e7Smilek7 uint64_t tableMask = (1UL << tableBits) - 1; 608a25542e7Smilek7 609a25542e7Smilek7 int shift = tableBits * (3 - level) + fPageBits; 610a25542e7Smilek7 uint64_t entrySize = 1UL << shift; 611a25542e7Smilek7 uint64_t entryMask = entrySize - 1; 612a25542e7Smilek7 613a25542e7Smilek7 int index = (va >> shift) & tableMask; 614a25542e7Smilek7 615a25542e7Smilek7 uint64_t pte = TableFromPa(ptPa)[index]; 616a25542e7Smilek7 int type = pte & 0x3; 617a25542e7Smilek7 618a25542e7Smilek7 if ((type & 0x1) == 0) 619a25542e7Smilek7 return false; 620a25542e7Smilek7 621a25542e7Smilek7 uint64_t addr = pte & kPteAddrMask; 622a25542e7Smilek7 if (level < 3) { 623a25542e7Smilek7 if (type == 0x3) { 624a25542e7Smilek7 return WalkTable(addr, level + 1, va, pa, rpte); 625a25542e7Smilek7 } else { 626a25542e7Smilek7 *pa = addr | (va & entryMask); 627a25542e7Smilek7 *rpte = pte; 628a25542e7Smilek7 } 629a25542e7Smilek7 } else { 630a25542e7Smilek7 ASSERT(type == 0x3); 631a25542e7Smilek7 *pa = addr; 632a25542e7Smilek7 *rpte = pte; 633a25542e7Smilek7 } 634a25542e7Smilek7 635a25542e7Smilek7 return true; 636a25542e7Smilek7 } 637a25542e7Smilek7 638a25542e7Smilek7 639a25542e7Smilek7 bool 640a25542e7Smilek7 VMSAv8TranslationMap::ValidateVa(addr_t va) 641a25542e7Smilek7 { 642a25542e7Smilek7 uint64_t vaMask = (1UL << fVaBits) - 1; 643a25542e7Smilek7 bool kernelAddr = (va & (1UL << 63)) != 0; 644a25542e7Smilek7 if (kernelAddr != fIsKernel) 645a25542e7Smilek7 return false; 646a25542e7Smilek7 if ((va & ~vaMask) != (fIsKernel ? ~vaMask : 0)) 647a25542e7Smilek7 return false; 648a25542e7Smilek7 return true; 649a25542e7Smilek7 } 650a25542e7Smilek7 651a25542e7Smilek7 652a25542e7Smilek7 status_t 653a25542e7Smilek7 VMSAv8TranslationMap::Query(addr_t va, phys_addr_t* pa, uint32* flags) 654a25542e7Smilek7 { 655a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 656a25542e7Smilek7 657a25542e7Smilek7 ASSERT(ValidateVa(va)); 658a25542e7Smilek7 659a25542e7Smilek7 uint64_t pte = 0; 660a25542e7Smilek7 bool ret = WalkTable(fPageTable, fInitialLevel, va, pa, &pte); 661a25542e7Smilek7 662a25542e7Smilek7 uint32 result = 0; 663a25542e7Smilek7 664a25542e7Smilek7 if (ret) { 665a25542e7Smilek7 result |= PAGE_PRESENT; 666a25542e7Smilek7 667a25542e7Smilek7 if ((pte & kAttrAF) != 0) 668a25542e7Smilek7 result |= PAGE_ACCESSED; 669*108f6fdcSOwen Anderson if ((pte & kAttrAPReadOnly) == 0) 670a25542e7Smilek7 result |= PAGE_MODIFIED; 671a25542e7Smilek7 672a25542e7Smilek7 if ((pte & kAttrUXN) == 0) 673a25542e7Smilek7 result |= B_EXECUTE_AREA; 674a25542e7Smilek7 if ((pte & kAttrPXN) == 0) 675a25542e7Smilek7 result |= B_KERNEL_EXECUTE_AREA; 676a25542e7Smilek7 677a25542e7Smilek7 result |= B_KERNEL_READ_AREA; 678a25542e7Smilek7 679*108f6fdcSOwen Anderson if ((pte & kAttrAPUserAccess) != 0) 680a25542e7Smilek7 result |= B_READ_AREA; 681a25542e7Smilek7 682*108f6fdcSOwen Anderson if ((pte & kAttrAPReadOnly) == 0 || (pte & kAttrSWDBM) != 0) { 683a25542e7Smilek7 result |= B_KERNEL_WRITE_AREA; 684a25542e7Smilek7 685*108f6fdcSOwen Anderson if ((pte & kAttrAPUserAccess) != 0) 686a25542e7Smilek7 result |= B_WRITE_AREA; 687a25542e7Smilek7 } 688a25542e7Smilek7 } 689a25542e7Smilek7 690a25542e7Smilek7 *flags = result; 691a25542e7Smilek7 return B_OK; 692a25542e7Smilek7 } 693a25542e7Smilek7 694a25542e7Smilek7 695a25542e7Smilek7 status_t 696a25542e7Smilek7 VMSAv8TranslationMap::QueryInterrupt( 697a25542e7Smilek7 addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags) 698a25542e7Smilek7 { 699a25542e7Smilek7 return Query(virtualAddress, _physicalAddress, _flags); 700a25542e7Smilek7 } 701a25542e7Smilek7 702a25542e7Smilek7 703a25542e7Smilek7 status_t 704a25542e7Smilek7 VMSAv8TranslationMap::Protect(addr_t start, addr_t end, uint32 attributes, uint32 memoryType) 705a25542e7Smilek7 { 706a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 707a25542e7Smilek7 708a25542e7Smilek7 size_t size = end - start + 1; 709a25542e7Smilek7 710a25542e7Smilek7 uint64_t pageMask = (1UL << fPageBits) - 1; 711a25542e7Smilek7 uint64_t vaMask = (1UL << fVaBits) - 1; 712a25542e7Smilek7 713a25542e7Smilek7 ASSERT((start & pageMask) == 0); 714a25542e7Smilek7 ASSERT((size & pageMask) == 0); 715a25542e7Smilek7 ASSERT(ValidateVa(start)); 716a25542e7Smilek7 717a25542e7Smilek7 uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel); 718a25542e7Smilek7 MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::SET_ATTR, attr, NULL); 719a25542e7Smilek7 720a25542e7Smilek7 return B_OK; 721a25542e7Smilek7 } 722a25542e7Smilek7 723a25542e7Smilek7 724a25542e7Smilek7 status_t 725a25542e7Smilek7 VMSAv8TranslationMap::ClearFlags(addr_t va, uint32 flags) 726a25542e7Smilek7 { 727a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 728a25542e7Smilek7 729a25542e7Smilek7 uint64_t pageMask = (1UL << fPageBits) - 1; 730a25542e7Smilek7 uint64_t vaMask = (1UL << fVaBits) - 1; 731a25542e7Smilek7 732a25542e7Smilek7 ASSERT((va & pageMask) == 0); 733a25542e7Smilek7 ASSERT(ValidateVa(va)); 734a25542e7Smilek7 735a25542e7Smilek7 MapRange( 736a25542e7Smilek7 fPageTable, fInitialLevel, va & vaMask, 0, B_PAGE_SIZE, VMAction::CLEAR_FLAGS, flags, NULL); 737a25542e7Smilek7 738a25542e7Smilek7 return B_OK; 739a25542e7Smilek7 } 740a25542e7Smilek7 741a25542e7Smilek7 742a25542e7Smilek7 bool 743a25542e7Smilek7 VMSAv8TranslationMap::ClearAccessedAndModified( 744a25542e7Smilek7 VMArea* area, addr_t address, bool unmapIfUnaccessed, bool& _modified) 745a25542e7Smilek7 { 746a25542e7Smilek7 panic("VMSAv8TranslationMap::ClearAccessedAndModified not implemented\n"); 747a25542e7Smilek7 return B_OK; 748a25542e7Smilek7 } 749a25542e7Smilek7 750a25542e7Smilek7 751a25542e7Smilek7 void 752a25542e7Smilek7 VMSAv8TranslationMap::Flush() 753a25542e7Smilek7 { 754a25542e7Smilek7 ThreadCPUPinner pinner(thread_get_current_thread()); 755a25542e7Smilek7 756a25542e7Smilek7 arch_cpu_global_TLB_invalidate(); 757a25542e7Smilek7 } 758