xref: /haiku/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp (revision 108f6fdc2f4ae409d268c99ac084326d95934c0a)
1a25542e7Smilek7 /*
2a25542e7Smilek7  * Copyright 2022 Haiku, Inc. All Rights Reserved.
3a25542e7Smilek7  * Distributed under the terms of the MIT License.
4a25542e7Smilek7  */
5a25542e7Smilek7 #include "VMSAv8TranslationMap.h"
6a25542e7Smilek7 
7a25542e7Smilek7 #include <util/AutoLock.h>
8a25542e7Smilek7 #include <util/ThreadAutoLock.h>
9a25542e7Smilek7 #include <vm/vm_page.h>
10a25542e7Smilek7 #include <vm/vm_priv.h>
11a25542e7Smilek7 
12a25542e7Smilek7 
13a25542e7Smilek7 static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12);
14a25542e7Smilek7 static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3);
15a25542e7Smilek7 
1618a27fe0SOwen Anderson static constexpr uint64_t kPteTypeMask = 0x3;
1718a27fe0SOwen Anderson static constexpr uint64_t kPteTypeL012Table = 0x3;
1818a27fe0SOwen Anderson static constexpr uint64_t kPteTypeL012Block = 0x1;
1918a27fe0SOwen Anderson static constexpr uint64_t kPteTypeL3Page = 0x3;
2018a27fe0SOwen Anderson 
21a25542e7Smilek7 static constexpr uint64_t kAttrSWDBM = (1UL << 55);
22a25542e7Smilek7 static constexpr uint64_t kAttrUXN = (1UL << 54);
23a25542e7Smilek7 static constexpr uint64_t kAttrPXN = (1UL << 53);
24a25542e7Smilek7 static constexpr uint64_t kAttrDBM = (1UL << 51);
25a25542e7Smilek7 static constexpr uint64_t kAttrNG = (1UL << 11);
26a25542e7Smilek7 static constexpr uint64_t kAttrAF = (1UL << 10);
27*108f6fdcSOwen Anderson static constexpr uint64_t kAttrSHInnerShareable = (3UL << 8);
28*108f6fdcSOwen Anderson static constexpr uint64_t kAttrAPReadOnly = (1UL << 7);
29*108f6fdcSOwen Anderson static constexpr uint64_t kAttrAPUserAccess = (1UL << 6);
30a25542e7Smilek7 
317908993dSOwen Anderson static constexpr uint64_t kTLBIMask = ((1UL << 44) - 1);
327908993dSOwen Anderson 
33a25542e7Smilek7 uint32_t VMSAv8TranslationMap::fHwFeature;
34a25542e7Smilek7 uint64_t VMSAv8TranslationMap::fMair;
35a25542e7Smilek7 
369fad0a5cSOwen Anderson // ASID Management
379fad0a5cSOwen Anderson static constexpr size_t kAsidBits = 8;
389fad0a5cSOwen Anderson static constexpr size_t kNumAsids = (1 << kAsidBits);
397908993dSOwen Anderson static spinlock sAsidLock = B_SPINLOCK_INITIALIZER;
409fad0a5cSOwen Anderson // A bitmap to track which ASIDs are in use.
419fad0a5cSOwen Anderson static uint64 sAsidBitMap[kNumAsids / 64] = {};
429fad0a5cSOwen Anderson // A mapping from ASID to translation map.
439fad0a5cSOwen Anderson static VMSAv8TranslationMap* sAsidMapping[kNumAsids] = {};
449fad0a5cSOwen Anderson 
459fad0a5cSOwen Anderson 
469fad0a5cSOwen Anderson static void
479fad0a5cSOwen Anderson free_asid(size_t asid)
489fad0a5cSOwen Anderson {
499fad0a5cSOwen Anderson 	for (size_t i = 0; i < B_COUNT_OF(sAsidBitMap); ++i) {
509fad0a5cSOwen Anderson 		if (asid < 64) {
519fad0a5cSOwen Anderson 			sAsidBitMap[i] &= ~(uint64_t{1} << asid);
529fad0a5cSOwen Anderson 			return;
539fad0a5cSOwen Anderson 		}
549fad0a5cSOwen Anderson 		asid -= 64;
559fad0a5cSOwen Anderson 	}
569fad0a5cSOwen Anderson 
579fad0a5cSOwen Anderson 	panic("Could not free ASID!");
589fad0a5cSOwen Anderson }
599fad0a5cSOwen Anderson 
609fad0a5cSOwen Anderson 
619fad0a5cSOwen Anderson static size_t
629fad0a5cSOwen Anderson alloc_first_free_asid(void)
639fad0a5cSOwen Anderson {
649fad0a5cSOwen Anderson 	int asid = 0;
659fad0a5cSOwen Anderson 	for (size_t i = 0; i < B_COUNT_OF(sAsidBitMap); ++i) {
669fad0a5cSOwen Anderson 		int avail = __builtin_ffsll(~sAsidBitMap[i]);
679fad0a5cSOwen Anderson 		if (avail != 0) {
689fad0a5cSOwen Anderson 			sAsidBitMap[i] |= (uint64_t{1} << (avail-1));
699fad0a5cSOwen Anderson 			asid += (avail - 1);
709fad0a5cSOwen Anderson 			return asid;
719fad0a5cSOwen Anderson 		}
729fad0a5cSOwen Anderson 		asid += 64;
739fad0a5cSOwen Anderson 	}
749fad0a5cSOwen Anderson 
759fad0a5cSOwen Anderson 	return kNumAsids;
769fad0a5cSOwen Anderson }
777908993dSOwen Anderson 
78a25542e7Smilek7 
79a25542e7Smilek7 VMSAv8TranslationMap::VMSAv8TranslationMap(
80a25542e7Smilek7 	bool kernel, phys_addr_t pageTable, int pageBits, int vaBits, int minBlockLevel)
81a25542e7Smilek7 	:
82a25542e7Smilek7 	fIsKernel(kernel),
83a25542e7Smilek7 	fPageTable(pageTable),
84a25542e7Smilek7 	fPageBits(pageBits),
85a25542e7Smilek7 	fVaBits(vaBits),
867908993dSOwen Anderson 	fMinBlockLevel(minBlockLevel),
879fad0a5cSOwen Anderson 	fASID(-1),
889fad0a5cSOwen Anderson 	fRefcount(0)
89a25542e7Smilek7 {
90a25542e7Smilek7 	dprintf("VMSAv8TranslationMap\n");
91a25542e7Smilek7 
92a25542e7Smilek7 	fInitialLevel = CalcStartLevel(fVaBits, fPageBits);
93a25542e7Smilek7 }
94a25542e7Smilek7 
95a25542e7Smilek7 
96a25542e7Smilek7 VMSAv8TranslationMap::~VMSAv8TranslationMap()
97a25542e7Smilek7 {
987908993dSOwen Anderson 	ASSERT(!fIsKernel);
999fad0a5cSOwen Anderson 	ASSERT(fRefcount == 0);
1007908993dSOwen Anderson 	{
1017908993dSOwen Anderson 		ThreadCPUPinner pinner(thread_get_current_thread());
1027908993dSOwen Anderson 		FreeTable(fPageTable, 0, fInitialLevel, [](int level, uint64_t oldPte) {});
1037908993dSOwen Anderson 	}
104a25542e7Smilek7 
1057908993dSOwen Anderson 	{
1067908993dSOwen Anderson 		InterruptsSpinLocker locker(sAsidLock);
1077908993dSOwen Anderson 
1089fad0a5cSOwen Anderson 		if (fASID != -1) {
1097908993dSOwen Anderson 			sAsidMapping[fASID] = NULL;
1109fad0a5cSOwen Anderson 			free_asid(fASID);
1117908993dSOwen Anderson 		}
112a25542e7Smilek7 	}
1139fad0a5cSOwen Anderson }
1149fad0a5cSOwen Anderson 
1159fad0a5cSOwen Anderson 
1169fad0a5cSOwen Anderson // Switch user map into TTBR0.
1179fad0a5cSOwen Anderson // Passing kernel map here configures empty page table.
1189fad0a5cSOwen Anderson void
1199fad0a5cSOwen Anderson VMSAv8TranslationMap::SwitchUserMap(VMSAv8TranslationMap *from, VMSAv8TranslationMap *to)
1209fad0a5cSOwen Anderson {
1219fad0a5cSOwen Anderson 	SpinLocker locker(sAsidLock);
1229fad0a5cSOwen Anderson 
1239fad0a5cSOwen Anderson 	if (!from->fIsKernel) {
1249fad0a5cSOwen Anderson 		from->fRefcount--;
1259fad0a5cSOwen Anderson 	}
1269fad0a5cSOwen Anderson 
1279fad0a5cSOwen Anderson 	if (!to->fIsKernel) {
1289fad0a5cSOwen Anderson 		to->fRefcount++;
1299fad0a5cSOwen Anderson 	} else {
1309fad0a5cSOwen Anderson 		arch_vm_install_empty_table_ttbr0();
1319fad0a5cSOwen Anderson 		return;
1329fad0a5cSOwen Anderson 	}
1339fad0a5cSOwen Anderson 
1349fad0a5cSOwen Anderson 	ASSERT(to->fPageTable != 0);
1359fad0a5cSOwen Anderson 	uint64_t ttbr = to->fPageTable | ((fHwFeature & HW_COMMON_NOT_PRIVATE) != 0 ? 1 : 0);
1369fad0a5cSOwen Anderson 
1379fad0a5cSOwen Anderson 	if (to->fASID != -1) {
1389fad0a5cSOwen Anderson 		WRITE_SPECIALREG(TTBR0_EL1, ((uint64_t)to->fASID << 48) | ttbr);
1399fad0a5cSOwen Anderson 		asm("isb");
1409fad0a5cSOwen Anderson 		return;
1419fad0a5cSOwen Anderson 	}
1429fad0a5cSOwen Anderson 
1439fad0a5cSOwen Anderson 	size_t allocatedAsid = alloc_first_free_asid();
1449fad0a5cSOwen Anderson 	if (allocatedAsid != kNumAsids) {
1459fad0a5cSOwen Anderson 		to->fASID = allocatedAsid;
1469fad0a5cSOwen Anderson 		sAsidMapping[allocatedAsid] = to;
1479fad0a5cSOwen Anderson 
1489fad0a5cSOwen Anderson 		WRITE_SPECIALREG(TTBR0_EL1, (allocatedAsid << 48) | ttbr);
1499fad0a5cSOwen Anderson 		asm("isb");
1509fad0a5cSOwen Anderson 		return;
1519fad0a5cSOwen Anderson 	}
1529fad0a5cSOwen Anderson 
1539fad0a5cSOwen Anderson 	for (size_t i = 0; i < kNumAsids; ++i) {
1549fad0a5cSOwen Anderson 		if (sAsidMapping[i]->fRefcount == 0) {
1559fad0a5cSOwen Anderson 			sAsidMapping[i]->fASID = -1;
1569fad0a5cSOwen Anderson 			to->fASID = i;
1579fad0a5cSOwen Anderson 			sAsidMapping[i] = to;
1589fad0a5cSOwen Anderson 
1599fad0a5cSOwen Anderson 			WRITE_SPECIALREG(TTBR0_EL1, (i << 48) | ttbr);
1609fad0a5cSOwen Anderson 			asm("dsb ishst");
1619fad0a5cSOwen Anderson 			asm("tlbi aside1is, %0" :: "r" (i << 48));
1629fad0a5cSOwen Anderson 			asm("dsb ish");
1639fad0a5cSOwen Anderson 			asm("isb");
1649fad0a5cSOwen Anderson 			return;
1659fad0a5cSOwen Anderson 		}
1669fad0a5cSOwen Anderson 	}
1679fad0a5cSOwen Anderson 
1689fad0a5cSOwen Anderson 	panic("cannot assign ASID");
1699fad0a5cSOwen Anderson }
170a25542e7Smilek7 
171a25542e7Smilek7 
172a25542e7Smilek7 int
173a25542e7Smilek7 VMSAv8TranslationMap::CalcStartLevel(int vaBits, int pageBits)
174a25542e7Smilek7 {
175a25542e7Smilek7 	int level = 4;
176a25542e7Smilek7 
177a25542e7Smilek7 	int bitsLeft = vaBits - pageBits;
178a25542e7Smilek7 	while (bitsLeft > 0) {
179a25542e7Smilek7 		int tableBits = pageBits - 3;
180a25542e7Smilek7 		bitsLeft -= tableBits;
181a25542e7Smilek7 		level--;
182a25542e7Smilek7 	}
183a25542e7Smilek7 
184a25542e7Smilek7 	ASSERT(level >= 0);
185a25542e7Smilek7 
186a25542e7Smilek7 	return level;
187a25542e7Smilek7 }
188a25542e7Smilek7 
189a25542e7Smilek7 
190a25542e7Smilek7 bool
191a25542e7Smilek7 VMSAv8TranslationMap::Lock()
192a25542e7Smilek7 {
193a25542e7Smilek7 	recursive_lock_lock(&fLock);
194a25542e7Smilek7 	return true;
195a25542e7Smilek7 }
196a25542e7Smilek7 
197a25542e7Smilek7 
198a25542e7Smilek7 void
199a25542e7Smilek7 VMSAv8TranslationMap::Unlock()
200a25542e7Smilek7 {
201a25542e7Smilek7 	if (recursive_lock_get_recursion(&fLock) == 1) {
202a25542e7Smilek7 		// we're about to release it for the last time
203a25542e7Smilek7 		Flush();
204a25542e7Smilek7 	}
205a25542e7Smilek7 	recursive_lock_unlock(&fLock);
206a25542e7Smilek7 }
207a25542e7Smilek7 
208a25542e7Smilek7 
209a25542e7Smilek7 addr_t
210a25542e7Smilek7 VMSAv8TranslationMap::MappedSize() const
211a25542e7Smilek7 {
212a25542e7Smilek7 	panic("VMSAv8TranslationMap::MappedSize not implemented");
213a25542e7Smilek7 	return 0;
214a25542e7Smilek7 }
215a25542e7Smilek7 
216a25542e7Smilek7 
217a25542e7Smilek7 size_t
218a25542e7Smilek7 VMSAv8TranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const
219a25542e7Smilek7 {
220a25542e7Smilek7 	size_t result = 0;
221a25542e7Smilek7 	size_t size = end - start + 1;
222a25542e7Smilek7 
223a25542e7Smilek7 	for (int i = fInitialLevel; i < 3; i++) {
224a25542e7Smilek7 		int tableBits = fPageBits - 3;
225a25542e7Smilek7 		int shift = tableBits * (3 - i) + fPageBits;
226a25542e7Smilek7 		uint64_t entrySize = 1UL << shift;
227a25542e7Smilek7 
228a25542e7Smilek7 		result += size / entrySize + 2;
229a25542e7Smilek7 	}
230a25542e7Smilek7 
231a25542e7Smilek7 	return result;
232a25542e7Smilek7 }
233a25542e7Smilek7 
234a25542e7Smilek7 
235a25542e7Smilek7 uint64_t*
236a25542e7Smilek7 VMSAv8TranslationMap::TableFromPa(phys_addr_t pa)
237a25542e7Smilek7 {
238a25542e7Smilek7 	return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa);
239a25542e7Smilek7 }
240a25542e7Smilek7 
241a25542e7Smilek7 
242a25542e7Smilek7 uint64_t
243a25542e7Smilek7 VMSAv8TranslationMap::MakeBlock(phys_addr_t pa, int level, uint64_t attr)
244a25542e7Smilek7 {
245a25542e7Smilek7 	ASSERT(level >= fMinBlockLevel && level < 4);
246a25542e7Smilek7 
247a25542e7Smilek7 	return pa | attr | (level == 3 ? 0x3 : 0x1);
248a25542e7Smilek7 }
249a25542e7Smilek7 
250a25542e7Smilek7 
2517908993dSOwen Anderson template<typename EntryRemoved>
252a25542e7Smilek7 void
2537908993dSOwen Anderson VMSAv8TranslationMap::FreeTable(phys_addr_t ptPa, uint64_t va, int level,
2547908993dSOwen Anderson 	EntryRemoved &&entryRemoved)
255a25542e7Smilek7 {
2567908993dSOwen Anderson 	ASSERT(level < 4);
257a25542e7Smilek7 
258a25542e7Smilek7 	int tableBits = fPageBits - 3;
259a25542e7Smilek7 	uint64_t tableSize = 1UL << tableBits;
2607908993dSOwen Anderson 	uint64_t vaMask = (1UL << fVaBits) - 1;
261a25542e7Smilek7 
2627908993dSOwen Anderson 	int shift = tableBits * (3 - level) + fPageBits;
2637908993dSOwen Anderson 	uint64_t entrySize = 1UL << shift;
2647908993dSOwen Anderson 
2657908993dSOwen Anderson 	uint64_t nextVa = va;
266a25542e7Smilek7 	uint64_t* pt = TableFromPa(ptPa);
267a25542e7Smilek7 	for (uint64_t i = 0; i < tableSize; i++) {
2687908993dSOwen Anderson 		uint64_t oldPte = (uint64_t) atomic_get_and_set64((int64*) &pt[i], 0);
2697908993dSOwen Anderson 
27018a27fe0SOwen Anderson 		if (level < 3 && (oldPte & kPteTypeMask) == kPteTypeL012Table) {
2717908993dSOwen Anderson 			FreeTable(oldPte & kPteAddrMask, nextVa, level + 1, entryRemoved);
27218a27fe0SOwen Anderson 		} else if ((oldPte & kPteTypeMask) != 0) {
2737908993dSOwen Anderson 			uint64_t fullVa = (fIsKernel ? ~vaMask : 0) | nextVa;
2747908993dSOwen Anderson 			asm("dsb ishst");
2757908993dSOwen Anderson 			asm("tlbi vaae1is, %0" :: "r" ((fullVa >> 12) & kTLBIMask));
2767908993dSOwen Anderson 			// Does it correctly flush block entries at level < 3? We don't use them anyway though.
2777908993dSOwen Anderson 			// TODO: Flush only currently used ASID (using vae1is)
2787908993dSOwen Anderson 			entryRemoved(level, oldPte);
279a25542e7Smilek7 		}
280a25542e7Smilek7 
2817908993dSOwen Anderson 		nextVa += entrySize;
2827908993dSOwen Anderson 	}
2837908993dSOwen Anderson 
2847908993dSOwen Anderson 	asm("dsb ish");
2857908993dSOwen Anderson 
286a25542e7Smilek7 	vm_page* page = vm_lookup_page(ptPa >> fPageBits);
2877908993dSOwen Anderson 	DEBUG_PAGE_ACCESS_START(page);
288a25542e7Smilek7 	vm_page_set_state(page, PAGE_STATE_FREE);
289a25542e7Smilek7 }
290a25542e7Smilek7 
291a25542e7Smilek7 
29218a27fe0SOwen Anderson // Make a new page sub-table.
29318a27fe0SOwen Anderson // The parent table is `ptPa`, and the new sub-table's PTE will be at `index`
29418a27fe0SOwen Anderson // in it.
29518a27fe0SOwen Anderson // Returns the physical address of the new table, or the address of the existing
29618a27fe0SOwen Anderson // one if the PTE is already filled.
297a25542e7Smilek7 phys_addr_t
298a25542e7Smilek7 VMSAv8TranslationMap::MakeTable(
299a25542e7Smilek7 	phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation)
300a25542e7Smilek7 {
30118a27fe0SOwen Anderson 	ASSERT(level < 3);
302a25542e7Smilek7 
30318a27fe0SOwen Anderson 	uint64_t* ptePtr = TableFromPa(ptPa) + index;
30418a27fe0SOwen Anderson 	uint64_t oldPte = atomic_get64((int64*) ptePtr);
305a25542e7Smilek7 
30618a27fe0SOwen Anderson 	int type = oldPte & kPteTypeMask;
30718a27fe0SOwen Anderson 	if (type == kPteTypeL012Table) {
30818a27fe0SOwen Anderson 		// This is table entry already, just return it
309a25542e7Smilek7 		return oldPte & kPteAddrMask;
31018a27fe0SOwen Anderson 	} else if (reservation != nullptr) {
31118a27fe0SOwen Anderson 		// Create new table there
31218a27fe0SOwen Anderson 		vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
313a25542e7Smilek7 		phys_addr_t newTablePa = page->physical_page_number << fPageBits;
31418a27fe0SOwen Anderson 		DEBUG_PAGE_ACCESS_END(page);
315a25542e7Smilek7 
31618a27fe0SOwen Anderson 		// We only create mappings at the final level so we don't need to handle
31718a27fe0SOwen Anderson 		// splitting block mappings
31818a27fe0SOwen Anderson 		ASSERT(type != kPteTypeL012Block);
319a25542e7Smilek7 
32018a27fe0SOwen Anderson 		// Ensure that writes to page being attached have completed
32118a27fe0SOwen Anderson 		asm("dsb ishst");
322a25542e7Smilek7 
32318a27fe0SOwen Anderson 		uint64_t oldPteRefetch = (uint64_t)atomic_test_and_set64((int64*) ptePtr,
32418a27fe0SOwen Anderson 			newTablePa | kPteTypeL012Table, oldPte);
32518a27fe0SOwen Anderson 		if (oldPteRefetch != oldPte) {
32618a27fe0SOwen Anderson 			// If the old PTE has mutated, it must be because another thread has allocated the
32718a27fe0SOwen Anderson 			// sub-table at the same time as us. If that has happened, deallocate the page we
32818a27fe0SOwen Anderson 			// setup and use the one they installed instead.
32918a27fe0SOwen Anderson 			ASSERT((oldPteRefetch & kPteTypeMask) == kPteTypeL012Table);
33018a27fe0SOwen Anderson 			DEBUG_PAGE_ACCESS_START(page);
33118a27fe0SOwen Anderson 			vm_page_set_state(page, PAGE_STATE_FREE);
33218a27fe0SOwen Anderson 			return oldPteRefetch & kPteAddrMask;
333a25542e7Smilek7 		}
334a25542e7Smilek7 
335a25542e7Smilek7 		return newTablePa;
336a25542e7Smilek7 	}
337a25542e7Smilek7 
33818a27fe0SOwen Anderson 	// There's no existing table and we have no reservation
339a25542e7Smilek7 	return 0;
340a25542e7Smilek7 }
341a25542e7Smilek7 
342a25542e7Smilek7 
343a25542e7Smilek7 void
344a25542e7Smilek7 VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size,
345a25542e7Smilek7 	VMSAv8TranslationMap::VMAction action, uint64_t attr, vm_page_reservation* reservation)
346a25542e7Smilek7 {
347a25542e7Smilek7 	ASSERT(level < 4);
348a25542e7Smilek7 	ASSERT(ptPa != 0);
349a25542e7Smilek7 	ASSERT(reservation != NULL || action != VMAction::MAP);
350a25542e7Smilek7 
351a25542e7Smilek7 	int tableBits = fPageBits - 3;
352a25542e7Smilek7 	uint64_t tableMask = (1UL << tableBits) - 1;
353a25542e7Smilek7 
354a25542e7Smilek7 	int shift = tableBits * (3 - level) + fPageBits;
355a25542e7Smilek7 	uint64_t entrySize = 1UL << shift;
356a25542e7Smilek7 
357a25542e7Smilek7 	uint64_t entryMask = entrySize - 1;
358a25542e7Smilek7 	uint64_t nextVa = va;
359a25542e7Smilek7 	uint64_t end = va + size;
360a25542e7Smilek7 	int index;
361a25542e7Smilek7 
362a25542e7Smilek7 	// Handle misaligned header that straddles entry boundary in next-level table
363a25542e7Smilek7 	if ((va & entryMask) != 0) {
364a25542e7Smilek7 		uint64_t aligned = (va & ~entryMask) + entrySize;
365a25542e7Smilek7 		if (end > aligned) {
366a25542e7Smilek7 			index = (va >> shift) & tableMask;
367a25542e7Smilek7 			phys_addr_t table = MakeTable(ptPa, level, index, reservation);
368a25542e7Smilek7 			MapRange(table, level + 1, va, pa, aligned - va, action, attr, reservation);
369a25542e7Smilek7 			nextVa = aligned;
370a25542e7Smilek7 		}
371a25542e7Smilek7 	}
372a25542e7Smilek7 
373a25542e7Smilek7 	// Handle fully aligned and appropriately sized chunks
374a25542e7Smilek7 	while (nextVa + entrySize <= end) {
375a25542e7Smilek7 		phys_addr_t targetPa = pa + (nextVa - va);
376a25542e7Smilek7 		index = (nextVa >> shift) & tableMask;
377a25542e7Smilek7 
378a25542e7Smilek7 		bool blockAllowed = false;
379a25542e7Smilek7 		if (action == VMAction::MAP)
380a25542e7Smilek7 			blockAllowed = (level >= fMinBlockLevel && (targetPa & entryMask) == 0);
381a25542e7Smilek7 		if (action == VMAction::SET_ATTR || action == VMAction::CLEAR_FLAGS)
382a25542e7Smilek7 			blockAllowed = (MakeTable(ptPa, level, index, NULL) == 0);
383a25542e7Smilek7 		if (action == VMAction::UNMAP)
384a25542e7Smilek7 			blockAllowed = true;
385a25542e7Smilek7 
386a25542e7Smilek7 		if (blockAllowed) {
387a25542e7Smilek7 			// Everything is aligned, we can make block mapping there
38818a27fe0SOwen Anderson 			uint64_t* pte = TableFromPa(ptPa) + index;
389a25542e7Smilek7 
390a25542e7Smilek7 		retry:
391a25542e7Smilek7 			uint64_t oldPte = atomic_get64((int64*) pte);
392a25542e7Smilek7 
393a25542e7Smilek7 			if (action == VMAction::MAP || (oldPte & 0x1) != 0) {
394a25542e7Smilek7 				uint64_t newPte = 0;
395a25542e7Smilek7 				if (action == VMAction::MAP) {
396a25542e7Smilek7 					newPte = MakeBlock(targetPa, level, attr);
397a25542e7Smilek7 				} else if (action == VMAction::SET_ATTR) {
398a25542e7Smilek7 					newPte = MakeBlock(oldPte & kPteAddrMask, level, MoveAttrFlags(attr, oldPte));
399a25542e7Smilek7 				} else if (action == VMAction::CLEAR_FLAGS) {
400a25542e7Smilek7 					newPte = MakeBlock(oldPte & kPteAddrMask, level, ClearAttrFlags(oldPte, attr));
401a25542e7Smilek7 				} else if (action == VMAction::UNMAP) {
402a25542e7Smilek7 					newPte = 0;
403a25542e7Smilek7 					tmp_pte = oldPte;
404a25542e7Smilek7 				}
405a25542e7Smilek7 
406a25542e7Smilek7 				// FIXME: this might not be enough on real hardware with SMP for some cases
407a25542e7Smilek7 				if ((uint64_t) atomic_test_and_set64((int64*) pte, newPte, oldPte) != oldPte)
408a25542e7Smilek7 					goto retry;
409a25542e7Smilek7 
410a25542e7Smilek7 				if (level < 3 && (oldPte & 0x3) == 0x3) {
411a25542e7Smilek7 					// If we're replacing existing pagetable clean it up
4127908993dSOwen Anderson 					FreeTable(oldPte & kPteAddrMask, nextVa, level + 1,
4137908993dSOwen Anderson 						[](int level, uint64_t oldPte) {});
414a25542e7Smilek7 				}
415a25542e7Smilek7 			}
416a25542e7Smilek7 		} else {
417a25542e7Smilek7 			// Otherwise handle mapping in next-level table
418a25542e7Smilek7 			phys_addr_t table = MakeTable(ptPa, level, index, reservation);
419a25542e7Smilek7 			MapRange(table, level + 1, nextVa, targetPa, entrySize, action, attr, reservation);
420a25542e7Smilek7 		}
421a25542e7Smilek7 		nextVa += entrySize;
422a25542e7Smilek7 	}
423a25542e7Smilek7 
424a25542e7Smilek7 	// Handle misaligned tail area (or entirety of small area) in next-level table
425a25542e7Smilek7 	if (nextVa < end) {
426a25542e7Smilek7 		index = (nextVa >> shift) & tableMask;
427a25542e7Smilek7 		phys_addr_t table = MakeTable(ptPa, level, index, reservation);
428a25542e7Smilek7 		MapRange(
429a25542e7Smilek7 			table, level + 1, nextVa, pa + (nextVa - va), end - nextVa, action, attr, reservation);
430a25542e7Smilek7 	}
431a25542e7Smilek7 }
432a25542e7Smilek7 
433a25542e7Smilek7 
434a25542e7Smilek7 uint8_t
435a25542e7Smilek7 VMSAv8TranslationMap::MairIndex(uint8_t type)
436a25542e7Smilek7 {
437a25542e7Smilek7 	for (int i = 0; i < 8; i++)
438a25542e7Smilek7 		if (((fMair >> (i * 8)) & 0xff) == type)
439a25542e7Smilek7 			return i;
440a25542e7Smilek7 
441a25542e7Smilek7 	panic("MAIR entry not found");
442a25542e7Smilek7 	return 0;
443a25542e7Smilek7 }
444a25542e7Smilek7 
445a25542e7Smilek7 
446a25542e7Smilek7 uint64_t
447a25542e7Smilek7 VMSAv8TranslationMap::ClearAttrFlags(uint64_t attr, uint32 flags)
448a25542e7Smilek7 {
449a25542e7Smilek7 	attr &= kPteAttrMask;
450a25542e7Smilek7 
451a25542e7Smilek7 	if ((flags & PAGE_ACCESSED) != 0)
452a25542e7Smilek7 		attr &= ~kAttrAF;
453a25542e7Smilek7 
454a25542e7Smilek7 	if ((flags & PAGE_MODIFIED) != 0 && (attr & kAttrSWDBM) != 0)
455*108f6fdcSOwen Anderson 		attr |= kAttrAPReadOnly;
456a25542e7Smilek7 
457a25542e7Smilek7 	return attr;
458a25542e7Smilek7 }
459a25542e7Smilek7 
460a25542e7Smilek7 
461a25542e7Smilek7 uint64_t
462a25542e7Smilek7 VMSAv8TranslationMap::MoveAttrFlags(uint64_t newAttr, uint64_t oldAttr)
463a25542e7Smilek7 {
464a25542e7Smilek7 	if ((oldAttr & kAttrAF) != 0)
465a25542e7Smilek7 		newAttr |= kAttrAF;
466*108f6fdcSOwen Anderson 	if (((newAttr & oldAttr) & kAttrSWDBM) != 0 && (oldAttr & kAttrAPReadOnly) == 0)
467*108f6fdcSOwen Anderson 		newAttr &= ~kAttrAPReadOnly;
468a25542e7Smilek7 
469a25542e7Smilek7 	return newAttr;
470a25542e7Smilek7 }
471a25542e7Smilek7 
472a25542e7Smilek7 
473a25542e7Smilek7 uint64_t
474a25542e7Smilek7 VMSAv8TranslationMap::GetMemoryAttr(uint32 attributes, uint32 memoryType, bool isKernel)
475a25542e7Smilek7 {
476a25542e7Smilek7 	uint64_t attr = 0;
477a25542e7Smilek7 
478a25542e7Smilek7 	if (!isKernel)
479a25542e7Smilek7 		attr |= kAttrNG;
480a25542e7Smilek7 
481a25542e7Smilek7 	if ((attributes & B_EXECUTE_AREA) == 0)
482a25542e7Smilek7 		attr |= kAttrUXN;
483a25542e7Smilek7 	if ((attributes & B_KERNEL_EXECUTE_AREA) == 0)
484a25542e7Smilek7 		attr |= kAttrPXN;
485a25542e7Smilek7 
486*108f6fdcSOwen Anderson 	// SWDBM is software reserved bit that we use to mark that
487*108f6fdcSOwen Anderson 	// writes are allowed, and fault handler should clear kAttrAPReadOnly.
488*108f6fdcSOwen Anderson 	// In that case kAttrAPReadOnly doubles as not-dirty bit.
489*108f6fdcSOwen Anderson 	// Additionally dirty state can be stored in SWDIRTY, in order not to lose
490*108f6fdcSOwen Anderson 	// dirty state when changing protection from RW to RO.
491a25542e7Smilek7 
492*108f6fdcSOwen Anderson 	// All page permissions begin life in RO state.
493*108f6fdcSOwen Anderson 	attr |= kAttrAPReadOnly;
494*108f6fdcSOwen Anderson 
495*108f6fdcSOwen Anderson 	// User-Execute implies User-Read, because it would break PAN otherwise
496*108f6fdcSOwen Anderson 	if ((attributes & B_READ_AREA) != 0 || (attributes & B_EXECUTE_AREA) != 0)
497*108f6fdcSOwen Anderson 		attr |= kAttrAPUserAccess; // Allow user reads
498*108f6fdcSOwen Anderson 
499*108f6fdcSOwen Anderson 	if ((attributes & B_WRITE_AREA) != 0 || (attributes & B_KERNEL_WRITE_AREA) != 0)
500*108f6fdcSOwen Anderson 		attr |= kAttrSWDBM; // Mark as writeable
501*108f6fdcSOwen Anderson 
502*108f6fdcSOwen Anderson 	// When supported by hardware copy our SWDBM bit into DBM,
503*108f6fdcSOwen Anderson 	// so that kAttrAPReadOnly is cleared on write attempt automatically
504*108f6fdcSOwen Anderson 	// without going through fault handler.
505*108f6fdcSOwen Anderson 	if ((fHwFeature & HW_DIRTY) != 0 && (attr & kAttrSWDBM) != 0)
506a25542e7Smilek7 		attr |= kAttrDBM;
507a25542e7Smilek7 
508*108f6fdcSOwen Anderson 	attr |= kAttrSHInnerShareable; // Inner Shareable
509a25542e7Smilek7 
510*108f6fdcSOwen Anderson 	uint8_t type = MAIR_NORMAL_WB;
511*108f6fdcSOwen Anderson 
512*108f6fdcSOwen Anderson 	if (memoryType & B_MTR_UC)
513*108f6fdcSOwen Anderson 		type = MAIR_DEVICE_nGnRnE; // TODO: This probably should be nGnRE for PCI
514*108f6fdcSOwen Anderson 	else if (memoryType & B_MTR_WC)
515*108f6fdcSOwen Anderson 		type = MAIR_NORMAL_NC;
516*108f6fdcSOwen Anderson 	else if (memoryType & B_MTR_WT)
517*108f6fdcSOwen Anderson 		type = MAIR_NORMAL_WT;
518*108f6fdcSOwen Anderson 	else if (memoryType & B_MTR_WP)
519*108f6fdcSOwen Anderson 		type = MAIR_NORMAL_WT;
520*108f6fdcSOwen Anderson 	else if (memoryType & B_MTR_WB)
521*108f6fdcSOwen Anderson 		type = MAIR_NORMAL_WB;
522*108f6fdcSOwen Anderson 
523*108f6fdcSOwen Anderson 	attr |= MairIndex(type) << 2;
524a25542e7Smilek7 
525a25542e7Smilek7 	return attr;
526a25542e7Smilek7 }
527a25542e7Smilek7 
528a25542e7Smilek7 
529a25542e7Smilek7 status_t
530a25542e7Smilek7 VMSAv8TranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, uint32 memoryType,
531a25542e7Smilek7 	vm_page_reservation* reservation)
532a25542e7Smilek7 {
533a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
534a25542e7Smilek7 
535a25542e7Smilek7 	uint64_t pageMask = (1UL << fPageBits) - 1;
536a25542e7Smilek7 	uint64_t vaMask = (1UL << fVaBits) - 1;
537a25542e7Smilek7 
538a25542e7Smilek7 	ASSERT((va & pageMask) == 0);
539a25542e7Smilek7 	ASSERT((pa & pageMask) == 0);
540a25542e7Smilek7 	ASSERT(ValidateVa(va));
541a25542e7Smilek7 
542a25542e7Smilek7 	uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel);
543a25542e7Smilek7 
544a25542e7Smilek7 	if (!fPageTable) {
545a25542e7Smilek7 		vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
546a25542e7Smilek7 		fPageTable = page->physical_page_number << fPageBits;
547a25542e7Smilek7 	}
548a25542e7Smilek7 
549a25542e7Smilek7 	MapRange(
550a25542e7Smilek7 		fPageTable, fInitialLevel, va & vaMask, pa, B_PAGE_SIZE, VMAction::MAP, attr, reservation);
551a25542e7Smilek7 
552a25542e7Smilek7 	return B_OK;
553a25542e7Smilek7 }
554a25542e7Smilek7 
555a25542e7Smilek7 
556a25542e7Smilek7 status_t
557a25542e7Smilek7 VMSAv8TranslationMap::Unmap(addr_t start, addr_t end)
558a25542e7Smilek7 {
559a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
560a25542e7Smilek7 
561a25542e7Smilek7 	size_t size = end - start + 1;
562a25542e7Smilek7 
563a25542e7Smilek7 	uint64_t pageMask = (1UL << fPageBits) - 1;
564a25542e7Smilek7 	uint64_t vaMask = (1UL << fVaBits) - 1;
565a25542e7Smilek7 
566a25542e7Smilek7 	ASSERT((start & pageMask) == 0);
567a25542e7Smilek7 	ASSERT((size & pageMask) == 0);
568a25542e7Smilek7 	ASSERT(ValidateVa(start));
569a25542e7Smilek7 
570a25542e7Smilek7 	MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::UNMAP, 0, NULL);
571a25542e7Smilek7 
572a25542e7Smilek7 	return B_OK;
573a25542e7Smilek7 }
574a25542e7Smilek7 
575a25542e7Smilek7 
576a25542e7Smilek7 status_t
577a25542e7Smilek7 VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, bool updatePageQueue)
578a25542e7Smilek7 {
579a25542e7Smilek7 
580a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
581a25542e7Smilek7 	RecursiveLocker locker(fLock);
582a25542e7Smilek7 
583a25542e7Smilek7 	// TODO: replace this kludge
584a25542e7Smilek7 
585a25542e7Smilek7 	phys_addr_t pa;
586a25542e7Smilek7 	uint64_t pte;
587a25542e7Smilek7 	if (!WalkTable(fPageTable, fInitialLevel, address, &pa, &pte))
588a25542e7Smilek7 		return B_ENTRY_NOT_FOUND;
589a25542e7Smilek7 
590a25542e7Smilek7 	uint64_t vaMask = (1UL << fVaBits) - 1;
591a25542e7Smilek7 	MapRange(fPageTable, fInitialLevel, address & vaMask, 0, B_PAGE_SIZE, VMAction::UNMAP, 0, NULL);
592a25542e7Smilek7 
593a25542e7Smilek7 	pinner.Unlock();
594a25542e7Smilek7 	locker.Detach();
595*108f6fdcSOwen Anderson 	PageUnmapped(area, pa >> fPageBits, (tmp_pte & kAttrAF) != 0, (tmp_pte & kAttrAPReadOnly) == 0,
596a25542e7Smilek7 		updatePageQueue);
597a25542e7Smilek7 
598a25542e7Smilek7 	return B_OK;
599a25542e7Smilek7 }
600a25542e7Smilek7 
601a25542e7Smilek7 
602a25542e7Smilek7 bool
603a25542e7Smilek7 VMSAv8TranslationMap::WalkTable(
604a25542e7Smilek7 	phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* rpte)
605a25542e7Smilek7 {
606a25542e7Smilek7 	int tableBits = fPageBits - 3;
607a25542e7Smilek7 	uint64_t tableMask = (1UL << tableBits) - 1;
608a25542e7Smilek7 
609a25542e7Smilek7 	int shift = tableBits * (3 - level) + fPageBits;
610a25542e7Smilek7 	uint64_t entrySize = 1UL << shift;
611a25542e7Smilek7 	uint64_t entryMask = entrySize - 1;
612a25542e7Smilek7 
613a25542e7Smilek7 	int index = (va >> shift) & tableMask;
614a25542e7Smilek7 
615a25542e7Smilek7 	uint64_t pte = TableFromPa(ptPa)[index];
616a25542e7Smilek7 	int type = pte & 0x3;
617a25542e7Smilek7 
618a25542e7Smilek7 	if ((type & 0x1) == 0)
619a25542e7Smilek7 		return false;
620a25542e7Smilek7 
621a25542e7Smilek7 	uint64_t addr = pte & kPteAddrMask;
622a25542e7Smilek7 	if (level < 3) {
623a25542e7Smilek7 		if (type == 0x3) {
624a25542e7Smilek7 			return WalkTable(addr, level + 1, va, pa, rpte);
625a25542e7Smilek7 		} else {
626a25542e7Smilek7 			*pa = addr | (va & entryMask);
627a25542e7Smilek7 			*rpte = pte;
628a25542e7Smilek7 		}
629a25542e7Smilek7 	} else {
630a25542e7Smilek7 		ASSERT(type == 0x3);
631a25542e7Smilek7 		*pa = addr;
632a25542e7Smilek7 		*rpte = pte;
633a25542e7Smilek7 	}
634a25542e7Smilek7 
635a25542e7Smilek7 	return true;
636a25542e7Smilek7 }
637a25542e7Smilek7 
638a25542e7Smilek7 
639a25542e7Smilek7 bool
640a25542e7Smilek7 VMSAv8TranslationMap::ValidateVa(addr_t va)
641a25542e7Smilek7 {
642a25542e7Smilek7 	uint64_t vaMask = (1UL << fVaBits) - 1;
643a25542e7Smilek7 	bool kernelAddr = (va & (1UL << 63)) != 0;
644a25542e7Smilek7 	if (kernelAddr != fIsKernel)
645a25542e7Smilek7 		return false;
646a25542e7Smilek7 	if ((va & ~vaMask) != (fIsKernel ? ~vaMask : 0))
647a25542e7Smilek7 		return false;
648a25542e7Smilek7 	return true;
649a25542e7Smilek7 }
650a25542e7Smilek7 
651a25542e7Smilek7 
652a25542e7Smilek7 status_t
653a25542e7Smilek7 VMSAv8TranslationMap::Query(addr_t va, phys_addr_t* pa, uint32* flags)
654a25542e7Smilek7 {
655a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
656a25542e7Smilek7 
657a25542e7Smilek7 	ASSERT(ValidateVa(va));
658a25542e7Smilek7 
659a25542e7Smilek7 	uint64_t pte = 0;
660a25542e7Smilek7 	bool ret = WalkTable(fPageTable, fInitialLevel, va, pa, &pte);
661a25542e7Smilek7 
662a25542e7Smilek7 	uint32 result = 0;
663a25542e7Smilek7 
664a25542e7Smilek7 	if (ret) {
665a25542e7Smilek7 		result |= PAGE_PRESENT;
666a25542e7Smilek7 
667a25542e7Smilek7 		if ((pte & kAttrAF) != 0)
668a25542e7Smilek7 			result |= PAGE_ACCESSED;
669*108f6fdcSOwen Anderson 		if ((pte & kAttrAPReadOnly) == 0)
670a25542e7Smilek7 			result |= PAGE_MODIFIED;
671a25542e7Smilek7 
672a25542e7Smilek7 		if ((pte & kAttrUXN) == 0)
673a25542e7Smilek7 			result |= B_EXECUTE_AREA;
674a25542e7Smilek7 		if ((pte & kAttrPXN) == 0)
675a25542e7Smilek7 			result |= B_KERNEL_EXECUTE_AREA;
676a25542e7Smilek7 
677a25542e7Smilek7 		result |= B_KERNEL_READ_AREA;
678a25542e7Smilek7 
679*108f6fdcSOwen Anderson 		if ((pte & kAttrAPUserAccess) != 0)
680a25542e7Smilek7 			result |= B_READ_AREA;
681a25542e7Smilek7 
682*108f6fdcSOwen Anderson 		if ((pte & kAttrAPReadOnly) == 0 || (pte & kAttrSWDBM) != 0) {
683a25542e7Smilek7 			result |= B_KERNEL_WRITE_AREA;
684a25542e7Smilek7 
685*108f6fdcSOwen Anderson 			if ((pte & kAttrAPUserAccess) != 0)
686a25542e7Smilek7 				result |= B_WRITE_AREA;
687a25542e7Smilek7 		}
688a25542e7Smilek7 	}
689a25542e7Smilek7 
690a25542e7Smilek7 	*flags = result;
691a25542e7Smilek7 	return B_OK;
692a25542e7Smilek7 }
693a25542e7Smilek7 
694a25542e7Smilek7 
695a25542e7Smilek7 status_t
696a25542e7Smilek7 VMSAv8TranslationMap::QueryInterrupt(
697a25542e7Smilek7 	addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags)
698a25542e7Smilek7 {
699a25542e7Smilek7 	return Query(virtualAddress, _physicalAddress, _flags);
700a25542e7Smilek7 }
701a25542e7Smilek7 
702a25542e7Smilek7 
703a25542e7Smilek7 status_t
704a25542e7Smilek7 VMSAv8TranslationMap::Protect(addr_t start, addr_t end, uint32 attributes, uint32 memoryType)
705a25542e7Smilek7 {
706a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
707a25542e7Smilek7 
708a25542e7Smilek7 	size_t size = end - start + 1;
709a25542e7Smilek7 
710a25542e7Smilek7 	uint64_t pageMask = (1UL << fPageBits) - 1;
711a25542e7Smilek7 	uint64_t vaMask = (1UL << fVaBits) - 1;
712a25542e7Smilek7 
713a25542e7Smilek7 	ASSERT((start & pageMask) == 0);
714a25542e7Smilek7 	ASSERT((size & pageMask) == 0);
715a25542e7Smilek7 	ASSERT(ValidateVa(start));
716a25542e7Smilek7 
717a25542e7Smilek7 	uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel);
718a25542e7Smilek7 	MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::SET_ATTR, attr, NULL);
719a25542e7Smilek7 
720a25542e7Smilek7 	return B_OK;
721a25542e7Smilek7 }
722a25542e7Smilek7 
723a25542e7Smilek7 
724a25542e7Smilek7 status_t
725a25542e7Smilek7 VMSAv8TranslationMap::ClearFlags(addr_t va, uint32 flags)
726a25542e7Smilek7 {
727a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
728a25542e7Smilek7 
729a25542e7Smilek7 	uint64_t pageMask = (1UL << fPageBits) - 1;
730a25542e7Smilek7 	uint64_t vaMask = (1UL << fVaBits) - 1;
731a25542e7Smilek7 
732a25542e7Smilek7 	ASSERT((va & pageMask) == 0);
733a25542e7Smilek7 	ASSERT(ValidateVa(va));
734a25542e7Smilek7 
735a25542e7Smilek7 	MapRange(
736a25542e7Smilek7 		fPageTable, fInitialLevel, va & vaMask, 0, B_PAGE_SIZE, VMAction::CLEAR_FLAGS, flags, NULL);
737a25542e7Smilek7 
738a25542e7Smilek7 	return B_OK;
739a25542e7Smilek7 }
740a25542e7Smilek7 
741a25542e7Smilek7 
742a25542e7Smilek7 bool
743a25542e7Smilek7 VMSAv8TranslationMap::ClearAccessedAndModified(
744a25542e7Smilek7 	VMArea* area, addr_t address, bool unmapIfUnaccessed, bool& _modified)
745a25542e7Smilek7 {
746a25542e7Smilek7 	panic("VMSAv8TranslationMap::ClearAccessedAndModified not implemented\n");
747a25542e7Smilek7 	return B_OK;
748a25542e7Smilek7 }
749a25542e7Smilek7 
750a25542e7Smilek7 
751a25542e7Smilek7 void
752a25542e7Smilek7 VMSAv8TranslationMap::Flush()
753a25542e7Smilek7 {
754a25542e7Smilek7 	ThreadCPUPinner pinner(thread_get_current_thread());
755a25542e7Smilek7 
756a25542e7Smilek7 	arch_cpu_global_TLB_invalidate();
757a25542e7Smilek7 }
758