1 /*
2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk
3 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
4 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
5 * Distributed under the terms of the MIT License.
6 *
7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
8 * Distributed under the terms of the NewOS License.
9 */
10
11
12 #include "paging/64bit/X86VMTranslationMap64Bit.h"
13
14 #include <int.h>
15 #include <slab/Slab.h>
16 #include <thread.h>
17 #include <util/AutoLock.h>
18 #include <util/ThreadAutoLock.h>
19 #include <vm/vm_page.h>
20 #include <vm/VMAddressSpace.h>
21 #include <vm/VMCache.h>
22
23 #include "paging/64bit/X86PagingMethod64Bit.h"
24 #include "paging/64bit/X86PagingStructures64Bit.h"
25 #include "paging/x86_physical_page_mapper.h"
26
27
28 //#define TRACE_X86_VM_TRANSLATION_MAP_64BIT
29 #ifdef TRACE_X86_VM_TRANSLATION_MAP_64BIT
30 # define TRACE(x...) dprintf(x)
31 #else
32 # define TRACE(x...) ;
33 #endif
34
35
36 // #pragma mark - X86VMTranslationMap64Bit
37
38
X86VMTranslationMap64Bit(bool la57)39 X86VMTranslationMap64Bit::X86VMTranslationMap64Bit(bool la57)
40 :
41 fPagingStructures(NULL),
42 fLA57(la57)
43 {
44 }
45
46
~X86VMTranslationMap64Bit()47 X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()
48 {
49 TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n");
50
51 if (fPagingStructures == NULL)
52 return;
53
54 if (fPageMapper != NULL) {
55 phys_addr_t address;
56 vm_page* page;
57
58 // Free all structures in the bottom half of the PMLTop (user memory).
59 uint64* virtualPML4 = fPagingStructures->VirtualPMLTop();
60 for (uint32 i = 0; i < 256; i++) {
61 if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0)
62 continue;
63
64 uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt(
65 virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK);
66 for (uint32 j = 0; j < 512; j++) {
67 if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0)
68 continue;
69
70 uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt(
71 virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK);
72 for (uint32 k = 0; k < 512; k++) {
73 if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0)
74 continue;
75
76 address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK;
77 page = vm_lookup_page(address / B_PAGE_SIZE);
78 if (page == NULL) {
79 panic("page table %u %u %u on invalid page %#"
80 B_PRIxPHYSADDR "\n", i, j, k, address);
81 }
82
83 DEBUG_PAGE_ACCESS_START(page);
84 vm_page_set_state(page, PAGE_STATE_FREE);
85 }
86
87 address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK;
88 page = vm_lookup_page(address / B_PAGE_SIZE);
89 if (page == NULL) {
90 panic("page directory %u %u on invalid page %#"
91 B_PRIxPHYSADDR "\n", i, j, address);
92 }
93
94 DEBUG_PAGE_ACCESS_START(page);
95 vm_page_set_state(page, PAGE_STATE_FREE);
96 }
97
98 address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK;
99 page = vm_lookup_page(address / B_PAGE_SIZE);
100 if (page == NULL) {
101 panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i,
102 address);
103 }
104
105 DEBUG_PAGE_ACCESS_START(page);
106 vm_page_set_state(page, PAGE_STATE_FREE);
107 }
108
109 fPageMapper->Delete();
110 }
111
112 fPagingStructures->RemoveReference();
113 }
114
115
116 status_t
Init(bool kernel)117 X86VMTranslationMap64Bit::Init(bool kernel)
118 {
119 TRACE("X86VMTranslationMap64Bit::Init()\n");
120
121 X86VMTranslationMap::Init(kernel);
122
123 fPagingStructures = new(std::nothrow) X86PagingStructures64Bit;
124 if (fPagingStructures == NULL)
125 return B_NO_MEMORY;
126
127 X86PagingMethod64Bit* method = X86PagingMethod64Bit::Method();
128
129 if (kernel) {
130 // Get the page mapper.
131 fPageMapper = method->KernelPhysicalPageMapper();
132
133 // Kernel PMLTop is already mapped.
134 fPagingStructures->Init(method->KernelVirtualPMLTop(),
135 method->KernelPhysicalPMLTop());
136 } else {
137 // Allocate a physical page mapper.
138 status_t error = method->PhysicalPageMapper()
139 ->CreateTranslationMapPhysicalPageMapper(&fPageMapper);
140 if (error != B_OK)
141 return error;
142
143 // Assuming that only the top 2 PMLTop entries are occupied for the
144 // kernel.
145 STATIC_ASSERT(KERNEL_PMAP_BASE == 0xffffff0000000000);
146 STATIC_ASSERT(KERNEL_BASE == 0xffffff0000000000);
147
148 // Allocate and clear the PMLTop.
149 uint64* virtualPMLTop = (uint64*)memalign(B_PAGE_SIZE, B_PAGE_SIZE);
150 if (virtualPMLTop == NULL)
151 return B_NO_MEMORY;
152 memset(virtualPMLTop, 0, B_PAGE_SIZE);
153
154 // Copy the top 2 PMLTop entries.
155 virtualPMLTop[510] = method->KernelVirtualPMLTop()[510];
156 virtualPMLTop[511] = method->KernelVirtualPMLTop()[511];
157
158 // Look up the PMLTop physical address.
159 phys_addr_t physicalPMLTop;
160 vm_get_page_mapping(VMAddressSpace::KernelID(), (addr_t)virtualPMLTop,
161 &physicalPMLTop);
162
163 // Initialize the paging structures.
164 fPagingStructures->Init(virtualPMLTop, physicalPMLTop);
165 }
166
167 return B_OK;
168 }
169
170
171 size_t
MaxPagesNeededToMap(addr_t start,addr_t end) const172 X86VMTranslationMap64Bit::MaxPagesNeededToMap(addr_t start, addr_t end) const
173 {
174 // If start == 0, the actual base address is not yet known to the caller and
175 // we shall assume the worst case, which is where the start address is the
176 // last page covered by a PDPT or PML4.
177 if (start == 0) {
178 start = (fLA57 ? k64BitPML4TRange : k64BitPDPTRange) - B_PAGE_SIZE;
179 end += start;
180 }
181
182 size_t requiredPML4s = 0;
183 if (fLA57) {
184 requiredPML4s = end / k64BitPML4TRange + 1
185 - start / k64BitPML4TRange;
186 }
187 size_t requiredPDPTs = end / k64BitPDPTRange + 1
188 - start / k64BitPDPTRange;
189 size_t requiredPageDirs = end / k64BitPageDirectoryRange + 1
190 - start / k64BitPageDirectoryRange;
191 size_t requiredPageTables = end / k64BitPageTableRange + 1
192 - start / k64BitPageTableRange;
193
194 return requiredPML4s + requiredPDPTs + requiredPageDirs
195 + requiredPageTables;
196 }
197
198
199 status_t
Map(addr_t virtualAddress,phys_addr_t physicalAddress,uint32 attributes,uint32 memoryType,vm_page_reservation * reservation)200 X86VMTranslationMap64Bit::Map(addr_t virtualAddress, phys_addr_t physicalAddress,
201 uint32 attributes, uint32 memoryType, vm_page_reservation* reservation)
202 {
203 TRACE("X86VMTranslationMap64Bit::Map(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR
204 ")\n", virtualAddress, physicalAddress);
205
206 ThreadCPUPinner pinner(thread_get_current_thread());
207
208 // Look up the page table for the virtual address, allocating new tables
209 // if required. Shouldn't fail.
210 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
211 fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap,
212 true, reservation, fPageMapper, fMapCount);
213 ASSERT(entry != NULL);
214
215 // The entry should not already exist.
216 ASSERT_PRINT((*entry & X86_64_PTE_PRESENT) == 0,
217 "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64,
218 virtualAddress, *entry);
219
220 // Fill in the table entry.
221 X86PagingMethod64Bit::PutPageTableEntryInTable(entry, physicalAddress,
222 attributes, memoryType, fIsKernelMap);
223
224 // Note: We don't need to invalidate the TLB for this address, as previously
225 // the entry was not present and the TLB doesn't cache those entries.
226
227 fMapCount++;
228
229 return 0;
230 }
231
232
233 status_t
Unmap(addr_t start,addr_t end)234 X86VMTranslationMap64Bit::Unmap(addr_t start, addr_t end)
235 {
236 start = ROUNDDOWN(start, B_PAGE_SIZE);
237 if (start >= end)
238 return B_OK;
239
240 TRACE("X86VMTranslationMap64Bit::Unmap(%#" B_PRIxADDR ", %#" B_PRIxADDR
241 ")\n", start, end);
242
243 ThreadCPUPinner pinner(thread_get_current_thread());
244
245 do {
246 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
247 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
248 NULL, fPageMapper, fMapCount);
249 if (pageTable == NULL) {
250 // Move on to the next page table.
251 start = ROUNDUP(start + 1, k64BitPageTableRange);
252 continue;
253 }
254
255 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
256 index < k64BitTableEntryCount && start < end;
257 index++, start += B_PAGE_SIZE) {
258 if ((pageTable[index] & X86_64_PTE_PRESENT) == 0)
259 continue;
260
261 TRACE("X86VMTranslationMap64Bit::Unmap(): removing page %#"
262 B_PRIxADDR " (%#" B_PRIxPHYSADDR ")\n", start,
263 pageTable[index] & X86_64_PTE_ADDRESS_MASK);
264
265 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(
266 &pageTable[index], X86_64_PTE_PRESENT);
267 fMapCount--;
268
269 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
270 // Note, that we only need to invalidate the address, if the
271 // accessed flags was set, since only then the entry could have
272 // been in any TLB.
273 InvalidatePage(start);
274 }
275 }
276 } while (start != 0 && start < end);
277
278 return B_OK;
279 }
280
281
282 status_t
DebugMarkRangePresent(addr_t start,addr_t end,bool markPresent)283 X86VMTranslationMap64Bit::DebugMarkRangePresent(addr_t start, addr_t end,
284 bool markPresent)
285 {
286 start = ROUNDDOWN(start, B_PAGE_SIZE);
287 if (start >= end)
288 return B_OK;
289
290 TRACE("X86VMTranslationMap64Bit::DebugMarkRangePresent(%#" B_PRIxADDR
291 ", %#" B_PRIxADDR ")\n", start, end);
292
293 ThreadCPUPinner pinner(thread_get_current_thread());
294
295 do {
296 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
297 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
298 NULL, fPageMapper, fMapCount);
299 if (pageTable == NULL) {
300 // Move on to the next page table.
301 start = ROUNDUP(start + 1, k64BitPageTableRange);
302 continue;
303 }
304
305 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
306 index < k64BitTableEntryCount && start < end;
307 index++, start += B_PAGE_SIZE) {
308 if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) {
309 if (!markPresent)
310 continue;
311
312 X86PagingMethod64Bit::SetTableEntryFlags(&pageTable[index],
313 X86_64_PTE_PRESENT);
314 } else {
315 if (markPresent)
316 continue;
317
318 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(
319 &pageTable[index], X86_64_PTE_PRESENT);
320
321 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
322 // Note, that we only need to invalidate the address, if the
323 // accessed flags was set, since only then the entry could
324 // have been in any TLB.
325 InvalidatePage(start);
326 }
327 }
328 }
329 } while (start != 0 && start < end);
330
331 return B_OK;
332 }
333
334
335 status_t
UnmapPage(VMArea * area,addr_t address,bool updatePageQueue)336 X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address,
337 bool updatePageQueue)
338 {
339 ASSERT(address % B_PAGE_SIZE == 0);
340
341 TRACE("X86VMTranslationMap64Bit::UnmapPage(%#" B_PRIxADDR ")\n", address);
342
343 ThreadCPUPinner pinner(thread_get_current_thread());
344
345 // Look up the page table for the virtual address.
346 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
347 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
348 false, NULL, fPageMapper, fMapCount);
349 if (entry == NULL)
350 return B_ENTRY_NOT_FOUND;
351
352 RecursiveLocker locker(fLock);
353
354 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
355
356 pinner.Unlock();
357
358 if ((oldEntry & X86_64_PTE_PRESENT) == 0)
359 return B_ENTRY_NOT_FOUND;
360
361 fMapCount--;
362
363 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
364 // Note, that we only need to invalidate the address, if the
365 // accessed flags was set, since only then the entry could have been
366 // in any TLB.
367 InvalidatePage(address);
368
369 Flush();
370
371 // NOTE: Between clearing the page table entry and Flush() other
372 // processors (actually even this processor with another thread of the
373 // same team) could still access the page in question via their cached
374 // entry. We can obviously lose a modified flag in this case, with the
375 // effect that the page looks unmodified (and might thus be recycled),
376 // but is actually modified.
377 // In most cases this is harmless, but for vm_remove_all_page_mappings()
378 // this is actually a problem.
379 // Interestingly FreeBSD seems to ignore this problem as well
380 // (cf. pmap_remove_all()), unless I've missed something.
381 }
382
383 locker.Detach();
384 // PageUnmapped() will unlock for us
385
386 PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE,
387 (oldEntry & X86_64_PTE_ACCESSED) != 0,
388 (oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue);
389
390 return B_OK;
391 }
392
393
394 void
UnmapPages(VMArea * area,addr_t base,size_t size,bool updatePageQueue)395 X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
396 bool updatePageQueue)
397 {
398 if (size == 0)
399 return;
400
401 addr_t start = base;
402 addr_t end = base + size - 1;
403
404 TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
405 B_PRIxADDR ")\n", area, start, end);
406
407 VMAreaMappings queue;
408
409 RecursiveLocker locker(fLock);
410 ThreadCPUPinner pinner(thread_get_current_thread());
411
412 do {
413 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
414 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
415 NULL, fPageMapper, fMapCount);
416 if (pageTable == NULL) {
417 // Move on to the next page table.
418 start = ROUNDUP(start + 1, k64BitPageTableRange);
419 continue;
420 }
421
422 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
423 index < k64BitTableEntryCount && start < end;
424 index++, start += B_PAGE_SIZE) {
425 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(
426 &pageTable[index]);
427 if ((oldEntry & X86_64_PTE_PRESENT) == 0)
428 continue;
429
430 fMapCount--;
431
432 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
433 // Note, that we only need to invalidate the address, if the
434 // accessed flags was set, since only then the entry could have
435 // been in any TLB.
436 InvalidatePage(start);
437 }
438
439 if (area->cache_type != CACHE_TYPE_DEVICE) {
440 // get the page
441 vm_page* page = vm_lookup_page(
442 (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
443 ASSERT(page != NULL);
444
445 DEBUG_PAGE_ACCESS_START(page);
446
447 // transfer the accessed/dirty flags to the page
448 if ((oldEntry & X86_64_PTE_ACCESSED) != 0)
449 page->accessed = true;
450 if ((oldEntry & X86_64_PTE_DIRTY) != 0)
451 page->modified = true;
452
453 // remove the mapping object/decrement the wired_count of the
454 // page
455 if (area->wiring == B_NO_LOCK) {
456 vm_page_mapping* mapping = NULL;
457 vm_page_mappings::Iterator iterator
458 = page->mappings.GetIterator();
459 while ((mapping = iterator.Next()) != NULL) {
460 if (mapping->area == area)
461 break;
462 }
463
464 ASSERT(mapping != NULL);
465
466 area->mappings.Remove(mapping);
467 page->mappings.Remove(mapping);
468 queue.Add(mapping);
469 } else
470 page->DecrementWiredCount();
471
472 if (!page->IsMapped()) {
473 atomic_add(&gMappedPagesCount, -1);
474
475 if (updatePageQueue) {
476 if (page->Cache()->temporary)
477 vm_page_set_state(page, PAGE_STATE_INACTIVE);
478 else if (page->modified)
479 vm_page_set_state(page, PAGE_STATE_MODIFIED);
480 else
481 vm_page_set_state(page, PAGE_STATE_CACHED);
482 }
483 }
484
485 DEBUG_PAGE_ACCESS_END(page);
486 }
487 }
488
489 Flush();
490 // flush explicitly, since we directly use the lock
491 } while (start != 0 && start < end);
492
493 // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
494 // really critical here, as in all cases this method is used, the unmapped
495 // area range is unmapped for good (resized/cut) and the pages will likely
496 // be freed.
497
498 locker.Unlock();
499
500 // free removed mappings
501 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
502 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
503 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
504 while (vm_page_mapping* mapping = queue.RemoveHead())
505 vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
506 }
507
508
509 void
UnmapArea(VMArea * area,bool deletingAddressSpace,bool ignoreTopCachePageFlags)510 X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
511 bool ignoreTopCachePageFlags)
512 {
513 TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area);
514
515 if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
516 X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(),
517 true);
518 return;
519 }
520
521 bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
522
523 RecursiveLocker locker(fLock);
524 ThreadCPUPinner pinner(thread_get_current_thread());
525
526 VMAreaMappings mappings;
527 mappings.MoveFrom(&area->mappings);
528
529 for (VMAreaMappings::Iterator it = mappings.GetIterator();
530 vm_page_mapping* mapping = it.Next();) {
531 vm_page* page = mapping->page;
532 page->mappings.Remove(mapping);
533
534 VMCache* cache = page->Cache();
535
536 bool pageFullyUnmapped = false;
537 if (!page->IsMapped()) {
538 atomic_add(&gMappedPagesCount, -1);
539 pageFullyUnmapped = true;
540 }
541
542 if (unmapPages || cache != area->cache) {
543 addr_t address = area->Base()
544 + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
545
546 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
547 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
548 false, NULL, fPageMapper, fMapCount);
549 if (entry == NULL) {
550 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
551 "has no page table", page, area, address);
552 continue;
553 }
554
555 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
556
557 if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
558 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
559 "has no page table entry", page, area, address);
560 continue;
561 }
562
563 // transfer the accessed/dirty flags to the page and invalidate
564 // the mapping, if necessary
565 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
566 page->accessed = true;
567
568 if (!deletingAddressSpace)
569 InvalidatePage(address);
570 }
571
572 if ((oldEntry & X86_64_PTE_DIRTY) != 0)
573 page->modified = true;
574
575 if (pageFullyUnmapped) {
576 DEBUG_PAGE_ACCESS_START(page);
577
578 if (cache->temporary)
579 vm_page_set_state(page, PAGE_STATE_INACTIVE);
580 else if (page->modified)
581 vm_page_set_state(page, PAGE_STATE_MODIFIED);
582 else
583 vm_page_set_state(page, PAGE_STATE_CACHED);
584
585 DEBUG_PAGE_ACCESS_END(page);
586 }
587 }
588
589 fMapCount--;
590 }
591
592 Flush();
593 // flush explicitely, since we directly use the lock
594
595 locker.Unlock();
596
597 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
598 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
599 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
600 while (vm_page_mapping* mapping = mappings.RemoveHead())
601 vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
602 }
603
604
605 status_t
Query(addr_t virtualAddress,phys_addr_t * _physicalAddress,uint32 * _flags)606 X86VMTranslationMap64Bit::Query(addr_t virtualAddress,
607 phys_addr_t* _physicalAddress, uint32* _flags)
608 {
609 *_flags = 0;
610 *_physicalAddress = 0;
611
612 ThreadCPUPinner pinner(thread_get_current_thread());
613
614 // This function may be called on the physical map area, so we must handle
615 // large pages here. Look up the page directory entry for the virtual
616 // address.
617 uint64* pde = X86PagingMethod64Bit::PageDirectoryEntryForAddress(
618 fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap,
619 false, NULL, fPageMapper, fMapCount);
620 if (pde == NULL || (*pde & X86_64_PDE_PRESENT) == 0)
621 return B_OK;
622
623 uint64 entry;
624 if ((*pde & X86_64_PDE_LARGE_PAGE) != 0) {
625 entry = *pde;
626 *_physicalAddress = (entry & X86_64_PDE_ADDRESS_MASK)
627 + (virtualAddress % 0x200000);
628 } else {
629 uint64* virtualPageTable = (uint64*)fPageMapper->GetPageTableAt(
630 *pde & X86_64_PDE_ADDRESS_MASK);
631 entry = virtualPageTable[VADDR_TO_PTE(virtualAddress)];
632 *_physicalAddress = entry & X86_64_PTE_ADDRESS_MASK;
633 }
634
635 // Translate the page state flags.
636 if ((entry & X86_64_PTE_USER) != 0) {
637 *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0)
638 | B_READ_AREA
639 | ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_EXECUTE_AREA : 0);
640 }
641
642 *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0)
643 | B_KERNEL_READ_AREA
644 | ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_KERNEL_EXECUTE_AREA : 0)
645 | ((entry & X86_64_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0)
646 | ((entry & X86_64_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0)
647 | ((entry & X86_64_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0);
648
649 TRACE("X86VMTranslationMap64Bit::Query(%#" B_PRIxADDR ") -> %#"
650 B_PRIxPHYSADDR " %#" B_PRIx32 " (entry: %#" B_PRIx64 ")\n",
651 virtualAddress, *_physicalAddress, *_flags, entry);
652
653 return B_OK;
654 }
655
656
657 status_t
QueryInterrupt(addr_t virtualAddress,phys_addr_t * _physicalAddress,uint32 * _flags)658 X86VMTranslationMap64Bit::QueryInterrupt(addr_t virtualAddress,
659 phys_addr_t* _physicalAddress, uint32* _flags)
660 {
661 // With our page mapper, there is no difference in getting a page table
662 // when interrupts are enabled or disabled, so just call Query().
663 return Query(virtualAddress, _physicalAddress, _flags);
664 }
665
666
667 status_t
Protect(addr_t start,addr_t end,uint32 attributes,uint32 memoryType)668 X86VMTranslationMap64Bit::Protect(addr_t start, addr_t end, uint32 attributes,
669 uint32 memoryType)
670 {
671 start = ROUNDDOWN(start, B_PAGE_SIZE);
672 if (start >= end)
673 return B_OK;
674
675 TRACE("X86VMTranslationMap64Bit::Protect(%#" B_PRIxADDR ", %#" B_PRIxADDR
676 ", %#" B_PRIx32 ")\n", start, end, attributes);
677
678 // compute protection flags
679 uint64 newProtectionFlags = 0;
680 if ((attributes & B_USER_PROTECTION) != 0) {
681 newProtectionFlags = X86_64_PTE_USER;
682 if ((attributes & B_WRITE_AREA) != 0)
683 newProtectionFlags |= X86_64_PTE_WRITABLE;
684 if ((attributes & B_EXECUTE_AREA) == 0
685 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
686 newProtectionFlags |= X86_64_PTE_NOT_EXECUTABLE;
687 }
688 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
689 newProtectionFlags = X86_64_PTE_WRITABLE;
690
691 ThreadCPUPinner pinner(thread_get_current_thread());
692
693 do {
694 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
695 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
696 NULL, fPageMapper, fMapCount);
697 if (pageTable == NULL) {
698 // Move on to the next page table.
699 start = ROUNDUP(start + 1, k64BitPageTableRange);
700 continue;
701 }
702
703 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
704 index < k64BitTableEntryCount && start < end;
705 index++, start += B_PAGE_SIZE) {
706 uint64 entry = pageTable[index];
707 if ((entry & X86_64_PTE_PRESENT) == 0)
708 continue;
709
710 TRACE("X86VMTranslationMap64Bit::Protect(): protect page %#"
711 B_PRIxADDR "\n", start);
712
713 // set the new protection flags -- we want to do that atomically,
714 // without changing the accessed or dirty flag
715 uint64 oldEntry;
716 while (true) {
717 oldEntry = X86PagingMethod64Bit::TestAndSetTableEntry(
718 &pageTable[index],
719 (entry & ~(X86_64_PTE_PROTECTION_MASK
720 | X86_64_PTE_MEMORY_TYPE_MASK))
721 | newProtectionFlags
722 | X86PagingMethod64Bit::MemoryTypeToPageTableEntryFlags(
723 memoryType),
724 entry);
725 if (oldEntry == entry)
726 break;
727 entry = oldEntry;
728 }
729
730 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
731 // Note, that we only need to invalidate the address, if the
732 // accessed flag was set, since only then the entry could have
733 // been in any TLB.
734 InvalidatePage(start);
735 }
736 }
737 } while (start != 0 && start < end);
738
739 return B_OK;
740 }
741
742
743 status_t
ClearFlags(addr_t address,uint32 flags)744 X86VMTranslationMap64Bit::ClearFlags(addr_t address, uint32 flags)
745 {
746 TRACE("X86VMTranslationMap64Bit::ClearFlags(%#" B_PRIxADDR ", %#" B_PRIx32
747 ")\n", address, flags);
748
749 ThreadCPUPinner pinner(thread_get_current_thread());
750
751 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
752 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
753 false, NULL, fPageMapper, fMapCount);
754 if (entry == NULL)
755 return B_OK;
756
757 uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_64_PTE_DIRTY : 0)
758 | ((flags & PAGE_ACCESSED) ? X86_64_PTE_ACCESSED : 0);
759
760 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry,
761 flagsToClear);
762
763 if ((oldEntry & flagsToClear) != 0)
764 InvalidatePage(address);
765
766 return B_OK;
767 }
768
769
770 bool
ClearAccessedAndModified(VMArea * area,addr_t address,bool unmapIfUnaccessed,bool & _modified)771 X86VMTranslationMap64Bit::ClearAccessedAndModified(VMArea* area, addr_t address,
772 bool unmapIfUnaccessed, bool& _modified)
773 {
774 ASSERT(address % B_PAGE_SIZE == 0);
775
776 TRACE("X86VMTranslationMap64Bit::ClearAccessedAndModified(%#" B_PRIxADDR
777 ")\n", address);
778
779 RecursiveLocker locker(fLock);
780 ThreadCPUPinner pinner(thread_get_current_thread());
781
782 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
783 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
784 false, NULL, fPageMapper, fMapCount);
785 if (entry == NULL)
786 return false;
787
788 uint64 oldEntry;
789
790 if (unmapIfUnaccessed) {
791 while (true) {
792 oldEntry = *entry;
793 if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
794 // page mapping not valid
795 return false;
796 }
797
798 if (oldEntry & X86_64_PTE_ACCESSED) {
799 // page was accessed -- just clear the flags
800 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry,
801 X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY);
802 break;
803 }
804
805 // page hasn't been accessed -- unmap it
806 if (X86PagingMethod64Bit::TestAndSetTableEntry(entry, 0, oldEntry)
807 == oldEntry) {
808 break;
809 }
810
811 // something changed -- check again
812 }
813 } else {
814 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry,
815 X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY);
816 }
817
818 pinner.Unlock();
819
820 _modified = (oldEntry & X86_64_PTE_DIRTY) != 0;
821
822 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
823 // Note, that we only need to invalidate the address, if the
824 // accessed flags was set, since only then the entry could have been
825 // in any TLB.
826 InvalidatePage(address);
827 Flush();
828
829 return true;
830 }
831
832 if (!unmapIfUnaccessed)
833 return false;
834
835 // We have unmapped the address. Do the "high level" stuff.
836
837 fMapCount--;
838
839 locker.Detach();
840 // UnaccessedPageUnmapped() will unlock for us
841
842 UnaccessedPageUnmapped(area,
843 (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
844
845 return false;
846 }
847
848
849 bool
DebugGetReverseMappingInfo(phys_addr_t physicalAddress,ReverseMappingInfoCallback & callback)850 X86VMTranslationMap64Bit::DebugGetReverseMappingInfo(phys_addr_t physicalAddress,
851 ReverseMappingInfoCallback& callback)
852 {
853 if (fLA57) {
854 kprintf("X86VMTranslationMap64Bit::DebugGetReverseMappingInfo not implemented for LA57\n");
855 return false;
856 }
857
858 const uint64* virtualPML4 = fPagingStructures->VirtualPMLTop();
859 for (uint32 i = 0; i < (fIsKernelMap ? 512 : 256); i++) {
860 if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0)
861 continue;
862 const uint64 addressMask = (i >= 256) ? 0xffffff0000000000LL : 0;
863
864 const uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt(
865 virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK);
866 for (uint32 j = 0; j < 512; j++) {
867 if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0)
868 continue;
869
870 const uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt(
871 virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK);
872 for (uint32 k = 0; k < 512; k++) {
873 if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0)
874 continue;
875
876 if ((virtualPageDir[k] & X86_64_PDE_LARGE_PAGE) != 0) {
877 phys_addr_t largeAddress = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK;
878 if (physicalAddress >= largeAddress
879 && physicalAddress < (largeAddress + k64BitPageTableRange)) {
880 off_t offset = physicalAddress - largeAddress;
881 addr_t virtualAddress = i * k64BitPDPTRange
882 + j * k64BitPageDirectoryRange
883 + k * k64BitPageTableRange
884 + offset;
885 virtualAddress |= addressMask;
886 if (callback.HandleVirtualAddress(virtualAddress))
887 return true;
888 }
889 continue;
890 }
891
892 const uint64* virtualPageTable = (uint64*)fPageMapper->GetPageTableAt(
893 virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK);
894 for (uint32 l = 0; l < 512; l++) {
895 if ((virtualPageTable[l] & X86_64_PTE_PRESENT) == 0)
896 continue;
897
898 if ((virtualPageTable[l] & X86_64_PTE_ADDRESS_MASK) == physicalAddress) {
899 addr_t virtualAddress = i * k64BitPDPTRange
900 + j * k64BitPageDirectoryRange
901 + k * k64BitPageTableRange
902 + l * B_PAGE_SIZE;
903 virtualAddress |= addressMask;
904 if (callback.HandleVirtualAddress(virtualAddress))
905 return true;
906 }
907 }
908 }
909 }
910 }
911
912 return false;
913 }
914
915
916 X86PagingStructures*
PagingStructures() const917 X86VMTranslationMap64Bit::PagingStructures() const
918 {
919 return fPagingStructures;
920 }
921