1 /* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk 3 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. 4 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12 #include "paging/64bit/X86VMTranslationMap64Bit.h" 13 14 #include <int.h> 15 #include <slab/Slab.h> 16 #include <thread.h> 17 #include <util/AutoLock.h> 18 #include <vm/vm_page.h> 19 #include <vm/VMAddressSpace.h> 20 #include <vm/VMCache.h> 21 22 #include "paging/64bit/X86PagingMethod64Bit.h" 23 #include "paging/64bit/X86PagingStructures64Bit.h" 24 #include "paging/x86_physical_page_mapper.h" 25 26 27 //#define TRACE_X86_VM_TRANSLATION_MAP_64BIT 28 #ifdef TRACE_X86_VM_TRANSLATION_MAP_64BIT 29 # define TRACE(x...) dprintf(x) 30 #else 31 # define TRACE(x...) ; 32 #endif 33 34 35 // #pragma mark - X86VMTranslationMap64Bit 36 37 38 X86VMTranslationMap64Bit::X86VMTranslationMap64Bit() 39 : 40 fPagingStructures(NULL) 41 { 42 } 43 44 45 X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit() 46 { 47 TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n"); 48 49 if (fPagingStructures == NULL) 50 return; 51 52 if (fPageMapper != NULL) { 53 phys_addr_t address; 54 vm_page* page; 55 56 // Free all structures in the bottom half of the PML4 (user memory). 57 uint64* virtualPML4 = fPagingStructures->VirtualPML4(); 58 for (uint32 i = 0; i < 256; i++) { 59 if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0) 60 continue; 61 62 uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt( 63 virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK); 64 for (uint32 j = 0; j < 512; j++) { 65 if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0) 66 continue; 67 68 uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt( 69 virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK); 70 for (uint32 k = 0; k < 512; k++) { 71 if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0) 72 continue; 73 74 address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK; 75 page = vm_lookup_page(address / B_PAGE_SIZE); 76 if (page == NULL) { 77 panic("page table %u %u %u on invalid page %#" 78 B_PRIxPHYSADDR "\n", i, j, k, address); 79 } 80 81 DEBUG_PAGE_ACCESS_START(page); 82 vm_page_set_state(page, PAGE_STATE_FREE); 83 } 84 85 address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK; 86 page = vm_lookup_page(address / B_PAGE_SIZE); 87 if (page == NULL) { 88 panic("page directory %u %u on invalid page %#" 89 B_PRIxPHYSADDR "\n", i, j, address); 90 } 91 92 DEBUG_PAGE_ACCESS_START(page); 93 vm_page_set_state(page, PAGE_STATE_FREE); 94 } 95 96 address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK; 97 page = vm_lookup_page(address / B_PAGE_SIZE); 98 if (page == NULL) { 99 panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i, 100 address); 101 } 102 103 DEBUG_PAGE_ACCESS_START(page); 104 vm_page_set_state(page, PAGE_STATE_FREE); 105 } 106 107 fPageMapper->Delete(); 108 } 109 110 fPagingStructures->RemoveReference(); 111 } 112 113 114 status_t 115 X86VMTranslationMap64Bit::Init(bool kernel) 116 { 117 TRACE("X86VMTranslationMap64Bit::Init()\n"); 118 119 X86VMTranslationMap::Init(kernel); 120 121 fPagingStructures = new(std::nothrow) X86PagingStructures64Bit; 122 if (fPagingStructures == NULL) 123 return B_NO_MEMORY; 124 125 X86PagingMethod64Bit* method = X86PagingMethod64Bit::Method(); 126 127 if (kernel) { 128 // Get the page mapper. 129 fPageMapper = method->KernelPhysicalPageMapper(); 130 131 // Kernel PML4 is already mapped. 132 fPagingStructures->Init(method->KernelVirtualPML4(), 133 method->KernelPhysicalPML4()); 134 } else { 135 // Allocate a physical page mapper. 136 status_t error = method->PhysicalPageMapper() 137 ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); 138 if (error != B_OK) 139 return error; 140 141 // Assuming that only the top 2 PML4 entries are occupied for the 142 // kernel. 143 STATIC_ASSERT(KERNEL_PMAP_BASE == 0xffffff0000000000); 144 STATIC_ASSERT(KERNEL_BASE == 0xffffff0000000000); 145 146 // Allocate and clear the PML4. 147 uint64* virtualPML4 = (uint64*)memalign(B_PAGE_SIZE, B_PAGE_SIZE); 148 if (virtualPML4 == NULL) 149 return B_NO_MEMORY; 150 memset(virtualPML4, 0, B_PAGE_SIZE); 151 152 // Copy the top 2 PML4 entries. 153 virtualPML4[510] = method->KernelVirtualPML4()[510]; 154 virtualPML4[511] = method->KernelVirtualPML4()[511]; 155 156 // Look up the PML4 physical address. 157 phys_addr_t physicalPML4; 158 vm_get_page_mapping(VMAddressSpace::KernelID(), (addr_t)virtualPML4, 159 &physicalPML4); 160 161 // Initialize the paging structures. 162 fPagingStructures->Init(virtualPML4, physicalPML4); 163 } 164 165 return B_OK; 166 } 167 168 169 size_t 170 X86VMTranslationMap64Bit::MaxPagesNeededToMap(addr_t start, addr_t end) const 171 { 172 // If start == 0, the actual base address is not yet known to the caller and 173 // we shall assume the worst case, which is where the start address is the 174 // last page covered by a PDPT. 175 if (start == 0) { 176 start = k64BitPDPTRange - B_PAGE_SIZE; 177 end += start; 178 } 179 180 size_t requiredPDPTs = end / k64BitPDPTRange + 1 181 - start / k64BitPDPTRange; 182 size_t requiredPageDirs = end / k64BitPageDirectoryRange + 1 183 - start / k64BitPageDirectoryRange; 184 size_t requiredPageTables = end / k64BitPageTableRange + 1 185 - start / k64BitPageTableRange; 186 187 return requiredPDPTs + requiredPageDirs + requiredPageTables; 188 } 189 190 191 status_t 192 X86VMTranslationMap64Bit::Map(addr_t virtualAddress, phys_addr_t physicalAddress, 193 uint32 attributes, uint32 memoryType, vm_page_reservation* reservation) 194 { 195 TRACE("X86VMTranslationMap64Bit::Map(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR 196 ")\n", virtualAddress, physicalAddress); 197 198 ThreadCPUPinner pinner(thread_get_current_thread()); 199 200 // Look up the page table for the virtual address, allocating new tables 201 // if required. Shouldn't fail. 202 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 203 fPagingStructures->VirtualPML4(), virtualAddress, fIsKernelMap, 204 true, reservation, fPageMapper, fMapCount); 205 ASSERT(entry != NULL); 206 207 // The entry should not already exist. 208 ASSERT_PRINT((*entry & X86_64_PTE_PRESENT) == 0, 209 "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64, 210 virtualAddress, *entry); 211 212 // Fill in the table entry. 213 X86PagingMethod64Bit::PutPageTableEntryInTable(entry, physicalAddress, 214 attributes, memoryType, fIsKernelMap); 215 216 // Note: We don't need to invalidate the TLB for this address, as previously 217 // the entry was not present and the TLB doesn't cache those entries. 218 219 fMapCount++; 220 221 return 0; 222 } 223 224 225 status_t 226 X86VMTranslationMap64Bit::Unmap(addr_t start, addr_t end) 227 { 228 start = ROUNDDOWN(start, B_PAGE_SIZE); 229 if (start >= end) 230 return B_OK; 231 232 TRACE("X86VMTranslationMap64Bit::Unmap(%#" B_PRIxADDR ", %#" B_PRIxADDR 233 ")\n", start, end); 234 235 ThreadCPUPinner pinner(thread_get_current_thread()); 236 237 do { 238 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 239 fPagingStructures->VirtualPML4(), start, fIsKernelMap, false, 240 NULL, fPageMapper, fMapCount); 241 if (pageTable == NULL) { 242 // Move on to the next page table. 243 start = ROUNDUP(start + 1, k64BitPageTableRange); 244 continue; 245 } 246 247 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 248 index < k64BitTableEntryCount && start < end; 249 index++, start += B_PAGE_SIZE) { 250 if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) 251 continue; 252 253 TRACE("X86VMTranslationMap64Bit::Unmap(): removing page %#" 254 B_PRIxADDR " (%#" B_PRIxPHYSADDR ")\n", start, 255 pageTable[index] & X86_64_PTE_ADDRESS_MASK); 256 257 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags( 258 &pageTable[index], X86_64_PTE_PRESENT); 259 fMapCount--; 260 261 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 262 // Note, that we only need to invalidate the address, if the 263 // accessed flags was set, since only then the entry could have 264 // been in any TLB. 265 InvalidatePage(start); 266 } 267 } 268 } while (start != 0 && start < end); 269 270 return B_OK; 271 } 272 273 274 status_t 275 X86VMTranslationMap64Bit::DebugMarkRangePresent(addr_t start, addr_t end, 276 bool markPresent) 277 { 278 start = ROUNDDOWN(start, B_PAGE_SIZE); 279 if (start >= end) 280 return B_OK; 281 282 TRACE("X86VMTranslationMap64Bit::DebugMarkRangePresent(%#" B_PRIxADDR 283 ", %#" B_PRIxADDR ")\n", start, end); 284 285 ThreadCPUPinner pinner(thread_get_current_thread()); 286 287 do { 288 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 289 fPagingStructures->VirtualPML4(), start, fIsKernelMap, false, 290 NULL, fPageMapper, fMapCount); 291 if (pageTable == NULL) { 292 // Move on to the next page table. 293 start = ROUNDUP(start + 1, k64BitPageTableRange); 294 continue; 295 } 296 297 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 298 index < k64BitTableEntryCount && start < end; 299 index++, start += B_PAGE_SIZE) { 300 if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) { 301 if (!markPresent) 302 continue; 303 304 X86PagingMethod64Bit::SetTableEntryFlags(&pageTable[index], 305 X86_64_PTE_PRESENT); 306 } else { 307 if (markPresent) 308 continue; 309 310 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags( 311 &pageTable[index], X86_64_PTE_PRESENT); 312 313 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 314 // Note, that we only need to invalidate the address, if the 315 // accessed flags was set, since only then the entry could 316 // have been in any TLB. 317 InvalidatePage(start); 318 } 319 } 320 } 321 } while (start != 0 && start < end); 322 323 return B_OK; 324 } 325 326 327 status_t 328 X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address, 329 bool updatePageQueue) 330 { 331 ASSERT(address % B_PAGE_SIZE == 0); 332 333 TRACE("X86VMTranslationMap64Bit::UnmapPage(%#" B_PRIxADDR ")\n", address); 334 335 ThreadCPUPinner pinner(thread_get_current_thread()); 336 337 // Look up the page table for the virtual address. 338 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 339 fPagingStructures->VirtualPML4(), address, fIsKernelMap, 340 false, NULL, fPageMapper, fMapCount); 341 if (entry == NULL) 342 return B_ENTRY_NOT_FOUND; 343 344 RecursiveLocker locker(fLock); 345 346 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry); 347 348 pinner.Unlock(); 349 350 if ((oldEntry & X86_64_PTE_PRESENT) == 0) 351 return B_ENTRY_NOT_FOUND; 352 353 fMapCount--; 354 355 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 356 // Note, that we only need to invalidate the address, if the 357 // accessed flags was set, since only then the entry could have been 358 // in any TLB. 359 InvalidatePage(address); 360 361 Flush(); 362 363 // NOTE: Between clearing the page table entry and Flush() other 364 // processors (actually even this processor with another thread of the 365 // same team) could still access the page in question via their cached 366 // entry. We can obviously lose a modified flag in this case, with the 367 // effect that the page looks unmodified (and might thus be recycled), 368 // but is actually modified. 369 // In most cases this is harmless, but for vm_remove_all_page_mappings() 370 // this is actually a problem. 371 // Interestingly FreeBSD seems to ignore this problem as well 372 // (cf. pmap_remove_all()), unless I've missed something. 373 } 374 375 locker.Detach(); 376 // PageUnmapped() will unlock for us 377 378 PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE, 379 (oldEntry & X86_64_PTE_ACCESSED) != 0, 380 (oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue); 381 382 return B_OK; 383 } 384 385 386 void 387 X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, 388 bool updatePageQueue) 389 { 390 if (size == 0) 391 return; 392 393 addr_t start = base; 394 addr_t end = base + size - 1; 395 396 TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#" 397 B_PRIxADDR ")\n", area, start, end); 398 399 VMAreaMappings queue; 400 401 RecursiveLocker locker(fLock); 402 ThreadCPUPinner pinner(thread_get_current_thread()); 403 404 do { 405 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 406 fPagingStructures->VirtualPML4(), start, fIsKernelMap, false, 407 NULL, fPageMapper, fMapCount); 408 if (pageTable == NULL) { 409 // Move on to the next page table. 410 start = ROUNDUP(start + 1, k64BitPageTableRange); 411 continue; 412 } 413 414 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 415 index < k64BitTableEntryCount && start < end; 416 index++, start += B_PAGE_SIZE) { 417 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry( 418 &pageTable[index]); 419 if ((oldEntry & X86_64_PTE_PRESENT) == 0) 420 continue; 421 422 fMapCount--; 423 424 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 425 // Note, that we only need to invalidate the address, if the 426 // accessed flags was set, since only then the entry could have 427 // been in any TLB. 428 InvalidatePage(start); 429 } 430 431 if (area->cache_type != CACHE_TYPE_DEVICE) { 432 // get the page 433 vm_page* page = vm_lookup_page( 434 (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE); 435 ASSERT(page != NULL); 436 437 DEBUG_PAGE_ACCESS_START(page); 438 439 // transfer the accessed/dirty flags to the page 440 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) 441 page->accessed = true; 442 if ((oldEntry & X86_64_PTE_DIRTY) != 0) 443 page->modified = true; 444 445 // remove the mapping object/decrement the wired_count of the 446 // page 447 if (area->wiring == B_NO_LOCK) { 448 vm_page_mapping* mapping = NULL; 449 vm_page_mappings::Iterator iterator 450 = page->mappings.GetIterator(); 451 while ((mapping = iterator.Next()) != NULL) { 452 if (mapping->area == area) 453 break; 454 } 455 456 ASSERT(mapping != NULL); 457 458 area->mappings.Remove(mapping); 459 page->mappings.Remove(mapping); 460 queue.Add(mapping); 461 } else 462 page->DecrementWiredCount(); 463 464 if (!page->IsMapped()) { 465 atomic_add(&gMappedPagesCount, -1); 466 467 if (updatePageQueue) { 468 if (page->Cache()->temporary) 469 vm_page_set_state(page, PAGE_STATE_INACTIVE); 470 else if (page->modified) 471 vm_page_set_state(page, PAGE_STATE_MODIFIED); 472 else 473 vm_page_set_state(page, PAGE_STATE_CACHED); 474 } 475 } 476 477 DEBUG_PAGE_ACCESS_END(page); 478 } 479 } 480 481 Flush(); 482 // flush explicitly, since we directly use the lock 483 } while (start != 0 && start < end); 484 485 // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not 486 // really critical here, as in all cases this method is used, the unmapped 487 // area range is unmapped for good (resized/cut) and the pages will likely 488 // be freed. 489 490 locker.Unlock(); 491 492 // free removed mappings 493 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); 494 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY 495 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); 496 while (vm_page_mapping* mapping = queue.RemoveHead()) 497 object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); 498 } 499 500 501 void 502 X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, 503 bool ignoreTopCachePageFlags) 504 { 505 TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area); 506 507 if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { 508 X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(), 509 true); 510 return; 511 } 512 513 bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; 514 515 RecursiveLocker locker(fLock); 516 ThreadCPUPinner pinner(thread_get_current_thread()); 517 518 VMAreaMappings mappings; 519 mappings.MoveFrom(&area->mappings); 520 521 for (VMAreaMappings::Iterator it = mappings.GetIterator(); 522 vm_page_mapping* mapping = it.Next();) { 523 vm_page* page = mapping->page; 524 page->mappings.Remove(mapping); 525 526 VMCache* cache = page->Cache(); 527 528 bool pageFullyUnmapped = false; 529 if (!page->IsMapped()) { 530 atomic_add(&gMappedPagesCount, -1); 531 pageFullyUnmapped = true; 532 } 533 534 if (unmapPages || cache != area->cache) { 535 addr_t address = area->Base() 536 + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); 537 538 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 539 fPagingStructures->VirtualPML4(), address, fIsKernelMap, 540 false, NULL, fPageMapper, fMapCount); 541 if (entry == NULL) { 542 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " 543 "has no page table", page, area, address); 544 continue; 545 } 546 547 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry); 548 549 if ((oldEntry & X86_64_PTE_PRESENT) == 0) { 550 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " 551 "has no page table entry", page, area, address); 552 continue; 553 } 554 555 // transfer the accessed/dirty flags to the page and invalidate 556 // the mapping, if necessary 557 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 558 page->accessed = true; 559 560 if (!deletingAddressSpace) 561 InvalidatePage(address); 562 } 563 564 if ((oldEntry & X86_64_PTE_DIRTY) != 0) 565 page->modified = true; 566 567 if (pageFullyUnmapped) { 568 DEBUG_PAGE_ACCESS_START(page); 569 570 if (cache->temporary) 571 vm_page_set_state(page, PAGE_STATE_INACTIVE); 572 else if (page->modified) 573 vm_page_set_state(page, PAGE_STATE_MODIFIED); 574 else 575 vm_page_set_state(page, PAGE_STATE_CACHED); 576 577 DEBUG_PAGE_ACCESS_END(page); 578 } 579 } 580 581 fMapCount--; 582 } 583 584 Flush(); 585 // flush explicitely, since we directly use the lock 586 587 locker.Unlock(); 588 589 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); 590 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY 591 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); 592 while (vm_page_mapping* mapping = mappings.RemoveHead()) 593 object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); 594 } 595 596 597 status_t 598 X86VMTranslationMap64Bit::Query(addr_t virtualAddress, 599 phys_addr_t* _physicalAddress, uint32* _flags) 600 { 601 *_flags = 0; 602 *_physicalAddress = 0; 603 604 ThreadCPUPinner pinner(thread_get_current_thread()); 605 606 // This function may be called on the physical map area, so we must handle 607 // large pages here. Look up the page directory entry for the virtual 608 // address. 609 uint64* pde = X86PagingMethod64Bit::PageDirectoryEntryForAddress( 610 fPagingStructures->VirtualPML4(), virtualAddress, fIsKernelMap, 611 false, NULL, fPageMapper, fMapCount); 612 if (pde == NULL || (*pde & X86_64_PDE_PRESENT) == 0) 613 return B_OK; 614 615 uint64 entry; 616 if ((*pde & X86_64_PDE_LARGE_PAGE) != 0) { 617 entry = *pde; 618 *_physicalAddress = (entry & X86_64_PDE_ADDRESS_MASK) 619 + (virtualAddress % 0x200000); 620 } else { 621 uint64* virtualPageTable = (uint64*)fPageMapper->GetPageTableAt( 622 *pde & X86_64_PDE_ADDRESS_MASK); 623 entry = virtualPageTable[VADDR_TO_PTE(virtualAddress)]; 624 *_physicalAddress = entry & X86_64_PTE_ADDRESS_MASK; 625 } 626 627 // Translate the page state flags. 628 if ((entry & X86_64_PTE_USER) != 0) { 629 *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) 630 | B_READ_AREA 631 | ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_EXECUTE_AREA : 0); 632 } 633 634 *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) 635 | B_KERNEL_READ_AREA 636 | ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_KERNEL_EXECUTE_AREA : 0) 637 | ((entry & X86_64_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) 638 | ((entry & X86_64_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) 639 | ((entry & X86_64_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); 640 641 TRACE("X86VMTranslationMap64Bit::Query(%#" B_PRIxADDR ") -> %#" 642 B_PRIxPHYSADDR " %#" B_PRIx32 " (entry: %#" B_PRIx64 ")\n", 643 virtualAddress, *_physicalAddress, *_flags, entry); 644 645 return B_OK; 646 } 647 648 649 status_t 650 X86VMTranslationMap64Bit::QueryInterrupt(addr_t virtualAddress, 651 phys_addr_t* _physicalAddress, uint32* _flags) 652 { 653 // With our page mapper, there is no difference in getting a page table 654 // when interrupts are enabled or disabled, so just call Query(). 655 return Query(virtualAddress, _physicalAddress, _flags); 656 } 657 658 659 status_t 660 X86VMTranslationMap64Bit::Protect(addr_t start, addr_t end, uint32 attributes, 661 uint32 memoryType) 662 { 663 start = ROUNDDOWN(start, B_PAGE_SIZE); 664 if (start >= end) 665 return B_OK; 666 667 TRACE("X86VMTranslationMap64Bit::Protect(%#" B_PRIxADDR ", %#" B_PRIxADDR 668 ", %#" B_PRIx32 ")\n", start, end, attributes); 669 670 // compute protection flags 671 uint64 newProtectionFlags = 0; 672 if ((attributes & B_USER_PROTECTION) != 0) { 673 newProtectionFlags = X86_64_PTE_USER; 674 if ((attributes & B_WRITE_AREA) != 0) 675 newProtectionFlags |= X86_64_PTE_WRITABLE; 676 if ((attributes & B_EXECUTE_AREA) == 0 677 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 678 newProtectionFlags |= X86_64_PTE_NOT_EXECUTABLE; 679 } 680 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 681 newProtectionFlags = X86_64_PTE_WRITABLE; 682 683 ThreadCPUPinner pinner(thread_get_current_thread()); 684 685 do { 686 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 687 fPagingStructures->VirtualPML4(), start, fIsKernelMap, false, 688 NULL, fPageMapper, fMapCount); 689 if (pageTable == NULL) { 690 // Move on to the next page table. 691 start = ROUNDUP(start + 1, k64BitPageTableRange); 692 continue; 693 } 694 695 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 696 index < k64BitTableEntryCount && start < end; 697 index++, start += B_PAGE_SIZE) { 698 uint64 entry = pageTable[index]; 699 if ((entry & X86_64_PTE_PRESENT) == 0) 700 continue; 701 702 TRACE("X86VMTranslationMap64Bit::Protect(): protect page %#" 703 B_PRIxADDR "\n", start); 704 705 // set the new protection flags -- we want to do that atomically, 706 // without changing the accessed or dirty flag 707 uint64 oldEntry; 708 while (true) { 709 oldEntry = X86PagingMethod64Bit::TestAndSetTableEntry( 710 &pageTable[index], 711 (entry & ~(X86_64_PTE_PROTECTION_MASK 712 | X86_64_PTE_MEMORY_TYPE_MASK)) 713 | newProtectionFlags 714 | X86PagingMethod64Bit::MemoryTypeToPageTableEntryFlags( 715 memoryType), 716 entry); 717 if (oldEntry == entry) 718 break; 719 entry = oldEntry; 720 } 721 722 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 723 // Note, that we only need to invalidate the address, if the 724 // accessed flag was set, since only then the entry could have 725 // been in any TLB. 726 InvalidatePage(start); 727 } 728 } 729 } while (start != 0 && start < end); 730 731 return B_OK; 732 } 733 734 735 status_t 736 X86VMTranslationMap64Bit::ClearFlags(addr_t address, uint32 flags) 737 { 738 TRACE("X86VMTranslationMap64Bit::ClearFlags(%#" B_PRIxADDR ", %#" B_PRIx32 739 ")\n", address, flags); 740 741 ThreadCPUPinner pinner(thread_get_current_thread()); 742 743 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 744 fPagingStructures->VirtualPML4(), address, fIsKernelMap, 745 false, NULL, fPageMapper, fMapCount); 746 if (entry == NULL) 747 return B_OK; 748 749 uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_64_PTE_DIRTY : 0) 750 | ((flags & PAGE_ACCESSED) ? X86_64_PTE_ACCESSED : 0); 751 752 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry, 753 flagsToClear); 754 755 if ((oldEntry & flagsToClear) != 0) 756 InvalidatePage(address); 757 758 return B_OK; 759 } 760 761 762 bool 763 X86VMTranslationMap64Bit::ClearAccessedAndModified(VMArea* area, addr_t address, 764 bool unmapIfUnaccessed, bool& _modified) 765 { 766 ASSERT(address % B_PAGE_SIZE == 0); 767 768 TRACE("X86VMTranslationMap64Bit::ClearAccessedAndModified(%#" B_PRIxADDR 769 ")\n", address); 770 771 RecursiveLocker locker(fLock); 772 ThreadCPUPinner pinner(thread_get_current_thread()); 773 774 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 775 fPagingStructures->VirtualPML4(), address, fIsKernelMap, 776 false, NULL, fPageMapper, fMapCount); 777 if (entry == NULL) 778 return false; 779 780 uint64 oldEntry; 781 782 if (unmapIfUnaccessed) { 783 while (true) { 784 oldEntry = *entry; 785 if ((oldEntry & X86_64_PTE_PRESENT) == 0) { 786 // page mapping not valid 787 return false; 788 } 789 790 if (oldEntry & X86_64_PTE_ACCESSED) { 791 // page was accessed -- just clear the flags 792 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry, 793 X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY); 794 break; 795 } 796 797 // page hasn't been accessed -- unmap it 798 if (X86PagingMethod64Bit::TestAndSetTableEntry(entry, 0, oldEntry) 799 == oldEntry) { 800 break; 801 } 802 803 // something changed -- check again 804 } 805 } else { 806 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry, 807 X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY); 808 } 809 810 pinner.Unlock(); 811 812 _modified = (oldEntry & X86_64_PTE_DIRTY) != 0; 813 814 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 815 // Note, that we only need to invalidate the address, if the 816 // accessed flags was set, since only then the entry could have been 817 // in any TLB. 818 InvalidatePage(address); 819 Flush(); 820 821 return true; 822 } 823 824 if (!unmapIfUnaccessed) 825 return false; 826 827 // We have unmapped the address. Do the "high level" stuff. 828 829 fMapCount--; 830 831 locker.Detach(); 832 // UnaccessedPageUnmapped() will unlock for us 833 834 UnaccessedPageUnmapped(area, 835 (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE); 836 837 return false; 838 } 839 840 841 X86PagingStructures* 842 X86VMTranslationMap64Bit::PagingStructures() const 843 { 844 return fPagingStructures; 845 } 846