1 /* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk 3 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. 4 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12 #include "paging/64bit/X86VMTranslationMap64Bit.h" 13 14 #include <int.h> 15 #include <slab/Slab.h> 16 #include <thread.h> 17 #include <util/AutoLock.h> 18 #include <util/ThreadAutoLock.h> 19 #include <vm/vm_page.h> 20 #include <vm/VMAddressSpace.h> 21 #include <vm/VMCache.h> 22 23 #include "paging/64bit/X86PagingMethod64Bit.h" 24 #include "paging/64bit/X86PagingStructures64Bit.h" 25 #include "paging/x86_physical_page_mapper.h" 26 27 28 //#define TRACE_X86_VM_TRANSLATION_MAP_64BIT 29 #ifdef TRACE_X86_VM_TRANSLATION_MAP_64BIT 30 # define TRACE(x...) dprintf(x) 31 #else 32 # define TRACE(x...) ; 33 #endif 34 35 36 // #pragma mark - X86VMTranslationMap64Bit 37 38 39 X86VMTranslationMap64Bit::X86VMTranslationMap64Bit(bool la57) 40 : 41 fPagingStructures(NULL), 42 fLA57(la57) 43 { 44 } 45 46 47 X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit() 48 { 49 TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n"); 50 51 if (fPagingStructures == NULL) 52 return; 53 54 if (fPageMapper != NULL) { 55 phys_addr_t address; 56 vm_page* page; 57 58 // Free all structures in the bottom half of the PMLTop (user memory). 59 uint64* virtualPML4 = fPagingStructures->VirtualPMLTop(); 60 for (uint32 i = 0; i < 256; i++) { 61 if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0) 62 continue; 63 64 uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt( 65 virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK); 66 for (uint32 j = 0; j < 512; j++) { 67 if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0) 68 continue; 69 70 uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt( 71 virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK); 72 for (uint32 k = 0; k < 512; k++) { 73 if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0) 74 continue; 75 76 address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK; 77 page = vm_lookup_page(address / B_PAGE_SIZE); 78 if (page == NULL) { 79 panic("page table %u %u %u on invalid page %#" 80 B_PRIxPHYSADDR "\n", i, j, k, address); 81 } 82 83 DEBUG_PAGE_ACCESS_START(page); 84 vm_page_set_state(page, PAGE_STATE_FREE); 85 } 86 87 address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK; 88 page = vm_lookup_page(address / B_PAGE_SIZE); 89 if (page == NULL) { 90 panic("page directory %u %u on invalid page %#" 91 B_PRIxPHYSADDR "\n", i, j, address); 92 } 93 94 DEBUG_PAGE_ACCESS_START(page); 95 vm_page_set_state(page, PAGE_STATE_FREE); 96 } 97 98 address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK; 99 page = vm_lookup_page(address / B_PAGE_SIZE); 100 if (page == NULL) { 101 panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i, 102 address); 103 } 104 105 DEBUG_PAGE_ACCESS_START(page); 106 vm_page_set_state(page, PAGE_STATE_FREE); 107 } 108 109 fPageMapper->Delete(); 110 } 111 112 fPagingStructures->RemoveReference(); 113 } 114 115 116 status_t 117 X86VMTranslationMap64Bit::Init(bool kernel) 118 { 119 TRACE("X86VMTranslationMap64Bit::Init()\n"); 120 121 X86VMTranslationMap::Init(kernel); 122 123 fPagingStructures = new(std::nothrow) X86PagingStructures64Bit; 124 if (fPagingStructures == NULL) 125 return B_NO_MEMORY; 126 127 X86PagingMethod64Bit* method = X86PagingMethod64Bit::Method(); 128 129 if (kernel) { 130 // Get the page mapper. 131 fPageMapper = method->KernelPhysicalPageMapper(); 132 133 // Kernel PMLTop is already mapped. 134 fPagingStructures->Init(method->KernelVirtualPMLTop(), 135 method->KernelPhysicalPMLTop()); 136 } else { 137 // Allocate a physical page mapper. 138 status_t error = method->PhysicalPageMapper() 139 ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); 140 if (error != B_OK) 141 return error; 142 143 // Assuming that only the top 2 PMLTop entries are occupied for the 144 // kernel. 145 STATIC_ASSERT(KERNEL_PMAP_BASE == 0xffffff0000000000); 146 STATIC_ASSERT(KERNEL_BASE == 0xffffff0000000000); 147 148 // Allocate and clear the PMLTop. 149 uint64* virtualPMLTop = (uint64*)memalign(B_PAGE_SIZE, B_PAGE_SIZE); 150 if (virtualPMLTop == NULL) 151 return B_NO_MEMORY; 152 memset(virtualPMLTop, 0, B_PAGE_SIZE); 153 154 // Copy the top 2 PMLTop entries. 155 virtualPMLTop[510] = method->KernelVirtualPMLTop()[510]; 156 virtualPMLTop[511] = method->KernelVirtualPMLTop()[511]; 157 158 // Look up the PMLTop physical address. 159 phys_addr_t physicalPMLTop; 160 vm_get_page_mapping(VMAddressSpace::KernelID(), (addr_t)virtualPMLTop, 161 &physicalPMLTop); 162 163 // Initialize the paging structures. 164 fPagingStructures->Init(virtualPMLTop, physicalPMLTop); 165 } 166 167 return B_OK; 168 } 169 170 171 size_t 172 X86VMTranslationMap64Bit::MaxPagesNeededToMap(addr_t start, addr_t end) const 173 { 174 // If start == 0, the actual base address is not yet known to the caller and 175 // we shall assume the worst case, which is where the start address is the 176 // last page covered by a PDPT or PML4. 177 if (start == 0) { 178 start = (fLA57 ? k64BitPML4TRange : k64BitPDPTRange) - B_PAGE_SIZE; 179 end += start; 180 } 181 182 size_t requiredPML4s = 0; 183 if (fLA57) { 184 requiredPML4s = end / k64BitPML4TRange + 1 185 - start / k64BitPML4TRange; 186 } 187 size_t requiredPDPTs = end / k64BitPDPTRange + 1 188 - start / k64BitPDPTRange; 189 size_t requiredPageDirs = end / k64BitPageDirectoryRange + 1 190 - start / k64BitPageDirectoryRange; 191 size_t requiredPageTables = end / k64BitPageTableRange + 1 192 - start / k64BitPageTableRange; 193 194 return requiredPML4s + requiredPDPTs + requiredPageDirs 195 + requiredPageTables; 196 } 197 198 199 status_t 200 X86VMTranslationMap64Bit::Map(addr_t virtualAddress, phys_addr_t physicalAddress, 201 uint32 attributes, uint32 memoryType, vm_page_reservation* reservation) 202 { 203 TRACE("X86VMTranslationMap64Bit::Map(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR 204 ")\n", virtualAddress, physicalAddress); 205 206 ThreadCPUPinner pinner(thread_get_current_thread()); 207 208 // Look up the page table for the virtual address, allocating new tables 209 // if required. Shouldn't fail. 210 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 211 fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap, 212 true, reservation, fPageMapper, fMapCount); 213 ASSERT(entry != NULL); 214 215 // The entry should not already exist. 216 ASSERT_PRINT((*entry & X86_64_PTE_PRESENT) == 0, 217 "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64, 218 virtualAddress, *entry); 219 220 // Fill in the table entry. 221 X86PagingMethod64Bit::PutPageTableEntryInTable(entry, physicalAddress, 222 attributes, memoryType, fIsKernelMap); 223 224 // Note: We don't need to invalidate the TLB for this address, as previously 225 // the entry was not present and the TLB doesn't cache those entries. 226 227 fMapCount++; 228 229 return 0; 230 } 231 232 233 status_t 234 X86VMTranslationMap64Bit::Unmap(addr_t start, addr_t end) 235 { 236 start = ROUNDDOWN(start, B_PAGE_SIZE); 237 if (start >= end) 238 return B_OK; 239 240 TRACE("X86VMTranslationMap64Bit::Unmap(%#" B_PRIxADDR ", %#" B_PRIxADDR 241 ")\n", start, end); 242 243 ThreadCPUPinner pinner(thread_get_current_thread()); 244 245 do { 246 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 247 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false, 248 NULL, fPageMapper, fMapCount); 249 if (pageTable == NULL) { 250 // Move on to the next page table. 251 start = ROUNDUP(start + 1, k64BitPageTableRange); 252 continue; 253 } 254 255 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 256 index < k64BitTableEntryCount && start < end; 257 index++, start += B_PAGE_SIZE) { 258 if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) 259 continue; 260 261 TRACE("X86VMTranslationMap64Bit::Unmap(): removing page %#" 262 B_PRIxADDR " (%#" B_PRIxPHYSADDR ")\n", start, 263 pageTable[index] & X86_64_PTE_ADDRESS_MASK); 264 265 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags( 266 &pageTable[index], X86_64_PTE_PRESENT); 267 fMapCount--; 268 269 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 270 // Note, that we only need to invalidate the address, if the 271 // accessed flags was set, since only then the entry could have 272 // been in any TLB. 273 InvalidatePage(start); 274 } 275 } 276 } while (start != 0 && start < end); 277 278 return B_OK; 279 } 280 281 282 status_t 283 X86VMTranslationMap64Bit::DebugMarkRangePresent(addr_t start, addr_t end, 284 bool markPresent) 285 { 286 start = ROUNDDOWN(start, B_PAGE_SIZE); 287 if (start >= end) 288 return B_OK; 289 290 TRACE("X86VMTranslationMap64Bit::DebugMarkRangePresent(%#" B_PRIxADDR 291 ", %#" B_PRIxADDR ")\n", start, end); 292 293 ThreadCPUPinner pinner(thread_get_current_thread()); 294 295 do { 296 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 297 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false, 298 NULL, fPageMapper, fMapCount); 299 if (pageTable == NULL) { 300 // Move on to the next page table. 301 start = ROUNDUP(start + 1, k64BitPageTableRange); 302 continue; 303 } 304 305 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 306 index < k64BitTableEntryCount && start < end; 307 index++, start += B_PAGE_SIZE) { 308 if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) { 309 if (!markPresent) 310 continue; 311 312 X86PagingMethod64Bit::SetTableEntryFlags(&pageTable[index], 313 X86_64_PTE_PRESENT); 314 } else { 315 if (markPresent) 316 continue; 317 318 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags( 319 &pageTable[index], X86_64_PTE_PRESENT); 320 321 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 322 // Note, that we only need to invalidate the address, if the 323 // accessed flags was set, since only then the entry could 324 // have been in any TLB. 325 InvalidatePage(start); 326 } 327 } 328 } 329 } while (start != 0 && start < end); 330 331 return B_OK; 332 } 333 334 335 status_t 336 X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address, 337 bool updatePageQueue) 338 { 339 ASSERT(address % B_PAGE_SIZE == 0); 340 341 TRACE("X86VMTranslationMap64Bit::UnmapPage(%#" B_PRIxADDR ")\n", address); 342 343 ThreadCPUPinner pinner(thread_get_current_thread()); 344 345 // Look up the page table for the virtual address. 346 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 347 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap, 348 false, NULL, fPageMapper, fMapCount); 349 if (entry == NULL) 350 return B_ENTRY_NOT_FOUND; 351 352 RecursiveLocker locker(fLock); 353 354 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry); 355 356 pinner.Unlock(); 357 358 if ((oldEntry & X86_64_PTE_PRESENT) == 0) 359 return B_ENTRY_NOT_FOUND; 360 361 fMapCount--; 362 363 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 364 // Note, that we only need to invalidate the address, if the 365 // accessed flags was set, since only then the entry could have been 366 // in any TLB. 367 InvalidatePage(address); 368 369 Flush(); 370 371 // NOTE: Between clearing the page table entry and Flush() other 372 // processors (actually even this processor with another thread of the 373 // same team) could still access the page in question via their cached 374 // entry. We can obviously lose a modified flag in this case, with the 375 // effect that the page looks unmodified (and might thus be recycled), 376 // but is actually modified. 377 // In most cases this is harmless, but for vm_remove_all_page_mappings() 378 // this is actually a problem. 379 // Interestingly FreeBSD seems to ignore this problem as well 380 // (cf. pmap_remove_all()), unless I've missed something. 381 } 382 383 locker.Detach(); 384 // PageUnmapped() will unlock for us 385 386 PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE, 387 (oldEntry & X86_64_PTE_ACCESSED) != 0, 388 (oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue); 389 390 return B_OK; 391 } 392 393 394 void 395 X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, 396 bool updatePageQueue) 397 { 398 if (size == 0) 399 return; 400 401 addr_t start = base; 402 addr_t end = base + size - 1; 403 404 TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#" 405 B_PRIxADDR ")\n", area, start, end); 406 407 VMAreaMappings queue; 408 409 RecursiveLocker locker(fLock); 410 ThreadCPUPinner pinner(thread_get_current_thread()); 411 412 do { 413 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 414 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false, 415 NULL, fPageMapper, fMapCount); 416 if (pageTable == NULL) { 417 // Move on to the next page table. 418 start = ROUNDUP(start + 1, k64BitPageTableRange); 419 continue; 420 } 421 422 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 423 index < k64BitTableEntryCount && start < end; 424 index++, start += B_PAGE_SIZE) { 425 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry( 426 &pageTable[index]); 427 if ((oldEntry & X86_64_PTE_PRESENT) == 0) 428 continue; 429 430 fMapCount--; 431 432 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 433 // Note, that we only need to invalidate the address, if the 434 // accessed flags was set, since only then the entry could have 435 // been in any TLB. 436 InvalidatePage(start); 437 } 438 439 if (area->cache_type != CACHE_TYPE_DEVICE) { 440 // get the page 441 vm_page* page = vm_lookup_page( 442 (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE); 443 ASSERT(page != NULL); 444 445 DEBUG_PAGE_ACCESS_START(page); 446 447 // transfer the accessed/dirty flags to the page 448 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) 449 page->accessed = true; 450 if ((oldEntry & X86_64_PTE_DIRTY) != 0) 451 page->modified = true; 452 453 // remove the mapping object/decrement the wired_count of the 454 // page 455 if (area->wiring == B_NO_LOCK) { 456 vm_page_mapping* mapping = NULL; 457 vm_page_mappings::Iterator iterator 458 = page->mappings.GetIterator(); 459 while ((mapping = iterator.Next()) != NULL) { 460 if (mapping->area == area) 461 break; 462 } 463 464 ASSERT(mapping != NULL); 465 466 area->mappings.Remove(mapping); 467 page->mappings.Remove(mapping); 468 queue.Add(mapping); 469 } else 470 page->DecrementWiredCount(); 471 472 if (!page->IsMapped()) { 473 atomic_add(&gMappedPagesCount, -1); 474 475 if (updatePageQueue) { 476 if (page->Cache()->temporary) 477 vm_page_set_state(page, PAGE_STATE_INACTIVE); 478 else if (page->modified) 479 vm_page_set_state(page, PAGE_STATE_MODIFIED); 480 else 481 vm_page_set_state(page, PAGE_STATE_CACHED); 482 } 483 } 484 485 DEBUG_PAGE_ACCESS_END(page); 486 } 487 } 488 489 Flush(); 490 // flush explicitly, since we directly use the lock 491 } while (start != 0 && start < end); 492 493 // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not 494 // really critical here, as in all cases this method is used, the unmapped 495 // area range is unmapped for good (resized/cut) and the pages will likely 496 // be freed. 497 498 locker.Unlock(); 499 500 // free removed mappings 501 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); 502 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY 503 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); 504 while (vm_page_mapping* mapping = queue.RemoveHead()) 505 vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); 506 } 507 508 509 void 510 X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, 511 bool ignoreTopCachePageFlags) 512 { 513 TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area); 514 515 if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { 516 X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(), 517 true); 518 return; 519 } 520 521 bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; 522 523 RecursiveLocker locker(fLock); 524 ThreadCPUPinner pinner(thread_get_current_thread()); 525 526 VMAreaMappings mappings; 527 mappings.MoveFrom(&area->mappings); 528 529 for (VMAreaMappings::Iterator it = mappings.GetIterator(); 530 vm_page_mapping* mapping = it.Next();) { 531 vm_page* page = mapping->page; 532 page->mappings.Remove(mapping); 533 534 VMCache* cache = page->Cache(); 535 536 bool pageFullyUnmapped = false; 537 if (!page->IsMapped()) { 538 atomic_add(&gMappedPagesCount, -1); 539 pageFullyUnmapped = true; 540 } 541 542 if (unmapPages || cache != area->cache) { 543 addr_t address = area->Base() 544 + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); 545 546 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 547 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap, 548 false, NULL, fPageMapper, fMapCount); 549 if (entry == NULL) { 550 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " 551 "has no page table", page, area, address); 552 continue; 553 } 554 555 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry); 556 557 if ((oldEntry & X86_64_PTE_PRESENT) == 0) { 558 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " 559 "has no page table entry", page, area, address); 560 continue; 561 } 562 563 // transfer the accessed/dirty flags to the page and invalidate 564 // the mapping, if necessary 565 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 566 page->accessed = true; 567 568 if (!deletingAddressSpace) 569 InvalidatePage(address); 570 } 571 572 if ((oldEntry & X86_64_PTE_DIRTY) != 0) 573 page->modified = true; 574 575 if (pageFullyUnmapped) { 576 DEBUG_PAGE_ACCESS_START(page); 577 578 if (cache->temporary) 579 vm_page_set_state(page, PAGE_STATE_INACTIVE); 580 else if (page->modified) 581 vm_page_set_state(page, PAGE_STATE_MODIFIED); 582 else 583 vm_page_set_state(page, PAGE_STATE_CACHED); 584 585 DEBUG_PAGE_ACCESS_END(page); 586 } 587 } 588 589 fMapCount--; 590 } 591 592 Flush(); 593 // flush explicitely, since we directly use the lock 594 595 locker.Unlock(); 596 597 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); 598 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY 599 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); 600 while (vm_page_mapping* mapping = mappings.RemoveHead()) 601 vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); 602 } 603 604 605 status_t 606 X86VMTranslationMap64Bit::Query(addr_t virtualAddress, 607 phys_addr_t* _physicalAddress, uint32* _flags) 608 { 609 *_flags = 0; 610 *_physicalAddress = 0; 611 612 ThreadCPUPinner pinner(thread_get_current_thread()); 613 614 // This function may be called on the physical map area, so we must handle 615 // large pages here. Look up the page directory entry for the virtual 616 // address. 617 uint64* pde = X86PagingMethod64Bit::PageDirectoryEntryForAddress( 618 fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap, 619 false, NULL, fPageMapper, fMapCount); 620 if (pde == NULL || (*pde & X86_64_PDE_PRESENT) == 0) 621 return B_OK; 622 623 uint64 entry; 624 if ((*pde & X86_64_PDE_LARGE_PAGE) != 0) { 625 entry = *pde; 626 *_physicalAddress = (entry & X86_64_PDE_ADDRESS_MASK) 627 + (virtualAddress % 0x200000); 628 } else { 629 uint64* virtualPageTable = (uint64*)fPageMapper->GetPageTableAt( 630 *pde & X86_64_PDE_ADDRESS_MASK); 631 entry = virtualPageTable[VADDR_TO_PTE(virtualAddress)]; 632 *_physicalAddress = entry & X86_64_PTE_ADDRESS_MASK; 633 } 634 635 // Translate the page state flags. 636 if ((entry & X86_64_PTE_USER) != 0) { 637 *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) 638 | B_READ_AREA 639 | ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_EXECUTE_AREA : 0); 640 } 641 642 *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) 643 | B_KERNEL_READ_AREA 644 | ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_KERNEL_EXECUTE_AREA : 0) 645 | ((entry & X86_64_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) 646 | ((entry & X86_64_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) 647 | ((entry & X86_64_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); 648 649 TRACE("X86VMTranslationMap64Bit::Query(%#" B_PRIxADDR ") -> %#" 650 B_PRIxPHYSADDR " %#" B_PRIx32 " (entry: %#" B_PRIx64 ")\n", 651 virtualAddress, *_physicalAddress, *_flags, entry); 652 653 return B_OK; 654 } 655 656 657 status_t 658 X86VMTranslationMap64Bit::QueryInterrupt(addr_t virtualAddress, 659 phys_addr_t* _physicalAddress, uint32* _flags) 660 { 661 // With our page mapper, there is no difference in getting a page table 662 // when interrupts are enabled or disabled, so just call Query(). 663 return Query(virtualAddress, _physicalAddress, _flags); 664 } 665 666 667 status_t 668 X86VMTranslationMap64Bit::Protect(addr_t start, addr_t end, uint32 attributes, 669 uint32 memoryType) 670 { 671 start = ROUNDDOWN(start, B_PAGE_SIZE); 672 if (start >= end) 673 return B_OK; 674 675 TRACE("X86VMTranslationMap64Bit::Protect(%#" B_PRIxADDR ", %#" B_PRIxADDR 676 ", %#" B_PRIx32 ")\n", start, end, attributes); 677 678 // compute protection flags 679 uint64 newProtectionFlags = 0; 680 if ((attributes & B_USER_PROTECTION) != 0) { 681 newProtectionFlags = X86_64_PTE_USER; 682 if ((attributes & B_WRITE_AREA) != 0) 683 newProtectionFlags |= X86_64_PTE_WRITABLE; 684 if ((attributes & B_EXECUTE_AREA) == 0 685 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 686 newProtectionFlags |= X86_64_PTE_NOT_EXECUTABLE; 687 } 688 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 689 newProtectionFlags = X86_64_PTE_WRITABLE; 690 691 ThreadCPUPinner pinner(thread_get_current_thread()); 692 693 do { 694 uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( 695 fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false, 696 NULL, fPageMapper, fMapCount); 697 if (pageTable == NULL) { 698 // Move on to the next page table. 699 start = ROUNDUP(start + 1, k64BitPageTableRange); 700 continue; 701 } 702 703 for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; 704 index < k64BitTableEntryCount && start < end; 705 index++, start += B_PAGE_SIZE) { 706 uint64 entry = pageTable[index]; 707 if ((entry & X86_64_PTE_PRESENT) == 0) 708 continue; 709 710 TRACE("X86VMTranslationMap64Bit::Protect(): protect page %#" 711 B_PRIxADDR "\n", start); 712 713 // set the new protection flags -- we want to do that atomically, 714 // without changing the accessed or dirty flag 715 uint64 oldEntry; 716 while (true) { 717 oldEntry = X86PagingMethod64Bit::TestAndSetTableEntry( 718 &pageTable[index], 719 (entry & ~(X86_64_PTE_PROTECTION_MASK 720 | X86_64_PTE_MEMORY_TYPE_MASK)) 721 | newProtectionFlags 722 | X86PagingMethod64Bit::MemoryTypeToPageTableEntryFlags( 723 memoryType), 724 entry); 725 if (oldEntry == entry) 726 break; 727 entry = oldEntry; 728 } 729 730 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 731 // Note, that we only need to invalidate the address, if the 732 // accessed flag was set, since only then the entry could have 733 // been in any TLB. 734 InvalidatePage(start); 735 } 736 } 737 } while (start != 0 && start < end); 738 739 return B_OK; 740 } 741 742 743 status_t 744 X86VMTranslationMap64Bit::ClearFlags(addr_t address, uint32 flags) 745 { 746 TRACE("X86VMTranslationMap64Bit::ClearFlags(%#" B_PRIxADDR ", %#" B_PRIx32 747 ")\n", address, flags); 748 749 ThreadCPUPinner pinner(thread_get_current_thread()); 750 751 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 752 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap, 753 false, NULL, fPageMapper, fMapCount); 754 if (entry == NULL) 755 return B_OK; 756 757 uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_64_PTE_DIRTY : 0) 758 | ((flags & PAGE_ACCESSED) ? X86_64_PTE_ACCESSED : 0); 759 760 uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry, 761 flagsToClear); 762 763 if ((oldEntry & flagsToClear) != 0) 764 InvalidatePage(address); 765 766 return B_OK; 767 } 768 769 770 bool 771 X86VMTranslationMap64Bit::ClearAccessedAndModified(VMArea* area, addr_t address, 772 bool unmapIfUnaccessed, bool& _modified) 773 { 774 ASSERT(address % B_PAGE_SIZE == 0); 775 776 TRACE("X86VMTranslationMap64Bit::ClearAccessedAndModified(%#" B_PRIxADDR 777 ")\n", address); 778 779 RecursiveLocker locker(fLock); 780 ThreadCPUPinner pinner(thread_get_current_thread()); 781 782 uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( 783 fPagingStructures->VirtualPMLTop(), address, fIsKernelMap, 784 false, NULL, fPageMapper, fMapCount); 785 if (entry == NULL) 786 return false; 787 788 uint64 oldEntry; 789 790 if (unmapIfUnaccessed) { 791 while (true) { 792 oldEntry = *entry; 793 if ((oldEntry & X86_64_PTE_PRESENT) == 0) { 794 // page mapping not valid 795 return false; 796 } 797 798 if (oldEntry & X86_64_PTE_ACCESSED) { 799 // page was accessed -- just clear the flags 800 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry, 801 X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY); 802 break; 803 } 804 805 // page hasn't been accessed -- unmap it 806 if (X86PagingMethod64Bit::TestAndSetTableEntry(entry, 0, oldEntry) 807 == oldEntry) { 808 break; 809 } 810 811 // something changed -- check again 812 } 813 } else { 814 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry, 815 X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY); 816 } 817 818 pinner.Unlock(); 819 820 _modified = (oldEntry & X86_64_PTE_DIRTY) != 0; 821 822 if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { 823 // Note, that we only need to invalidate the address, if the 824 // accessed flags was set, since only then the entry could have been 825 // in any TLB. 826 InvalidatePage(address); 827 Flush(); 828 829 return true; 830 } 831 832 if (!unmapIfUnaccessed) 833 return false; 834 835 // We have unmapped the address. Do the "high level" stuff. 836 837 fMapCount--; 838 839 locker.Detach(); 840 // UnaccessedPageUnmapped() will unlock for us 841 842 UnaccessedPageUnmapped(area, 843 (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE); 844 845 return false; 846 } 847 848 849 X86PagingStructures* 850 X86VMTranslationMap64Bit::PagingStructures() const 851 { 852 return fPagingStructures; 853 } 854