1 /* 2 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "paging/pae/X86VMTranslationMapPAE.h" 12 13 #include <int.h> 14 #include <slab/Slab.h> 15 #include <thread.h> 16 #include <tracing.h> 17 #include <util/AutoLock.h> 18 #include <vm/vm_page.h> 19 #include <vm/VMAddressSpace.h> 20 #include <vm/VMCache.h> 21 22 #include "paging/pae/X86PagingMethodPAE.h" 23 #include "paging/pae/X86PagingStructuresPAE.h" 24 #include "paging/x86_physical_page_mapper.h" 25 26 27 //#define TRACE_X86_VM_TRANSLATION_MAP_PAE 28 #ifdef TRACE_X86_VM_TRANSLATION_MAP_PAE 29 # define TRACE(x...) dprintf(x) 30 #else 31 # define TRACE(x...) ; 32 #endif 33 34 35 #if B_HAIKU_PHYSICAL_BITS == 64 36 37 38 #if TRANSLATION_MAP_TRACING 39 40 41 namespace TranslationMapTracing { 42 43 44 class TraceEntryBase : public AbstractTraceEntry { 45 public: 46 TraceEntryBase() 47 { 48 #if TRANSLATION_MAP_TRACING_STACK_TRACE 49 fStackTrace = capture_tracing_stack_trace( 50 TRANSLATION_MAP_TRACING_STACK_TRACE, 0, true); 51 // Don't capture userland stack trace to avoid potential 52 // deadlocks. 53 #endif 54 } 55 56 #if TRANSLATION_MAP_TRACING_STACK_TRACE 57 virtual void DumpStackTrace(TraceOutput& out) 58 { 59 out.PrintStackTrace(fStackTrace); 60 } 61 #endif 62 63 private: 64 #if TRANSLATION_MAP_TRACING_STACK_TRACE 65 tracing_stack_trace* fStackTrace; 66 #endif 67 }; 68 69 70 class Map : public TraceEntryBase { 71 public: 72 Map(X86VMTranslationMapPAE* map, addr_t virtualAddress, 73 phys_addr_t physicalAddress) 74 : 75 TraceEntryBase(), 76 fMap(map), 77 fVirtualAddress(virtualAddress), 78 fPhysicalAddress(physicalAddress) 79 { 80 Initialized(); 81 } 82 83 virtual void AddDump(TraceOutput& out) 84 { 85 out.Print("translation map map: %p: %#" B_PRIxADDR 86 " -> %#" B_PRIxPHYSADDR, fMap, fVirtualAddress, fPhysicalAddress); 87 } 88 89 private: 90 X86VMTranslationMapPAE* fMap; 91 addr_t fVirtualAddress, 92 phys_addr_t fPhysicalAddress; 93 }; 94 95 96 class Unmap : public TraceEntryBase { 97 public: 98 Unmap(X86VMTranslationMapPAE* map, addr_t virtualAddress, 99 phys_addr_t physicalAddress) 100 : 101 TraceEntryBase(), 102 fMap(map), 103 fVirtualAddress(virtualAddress), 104 fPhysicalAddress(physicalAddress) 105 { 106 Initialized(); 107 } 108 109 virtual void AddDump(TraceOutput& out) 110 { 111 out.Print("translation map unmap: %p: %#" B_PRIxADDR 112 " -> %#" B_PRIxPHYSADDR, fMap, fVirtualAddress, fPhysicalAddress); 113 } 114 115 private: 116 X86VMTranslationMapPAE* fMap; 117 addr_t fVirtualAddress, 118 phys_addr_t fPhysicalAddress; 119 }; 120 121 122 } // namespace TranslationMapTracing 123 124 # define T(x) new(std::nothrow) TranslationMapTracing::x 125 126 #else 127 # define T(x) 128 #endif // TRANSLATION_MAP_TRACING 129 130 131 132 X86VMTranslationMapPAE::X86VMTranslationMapPAE() 133 : 134 fPagingStructures(NULL) 135 { 136 } 137 138 139 X86VMTranslationMapPAE::~X86VMTranslationMapPAE() 140 { 141 if (fPagingStructures == NULL) 142 return; 143 144 if (fPageMapper != NULL) 145 fPageMapper->Delete(); 146 147 // cycle through and free all of the user space page tables 148 149 STATIC_ASSERT(KERNEL_BASE == 0x80000000 && KERNEL_SIZE == 0x80000000); 150 // assuming 1-1 split of the address space 151 152 for (uint32 k = 0; k < 2; k++) { 153 pae_page_directory_entry* pageDir 154 = fPagingStructures->VirtualPageDirs()[k]; 155 if (pageDir == NULL) 156 continue; 157 158 for (uint32 i = 0; i < kPAEPageDirEntryCount; i++) { 159 if ((pageDir[i] & X86_PAE_PDE_PRESENT) != 0) { 160 phys_addr_t address = pageDir[i] & X86_PAE_PDE_ADDRESS_MASK; 161 vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); 162 if (page == NULL) 163 panic("X86VMTranslationMapPAE::~X86VMTranslationMapPAE: " 164 "didn't find page table page: page address: %#" 165 B_PRIxPHYSADDR ", virtual base: %#" B_PRIxADDR "\n", 166 address, 167 (k * kPAEPageDirEntryCount + i) * kPAEPageTableRange); 168 DEBUG_PAGE_ACCESS_START(page); 169 vm_page_set_state(page, PAGE_STATE_FREE); 170 } 171 } 172 } 173 174 fPagingStructures->RemoveReference(); 175 } 176 177 178 status_t 179 X86VMTranslationMapPAE::Init(bool kernel) 180 { 181 TRACE("X86VMTranslationMapPAE::Init()\n"); 182 183 X86VMTranslationMap::Init(kernel); 184 185 fPagingStructures = new(std::nothrow) X86PagingStructuresPAE; 186 if (fPagingStructures == NULL) 187 return B_NO_MEMORY; 188 189 X86PagingMethodPAE* method = X86PagingMethodPAE::Method(); 190 191 if (kernel) { 192 // kernel 193 // get the physical page mapper 194 fPageMapper = method->KernelPhysicalPageMapper(); 195 196 // we already know the kernel pgdir mapping 197 fPagingStructures->Init(method->KernelVirtualPageDirPointerTable(), 198 method->KernelPhysicalPageDirPointerTable(), NULL, 199 method->KernelVirtualPageDirs(), method->KernelPhysicalPageDirs()); 200 } else { 201 // user 202 // allocate a physical page mapper 203 status_t error = method->PhysicalPageMapper() 204 ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); 205 if (error != B_OK) 206 return error; 207 208 // The following code assumes that the kernel address space occupies the 209 // upper half of the virtual address space. This simplifies things a 210 // lot, since it allows us to just use the upper two page directories 211 // of the kernel and create two new lower page directories for the 212 // userland. 213 STATIC_ASSERT(KERNEL_BASE == 0x80000000 && KERNEL_SIZE == 0x80000000); 214 215 // allocate the page directories (both at once) 216 pae_page_directory_entry* virtualPageDirs[4]; 217 phys_addr_t physicalPageDirs[4]; 218 virtualPageDirs[0] = (pae_page_directory_entry*)memalign(B_PAGE_SIZE, 219 2 * B_PAGE_SIZE); 220 if (virtualPageDirs[0] == NULL) 221 return B_NO_MEMORY; 222 virtualPageDirs[1] = virtualPageDirs[0] + kPAEPageTableEntryCount; 223 224 // clear the userland page directories 225 memset(virtualPageDirs[0], 0, 2 * B_PAGE_SIZE); 226 227 // use the upper two kernel page directories 228 for (int32 i = 2; i < 4; i++) { 229 virtualPageDirs[i] = method->KernelVirtualPageDirs()[i]; 230 physicalPageDirs[i] = method->KernelPhysicalPageDirs()[i]; 231 } 232 233 // look up the page directories' physical addresses 234 for (int32 i = 0; i < 2; i++) { 235 vm_get_page_mapping(VMAddressSpace::KernelID(), 236 (addr_t)virtualPageDirs[i], &physicalPageDirs[i]); 237 } 238 239 // allocate the PDPT -- needs to have a 32 bit physical address 240 phys_addr_t physicalPDPT; 241 void* pdptHandle; 242 pae_page_directory_pointer_table_entry* pdpt 243 = (pae_page_directory_pointer_table_entry*) 244 method->Allocate32BitPage(physicalPDPT, pdptHandle); 245 if (pdpt == NULL) { 246 free(virtualPageDirs[0]); 247 return B_NO_MEMORY; 248 } 249 250 // init the PDPT entries 251 for (int32 i = 0; i < 4; i++) { 252 pdpt[i] = (physicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK) 253 | X86_PAE_PDPTE_PRESENT; 254 } 255 256 // init the paging structures 257 fPagingStructures->Init(pdpt, physicalPDPT, pdptHandle, virtualPageDirs, 258 physicalPageDirs); 259 } 260 261 return B_OK; 262 } 263 264 265 size_t 266 X86VMTranslationMapPAE::MaxPagesNeededToMap(addr_t start, addr_t end) const 267 { 268 // If start == 0, the actual base address is not yet known to the caller and 269 // we shall assume the worst case. 270 if (start == 0) { 271 // offset the range so it has the worst possible alignment 272 start = kPAEPageTableRange - B_PAGE_SIZE; 273 end += kPAEPageTableRange - B_PAGE_SIZE; 274 } 275 276 return end / kPAEPageTableRange + 1 - start / kPAEPageTableRange; 277 } 278 279 280 status_t 281 X86VMTranslationMapPAE::Map(addr_t virtualAddress, phys_addr_t physicalAddress, 282 uint32 attributes, uint32 memoryType, vm_page_reservation* reservation) 283 { 284 TRACE("X86VMTranslationMapPAE::Map(): %#" B_PRIxADDR " -> %#" B_PRIxPHYSADDR 285 "\n", virtualAddress, physicalAddress); 286 T(Map(this, virtualAddress, physicalAddress)); 287 288 // check to see if a page table exists for this range 289 pae_page_directory_entry* pageDirEntry 290 = X86PagingMethodPAE::PageDirEntryForAddress( 291 fPagingStructures->VirtualPageDirs(), virtualAddress); 292 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 293 // we need to allocate a page table 294 vm_page *page = vm_page_allocate_page(reservation, 295 PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); 296 297 DEBUG_PAGE_ACCESS_END(page); 298 299 phys_addr_t physicalPageTable 300 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 301 302 TRACE("X86VMTranslationMapPAE::Map(): asked for free page for " 303 "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable); 304 305 // put it in the page dir 306 X86PagingMethodPAE::PutPageTableInPageDir(pageDirEntry, 307 physicalPageTable, 308 attributes 309 | ((attributes & B_USER_PROTECTION) != 0 310 ? B_WRITE_AREA : B_KERNEL_WRITE_AREA)); 311 312 fMapCount++; 313 } 314 315 // now, fill in the page table entry 316 Thread* thread = thread_get_current_thread(); 317 ThreadCPUPinner pinner(thread); 318 319 pae_page_table_entry* pageTable 320 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 321 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 322 pae_page_table_entry* entry = pageTable 323 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 324 325 ASSERT_PRINT((*entry & X86_PAE_PTE_PRESENT) == 0, 326 "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64 " @ %p", 327 virtualAddress, *entry, entry); 328 329 X86PagingMethodPAE::PutPageTableEntryInTable(entry, physicalAddress, 330 attributes, memoryType, fIsKernelMap); 331 332 pinner.Unlock(); 333 334 // Note: We don't need to invalidate the TLB for this address, as previously 335 // the entry was not present and the TLB doesn't cache those entries. 336 337 fMapCount++; 338 339 return 0; 340 } 341 342 343 status_t 344 X86VMTranslationMapPAE::Unmap(addr_t start, addr_t end) 345 { 346 start = ROUNDDOWN(start, B_PAGE_SIZE); 347 if (start >= end) 348 return B_OK; 349 350 TRACE("X86VMTranslationMapPAE::Unmap(): %#" B_PRIxADDR " - %#" B_PRIxADDR 351 "\n", start, end); 352 353 do { 354 pae_page_directory_entry* pageDirEntry 355 = X86PagingMethodPAE::PageDirEntryForAddress( 356 fPagingStructures->VirtualPageDirs(), start); 357 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 358 // no page table here, move the start up to access the next page 359 // table 360 start = ROUNDUP(start + 1, kPAEPageTableRange); 361 continue; 362 } 363 364 Thread* thread = thread_get_current_thread(); 365 ThreadCPUPinner pinner(thread); 366 367 pae_page_table_entry* pageTable 368 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 369 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 370 371 uint32 index = start / B_PAGE_SIZE % kPAEPageTableEntryCount; 372 for (; index < kPAEPageTableEntryCount && start < end; 373 index++, start += B_PAGE_SIZE) { 374 if ((pageTable[index] & X86_PAE_PTE_PRESENT) == 0) { 375 // page mapping not valid 376 continue; 377 } 378 379 TRACE("X86VMTranslationMapPAE::Unmap(): removing page %#" 380 B_PRIxADDR "\n", start); 381 382 pae_page_table_entry oldEntry 383 = X86PagingMethodPAE::ClearPageTableEntryFlags( 384 &pageTable[index], X86_PAE_PTE_PRESENT); 385 386 T(Unmap(this, start, oldEntry & X86_PAE_PDE_ADDRESS_MASK)); 387 388 fMapCount--; 389 390 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 391 // Note, that we only need to invalidate the address, if the 392 // accessed flags was set, since only then the entry could have 393 // been in any TLB. 394 InvalidatePage(start); 395 } 396 } 397 } while (start != 0 && start < end); 398 399 return B_OK; 400 } 401 402 403 status_t 404 X86VMTranslationMapPAE::DebugMarkRangePresent(addr_t start, addr_t end, 405 bool markPresent) 406 { 407 start = ROUNDDOWN(start, B_PAGE_SIZE); 408 if (start >= end) 409 return B_OK; 410 411 do { 412 pae_page_directory_entry* pageDirEntry 413 = X86PagingMethodPAE::PageDirEntryForAddress( 414 fPagingStructures->VirtualPageDirs(), start); 415 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 416 // no page table here, move the start up to access the next page 417 // table 418 start = ROUNDUP(start + 1, kPAEPageTableRange); 419 continue; 420 } 421 422 Thread* thread = thread_get_current_thread(); 423 ThreadCPUPinner pinner(thread); 424 425 pae_page_table_entry* pageTable 426 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 427 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 428 429 uint32 index = start / B_PAGE_SIZE % kPAEPageTableEntryCount; 430 for (; index < kPAEPageTableEntryCount && start < end; 431 index++, start += B_PAGE_SIZE) { 432 433 if ((pageTable[index] & X86_PAE_PTE_PRESENT) == 0) { 434 if (!markPresent) 435 continue; 436 437 X86PagingMethodPAE::SetPageTableEntryFlags( 438 &pageTable[index], X86_PAE_PTE_PRESENT); 439 } else { 440 if (markPresent) 441 continue; 442 443 pae_page_table_entry oldEntry 444 = X86PagingMethodPAE::ClearPageTableEntryFlags( 445 &pageTable[index], X86_PAE_PTE_PRESENT); 446 447 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 448 // Note, that we only need to invalidate the address, if the 449 // accessed flags was set, since only then the entry could 450 // have been in any TLB. 451 InvalidatePage(start); 452 } 453 } 454 } 455 } while (start != 0 && start < end); 456 457 return B_OK; 458 } 459 460 461 /*! Caller must have locked the cache of the page to be unmapped. 462 This object shouldn't be locked. 463 */ 464 status_t 465 X86VMTranslationMapPAE::UnmapPage(VMArea* area, addr_t address, 466 bool updatePageQueue) 467 { 468 ASSERT(address % B_PAGE_SIZE == 0); 469 470 pae_page_directory_entry* pageDirEntry 471 = X86PagingMethodPAE::PageDirEntryForAddress( 472 fPagingStructures->VirtualPageDirs(), address); 473 474 TRACE("X86VMTranslationMapPAE::UnmapPage(%#" B_PRIxADDR ")\n", address); 475 476 RecursiveLocker locker(fLock); 477 478 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) 479 return B_ENTRY_NOT_FOUND; 480 481 ThreadCPUPinner pinner(thread_get_current_thread()); 482 483 pae_page_table_entry* pageTable 484 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 485 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 486 487 pae_page_table_entry oldEntry = X86PagingMethodPAE::ClearPageTableEntry( 488 &pageTable[address / B_PAGE_SIZE % kPAEPageTableEntryCount]); 489 490 T(Unmap(this, address, oldEntry & X86_PAE_PDE_ADDRESS_MASK)); 491 492 pinner.Unlock(); 493 494 if ((oldEntry & X86_PAE_PTE_PRESENT) == 0) { 495 // page mapping not valid 496 return B_ENTRY_NOT_FOUND; 497 } 498 499 fMapCount--; 500 501 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 502 // Note, that we only need to invalidate the address, if the 503 // accessed flags was set, since only then the entry could have been 504 // in any TLB. 505 InvalidatePage(address); 506 507 Flush(); 508 509 // NOTE: Between clearing the page table entry and Flush() other 510 // processors (actually even this processor with another thread of the 511 // same team) could still access the page in question via their cached 512 // entry. We can obviously lose a modified flag in this case, with the 513 // effect that the page looks unmodified (and might thus be recycled), 514 // but is actually modified. 515 // In most cases this is harmless, but for vm_remove_all_page_mappings() 516 // this is actually a problem. 517 // Interestingly FreeBSD seems to ignore this problem as well 518 // (cf. pmap_remove_all()), unless I've missed something. 519 } 520 521 locker.Detach(); 522 // PageUnmapped() will unlock for us 523 524 PageUnmapped(area, (oldEntry & X86_PAE_PTE_ADDRESS_MASK) / B_PAGE_SIZE, 525 (oldEntry & X86_PAE_PTE_ACCESSED) != 0, 526 (oldEntry & X86_PAE_PTE_DIRTY) != 0, updatePageQueue); 527 528 return B_OK; 529 } 530 531 532 void 533 X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size, 534 bool updatePageQueue) 535 { 536 if (size == 0) 537 return; 538 539 addr_t start = base; 540 addr_t end = base + size - 1; 541 542 TRACE("X86VMTranslationMapPAE::UnmapPages(%p, %#" B_PRIxADDR ", %#" 543 B_PRIxADDR ")\n", area, start, end); 544 545 VMAreaMappings queue; 546 547 RecursiveLocker locker(fLock); 548 549 do { 550 pae_page_directory_entry* pageDirEntry 551 = X86PagingMethodPAE::PageDirEntryForAddress( 552 fPagingStructures->VirtualPageDirs(), start); 553 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 554 // no page table here, move the start up to access the next page 555 // table 556 start = ROUNDUP(start + 1, kPAEPageTableRange); 557 continue; 558 } 559 560 Thread* thread = thread_get_current_thread(); 561 ThreadCPUPinner pinner(thread); 562 563 pae_page_table_entry* pageTable 564 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 565 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 566 567 uint32 index = start / B_PAGE_SIZE % kPAEPageTableEntryCount; 568 for (; index < kPAEPageTableEntryCount && start < end; 569 index++, start += B_PAGE_SIZE) { 570 pae_page_table_entry oldEntry 571 = X86PagingMethodPAE::ClearPageTableEntry(&pageTable[index]); 572 if ((oldEntry & X86_PAE_PTE_PRESENT) == 0) 573 continue; 574 575 T(Unmap(this, start, oldEntry & X86_PAE_PDE_ADDRESS_MASK)); 576 577 fMapCount--; 578 579 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 580 // Note, that we only need to invalidate the address, if the 581 // accessed flags was set, since only then the entry could have 582 // been in any TLB. 583 InvalidatePage(start); 584 } 585 586 if (area->cache_type != CACHE_TYPE_DEVICE) { 587 // get the page 588 vm_page* page = vm_lookup_page( 589 (oldEntry & X86_PAE_PTE_ADDRESS_MASK) / B_PAGE_SIZE); 590 ASSERT(page != NULL); 591 592 DEBUG_PAGE_ACCESS_START(page); 593 594 // transfer the accessed/dirty flags to the page 595 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) 596 page->accessed = true; 597 if ((oldEntry & X86_PAE_PTE_DIRTY) != 0) 598 page->modified = true; 599 600 // remove the mapping object/decrement the wired_count of the 601 // page 602 if (area->wiring == B_NO_LOCK) { 603 vm_page_mapping* mapping = NULL; 604 vm_page_mappings::Iterator iterator 605 = page->mappings.GetIterator(); 606 while ((mapping = iterator.Next()) != NULL) { 607 if (mapping->area == area) 608 break; 609 } 610 611 ASSERT(mapping != NULL); 612 613 area->mappings.Remove(mapping); 614 page->mappings.Remove(mapping); 615 queue.Add(mapping); 616 } else 617 page->DecrementWiredCount(); 618 619 if (!page->IsMapped()) { 620 atomic_add(&gMappedPagesCount, -1); 621 622 if (updatePageQueue) { 623 if (page->Cache()->temporary) 624 vm_page_set_state(page, PAGE_STATE_INACTIVE); 625 else if (page->modified) 626 vm_page_set_state(page, PAGE_STATE_MODIFIED); 627 else 628 vm_page_set_state(page, PAGE_STATE_CACHED); 629 } 630 } 631 632 DEBUG_PAGE_ACCESS_END(page); 633 } 634 } 635 636 Flush(); 637 // flush explicitly, since we directly use the lock 638 } while (start != 0 && start < end); 639 640 // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not 641 // really critical here, as in all cases this method is used, the unmapped 642 // area range is unmapped for good (resized/cut) and the pages will likely 643 // be freed. 644 645 locker.Unlock(); 646 647 // free removed mappings 648 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); 649 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY 650 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); 651 while (vm_page_mapping* mapping = queue.RemoveHead()) 652 object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); 653 } 654 655 656 void 657 X86VMTranslationMapPAE::UnmapArea(VMArea* area, bool deletingAddressSpace, 658 bool ignoreTopCachePageFlags) 659 { 660 if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { 661 X86VMTranslationMapPAE::UnmapPages(area, area->Base(), area->Size(), 662 true); 663 return; 664 } 665 666 bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; 667 668 RecursiveLocker locker(fLock); 669 670 VMAreaMappings mappings; 671 mappings.MoveFrom(&area->mappings); 672 673 for (VMAreaMappings::Iterator it = mappings.GetIterator(); 674 vm_page_mapping* mapping = it.Next();) { 675 vm_page* page = mapping->page; 676 page->mappings.Remove(mapping); 677 678 VMCache* cache = page->Cache(); 679 680 bool pageFullyUnmapped = false; 681 if (!page->IsMapped()) { 682 atomic_add(&gMappedPagesCount, -1); 683 pageFullyUnmapped = true; 684 } 685 686 if (unmapPages || cache != area->cache) { 687 addr_t address = area->Base() 688 + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); 689 690 pae_page_directory_entry* pageDirEntry 691 = X86PagingMethodPAE::PageDirEntryForAddress( 692 fPagingStructures->VirtualPageDirs(), address); 693 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 694 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " 695 "has no page dir entry", page, area, address); 696 continue; 697 } 698 699 ThreadCPUPinner pinner(thread_get_current_thread()); 700 701 pae_page_table_entry* pageTable 702 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 703 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 704 pae_page_table_entry oldEntry 705 = X86PagingMethodPAE::ClearPageTableEntry( 706 &pageTable[address / B_PAGE_SIZE 707 % kPAEPageTableEntryCount]); 708 709 pinner.Unlock(); 710 711 if ((oldEntry & X86_PAE_PTE_PRESENT) == 0) { 712 panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " 713 "has no page table entry", page, area, address); 714 continue; 715 } 716 717 T(Unmap(this, address, oldEntry & X86_PAE_PDE_ADDRESS_MASK)); 718 719 // transfer the accessed/dirty flags to the page and invalidate 720 // the mapping, if necessary 721 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 722 page->accessed = true; 723 724 if (!deletingAddressSpace) 725 InvalidatePage(address); 726 } 727 728 if ((oldEntry & X86_PAE_PTE_DIRTY) != 0) 729 page->modified = true; 730 731 if (pageFullyUnmapped) { 732 DEBUG_PAGE_ACCESS_START(page); 733 734 if (cache->temporary) 735 vm_page_set_state(page, PAGE_STATE_INACTIVE); 736 else if (page->modified) 737 vm_page_set_state(page, PAGE_STATE_MODIFIED); 738 else 739 vm_page_set_state(page, PAGE_STATE_CACHED); 740 741 DEBUG_PAGE_ACCESS_END(page); 742 } 743 } else { 744 #if TRANSLATION_MAP_TRACING 745 addr_t address = area->Base() 746 + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); 747 748 ThreadCPUPinner pinner(thread_get_current_thread()); 749 750 pae_page_directory_entry* pageDirEntry 751 = X86PagingMethodPAE::PageDirEntryForAddress( 752 fPagingStructures->VirtualPageDirs(), address); 753 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) != 0) { 754 pae_page_table_entry* pageTable 755 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 756 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 757 pae_page_table_entry oldEntry = pageTable[ 758 address / B_PAGE_SIZE % kPAEPageTableEntryCount]; 759 760 pinner.Unlock(); 761 762 if ((oldEntry & X86_PAE_PTE_PRESENT) != 0) { 763 T(Unmap(this, address, 764 oldEntry & X86_PAE_PDE_ADDRESS_MASK)); 765 } 766 } 767 #endif 768 } 769 770 fMapCount--; 771 } 772 773 Flush(); 774 // flush explicitely, since we directly use the lock 775 776 locker.Unlock(); 777 778 bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); 779 uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY 780 | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); 781 while (vm_page_mapping* mapping = mappings.RemoveHead()) 782 object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); 783 } 784 785 786 status_t 787 X86VMTranslationMapPAE::Query(addr_t virtualAddress, 788 phys_addr_t* _physicalAddress, uint32* _flags) 789 { 790 // default the flags to not present 791 *_flags = 0; 792 *_physicalAddress = 0; 793 794 // get the page directory entry 795 pae_page_directory_entry* pageDirEntry 796 = X86PagingMethodPAE::PageDirEntryForAddress( 797 fPagingStructures->VirtualPageDirs(), virtualAddress); 798 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 799 // no pagetable here 800 return B_OK; 801 } 802 803 // get the page table entry 804 Thread* thread = thread_get_current_thread(); 805 ThreadCPUPinner pinner(thread); 806 807 pae_page_table_entry* pageTable 808 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 809 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 810 pae_page_table_entry entry 811 = pageTable[virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount]; 812 813 pinner.Unlock(); 814 815 *_physicalAddress = entry & X86_PAE_PTE_ADDRESS_MASK; 816 817 // translate the page state flags 818 if ((entry & X86_PAE_PTE_USER) != 0) { 819 *_flags |= ((entry & X86_PAE_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) 820 | B_READ_AREA 821 | ((entry & X86_PAE_PTE_NOT_EXECUTABLE) == 0 ? B_EXECUTE_AREA : 0); 822 } 823 824 *_flags |= ((entry & X86_PAE_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) 825 | B_KERNEL_READ_AREA 826 | ((entry & X86_PAE_PTE_NOT_EXECUTABLE) == 0 827 ? B_KERNEL_EXECUTE_AREA : 0) 828 | ((entry & X86_PAE_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) 829 | ((entry & X86_PAE_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) 830 | ((entry & X86_PAE_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); 831 832 TRACE("X86VMTranslationMapPAE::Query(%#" B_PRIxADDR ") -> %#" 833 B_PRIxPHYSADDR ":\n", virtualAddress, *_physicalAddress); 834 835 return B_OK; 836 } 837 838 839 status_t 840 X86VMTranslationMapPAE::QueryInterrupt(addr_t virtualAddress, 841 phys_addr_t* _physicalAddress, uint32* _flags) 842 { 843 // default the flags to not present 844 *_flags = 0; 845 *_physicalAddress = 0; 846 847 // get the page directory entry 848 pae_page_directory_entry* pageDirEntry 849 = X86PagingMethodPAE::PageDirEntryForAddress( 850 fPagingStructures->VirtualPageDirs(), virtualAddress); 851 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 852 // no pagetable here 853 return B_OK; 854 } 855 856 // get the page table entry 857 pae_page_table_entry* pageTable 858 = (pae_page_table_entry*)X86PagingMethodPAE::Method() 859 ->PhysicalPageMapper()->InterruptGetPageTableAt( 860 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 861 pae_page_table_entry entry 862 = pageTable[virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount]; 863 864 *_physicalAddress = entry & X86_PAE_PTE_ADDRESS_MASK; 865 866 // translate the page state flags 867 if ((entry & X86_PAE_PTE_USER) != 0) { 868 *_flags |= ((entry & X86_PAE_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) 869 | B_READ_AREA 870 | ((entry & X86_PAE_PTE_NOT_EXECUTABLE) == 0 ? B_EXECUTE_AREA : 0); 871 } 872 873 *_flags |= ((entry & X86_PAE_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) 874 | B_KERNEL_READ_AREA 875 | ((entry & X86_PAE_PTE_NOT_EXECUTABLE) == 0 876 ? B_KERNEL_EXECUTE_AREA : 0) 877 | ((entry & X86_PAE_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) 878 | ((entry & X86_PAE_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) 879 | ((entry & X86_PAE_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); 880 881 TRACE("X86VMTranslationMapPAE::Query(%#" B_PRIxADDR ") -> %#" 882 B_PRIxPHYSADDR ":\n", *_physicalAddress, virtualAddress); 883 884 return B_OK; 885 } 886 887 888 status_t 889 X86VMTranslationMapPAE::Protect(addr_t start, addr_t end, uint32 attributes, 890 uint32 memoryType) 891 { 892 start = ROUNDDOWN(start, B_PAGE_SIZE); 893 if (start >= end) 894 return B_OK; 895 896 TRACE("X86VMTranslationMapPAE::Protect(): %#" B_PRIxADDR " - %#" B_PRIxADDR 897 ", attributes: %#" B_PRIx32 "\n", start, end, attributes); 898 899 // compute protection flags 900 uint64 newProtectionFlags = 0; 901 if ((attributes & B_USER_PROTECTION) != 0) { 902 newProtectionFlags = X86_PAE_PTE_USER; 903 if ((attributes & B_WRITE_AREA) != 0) 904 newProtectionFlags |= X86_PAE_PTE_WRITABLE; 905 if ((attributes & B_EXECUTE_AREA) == 0 906 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 907 newProtectionFlags |= X86_PAE_PTE_NOT_EXECUTABLE; 908 } 909 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 910 newProtectionFlags = X86_PAE_PTE_WRITABLE; 911 912 do { 913 pae_page_directory_entry* pageDirEntry 914 = X86PagingMethodPAE::PageDirEntryForAddress( 915 fPagingStructures->VirtualPageDirs(), start); 916 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 917 // no page table here, move the start up to access the next page 918 // table 919 start = ROUNDUP(start + 1, kPAEPageTableRange); 920 continue; 921 } 922 923 Thread* thread = thread_get_current_thread(); 924 ThreadCPUPinner pinner(thread); 925 926 pae_page_table_entry* pageTable 927 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 928 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 929 930 uint32 index = start / B_PAGE_SIZE % kPAEPageTableEntryCount; 931 for (; index < kPAEPageTableEntryCount && start < end; 932 index++, start += B_PAGE_SIZE) { 933 pae_page_table_entry entry = pageTable[index]; 934 if ((pageTable[index] & X86_PAE_PTE_PRESENT) == 0) { 935 // page mapping not valid 936 continue; 937 } 938 939 TRACE("X86VMTranslationMapPAE::Protect(): protect page %#" 940 B_PRIxADDR "\n", start); 941 942 // set the new protection flags -- we want to do that atomically, 943 // without changing the accessed or dirty flag 944 pae_page_table_entry oldEntry; 945 while (true) { 946 oldEntry = X86PagingMethodPAE::TestAndSetPageTableEntry( 947 &pageTable[index], 948 (entry & ~(X86_PAE_PTE_PROTECTION_MASK 949 | X86_PAE_PTE_MEMORY_TYPE_MASK)) 950 | newProtectionFlags 951 | X86PagingMethodPAE::MemoryTypeToPageTableEntryFlags( 952 memoryType), 953 entry); 954 if (oldEntry == entry) 955 break; 956 entry = oldEntry; 957 } 958 959 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 960 // Note, that we only need to invalidate the address, if the 961 // accessed flag was set, since only then the entry could have been 962 // in any TLB. 963 InvalidatePage(start); 964 } 965 } 966 } while (start != 0 && start < end); 967 968 return B_OK; 969 } 970 971 972 status_t 973 X86VMTranslationMapPAE::ClearFlags(addr_t address, uint32 flags) 974 { 975 pae_page_directory_entry* pageDirEntry 976 = X86PagingMethodPAE::PageDirEntryForAddress( 977 fPagingStructures->VirtualPageDirs(), address); 978 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 979 // no pagetable here 980 return B_OK; 981 } 982 983 uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_PAE_PTE_DIRTY : 0) 984 | ((flags & PAGE_ACCESSED) ? X86_PAE_PTE_ACCESSED : 0); 985 986 Thread* thread = thread_get_current_thread(); 987 ThreadCPUPinner pinner(thread); 988 989 pae_page_table_entry* entry 990 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 991 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK) 992 + address / B_PAGE_SIZE % kPAEPageTableEntryCount; 993 994 // clear out the flags we've been requested to clear 995 pae_page_table_entry oldEntry 996 = X86PagingMethodPAE::ClearPageTableEntryFlags(entry, flagsToClear); 997 998 pinner.Unlock(); 999 1000 if ((oldEntry & flagsToClear) != 0) 1001 InvalidatePage(address); 1002 1003 return B_OK; 1004 } 1005 1006 1007 bool 1008 X86VMTranslationMapPAE::ClearAccessedAndModified(VMArea* area, addr_t address, 1009 bool unmapIfUnaccessed, bool& _modified) 1010 { 1011 ASSERT(address % B_PAGE_SIZE == 0); 1012 1013 TRACE("X86VMTranslationMapPAE::ClearAccessedAndModified(%#" B_PRIxADDR 1014 ")\n", address); 1015 1016 pae_page_directory_entry* pageDirEntry 1017 = X86PagingMethodPAE::PageDirEntryForAddress( 1018 fPagingStructures->VirtualPageDirs(), address); 1019 1020 RecursiveLocker locker(fLock); 1021 1022 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) 1023 return false; 1024 1025 ThreadCPUPinner pinner(thread_get_current_thread()); 1026 1027 pae_page_table_entry* entry 1028 = (pae_page_table_entry*)fPageMapper->GetPageTableAt( 1029 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK) 1030 + address / B_PAGE_SIZE % kPAEPageTableEntryCount; 1031 1032 // perform the deed 1033 pae_page_table_entry oldEntry; 1034 1035 if (unmapIfUnaccessed) { 1036 while (true) { 1037 oldEntry = *entry; 1038 if ((oldEntry & X86_PAE_PTE_PRESENT) == 0) { 1039 // page mapping not valid 1040 return false; 1041 } 1042 1043 if (oldEntry & X86_PAE_PTE_ACCESSED) { 1044 // page was accessed -- just clear the flags 1045 oldEntry = X86PagingMethodPAE::ClearPageTableEntryFlags(entry, 1046 X86_PAE_PTE_ACCESSED | X86_PAE_PTE_DIRTY); 1047 break; 1048 } 1049 1050 // page hasn't been accessed -- unmap it 1051 if (X86PagingMethodPAE::TestAndSetPageTableEntry(entry, 0, oldEntry) 1052 == oldEntry) { 1053 break; 1054 } 1055 1056 // something changed -- check again 1057 } 1058 } else { 1059 oldEntry = X86PagingMethodPAE::ClearPageTableEntryFlags(entry, 1060 X86_PAE_PTE_ACCESSED | X86_PAE_PTE_DIRTY); 1061 } 1062 1063 pinner.Unlock(); 1064 1065 _modified = (oldEntry & X86_PAE_PTE_DIRTY) != 0; 1066 1067 if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { 1068 // Note, that we only need to invalidate the address, if the 1069 // accessed flags was set, since only then the entry could have been 1070 // in any TLB. 1071 InvalidatePage(address); 1072 Flush(); 1073 1074 return true; 1075 } 1076 1077 if (!unmapIfUnaccessed) 1078 return false; 1079 1080 // We have unmapped the address. Do the "high level" stuff. 1081 1082 fMapCount--; 1083 1084 locker.Detach(); 1085 // UnaccessedPageUnmapped() will unlock for us 1086 1087 UnaccessedPageUnmapped(area, 1088 (oldEntry & X86_PAE_PTE_ADDRESS_MASK) / B_PAGE_SIZE); 1089 1090 return false; 1091 } 1092 1093 1094 X86PagingStructures* 1095 X86VMTranslationMapPAE::PagingStructures() const 1096 { 1097 return fPagingStructures; 1098 } 1099 1100 1101 #endif // B_HAIKU_PHYSICAL_BITS == 64 1102