1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "paging/pae/X86PagingMethodPAE.h" 12 13 #include <stdlib.h> 14 #include <string.h> 15 16 #include <AutoDeleter.h> 17 18 #include <boot/kernel_args.h> 19 #include <util/AutoLock.h> 20 #include <vm/vm.h> 21 #include <vm/vm_page.h> 22 #include <vm/VMAddressSpace.h> 23 24 #include "paging/32bit/paging.h" 25 #include "paging/32bit/X86PagingMethod32Bit.h" 26 #include "paging/pae/X86PagingStructuresPAE.h" 27 #include "paging/pae/X86VMTranslationMapPAE.h" 28 #include "paging/x86_physical_page_mapper.h" 29 #include "paging/x86_physical_page_mapper_large_memory.h" 30 31 32 //#define TRACE_X86_PAGING_METHOD_PAE 33 #ifdef TRACE_X86_PAGING_METHOD_PAE 34 # define TRACE(x...) dprintf(x) 35 #else 36 # define TRACE(x...) ; 37 #endif 38 39 40 #if B_HAIKU_PHYSICAL_BITS == 64 41 42 43 using X86LargePhysicalPageMapper::PhysicalPageSlot; 44 45 46 // number of 32 bit pages that will be cached 47 static const page_num_t kMaxFree32BitPagesCount = 32; 48 49 50 // #pragma mark - ToPAESwitcher 51 52 53 struct X86PagingMethodPAE::ToPAESwitcher { 54 ToPAESwitcher(kernel_args* args) 55 : 56 fKernelArgs(args) 57 { 58 // page hole set up in the boot loader 59 fPageHole = (page_table_entry*)fKernelArgs->arch_args.page_hole; 60 61 // calculate where the page dir would be 62 fPageHolePageDir = (page_directory_entry*) 63 (((addr_t)fKernelArgs->arch_args.page_hole) 64 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); 65 66 fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir; 67 68 TRACE("page hole: %p\n", fPageHole); 69 TRACE("page dir: %p (physical: %#" B_PRIxPHYSADDR ")\n", 70 fPageHolePageDir, fPhysicalPageDir); 71 } 72 73 void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT, 74 phys_addr_t& _physicalPDPT, void*& _pageStructures, 75 size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs, 76 phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot, 77 pae_page_table_entry*& _freeVirtualSlotPTE) 78 { 79 // count the page tables we have to translate 80 uint32 pageTableCount = 0; 81 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 82 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 83 page_directory_entry entry = fPageHolePageDir[i]; 84 if ((entry & X86_PDE_PRESENT) != 0) 85 pageTableCount++; 86 } 87 88 TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount); 89 90 // The pages we need to allocate to do our job: 91 // + 1 page dir pointer table 92 // + 4 page dirs 93 // + 2 * page tables (each has 512 instead of 1024 entries) 94 // + 1 page for the free virtual slot (no physical page needed) 95 uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1; 96 97 // We need additional PAE page tables for the new pages we're going to 98 // allocate: Two tables for every 1024 pages to map, i.e. 2 additional 99 // pages for every 1022 pages we want to allocate. We also need 32 bit 100 // page tables, but we don't need additional virtual space for them, 101 // since we can access then via the page hole. 102 pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2; 103 104 TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded); 105 106 // allocate the pages we need 107 _AllocateNeededPages(pagesNeeded); 108 109 // prepare the page directory pointer table 110 phys_addr_t physicalPDPT; 111 pae_page_directory_pointer_table_entry* pdpt 112 = (pae_page_directory_pointer_table_entry*)_NextPage(true, 113 physicalPDPT); 114 115 for (int32 i = 0; i < 4; i++) { 116 fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true, 117 fPhysicalPageDirs[i]); 118 119 pdpt[i] = X86_PAE_PDPTE_PRESENT 120 | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK); 121 } 122 123 // Since we have to enable PAE in two steps -- setting cr3 to the PDPT 124 // and setting the cr4 PAE bit -- we copy the kernel page dir entries to 125 // the PDPT page, so after setting cr3, we continue to have working 126 // kernel mappings. This requires that the PDPTE registers and the 127 // page dir entries don't interect, obviously. 128 ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry) 129 <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry)); 130 131 // translate the page tables 132 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 133 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 134 if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) { 135 // two PAE page tables per 32 bit page table 136 _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE); 137 _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE); 138 139 // copy the page directory entry to the PDPT page 140 ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i]; 141 } 142 } 143 144 TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n", 145 fFreeVirtualSlot, fFreeVirtualSlotPTE); 146 147 // enable PAE on all CPUs 148 call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT); 149 150 // set return values 151 _virtualPDPT = pdpt; 152 _physicalPDPT = physicalPDPT; 153 _pageStructures = fAllocatedPages; 154 _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE; 155 memcpy(pageDirs, fPageDirs, sizeof(fPageDirs)); 156 memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs)); 157 158 _freeVirtualSlot = fFreeVirtualSlot; 159 _freeVirtualSlotPTE = fFreeVirtualSlotPTE; 160 } 161 162 private: 163 static void _EnablePAE(void* physicalPDPT, int cpu) 164 { 165 write_cr3((addr_t)physicalPDPT); 166 x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES); 167 } 168 169 void _TranslatePageTable(addr_t virtualBase) 170 { 171 page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE]; 172 173 // allocate a PAE page table 174 phys_addr_t physicalTable; 175 pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false, 176 physicalTable); 177 178 // enter it into the page dir 179 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 180 fPageDirs, virtualBase); 181 PutPageTableInPageDir(pageDirEntry, physicalTable, 182 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 183 184 pae_page_table_entry* paeEntry = paeTable; 185 for (uint32 i = 0; i < kPAEPageTableEntryCount; 186 i++, entry++, paeEntry++) { 187 if ((*entry & X86_PTE_PRESENT) != 0) { 188 // Note, we use the fact that the PAE flags are defined to the 189 // same values. 190 *paeEntry = *entry & (X86_PTE_PRESENT 191 | X86_PTE_WRITABLE 192 | X86_PTE_USER 193 | X86_PTE_WRITE_THROUGH 194 | X86_PTE_CACHING_DISABLED 195 | X86_PTE_GLOBAL 196 | X86_PTE_ADDRESS_MASK); 197 } else 198 *paeEntry = 0; 199 } 200 201 if (fFreeVirtualSlot / kPAEPageTableRange 202 == virtualBase / kPAEPageTableRange) { 203 fFreeVirtualSlotPTE = paeTable 204 + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount; 205 } 206 } 207 208 void _AllocateNeededPages(uint32 pagesNeeded) 209 { 210 size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE; 211 addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0, 212 kPageTableAlignment); 213 if (virtualBase == 0) { 214 panic("Failed to reserve virtual address space for the switch to " 215 "PAE!"); 216 } 217 218 TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n", 219 virtualBase, virtualSize); 220 221 // allocate pages for the 32 bit page tables and prepare the tables 222 uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024; 223 for (uint32 i = 0; i < oldPageTableCount; i++) { 224 // allocate a page 225 phys_addr_t physicalTable =_AllocatePage32Bit(); 226 227 // put the page into the page dir 228 page_directory_entry* entry = &fPageHolePageDir[ 229 virtualBase / B_PAGE_SIZE / 1024 + i]; 230 X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable, 231 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 232 233 // clear the table 234 memset((void*)((addr_t)fPageHole 235 + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE), 236 0, B_PAGE_SIZE); 237 } 238 239 // We don't need a physical page for the free virtual slot. 240 pagesNeeded--; 241 242 // allocate and map the pages we need 243 for (uint32 i = 0; i < pagesNeeded; i++) { 244 // allocate a page 245 phys_addr_t physicalAddress =_AllocatePage32Bit(); 246 247 // put the page into the page table 248 page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i; 249 X86PagingMethod32Bit::PutPageTableEntryInTable(entry, 250 physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, 251 true); 252 253 // Write the page's physical address into the page itself, so we 254 // don't need to look it up later. 255 *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress; 256 } 257 258 fAllocatedPages = (uint8*)virtualBase; 259 fAllocatedPagesCount = pagesNeeded; 260 fUsedPagesCount = 0; 261 fFreeVirtualSlot 262 = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE); 263 } 264 265 phys_addr_t _AllocatePage() 266 { 267 phys_addr_t physicalAddress 268 = (phys_addr_t)vm_allocate_early_physical_page(fKernelArgs) 269 * B_PAGE_SIZE; 270 if (physicalAddress == 0) 271 panic("Failed to allocate page for the switch to PAE!"); 272 return physicalAddress; 273 } 274 275 phys_addr_t _AllocatePage32Bit() 276 { 277 phys_addr_t physicalAddress = _AllocatePage(); 278 if (physicalAddress > 0xffffffff) { 279 panic("Failed to allocate 32 bit addressable page for the switch " 280 "to PAE!"); 281 return 0; 282 } 283 return physicalAddress; 284 } 285 286 void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress) 287 { 288 if (fUsedPagesCount >= fAllocatedPagesCount) { 289 panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more " 290 "allocated pages!"); 291 return NULL; 292 } 293 294 void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE; 295 _physicalAddress = *((phys_addr_t*)page); 296 297 if (clearPage) 298 memset(page, 0, B_PAGE_SIZE); 299 300 return page; 301 } 302 303 private: 304 kernel_args* fKernelArgs; 305 page_table_entry* fPageHole; 306 page_directory_entry* fPageHolePageDir; 307 phys_addr_t fPhysicalPageDir; 308 uint8* fAllocatedPages; 309 uint32 fAllocatedPagesCount; 310 uint32 fUsedPagesCount; 311 addr_t fFreeVirtualSlot; 312 pae_page_table_entry* fFreeVirtualSlotPTE; 313 pae_page_directory_entry* fPageDirs[4]; 314 phys_addr_t fPhysicalPageDirs[4]; 315 }; 316 317 318 // #pragma mark - PhysicalPageSlotPool 319 320 321 struct X86PagingMethodPAE::PhysicalPageSlotPool 322 : X86LargePhysicalPageMapper::PhysicalPageSlotPool { 323 public: 324 virtual ~PhysicalPageSlotPool(); 325 326 status_t InitInitial(X86PagingMethodPAE* method, 327 kernel_args* args); 328 status_t InitInitialPostArea(kernel_args* args); 329 330 void Init(area_id dataArea, 331 pae_page_table_entry* pageTable, 332 area_id virtualArea, addr_t virtualBase); 333 334 virtual status_t AllocatePool( 335 X86LargePhysicalPageMapper 336 ::PhysicalPageSlotPool*& _pool); 337 virtual void Map(phys_addr_t physicalAddress, 338 addr_t virtualAddress); 339 340 public: 341 static PhysicalPageSlotPool sInitialPhysicalPagePool; 342 343 private: 344 area_id fDataArea; 345 area_id fVirtualArea; 346 addr_t fVirtualBase; 347 pae_page_table_entry* fPageTable; 348 }; 349 350 351 X86PagingMethodPAE::PhysicalPageSlotPool 352 X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool; 353 354 355 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool() 356 { 357 } 358 359 360 status_t 361 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial( 362 X86PagingMethodPAE* method, kernel_args* args) 363 { 364 // allocate a virtual address range for the pages to be mapped into 365 addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0, 366 kPAEPageTableRange); 367 if (virtualBase == 0) { 368 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve " 369 "physical page pool space in virtual address space!"); 370 return B_ERROR; 371 } 372 373 // allocate memory for the page table and data 374 size_t areaSize = B_PAGE_SIZE 375 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 376 pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early( 377 args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0); 378 379 // clear the page table and put it in the page dir 380 memset(pageTable, 0, B_PAGE_SIZE); 381 382 phys_addr_t physicalTable = 0; 383 method->_EarlyQuery((addr_t)pageTable, &physicalTable); 384 385 pae_page_directory_entry* entry = PageDirEntryForAddress( 386 method->KernelVirtualPageDirs(), virtualBase); 387 PutPageTableInPageDir(entry, physicalTable, 388 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 389 390 // init the pool structure and add the initial pool 391 Init(-1, pageTable, -1, (addr_t)virtualBase); 392 393 return B_OK; 394 } 395 396 397 status_t 398 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea( 399 kernel_args* args) 400 { 401 // create an area for the (already allocated) data 402 size_t areaSize = B_PAGE_SIZE 403 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 404 void* temp = fPageTable; 405 area_id area = create_area("physical page pool", &temp, 406 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED, 407 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 408 if (area < B_OK) { 409 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 410 "create area for physical page pool."); 411 return area; 412 } 413 fDataArea = area; 414 415 // create an area for the virtual address space 416 temp = (void*)fVirtualBase; 417 area = vm_create_null_area(VMAddressSpace::KernelID(), 418 "physical page pool space", &temp, B_EXACT_ADDRESS, 419 kPAEPageTableRange, 0); 420 if (area < B_OK) { 421 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 422 "create area for physical page pool space."); 423 return area; 424 } 425 fVirtualArea = area; 426 427 return B_OK; 428 } 429 430 431 void 432 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea, 433 pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase) 434 { 435 fDataArea = dataArea; 436 fVirtualArea = virtualArea; 437 fVirtualBase = virtualBase; 438 fPageTable = pageTable; 439 440 // init slot list 441 fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount); 442 addr_t slotAddress = virtualBase; 443 for (uint32 i = 0; i < kPAEPageTableEntryCount; 444 i++, slotAddress += B_PAGE_SIZE) { 445 PhysicalPageSlot* slot = &fSlots[i]; 446 slot->next = slot + 1; 447 slot->pool = this; 448 slot->address = slotAddress; 449 } 450 451 fSlots[kPAEPageTableEntryCount - 1].next = NULL; 452 // terminate list 453 } 454 455 456 void 457 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress, 458 addr_t virtualAddress) 459 { 460 pae_page_table_entry& pte = fPageTable[ 461 (virtualAddress - fVirtualBase) / B_PAGE_SIZE]; 462 pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 463 | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT; 464 465 invalidate_TLB(virtualAddress); 466 } 467 468 469 status_t 470 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool( 471 X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool) 472 { 473 // create the pool structure 474 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool; 475 if (pool == NULL) 476 return B_NO_MEMORY; 477 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool); 478 479 // create an area that can contain the page table and the slot 480 // structures 481 size_t areaSize = B_PAGE_SIZE 482 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 483 void* data; 484 virtual_address_restrictions virtualRestrictions = {}; 485 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 486 physical_address_restrictions physicalRestrictions = {}; 487 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool", 488 PAGE_ALIGN(areaSize), B_FULL_LOCK, 489 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 490 &virtualRestrictions, &physicalRestrictions, &data); 491 if (dataArea < 0) 492 return dataArea; 493 494 // create the null area for the virtual address space 495 void* virtualBase; 496 area_id virtualArea = vm_create_null_area( 497 VMAddressSpace::KernelID(), "physical page pool space", 498 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange, 499 CREATE_AREA_PRIORITY_VIP); 500 if (virtualArea < 0) { 501 delete_area(dataArea); 502 return virtualArea; 503 } 504 505 // prepare the page table 506 memset(data, 0, B_PAGE_SIZE); 507 508 // get the page table's physical address 509 phys_addr_t physicalTable; 510 X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>( 511 VMAddressSpace::Kernel()->TranslationMap()); 512 uint32 dummyFlags; 513 cpu_status state = disable_interrupts(); 514 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags); 515 restore_interrupts(state); 516 517 // put the page table into the page directory 518 pae_page_directory_entry* pageDirEntry 519 = X86PagingMethodPAE::PageDirEntryForAddress( 520 map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase); 521 PutPageTableInPageDir(pageDirEntry, physicalTable, 522 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 523 524 // init the pool structure 525 pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea, 526 (addr_t)virtualBase); 527 poolDeleter.Detach(); 528 _pool = pool; 529 return B_OK; 530 } 531 532 533 // #pragma mark - X86PagingMethodPAE 534 535 536 X86PagingMethodPAE::X86PagingMethodPAE() 537 : 538 fPhysicalPageMapper(NULL), 539 fKernelPhysicalPageMapper(NULL), 540 fFreePages(NULL), 541 fFreePagesCount(0) 542 { 543 mutex_init(&fFreePagesLock, "x86 PAE free pages"); 544 } 545 546 547 X86PagingMethodPAE::~X86PagingMethodPAE() 548 { 549 } 550 551 552 status_t 553 X86PagingMethodPAE::Init(kernel_args* args, 554 VMPhysicalPageMapper** _physicalPageMapper) 555 { 556 // switch to PAE 557 ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable, 558 fKernelPhysicalPageDirPointerTable, fEarlyPageStructures, 559 fEarlyPageStructuresSize, fKernelVirtualPageDirs, 560 fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE); 561 562 // create the initial pool for the physical page mapper 563 PhysicalPageSlotPool* pool 564 = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) 565 PhysicalPageSlotPool; 566 status_t error = pool->InitInitial(this, args); 567 if (error != B_OK) { 568 panic("X86PagingMethodPAE::Init(): Failed to create initial pool " 569 "for physical page mapper!"); 570 return error; 571 } 572 573 // create physical page mapper 574 large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper, 575 fKernelPhysicalPageMapper); 576 577 *_physicalPageMapper = fPhysicalPageMapper; 578 return B_OK; 579 } 580 581 582 status_t 583 X86PagingMethodPAE::InitPostArea(kernel_args* args) 584 { 585 // wrap the kernel paging structures in an area 586 area_id area = create_area("kernel paging structs", &fEarlyPageStructures, 587 B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED, 588 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 589 if (area < B_OK) 590 return area; 591 592 // let the initial page pool create areas for its structures 593 status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool 594 .InitInitialPostArea(args); 595 if (error != B_OK) 596 return error; 597 598 // The early physical page mapping mechanism is no longer needed. Unmap the 599 // slot. 600 *fFreeVirtualSlotPTE = 0; 601 invalidate_TLB(fFreeVirtualSlot); 602 603 fFreeVirtualSlotPTE = NULL; 604 fFreeVirtualSlot = 0; 605 606 return B_OK; 607 } 608 609 610 status_t 611 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 612 { 613 X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE; 614 if (map == NULL) 615 return B_NO_MEMORY; 616 617 status_t error = map->Init(kernel); 618 if (error != B_OK) { 619 delete map; 620 return error; 621 } 622 623 *_map = map; 624 return B_OK; 625 } 626 627 628 status_t 629 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress, 630 phys_addr_t physicalAddress, uint8 attributes, 631 phys_addr_t (*get_free_page)(kernel_args*)) 632 { 633 // check to see if a page table exists for this range 634 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 635 fKernelVirtualPageDirs, virtualAddress); 636 pae_page_table_entry* pageTable; 637 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 638 // we need to allocate a page table 639 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE; 640 641 TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for " 642 "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable); 643 644 // put it in the page dir 645 PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes); 646 647 // zero it out 648 pageTable = _EarlyGetPageTable(physicalPageTable); 649 memset(pageTable, 0, B_PAGE_SIZE); 650 } else { 651 // table already exists -- map it 652 pageTable = _EarlyGetPageTable( 653 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 654 } 655 656 pae_page_table_entry* entry = pageTable 657 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 658 659 ASSERT_PRINT( 660 (*entry & X86_PAE_PTE_PRESENT) == 0, 661 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64 662 ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry); 663 664 // now, fill in the pentry 665 PutPageTableEntryInTable(entry, physicalAddress, attributes, 0, 666 IS_KERNEL_ADDRESS(virtualAddress)); 667 668 return B_OK; 669 } 670 671 672 bool 673 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress, 674 uint32 protection) 675 { 676 // we can't check much without the physical page mapper 677 if (fPhysicalPageMapper == NULL) 678 return false; 679 680 // We only trust the kernel team's page directories. So switch to the 681 // kernel PDPT first. Always set it to make sure the TLBs don't contain 682 // obsolete data. 683 uint32 physicalPDPT; 684 read_cr3(physicalPDPT); 685 write_cr3(fKernelPhysicalPageDirPointerTable); 686 687 // get the PDPT entry for the address 688 pae_page_directory_pointer_table_entry pdptEntry = 0; 689 if (physicalPDPT == fKernelPhysicalPageDirPointerTable) { 690 pdptEntry = fKernelVirtualPageDirPointerTable[ 691 virtualAddress / kPAEPageDirRange]; 692 } else { 693 // map the original PDPT and get the entry 694 void* handle; 695 addr_t virtualPDPT; 696 status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT, 697 &virtualPDPT, &handle); 698 if (error == B_OK) { 699 pdptEntry = ((pae_page_directory_pointer_table_entry*) 700 virtualPDPT)[virtualAddress / kPAEPageDirRange]; 701 fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle); 702 } 703 } 704 705 // map the page dir and get the entry 706 pae_page_directory_entry pageDirEntry = 0; 707 if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) { 708 void* handle; 709 addr_t virtualPageDir; 710 status_t error = fPhysicalPageMapper->GetPageDebug( 711 pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle); 712 if (error == B_OK) { 713 pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[ 714 virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount]; 715 fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle); 716 } 717 } 718 719 // map the page table and get the entry 720 pae_page_table_entry pageTableEntry = 0; 721 if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) { 722 void* handle; 723 addr_t virtualPageTable; 724 status_t error = fPhysicalPageMapper->GetPageDebug( 725 pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable, 726 &handle); 727 if (error == B_OK) { 728 pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[ 729 virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount]; 730 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); 731 } 732 } 733 734 // switch back to the original page directory 735 if (physicalPDPT != fKernelPhysicalPageDirPointerTable) 736 write_cr3(physicalPDPT); 737 738 if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0) 739 return false; 740 741 // present means kernel-readable, so check for writable 742 return (protection & B_KERNEL_WRITE_AREA) == 0 743 || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0; 744 } 745 746 747 /*static*/ void 748 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry, 749 phys_addr_t physicalTable, uint32 attributes) 750 { 751 *entry = (physicalTable & X86_PAE_PDE_ADDRESS_MASK) 752 | X86_PAE_PDE_PRESENT 753 | X86_PAE_PDE_WRITABLE 754 | X86_PAE_PDE_USER; 755 // TODO: We ignore the attributes of the page table -- for compatibility 756 // with BeOS we allow having user accessible areas in the kernel address 757 // space. This is currently being used by some drivers, mainly for the 758 // frame buffer. Our current real time data implementation makes use of 759 // this fact, too. 760 // We might want to get rid of this possibility one day, especially if 761 // we intend to port it to a platform that does not support this. 762 } 763 764 765 /*static*/ void 766 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry, 767 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 768 bool globalPage) 769 { 770 pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 771 | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0) 772 | MemoryTypeToPageTableEntryFlags(memoryType); 773 774 // if the page is user accessible, it's automatically 775 // accessible in kernel space, too (but with the same 776 // protection) 777 if ((attributes & B_USER_PROTECTION) != 0) { 778 page |= X86_PAE_PTE_USER; 779 if ((attributes & B_WRITE_AREA) != 0) 780 page |= X86_PAE_PTE_WRITABLE; 781 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 782 page |= X86_PAE_PTE_WRITABLE; 783 784 // put it in the page table 785 *(volatile pae_page_table_entry*)entry = page; 786 } 787 788 789 void* 790 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress, 791 void*& _handle) 792 { 793 // get a free page 794 MutexLocker locker(fFreePagesLock); 795 vm_page* page; 796 if (fFreePages != NULL) { 797 page = fFreePages; 798 fFreePages = page->cache_next; 799 fFreePagesCount--; 800 locker.Unlock(); 801 } else { 802 // no pages -- allocate one 803 locker.Unlock(); 804 805 physical_address_restrictions restrictions = {}; 806 restrictions.high_address = 0x100000000LL; 807 page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions, 808 VM_PRIORITY_SYSTEM); 809 if (page == NULL) 810 return NULL; 811 812 DEBUG_PAGE_ACCESS_END(page); 813 } 814 815 // map the page 816 phys_addr_t physicalAddress 817 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 818 addr_t virtualAddress; 819 if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle) 820 != B_OK) { 821 // mapping failed -- free page 822 locker.Lock(); 823 page->cache_next = fFreePages; 824 fFreePages = page; 825 fFreePagesCount++; 826 return NULL; 827 } 828 829 _physicalAddress = physicalAddress; 830 return (void*)virtualAddress; 831 } 832 833 834 void 835 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress, 836 void* handle) 837 { 838 // unmap the page 839 fPhysicalPageMapper->PutPage((addr_t)address, handle); 840 841 // free it 842 vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); 843 MutexLocker locker(fFreePagesLock); 844 if (fFreePagesCount < kMaxFree32BitPagesCount) { 845 // cache not full yet -- cache it 846 page->cache_next = fFreePages; 847 fFreePages = page; 848 fFreePagesCount++; 849 } else { 850 // cache full -- free it 851 locker.Unlock(); 852 DEBUG_PAGE_ACCESS_START(page); 853 vm_page_free(NULL, page); 854 } 855 } 856 857 858 bool 859 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress, 860 phys_addr_t* _physicalAddress) 861 { 862 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 863 fKernelVirtualPageDirs, virtualAddress); 864 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 865 // no pagetable here 866 return false; 867 } 868 869 pae_page_table_entry* entry = _EarlyGetPageTable( 870 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK) 871 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 872 if ((*entry & X86_PAE_PTE_PRESENT) == 0) { 873 // page mapping not valid 874 return false; 875 } 876 877 *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK; 878 return true; 879 } 880 881 882 pae_page_table_entry* 883 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address) 884 { 885 *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK) 886 | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL; 887 888 invalidate_TLB(fFreeVirtualSlot); 889 890 return (pae_page_table_entry*)fFreeVirtualSlot; 891 } 892 893 894 #endif // B_HAIKU_PHYSICAL_BITS == 64 895