1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "paging/pae/X86PagingMethodPAE.h" 12 13 #include <stdlib.h> 14 #include <string.h> 15 16 #include <AutoDeleter.h> 17 18 #include <arch/smp.h> 19 #include <boot/kernel_args.h> 20 #include <util/AutoLock.h> 21 #include <vm/vm.h> 22 #include <vm/vm_page.h> 23 #include <vm/VMAddressSpace.h> 24 25 #include "paging/32bit/paging.h" 26 #include "paging/32bit/X86PagingMethod32Bit.h" 27 #include "paging/pae/X86PagingStructuresPAE.h" 28 #include "paging/pae/X86VMTranslationMapPAE.h" 29 #include "paging/x86_physical_page_mapper.h" 30 #include "paging/x86_physical_page_mapper_large_memory.h" 31 32 33 //#define TRACE_X86_PAGING_METHOD_PAE 34 #ifdef TRACE_X86_PAGING_METHOD_PAE 35 # define TRACE(x...) dprintf(x) 36 #else 37 # define TRACE(x...) ; 38 #endif 39 40 41 #if B_HAIKU_PHYSICAL_BITS == 64 42 43 44 #define MAX_INITIAL_POOLS \ 45 (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, \ 46 kPAEPageTableEntryCount) \ 47 / kPAEPageTableEntryCount) 48 49 50 using X86LargePhysicalPageMapper::PhysicalPageSlot; 51 52 53 // number of 32 bit pages that will be cached 54 static const page_num_t kMaxFree32BitPagesCount = 32; 55 56 57 // #pragma mark - ToPAESwitcher 58 59 60 struct X86PagingMethodPAE::ToPAESwitcher { 61 ToPAESwitcher(kernel_args* args) 62 : 63 fKernelArgs(args) 64 { 65 // page hole set up in the boot loader 66 fPageHole = (page_table_entry*) 67 (addr_t)fKernelArgs->arch_args.page_hole; 68 69 // calculate where the page dir would be 70 fPageHolePageDir = (page_directory_entry*) 71 (((addr_t)fKernelArgs->arch_args.page_hole) 72 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); 73 74 fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir; 75 76 TRACE("page hole: %p\n", fPageHole); 77 TRACE("page dir: %p (physical: %#" B_PRIxPHYSADDR ")\n", 78 fPageHolePageDir, fPhysicalPageDir); 79 } 80 81 void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT, 82 phys_addr_t& _physicalPDPT, void*& _pageStructures, 83 size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs, 84 phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot, 85 pae_page_table_entry*& _freeVirtualSlotPTE) 86 { 87 // count the page tables we have to translate 88 uint32 pageTableCount = 0; 89 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 90 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 91 page_directory_entry entry = fPageHolePageDir[i]; 92 if ((entry & X86_PDE_PRESENT) != 0) 93 pageTableCount++; 94 } 95 96 TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount); 97 98 // The pages we need to allocate to do our job: 99 // + 1 page dir pointer table 100 // + 4 page dirs 101 // + 2 * page tables (each has 512 instead of 1024 entries) 102 // + 1 page for the free virtual slot (no physical page needed) 103 uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1; 104 105 // We need additional PAE page tables for the new pages we're going to 106 // allocate: Two tables for every 1024 pages to map, i.e. 2 additional 107 // pages for every 1022 pages we want to allocate. We also need 32 bit 108 // page tables, but we don't need additional virtual space for them, 109 // since we can access then via the page hole. 110 pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2; 111 112 TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded); 113 114 // allocate the pages we need 115 _AllocateNeededPages(pagesNeeded); 116 117 // prepare the page directory pointer table 118 phys_addr_t physicalPDPT = 0; 119 pae_page_directory_pointer_table_entry* pdpt 120 = (pae_page_directory_pointer_table_entry*)_NextPage(true, 121 physicalPDPT); 122 123 for (int32 i = 0; i < 4; i++) { 124 fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true, 125 fPhysicalPageDirs[i]); 126 127 pdpt[i] = X86_PAE_PDPTE_PRESENT 128 | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK); 129 } 130 131 // Since we have to enable PAE in two steps -- setting cr3 to the PDPT 132 // and setting the cr4 PAE bit -- we copy the kernel page dir entries to 133 // the PDPT page, so after setting cr3, we continue to have working 134 // kernel mappings. This requires that the PDPTE registers and the 135 // page dir entries don't interect, obviously. 136 ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry) 137 <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry)); 138 139 // translate the page tables 140 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 141 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 142 if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) { 143 // two PAE page tables per 32 bit page table 144 _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE); 145 _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE); 146 147 // copy the page directory entry to the PDPT page 148 ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i]; 149 } 150 } 151 152 TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n", 153 fFreeVirtualSlot, fFreeVirtualSlotPTE); 154 155 // enable PAE on all CPUs 156 call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT); 157 158 // if availalbe enable NX-bit (No eXecute) 159 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) 160 call_all_cpus_sync(&_EnableExecutionDisable, NULL); 161 162 // set return values 163 _virtualPDPT = pdpt; 164 _physicalPDPT = physicalPDPT; 165 _pageStructures = fAllocatedPages; 166 _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE; 167 memcpy(pageDirs, fPageDirs, sizeof(fPageDirs)); 168 memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs)); 169 170 _freeVirtualSlot = fFreeVirtualSlot; 171 _freeVirtualSlotPTE = fFreeVirtualSlotPTE; 172 } 173 174 private: 175 static void _EnablePAE(void* physicalPDPT, int cpu) 176 { 177 x86_write_cr3((addr_t)physicalPDPT); 178 x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES); 179 } 180 181 static void _EnableExecutionDisable(void* dummy, int cpu) 182 { 183 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER) 184 | IA32_MSR_EFER_NX); 185 } 186 187 void _TranslatePageTable(addr_t virtualBase) 188 { 189 page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE]; 190 191 // allocate a PAE page table 192 phys_addr_t physicalTable = 0; 193 pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false, 194 physicalTable); 195 196 // enter it into the page dir 197 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 198 fPageDirs, virtualBase); 199 PutPageTableInPageDir(pageDirEntry, physicalTable, 200 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 201 202 pae_page_table_entry* paeEntry = paeTable; 203 for (uint32 i = 0; i < kPAEPageTableEntryCount; 204 i++, entry++, paeEntry++) { 205 if ((*entry & X86_PTE_PRESENT) != 0 206 && _IsVirtualAddressAllocated(virtualBase + i * B_PAGE_SIZE)) { 207 // Note, we use the fact that the PAE flags are defined to the 208 // same values. 209 *paeEntry = *entry & (X86_PTE_PRESENT 210 | X86_PTE_WRITABLE 211 | X86_PTE_USER 212 | X86_PTE_WRITE_THROUGH 213 | X86_PTE_CACHING_DISABLED 214 | X86_PTE_GLOBAL 215 | X86_PTE_ADDRESS_MASK); 216 } else 217 *paeEntry = 0; 218 } 219 220 if (fFreeVirtualSlot / kPAEPageTableRange 221 == virtualBase / kPAEPageTableRange) { 222 fFreeVirtualSlotPTE = paeTable 223 + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount; 224 } 225 } 226 227 void _AllocateNeededPages(uint32 pagesNeeded) 228 { 229 size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE; 230 addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0, 231 kPageTableAlignment); 232 if (virtualBase == 0) { 233 panic("Failed to reserve virtual address space for the switch to " 234 "PAE!"); 235 } 236 237 TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n", 238 virtualBase, virtualSize); 239 240 // allocate pages for the 32 bit page tables and prepare the tables 241 uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024; 242 for (uint32 i = 0; i < oldPageTableCount; i++) { 243 // allocate a page 244 phys_addr_t physicalTable =_AllocatePage32Bit(); 245 246 // put the page into the page dir 247 page_directory_entry* entry = &fPageHolePageDir[ 248 virtualBase / B_PAGE_SIZE / 1024 + i]; 249 X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable, 250 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 251 252 // clear the table 253 memset((void*)((addr_t)fPageHole 254 + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE), 255 0, B_PAGE_SIZE); 256 } 257 258 // We don't need a physical page for the free virtual slot. 259 pagesNeeded--; 260 261 // allocate and map the pages we need 262 for (uint32 i = 0; i < pagesNeeded; i++) { 263 // allocate a page 264 phys_addr_t physicalAddress =_AllocatePage32Bit(); 265 266 // put the page into the page table 267 page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i; 268 X86PagingMethod32Bit::PutPageTableEntryInTable(entry, 269 physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, 270 true); 271 272 // Write the page's physical address into the page itself, so we 273 // don't need to look it up later. 274 *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress; 275 } 276 277 fAllocatedPages = (uint8*)virtualBase; 278 fAllocatedPagesCount = pagesNeeded; 279 fUsedPagesCount = 0; 280 fFreeVirtualSlot 281 = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE); 282 } 283 284 phys_addr_t _AllocatePage() 285 { 286 phys_addr_t physicalAddress 287 = (phys_addr_t)vm_allocate_early_physical_page(fKernelArgs) 288 * B_PAGE_SIZE; 289 if (physicalAddress == 0) 290 panic("Failed to allocate page for the switch to PAE!"); 291 return physicalAddress; 292 } 293 294 phys_addr_t _AllocatePage32Bit() 295 { 296 phys_addr_t physicalAddress = _AllocatePage(); 297 if (physicalAddress > 0xffffffff) { 298 panic("Failed to allocate 32 bit addressable page for the switch " 299 "to PAE!"); 300 return 0; 301 } 302 return physicalAddress; 303 } 304 305 void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress) 306 { 307 if (fUsedPagesCount >= fAllocatedPagesCount) { 308 panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more " 309 "allocated pages!"); 310 return NULL; 311 } 312 313 void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE; 314 _physicalAddress = *((phys_addr_t*)page); 315 316 if (clearPage) 317 memset(page, 0, B_PAGE_SIZE); 318 319 return page; 320 } 321 322 bool _IsVirtualAddressAllocated(addr_t address) const 323 { 324 for (uint32 i = 0; i < fKernelArgs->num_virtual_allocated_ranges; i++) { 325 addr_t start = fKernelArgs->virtual_allocated_range[i].start; 326 addr_t end = start + fKernelArgs->virtual_allocated_range[i].size; 327 if (address < start) 328 return false; 329 if (address <= end - 1) 330 return true; 331 } 332 333 return false; 334 } 335 336 private: 337 kernel_args* fKernelArgs; 338 page_table_entry* fPageHole; 339 page_directory_entry* fPageHolePageDir; 340 phys_addr_t fPhysicalPageDir; 341 uint8* fAllocatedPages; 342 uint32 fAllocatedPagesCount; 343 uint32 fUsedPagesCount; 344 addr_t fFreeVirtualSlot; 345 pae_page_table_entry* fFreeVirtualSlotPTE; 346 pae_page_directory_entry* fPageDirs[4]; 347 phys_addr_t fPhysicalPageDirs[4]; 348 }; 349 350 351 // #pragma mark - PhysicalPageSlotPool 352 353 354 struct X86PagingMethodPAE::PhysicalPageSlotPool 355 : X86LargePhysicalPageMapper::PhysicalPageSlotPool { 356 public: 357 virtual ~PhysicalPageSlotPool(); 358 359 status_t InitInitial(X86PagingMethodPAE* method, 360 kernel_args* args); 361 status_t InitInitialPostArea(kernel_args* args); 362 363 void Init(area_id dataArea, 364 pae_page_table_entry* pageTable, 365 area_id virtualArea, addr_t virtualBase); 366 367 virtual status_t AllocatePool( 368 X86LargePhysicalPageMapper 369 ::PhysicalPageSlotPool*& _pool); 370 virtual void Map(phys_addr_t physicalAddress, 371 addr_t virtualAddress); 372 373 public: 374 static PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS]; 375 376 private: 377 area_id fDataArea; 378 area_id fVirtualArea; 379 addr_t fVirtualBase; 380 pae_page_table_entry* fPageTable; 381 }; 382 383 384 X86PagingMethodPAE::PhysicalPageSlotPool 385 X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[ 386 MAX_INITIAL_POOLS]; 387 388 389 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool() 390 { 391 } 392 393 394 status_t 395 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial( 396 X86PagingMethodPAE* method, kernel_args* args) 397 { 398 // allocate a virtual address range for the pages to be mapped into 399 addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0, 400 kPAEPageTableRange); 401 if (virtualBase == 0) { 402 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve " 403 "physical page pool space in virtual address space!"); 404 return B_ERROR; 405 } 406 407 // allocate memory for the page table and data 408 size_t areaSize = B_PAGE_SIZE 409 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 410 pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early( 411 args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0); 412 if (pageTable == 0) { 413 panic("X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(): Failed " 414 "to allocate memory for page table!"); 415 return B_ERROR; 416 } 417 418 // clear the page table and put it in the page dir 419 memset(pageTable, 0, B_PAGE_SIZE); 420 421 phys_addr_t physicalTable = 0; 422 method->_EarlyQuery((addr_t)pageTable, &physicalTable); 423 424 pae_page_directory_entry* entry = PageDirEntryForAddress( 425 method->KernelVirtualPageDirs(), virtualBase); 426 PutPageTableInPageDir(entry, physicalTable, 427 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 428 429 // init the pool structure and add the initial pool 430 Init(-1, pageTable, -1, (addr_t)virtualBase); 431 432 return B_OK; 433 } 434 435 436 status_t 437 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea( 438 kernel_args* args) 439 { 440 // create an area for the (already allocated) data 441 size_t areaSize = B_PAGE_SIZE 442 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 443 void* temp = fPageTable; 444 area_id area = create_area("physical page pool", &temp, 445 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED, 446 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 447 if (area < B_OK) { 448 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 449 "create area for physical page pool."); 450 return area; 451 } 452 fDataArea = area; 453 454 // create an area for the virtual address space 455 temp = (void*)fVirtualBase; 456 area = vm_create_null_area(VMAddressSpace::KernelID(), 457 "physical page pool space", &temp, B_EXACT_ADDRESS, 458 kPAEPageTableRange, 0); 459 if (area < B_OK) { 460 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 461 "create area for physical page pool space."); 462 return area; 463 } 464 fVirtualArea = area; 465 466 return B_OK; 467 } 468 469 470 void 471 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea, 472 pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase) 473 { 474 fDataArea = dataArea; 475 fVirtualArea = virtualArea; 476 fVirtualBase = virtualBase; 477 fPageTable = pageTable; 478 479 // init slot list 480 fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount); 481 addr_t slotAddress = virtualBase; 482 for (uint32 i = 0; i < kPAEPageTableEntryCount; 483 i++, slotAddress += B_PAGE_SIZE) { 484 PhysicalPageSlot* slot = &fSlots[i]; 485 slot->next = slot + 1; 486 slot->pool = this; 487 slot->address = slotAddress; 488 } 489 490 fSlots[kPAEPageTableEntryCount - 1].next = NULL; 491 // terminate list 492 } 493 494 495 void 496 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress, 497 addr_t virtualAddress) 498 { 499 pae_page_table_entry& pte = fPageTable[ 500 (virtualAddress - fVirtualBase) / B_PAGE_SIZE]; 501 pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 502 | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT; 503 504 invalidate_TLB(virtualAddress); 505 } 506 507 508 status_t 509 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool( 510 X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool) 511 { 512 // create the pool structure 513 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool; 514 if (pool == NULL) 515 return B_NO_MEMORY; 516 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool); 517 518 // create an area that can contain the page table and the slot 519 // structures 520 size_t areaSize = B_PAGE_SIZE 521 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 522 void* data; 523 virtual_address_restrictions virtualRestrictions = {}; 524 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 525 physical_address_restrictions physicalRestrictions = {}; 526 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool", 527 PAGE_ALIGN(areaSize), B_FULL_LOCK, 528 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 529 &virtualRestrictions, &physicalRestrictions, &data); 530 if (dataArea < 0) 531 return dataArea; 532 533 // create the null area for the virtual address space 534 void* virtualBase; 535 area_id virtualArea = vm_create_null_area( 536 VMAddressSpace::KernelID(), "physical page pool space", 537 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange, 538 CREATE_AREA_PRIORITY_VIP); 539 if (virtualArea < 0) { 540 delete_area(dataArea); 541 return virtualArea; 542 } 543 544 // prepare the page table 545 memset(data, 0, B_PAGE_SIZE); 546 547 // get the page table's physical address 548 phys_addr_t physicalTable; 549 X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>( 550 VMAddressSpace::Kernel()->TranslationMap()); 551 uint32 dummyFlags; 552 cpu_status state = disable_interrupts(); 553 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags); 554 restore_interrupts(state); 555 556 // put the page table into the page directory 557 pae_page_directory_entry* pageDirEntry 558 = X86PagingMethodPAE::PageDirEntryForAddress( 559 map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase); 560 PutPageTableInPageDir(pageDirEntry, physicalTable, 561 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 562 563 // init the pool structure 564 pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea, 565 (addr_t)virtualBase); 566 poolDeleter.Detach(); 567 _pool = pool; 568 return B_OK; 569 } 570 571 572 // #pragma mark - X86PagingMethodPAE 573 574 575 X86PagingMethodPAE::X86PagingMethodPAE() 576 : 577 fPhysicalPageMapper(NULL), 578 fKernelPhysicalPageMapper(NULL), 579 fFreePages(NULL), 580 fFreePagesCount(0) 581 { 582 mutex_init(&fFreePagesLock, "x86 PAE free pages"); 583 } 584 585 586 X86PagingMethodPAE::~X86PagingMethodPAE() 587 { 588 } 589 590 591 status_t 592 X86PagingMethodPAE::Init(kernel_args* args, 593 VMPhysicalPageMapper** _physicalPageMapper) 594 { 595 // switch to PAE 596 ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable, 597 fKernelPhysicalPageDirPointerTable, fEarlyPageStructures, 598 fEarlyPageStructuresSize, fKernelVirtualPageDirs, 599 fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE); 600 601 // create the initial pools for the physical page mapper 602 int32 poolCount = _GetInitialPoolCount(); 603 PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool; 604 605 for (int32 i = 0; i < poolCount; i++) { 606 new(&pool[i]) PhysicalPageSlotPool; 607 status_t error = pool[i].InitInitial(this, args); 608 if (error != B_OK) { 609 panic("X86PagingMethodPAE::Init(): Failed to create initial pool " 610 "for physical page mapper!"); 611 return error; 612 } 613 } 614 615 // create physical page mapper 616 large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool), 617 fPhysicalPageMapper, fKernelPhysicalPageMapper); 618 619 *_physicalPageMapper = fPhysicalPageMapper; 620 return B_OK; 621 } 622 623 624 status_t 625 X86PagingMethodPAE::InitPostArea(kernel_args* args) 626 { 627 // wrap the kernel paging structures in an area 628 area_id area = create_area("kernel paging structs", &fEarlyPageStructures, 629 B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED, 630 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 631 if (area < B_OK) 632 return area; 633 634 // let the initial page pools create areas for its structures 635 int32 poolCount = _GetInitialPoolCount(); 636 for (int32 i = 0; i < poolCount; i++) { 637 status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i] 638 .InitInitialPostArea(args); 639 if (error != B_OK) 640 return error; 641 } 642 643 // The early physical page mapping mechanism is no longer needed. Unmap the 644 // slot. 645 *fFreeVirtualSlotPTE = 0; 646 invalidate_TLB(fFreeVirtualSlot); 647 648 fFreeVirtualSlotPTE = NULL; 649 fFreeVirtualSlot = 0; 650 651 return B_OK; 652 } 653 654 655 status_t 656 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 657 { 658 X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE; 659 if (map == NULL) 660 return B_NO_MEMORY; 661 662 status_t error = map->Init(kernel); 663 if (error != B_OK) { 664 delete map; 665 return error; 666 } 667 668 *_map = map; 669 return B_OK; 670 } 671 672 673 status_t 674 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress, 675 phys_addr_t physicalAddress, uint8 attributes, 676 page_num_t (*get_free_page)(kernel_args*)) 677 { 678 // check to see if a page table exists for this range 679 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 680 fKernelVirtualPageDirs, virtualAddress); 681 pae_page_table_entry* pageTable; 682 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 683 // we need to allocate a page table 684 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE; 685 686 TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for " 687 "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable); 688 689 // put it in the page dir 690 PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes); 691 692 // zero it out 693 pageTable = _EarlyGetPageTable(physicalPageTable); 694 memset(pageTable, 0, B_PAGE_SIZE); 695 } else { 696 // table already exists -- map it 697 pageTable = _EarlyGetPageTable( 698 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 699 } 700 701 pae_page_table_entry* entry = pageTable 702 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 703 704 ASSERT_PRINT( 705 (*entry & X86_PAE_PTE_PRESENT) == 0, 706 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64 707 ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry); 708 709 // now, fill in the pentry 710 PutPageTableEntryInTable(entry, physicalAddress, attributes, 0, 711 IS_KERNEL_ADDRESS(virtualAddress)); 712 713 return B_OK; 714 } 715 716 717 bool 718 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress, 719 uint32 protection) 720 { 721 // we can't check much without the physical page mapper 722 if (fPhysicalPageMapper == NULL) 723 return false; 724 725 // We only trust the kernel team's page directories. So switch to the 726 // kernel PDPT first. Always set it to make sure the TLBs don't contain 727 // obsolete data. 728 uint32 physicalPDPT = x86_read_cr3(); 729 x86_write_cr3(fKernelPhysicalPageDirPointerTable); 730 731 // get the PDPT entry for the address 732 pae_page_directory_pointer_table_entry pdptEntry = 0; 733 if (physicalPDPT == fKernelPhysicalPageDirPointerTable) { 734 pdptEntry = fKernelVirtualPageDirPointerTable[ 735 virtualAddress / kPAEPageDirRange]; 736 } else { 737 // map the original PDPT and get the entry 738 void* handle; 739 addr_t virtualPDPT; 740 status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT, 741 &virtualPDPT, &handle); 742 if (error == B_OK) { 743 pdptEntry = ((pae_page_directory_pointer_table_entry*) 744 virtualPDPT)[virtualAddress / kPAEPageDirRange]; 745 fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle); 746 } 747 } 748 749 // map the page dir and get the entry 750 pae_page_directory_entry pageDirEntry = 0; 751 if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) { 752 void* handle; 753 addr_t virtualPageDir; 754 status_t error = fPhysicalPageMapper->GetPageDebug( 755 pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle); 756 if (error == B_OK) { 757 pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[ 758 virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount]; 759 fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle); 760 } 761 } 762 763 // map the page table and get the entry 764 pae_page_table_entry pageTableEntry = 0; 765 if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) { 766 void* handle; 767 addr_t virtualPageTable; 768 status_t error = fPhysicalPageMapper->GetPageDebug( 769 pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable, 770 &handle); 771 if (error == B_OK) { 772 pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[ 773 virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount]; 774 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); 775 } 776 } 777 778 // switch back to the original page directory 779 if (physicalPDPT != fKernelPhysicalPageDirPointerTable) 780 x86_write_cr3(physicalPDPT); 781 782 if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0) 783 return false; 784 785 // present means kernel-readable, so check for writable 786 return (protection & B_KERNEL_WRITE_AREA) == 0 787 || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0; 788 } 789 790 791 /*static*/ void 792 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry, 793 phys_addr_t physicalTable, uint32 attributes) 794 { 795 *entry = (physicalTable & X86_PAE_PDE_ADDRESS_MASK) 796 | X86_PAE_PDE_PRESENT 797 | X86_PAE_PDE_WRITABLE 798 | X86_PAE_PDE_USER; 799 // TODO: We ignore the attributes of the page table -- for compatibility 800 // with BeOS we allow having user accessible areas in the kernel address 801 // space. This is currently being used by some drivers, mainly for the 802 // frame buffer. Our current real time data implementation makes use of 803 // this fact, too. 804 // We might want to get rid of this possibility one day, especially if 805 // we intend to port it to a platform that does not support this. 806 } 807 808 809 /*static*/ void 810 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry, 811 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 812 bool globalPage) 813 { 814 pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 815 | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0) 816 | MemoryTypeToPageTableEntryFlags(memoryType); 817 818 // if the page is user accessible, it's automatically 819 // accessible in kernel space, too (but with the same 820 // protection) 821 if ((attributes & B_USER_PROTECTION) != 0) { 822 page |= X86_PAE_PTE_USER; 823 if ((attributes & B_WRITE_AREA) != 0) 824 page |= X86_PAE_PTE_WRITABLE; 825 if ((attributes & B_EXECUTE_AREA) == 0 826 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 827 page |= X86_PAE_PTE_NOT_EXECUTABLE; 828 } 829 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 830 page |= X86_PAE_PTE_WRITABLE; 831 832 // put it in the page table 833 *(volatile pae_page_table_entry*)entry = page; 834 } 835 836 837 void* 838 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress, 839 void*& _handle) 840 { 841 // get a free page 842 MutexLocker locker(fFreePagesLock); 843 vm_page* page; 844 if (fFreePages != NULL) { 845 page = fFreePages; 846 fFreePages = page->cache_next; 847 fFreePagesCount--; 848 locker.Unlock(); 849 } else { 850 // no pages -- allocate one 851 locker.Unlock(); 852 853 physical_address_restrictions restrictions = {}; 854 restrictions.high_address = 0x100000000LL; 855 page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions, 856 VM_PRIORITY_SYSTEM); 857 if (page == NULL) 858 return NULL; 859 860 DEBUG_PAGE_ACCESS_END(page); 861 } 862 863 // map the page 864 phys_addr_t physicalAddress 865 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 866 addr_t virtualAddress; 867 if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle) 868 != B_OK) { 869 // mapping failed -- free page 870 locker.Lock(); 871 page->cache_next = fFreePages; 872 fFreePages = page; 873 fFreePagesCount++; 874 return NULL; 875 } 876 877 _physicalAddress = physicalAddress; 878 return (void*)virtualAddress; 879 } 880 881 882 void 883 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress, 884 void* handle) 885 { 886 // unmap the page 887 fPhysicalPageMapper->PutPage((addr_t)address, handle); 888 889 // free it 890 vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); 891 MutexLocker locker(fFreePagesLock); 892 if (fFreePagesCount < kMaxFree32BitPagesCount) { 893 // cache not full yet -- cache it 894 page->cache_next = fFreePages; 895 fFreePages = page; 896 fFreePagesCount++; 897 } else { 898 // cache full -- free it 899 locker.Unlock(); 900 DEBUG_PAGE_ACCESS_START(page); 901 vm_page_free(NULL, page); 902 } 903 } 904 905 906 inline int32 907 X86PagingMethodPAE::_GetInitialPoolCount() 908 { 909 int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU 910 + EXTRA_SLOTS; 911 return (requiredSlots + kPAEPageTableEntryCount - 1) 912 / kPAEPageTableEntryCount; 913 } 914 915 916 bool 917 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress, 918 phys_addr_t* _physicalAddress) 919 { 920 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 921 fKernelVirtualPageDirs, virtualAddress); 922 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 923 // no pagetable here 924 return false; 925 } 926 927 pae_page_table_entry* entry = _EarlyGetPageTable( 928 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK) 929 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 930 if ((*entry & X86_PAE_PTE_PRESENT) == 0) { 931 // page mapping not valid 932 return false; 933 } 934 935 *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK; 936 return true; 937 } 938 939 940 pae_page_table_entry* 941 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address) 942 { 943 *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK) 944 | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL; 945 946 invalidate_TLB(fFreeVirtualSlot); 947 948 return (pae_page_table_entry*)fFreeVirtualSlot; 949 } 950 951 952 #endif // B_HAIKU_PHYSICAL_BITS == 64 953