1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "paging/pae/X86PagingMethodPAE.h" 12 13 #include <stdlib.h> 14 #include <string.h> 15 16 #include <AutoDeleter.h> 17 18 #include <boot/kernel_args.h> 19 #include <util/AutoLock.h> 20 #include <vm/vm.h> 21 #include <vm/vm_page.h> 22 #include <vm/VMAddressSpace.h> 23 24 #include "paging/32bit/paging.h" 25 #include "paging/32bit/X86PagingMethod32Bit.h" 26 #include "paging/pae/X86PagingStructuresPAE.h" 27 #include "paging/pae/X86VMTranslationMapPAE.h" 28 #include "paging/x86_physical_page_mapper.h" 29 #include "paging/x86_physical_page_mapper_large_memory.h" 30 31 32 //#define TRACE_X86_PAGING_METHOD_PAE 33 #ifdef TRACE_X86_PAGING_METHOD_PAE 34 # define TRACE(x...) dprintf(x) 35 #else 36 # define TRACE(x...) ; 37 #endif 38 39 40 #if B_HAIKU_PHYSICAL_BITS == 64 41 42 43 using X86LargePhysicalPageMapper::PhysicalPageSlot; 44 45 46 // number of 32 bit pages that will be cached 47 static const page_num_t kMaxFree32BitPagesCount = 32; 48 49 50 // #pragma mark - ToPAESwitcher 51 52 53 struct X86PagingMethodPAE::ToPAESwitcher { 54 ToPAESwitcher(kernel_args* args) 55 : 56 fKernelArgs(args) 57 { 58 // page hole set up in the boot loader 59 fPageHole = (page_table_entry*) 60 (addr_t)fKernelArgs->arch_args.page_hole; 61 62 // calculate where the page dir would be 63 fPageHolePageDir = (page_directory_entry*) 64 (((addr_t)fKernelArgs->arch_args.page_hole) 65 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); 66 67 fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir; 68 69 TRACE("page hole: %p\n", fPageHole); 70 TRACE("page dir: %p (physical: %#" B_PRIxPHYSADDR ")\n", 71 fPageHolePageDir, fPhysicalPageDir); 72 } 73 74 void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT, 75 phys_addr_t& _physicalPDPT, void*& _pageStructures, 76 size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs, 77 phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot, 78 pae_page_table_entry*& _freeVirtualSlotPTE) 79 { 80 // count the page tables we have to translate 81 uint32 pageTableCount = 0; 82 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 83 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 84 page_directory_entry entry = fPageHolePageDir[i]; 85 if ((entry & X86_PDE_PRESENT) != 0) 86 pageTableCount++; 87 } 88 89 TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount); 90 91 // The pages we need to allocate to do our job: 92 // + 1 page dir pointer table 93 // + 4 page dirs 94 // + 2 * page tables (each has 512 instead of 1024 entries) 95 // + 1 page for the free virtual slot (no physical page needed) 96 uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1; 97 98 // We need additional PAE page tables for the new pages we're going to 99 // allocate: Two tables for every 1024 pages to map, i.e. 2 additional 100 // pages for every 1022 pages we want to allocate. We also need 32 bit 101 // page tables, but we don't need additional virtual space for them, 102 // since we can access then via the page hole. 103 pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2; 104 105 TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded); 106 107 // allocate the pages we need 108 _AllocateNeededPages(pagesNeeded); 109 110 // prepare the page directory pointer table 111 phys_addr_t physicalPDPT = 0; 112 pae_page_directory_pointer_table_entry* pdpt 113 = (pae_page_directory_pointer_table_entry*)_NextPage(true, 114 physicalPDPT); 115 116 for (int32 i = 0; i < 4; i++) { 117 fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true, 118 fPhysicalPageDirs[i]); 119 120 pdpt[i] = X86_PAE_PDPTE_PRESENT 121 | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK); 122 } 123 124 // Since we have to enable PAE in two steps -- setting cr3 to the PDPT 125 // and setting the cr4 PAE bit -- we copy the kernel page dir entries to 126 // the PDPT page, so after setting cr3, we continue to have working 127 // kernel mappings. This requires that the PDPTE registers and the 128 // page dir entries don't interect, obviously. 129 ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry) 130 <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry)); 131 132 // translate the page tables 133 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 134 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 135 if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) { 136 // two PAE page tables per 32 bit page table 137 _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE); 138 _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE); 139 140 // copy the page directory entry to the PDPT page 141 ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i]; 142 } 143 } 144 145 TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n", 146 fFreeVirtualSlot, fFreeVirtualSlotPTE); 147 148 // enable PAE on all CPUs 149 call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT); 150 151 // if availalbe enable NX-bit (No eXecute) 152 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 153 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER) 154 | IA32_MSR_EFER_NX); 155 } 156 157 // set return values 158 _virtualPDPT = pdpt; 159 _physicalPDPT = physicalPDPT; 160 _pageStructures = fAllocatedPages; 161 _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE; 162 memcpy(pageDirs, fPageDirs, sizeof(fPageDirs)); 163 memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs)); 164 165 _freeVirtualSlot = fFreeVirtualSlot; 166 _freeVirtualSlotPTE = fFreeVirtualSlotPTE; 167 } 168 169 private: 170 static void _EnablePAE(void* physicalPDPT, int cpu) 171 { 172 x86_write_cr3((addr_t)physicalPDPT); 173 x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES); 174 } 175 176 void _TranslatePageTable(addr_t virtualBase) 177 { 178 page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE]; 179 180 // allocate a PAE page table 181 phys_addr_t physicalTable = 0; 182 pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false, 183 physicalTable); 184 185 // enter it into the page dir 186 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 187 fPageDirs, virtualBase); 188 PutPageTableInPageDir(pageDirEntry, physicalTable, 189 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 190 191 pae_page_table_entry* paeEntry = paeTable; 192 for (uint32 i = 0; i < kPAEPageTableEntryCount; 193 i++, entry++, paeEntry++) { 194 if ((*entry & X86_PTE_PRESENT) != 0) { 195 // Note, we use the fact that the PAE flags are defined to the 196 // same values. 197 *paeEntry = *entry & (X86_PTE_PRESENT 198 | X86_PTE_WRITABLE 199 | X86_PTE_USER 200 | X86_PTE_WRITE_THROUGH 201 | X86_PTE_CACHING_DISABLED 202 | X86_PTE_GLOBAL 203 | X86_PTE_ADDRESS_MASK); 204 } else 205 *paeEntry = 0; 206 } 207 208 if (fFreeVirtualSlot / kPAEPageTableRange 209 == virtualBase / kPAEPageTableRange) { 210 fFreeVirtualSlotPTE = paeTable 211 + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount; 212 } 213 } 214 215 void _AllocateNeededPages(uint32 pagesNeeded) 216 { 217 size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE; 218 addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0, 219 kPageTableAlignment); 220 if (virtualBase == 0) { 221 panic("Failed to reserve virtual address space for the switch to " 222 "PAE!"); 223 } 224 225 TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n", 226 virtualBase, virtualSize); 227 228 // allocate pages for the 32 bit page tables and prepare the tables 229 uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024; 230 for (uint32 i = 0; i < oldPageTableCount; i++) { 231 // allocate a page 232 phys_addr_t physicalTable =_AllocatePage32Bit(); 233 234 // put the page into the page dir 235 page_directory_entry* entry = &fPageHolePageDir[ 236 virtualBase / B_PAGE_SIZE / 1024 + i]; 237 X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable, 238 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 239 240 // clear the table 241 memset((void*)((addr_t)fPageHole 242 + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE), 243 0, B_PAGE_SIZE); 244 } 245 246 // We don't need a physical page for the free virtual slot. 247 pagesNeeded--; 248 249 // allocate and map the pages we need 250 for (uint32 i = 0; i < pagesNeeded; i++) { 251 // allocate a page 252 phys_addr_t physicalAddress =_AllocatePage32Bit(); 253 254 // put the page into the page table 255 page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i; 256 X86PagingMethod32Bit::PutPageTableEntryInTable(entry, 257 physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, 258 true); 259 260 // Write the page's physical address into the page itself, so we 261 // don't need to look it up later. 262 *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress; 263 } 264 265 fAllocatedPages = (uint8*)virtualBase; 266 fAllocatedPagesCount = pagesNeeded; 267 fUsedPagesCount = 0; 268 fFreeVirtualSlot 269 = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE); 270 } 271 272 phys_addr_t _AllocatePage() 273 { 274 phys_addr_t physicalAddress 275 = (phys_addr_t)vm_allocate_early_physical_page(fKernelArgs) 276 * B_PAGE_SIZE; 277 if (physicalAddress == 0) 278 panic("Failed to allocate page for the switch to PAE!"); 279 return physicalAddress; 280 } 281 282 phys_addr_t _AllocatePage32Bit() 283 { 284 phys_addr_t physicalAddress = _AllocatePage(); 285 if (physicalAddress > 0xffffffff) { 286 panic("Failed to allocate 32 bit addressable page for the switch " 287 "to PAE!"); 288 return 0; 289 } 290 return physicalAddress; 291 } 292 293 void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress) 294 { 295 if (fUsedPagesCount >= fAllocatedPagesCount) { 296 panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more " 297 "allocated pages!"); 298 return NULL; 299 } 300 301 void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE; 302 _physicalAddress = *((phys_addr_t*)page); 303 304 if (clearPage) 305 memset(page, 0, B_PAGE_SIZE); 306 307 return page; 308 } 309 310 private: 311 kernel_args* fKernelArgs; 312 page_table_entry* fPageHole; 313 page_directory_entry* fPageHolePageDir; 314 phys_addr_t fPhysicalPageDir; 315 uint8* fAllocatedPages; 316 uint32 fAllocatedPagesCount; 317 uint32 fUsedPagesCount; 318 addr_t fFreeVirtualSlot; 319 pae_page_table_entry* fFreeVirtualSlotPTE; 320 pae_page_directory_entry* fPageDirs[4]; 321 phys_addr_t fPhysicalPageDirs[4]; 322 }; 323 324 325 // #pragma mark - PhysicalPageSlotPool 326 327 328 struct X86PagingMethodPAE::PhysicalPageSlotPool 329 : X86LargePhysicalPageMapper::PhysicalPageSlotPool { 330 public: 331 virtual ~PhysicalPageSlotPool(); 332 333 status_t InitInitial(X86PagingMethodPAE* method, 334 kernel_args* args); 335 status_t InitInitialPostArea(kernel_args* args); 336 337 void Init(area_id dataArea, 338 pae_page_table_entry* pageTable, 339 area_id virtualArea, addr_t virtualBase); 340 341 virtual status_t AllocatePool( 342 X86LargePhysicalPageMapper 343 ::PhysicalPageSlotPool*& _pool); 344 virtual void Map(phys_addr_t physicalAddress, 345 addr_t virtualAddress); 346 347 public: 348 static PhysicalPageSlotPool sInitialPhysicalPagePool; 349 350 private: 351 area_id fDataArea; 352 area_id fVirtualArea; 353 addr_t fVirtualBase; 354 pae_page_table_entry* fPageTable; 355 }; 356 357 358 X86PagingMethodPAE::PhysicalPageSlotPool 359 X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool; 360 361 362 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool() 363 { 364 } 365 366 367 status_t 368 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial( 369 X86PagingMethodPAE* method, kernel_args* args) 370 { 371 // allocate a virtual address range for the pages to be mapped into 372 addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0, 373 kPAEPageTableRange); 374 if (virtualBase == 0) { 375 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve " 376 "physical page pool space in virtual address space!"); 377 return B_ERROR; 378 } 379 380 // allocate memory for the page table and data 381 size_t areaSize = B_PAGE_SIZE 382 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 383 pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early( 384 args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0); 385 386 // clear the page table and put it in the page dir 387 memset(pageTable, 0, B_PAGE_SIZE); 388 389 phys_addr_t physicalTable = 0; 390 method->_EarlyQuery((addr_t)pageTable, &physicalTable); 391 392 pae_page_directory_entry* entry = PageDirEntryForAddress( 393 method->KernelVirtualPageDirs(), virtualBase); 394 PutPageTableInPageDir(entry, physicalTable, 395 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 396 397 // init the pool structure and add the initial pool 398 Init(-1, pageTable, -1, (addr_t)virtualBase); 399 400 return B_OK; 401 } 402 403 404 status_t 405 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea( 406 kernel_args* args) 407 { 408 // create an area for the (already allocated) data 409 size_t areaSize = B_PAGE_SIZE 410 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 411 void* temp = fPageTable; 412 area_id area = create_area("physical page pool", &temp, 413 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED, 414 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 415 if (area < B_OK) { 416 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 417 "create area for physical page pool."); 418 return area; 419 } 420 fDataArea = area; 421 422 // create an area for the virtual address space 423 temp = (void*)fVirtualBase; 424 area = vm_create_null_area(VMAddressSpace::KernelID(), 425 "physical page pool space", &temp, B_EXACT_ADDRESS, 426 kPAEPageTableRange, 0); 427 if (area < B_OK) { 428 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 429 "create area for physical page pool space."); 430 return area; 431 } 432 fVirtualArea = area; 433 434 return B_OK; 435 } 436 437 438 void 439 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea, 440 pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase) 441 { 442 fDataArea = dataArea; 443 fVirtualArea = virtualArea; 444 fVirtualBase = virtualBase; 445 fPageTable = pageTable; 446 447 // init slot list 448 fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount); 449 addr_t slotAddress = virtualBase; 450 for (uint32 i = 0; i < kPAEPageTableEntryCount; 451 i++, slotAddress += B_PAGE_SIZE) { 452 PhysicalPageSlot* slot = &fSlots[i]; 453 slot->next = slot + 1; 454 slot->pool = this; 455 slot->address = slotAddress; 456 } 457 458 fSlots[kPAEPageTableEntryCount - 1].next = NULL; 459 // terminate list 460 } 461 462 463 void 464 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress, 465 addr_t virtualAddress) 466 { 467 pae_page_table_entry& pte = fPageTable[ 468 (virtualAddress - fVirtualBase) / B_PAGE_SIZE]; 469 pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 470 | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT; 471 472 invalidate_TLB(virtualAddress); 473 } 474 475 476 status_t 477 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool( 478 X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool) 479 { 480 // create the pool structure 481 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool; 482 if (pool == NULL) 483 return B_NO_MEMORY; 484 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool); 485 486 // create an area that can contain the page table and the slot 487 // structures 488 size_t areaSize = B_PAGE_SIZE 489 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 490 void* data; 491 virtual_address_restrictions virtualRestrictions = {}; 492 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 493 physical_address_restrictions physicalRestrictions = {}; 494 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool", 495 PAGE_ALIGN(areaSize), B_FULL_LOCK, 496 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 497 &virtualRestrictions, &physicalRestrictions, &data); 498 if (dataArea < 0) 499 return dataArea; 500 501 // create the null area for the virtual address space 502 void* virtualBase; 503 area_id virtualArea = vm_create_null_area( 504 VMAddressSpace::KernelID(), "physical page pool space", 505 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange, 506 CREATE_AREA_PRIORITY_VIP); 507 if (virtualArea < 0) { 508 delete_area(dataArea); 509 return virtualArea; 510 } 511 512 // prepare the page table 513 memset(data, 0, B_PAGE_SIZE); 514 515 // get the page table's physical address 516 phys_addr_t physicalTable; 517 X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>( 518 VMAddressSpace::Kernel()->TranslationMap()); 519 uint32 dummyFlags; 520 cpu_status state = disable_interrupts(); 521 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags); 522 restore_interrupts(state); 523 524 // put the page table into the page directory 525 pae_page_directory_entry* pageDirEntry 526 = X86PagingMethodPAE::PageDirEntryForAddress( 527 map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase); 528 PutPageTableInPageDir(pageDirEntry, physicalTable, 529 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 530 531 // init the pool structure 532 pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea, 533 (addr_t)virtualBase); 534 poolDeleter.Detach(); 535 _pool = pool; 536 return B_OK; 537 } 538 539 540 // #pragma mark - X86PagingMethodPAE 541 542 543 X86PagingMethodPAE::X86PagingMethodPAE() 544 : 545 fPhysicalPageMapper(NULL), 546 fKernelPhysicalPageMapper(NULL), 547 fFreePages(NULL), 548 fFreePagesCount(0) 549 { 550 mutex_init(&fFreePagesLock, "x86 PAE free pages"); 551 } 552 553 554 X86PagingMethodPAE::~X86PagingMethodPAE() 555 { 556 } 557 558 559 status_t 560 X86PagingMethodPAE::Init(kernel_args* args, 561 VMPhysicalPageMapper** _physicalPageMapper) 562 { 563 // switch to PAE 564 ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable, 565 fKernelPhysicalPageDirPointerTable, fEarlyPageStructures, 566 fEarlyPageStructuresSize, fKernelVirtualPageDirs, 567 fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE); 568 569 // create the initial pool for the physical page mapper 570 PhysicalPageSlotPool* pool 571 = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) 572 PhysicalPageSlotPool; 573 status_t error = pool->InitInitial(this, args); 574 if (error != B_OK) { 575 panic("X86PagingMethodPAE::Init(): Failed to create initial pool " 576 "for physical page mapper!"); 577 return error; 578 } 579 580 // create physical page mapper 581 large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper, 582 fKernelPhysicalPageMapper); 583 584 *_physicalPageMapper = fPhysicalPageMapper; 585 return B_OK; 586 } 587 588 589 status_t 590 X86PagingMethodPAE::InitPostArea(kernel_args* args) 591 { 592 // wrap the kernel paging structures in an area 593 area_id area = create_area("kernel paging structs", &fEarlyPageStructures, 594 B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED, 595 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 596 if (area < B_OK) 597 return area; 598 599 // let the initial page pool create areas for its structures 600 status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool 601 .InitInitialPostArea(args); 602 if (error != B_OK) 603 return error; 604 605 // The early physical page mapping mechanism is no longer needed. Unmap the 606 // slot. 607 *fFreeVirtualSlotPTE = 0; 608 invalidate_TLB(fFreeVirtualSlot); 609 610 fFreeVirtualSlotPTE = NULL; 611 fFreeVirtualSlot = 0; 612 613 return B_OK; 614 } 615 616 617 status_t 618 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 619 { 620 X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE; 621 if (map == NULL) 622 return B_NO_MEMORY; 623 624 status_t error = map->Init(kernel); 625 if (error != B_OK) { 626 delete map; 627 return error; 628 } 629 630 *_map = map; 631 return B_OK; 632 } 633 634 635 status_t 636 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress, 637 phys_addr_t physicalAddress, uint8 attributes, 638 page_num_t (*get_free_page)(kernel_args*)) 639 { 640 // check to see if a page table exists for this range 641 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 642 fKernelVirtualPageDirs, virtualAddress); 643 pae_page_table_entry* pageTable; 644 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 645 // we need to allocate a page table 646 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE; 647 648 TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for " 649 "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable); 650 651 // put it in the page dir 652 PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes); 653 654 // zero it out 655 pageTable = _EarlyGetPageTable(physicalPageTable); 656 memset(pageTable, 0, B_PAGE_SIZE); 657 } else { 658 // table already exists -- map it 659 pageTable = _EarlyGetPageTable( 660 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 661 } 662 663 pae_page_table_entry* entry = pageTable 664 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 665 666 ASSERT_PRINT( 667 (*entry & X86_PAE_PTE_PRESENT) == 0, 668 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64 669 ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry); 670 671 // now, fill in the pentry 672 PutPageTableEntryInTable(entry, physicalAddress, attributes, 0, 673 IS_KERNEL_ADDRESS(virtualAddress)); 674 675 return B_OK; 676 } 677 678 679 bool 680 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress, 681 uint32 protection) 682 { 683 // we can't check much without the physical page mapper 684 if (fPhysicalPageMapper == NULL) 685 return false; 686 687 // We only trust the kernel team's page directories. So switch to the 688 // kernel PDPT first. Always set it to make sure the TLBs don't contain 689 // obsolete data. 690 uint32 physicalPDPT = x86_read_cr3(); 691 x86_write_cr3(fKernelPhysicalPageDirPointerTable); 692 693 // get the PDPT entry for the address 694 pae_page_directory_pointer_table_entry pdptEntry = 0; 695 if (physicalPDPT == fKernelPhysicalPageDirPointerTable) { 696 pdptEntry = fKernelVirtualPageDirPointerTable[ 697 virtualAddress / kPAEPageDirRange]; 698 } else { 699 // map the original PDPT and get the entry 700 void* handle; 701 addr_t virtualPDPT; 702 status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT, 703 &virtualPDPT, &handle); 704 if (error == B_OK) { 705 pdptEntry = ((pae_page_directory_pointer_table_entry*) 706 virtualPDPT)[virtualAddress / kPAEPageDirRange]; 707 fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle); 708 } 709 } 710 711 // map the page dir and get the entry 712 pae_page_directory_entry pageDirEntry = 0; 713 if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) { 714 void* handle; 715 addr_t virtualPageDir; 716 status_t error = fPhysicalPageMapper->GetPageDebug( 717 pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle); 718 if (error == B_OK) { 719 pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[ 720 virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount]; 721 fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle); 722 } 723 } 724 725 // map the page table and get the entry 726 pae_page_table_entry pageTableEntry = 0; 727 if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) { 728 void* handle; 729 addr_t virtualPageTable; 730 status_t error = fPhysicalPageMapper->GetPageDebug( 731 pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable, 732 &handle); 733 if (error == B_OK) { 734 pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[ 735 virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount]; 736 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); 737 } 738 } 739 740 // switch back to the original page directory 741 if (physicalPDPT != fKernelPhysicalPageDirPointerTable) 742 x86_write_cr3(physicalPDPT); 743 744 if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0) 745 return false; 746 747 // present means kernel-readable, so check for writable 748 return (protection & B_KERNEL_WRITE_AREA) == 0 749 || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0; 750 } 751 752 753 /*static*/ void 754 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry, 755 phys_addr_t physicalTable, uint32 attributes) 756 { 757 *entry = (physicalTable & X86_PAE_PDE_ADDRESS_MASK) 758 | X86_PAE_PDE_PRESENT 759 | X86_PAE_PDE_WRITABLE 760 | X86_PAE_PDE_USER; 761 // TODO: We ignore the attributes of the page table -- for compatibility 762 // with BeOS we allow having user accessible areas in the kernel address 763 // space. This is currently being used by some drivers, mainly for the 764 // frame buffer. Our current real time data implementation makes use of 765 // this fact, too. 766 // We might want to get rid of this possibility one day, especially if 767 // we intend to port it to a platform that does not support this. 768 } 769 770 771 /*static*/ void 772 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry, 773 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 774 bool globalPage) 775 { 776 pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 777 | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0) 778 | MemoryTypeToPageTableEntryFlags(memoryType); 779 780 // if the page is user accessible, it's automatically 781 // accessible in kernel space, too (but with the same 782 // protection) 783 if ((attributes & B_USER_PROTECTION) != 0) { 784 page |= X86_PAE_PTE_USER; 785 if ((attributes & B_WRITE_AREA) != 0) 786 page |= X86_PAE_PTE_WRITABLE; 787 if ((attributes & B_EXECUTE_AREA) == 0 788 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 789 page |= X86_PAE_PTE_NOT_EXECUTABLE; 790 } 791 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 792 page |= X86_PAE_PTE_WRITABLE; 793 794 // put it in the page table 795 *(volatile pae_page_table_entry*)entry = page; 796 } 797 798 799 void* 800 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress, 801 void*& _handle) 802 { 803 // get a free page 804 MutexLocker locker(fFreePagesLock); 805 vm_page* page; 806 if (fFreePages != NULL) { 807 page = fFreePages; 808 fFreePages = page->cache_next; 809 fFreePagesCount--; 810 locker.Unlock(); 811 } else { 812 // no pages -- allocate one 813 locker.Unlock(); 814 815 physical_address_restrictions restrictions = {}; 816 restrictions.high_address = 0x100000000LL; 817 page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions, 818 VM_PRIORITY_SYSTEM); 819 if (page == NULL) 820 return NULL; 821 822 DEBUG_PAGE_ACCESS_END(page); 823 } 824 825 // map the page 826 phys_addr_t physicalAddress 827 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 828 addr_t virtualAddress; 829 if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle) 830 != B_OK) { 831 // mapping failed -- free page 832 locker.Lock(); 833 page->cache_next = fFreePages; 834 fFreePages = page; 835 fFreePagesCount++; 836 return NULL; 837 } 838 839 _physicalAddress = physicalAddress; 840 return (void*)virtualAddress; 841 } 842 843 844 void 845 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress, 846 void* handle) 847 { 848 // unmap the page 849 fPhysicalPageMapper->PutPage((addr_t)address, handle); 850 851 // free it 852 vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); 853 MutexLocker locker(fFreePagesLock); 854 if (fFreePagesCount < kMaxFree32BitPagesCount) { 855 // cache not full yet -- cache it 856 page->cache_next = fFreePages; 857 fFreePages = page; 858 fFreePagesCount++; 859 } else { 860 // cache full -- free it 861 locker.Unlock(); 862 DEBUG_PAGE_ACCESS_START(page); 863 vm_page_free(NULL, page); 864 } 865 } 866 867 868 bool 869 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress, 870 phys_addr_t* _physicalAddress) 871 { 872 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 873 fKernelVirtualPageDirs, virtualAddress); 874 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 875 // no pagetable here 876 return false; 877 } 878 879 pae_page_table_entry* entry = _EarlyGetPageTable( 880 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK) 881 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 882 if ((*entry & X86_PAE_PTE_PRESENT) == 0) { 883 // page mapping not valid 884 return false; 885 } 886 887 *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK; 888 return true; 889 } 890 891 892 pae_page_table_entry* 893 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address) 894 { 895 *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK) 896 | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL; 897 898 invalidate_TLB(fFreeVirtualSlot); 899 900 return (pae_page_table_entry*)fFreeVirtualSlot; 901 } 902 903 904 #endif // B_HAIKU_PHYSICAL_BITS == 64 905