1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "paging/pae/X86PagingMethodPAE.h" 12 13 #include <stdlib.h> 14 #include <string.h> 15 16 #include <AutoDeleter.h> 17 18 #include <arch/smp.h> 19 #include <boot/kernel_args.h> 20 #include <util/AutoLock.h> 21 #include <vm/vm.h> 22 #include <vm/vm_page.h> 23 #include <vm/VMAddressSpace.h> 24 25 #include "paging/32bit/paging.h" 26 #include "paging/32bit/X86PagingMethod32Bit.h" 27 #include "paging/pae/X86PagingStructuresPAE.h" 28 #include "paging/pae/X86VMTranslationMapPAE.h" 29 #include "paging/x86_physical_page_mapper.h" 30 #include "paging/x86_physical_page_mapper_large_memory.h" 31 32 33 //#define TRACE_X86_PAGING_METHOD_PAE 34 #ifdef TRACE_X86_PAGING_METHOD_PAE 35 # define TRACE(x...) dprintf(x) 36 #else 37 # define TRACE(x...) ; 38 #endif 39 40 41 #if B_HAIKU_PHYSICAL_BITS == 64 42 43 44 #define MAX_INITIAL_POOLS \ 45 (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, \ 46 kPAEPageTableEntryCount) \ 47 / kPAEPageTableEntryCount) 48 49 50 using X86LargePhysicalPageMapper::PhysicalPageSlot; 51 52 53 // number of 32 bit pages that will be cached 54 static const page_num_t kMaxFree32BitPagesCount = 32; 55 56 57 // #pragma mark - ToPAESwitcher 58 59 60 struct X86PagingMethodPAE::ToPAESwitcher { 61 ToPAESwitcher(kernel_args* args) 62 : 63 fKernelArgs(args) 64 { 65 // page hole set up in the boot loader 66 fPageHole = (page_table_entry*) 67 (addr_t)fKernelArgs->arch_args.page_hole; 68 69 // calculate where the page dir would be 70 fPageHolePageDir = (page_directory_entry*) 71 (((addr_t)fKernelArgs->arch_args.page_hole) 72 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); 73 74 fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir; 75 76 TRACE("page hole: %p\n", fPageHole); 77 TRACE("page dir: %p (physical: %#" B_PRIxPHYSADDR ")\n", 78 fPageHolePageDir, fPhysicalPageDir); 79 } 80 81 void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT, 82 phys_addr_t& _physicalPDPT, void*& _pageStructures, 83 size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs, 84 phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot, 85 pae_page_table_entry*& _freeVirtualSlotPTE) 86 { 87 // count the page tables we have to translate 88 uint32 pageTableCount = 0; 89 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 90 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 91 page_directory_entry entry = fPageHolePageDir[i]; 92 if ((entry & X86_PDE_PRESENT) != 0) 93 pageTableCount++; 94 } 95 96 TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount); 97 98 // The pages we need to allocate to do our job: 99 // + 1 page dir pointer table 100 // + 4 page dirs 101 // + 2 * page tables (each has 512 instead of 1024 entries) 102 // + 1 page for the free virtual slot (no physical page needed) 103 uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1; 104 105 // We need additional PAE page tables for the new pages we're going to 106 // allocate: Two tables for every 1024 pages to map, i.e. 2 additional 107 // pages for every 1022 pages we want to allocate. We also need 32 bit 108 // page tables, but we don't need additional virtual space for them, 109 // since we can access then via the page hole. 110 pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2; 111 112 TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded); 113 114 // allocate the pages we need 115 _AllocateNeededPages(pagesNeeded); 116 117 // prepare the page directory pointer table 118 phys_addr_t physicalPDPT = 0; 119 pae_page_directory_pointer_table_entry* pdpt 120 = (pae_page_directory_pointer_table_entry*)_NextPage(true, 121 physicalPDPT); 122 123 for (int32 i = 0; i < 4; i++) { 124 fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true, 125 fPhysicalPageDirs[i]); 126 127 pdpt[i] = X86_PAE_PDPTE_PRESENT 128 | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK); 129 } 130 131 // Since we have to enable PAE in two steps -- setting cr3 to the PDPT 132 // and setting the cr4 PAE bit -- we copy the kernel page dir entries to 133 // the PDPT page, so after setting cr3, we continue to have working 134 // kernel mappings. This requires that the PDPTE registers and the 135 // page dir entries don't interect, obviously. 136 ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry) 137 <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry)); 138 139 // translate the page tables 140 for (uint32 i = FIRST_KERNEL_PGDIR_ENT; 141 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) { 142 if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) { 143 // two PAE page tables per 32 bit page table 144 _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE); 145 _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE); 146 147 // copy the page directory entry to the PDPT page 148 ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i]; 149 } 150 } 151 152 TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n", 153 fFreeVirtualSlot, fFreeVirtualSlotPTE); 154 155 // enable PAE on all CPUs 156 call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT); 157 158 // if available enable NX-bit (No eXecute) 159 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) 160 call_all_cpus_sync(&_EnableExecutionDisable, NULL); 161 162 // set return values 163 _virtualPDPT = pdpt; 164 _physicalPDPT = physicalPDPT; 165 _pageStructures = fAllocatedPages; 166 _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE; 167 memcpy(pageDirs, fPageDirs, sizeof(fPageDirs)); 168 memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs)); 169 170 _freeVirtualSlot = fFreeVirtualSlot; 171 _freeVirtualSlotPTE = fFreeVirtualSlotPTE; 172 } 173 174 private: 175 static void _EnablePAE(void* physicalPDPT, int cpu) 176 { 177 x86_write_cr3((addr_t)physicalPDPT); 178 x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES); 179 } 180 181 static void _EnableExecutionDisable(void* dummy, int cpu) 182 { 183 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER) 184 | IA32_MSR_EFER_NX); 185 } 186 187 void _TranslatePageTable(addr_t virtualBase) 188 { 189 page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE]; 190 191 // allocate a PAE page table 192 phys_addr_t physicalTable = 0; 193 pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false, 194 physicalTable); 195 196 // enter it into the page dir 197 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 198 fPageDirs, virtualBase); 199 PutPageTableInPageDir(pageDirEntry, physicalTable, 200 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 201 202 pae_page_table_entry* paeEntry = paeTable; 203 for (uint32 i = 0; i < kPAEPageTableEntryCount; 204 i++, entry++, paeEntry++) { 205 if ((*entry & X86_PTE_PRESENT) != 0 206 && _IsVirtualAddressAllocated(virtualBase + i * B_PAGE_SIZE)) { 207 // Note, we use the fact that the PAE flags are defined to the 208 // same values. 209 *paeEntry = *entry & (X86_PTE_PRESENT 210 | X86_PTE_WRITABLE 211 | X86_PTE_USER 212 | X86_PTE_WRITE_THROUGH 213 | X86_PTE_CACHING_DISABLED 214 | X86_PTE_GLOBAL 215 | X86_PTE_ADDRESS_MASK); 216 } else 217 *paeEntry = 0; 218 } 219 220 if (fFreeVirtualSlot / kPAEPageTableRange 221 == virtualBase / kPAEPageTableRange) { 222 fFreeVirtualSlotPTE = paeTable 223 + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount; 224 } 225 } 226 227 void _AllocateNeededPages(uint32 pagesNeeded) 228 { 229 size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE; 230 addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0, 231 kPageTableAlignment); 232 if (virtualBase == 0) { 233 panic("Failed to reserve virtual address space for the switch to " 234 "PAE!"); 235 } 236 237 TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n", 238 virtualBase, virtualSize); 239 240 // allocate pages for the 32 bit page tables and prepare the tables 241 uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024; 242 for (uint32 i = 0; i < oldPageTableCount; i++) { 243 // allocate a page 244 phys_addr_t physicalTable = _Allocate32BitPage(); 245 246 // put the page into the page dir 247 page_directory_entry* entry = &fPageHolePageDir[ 248 virtualBase / B_PAGE_SIZE / 1024 + i]; 249 X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable, 250 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 251 252 // clear the table 253 memset((void*)((addr_t)fPageHole 254 + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE), 255 0, B_PAGE_SIZE); 256 } 257 258 // We don't need a physical page for the free virtual slot. 259 pagesNeeded--; 260 261 // allocate and map the pages we need 262 for (uint32 i = 0; i < pagesNeeded; i++) { 263 // allocate a page 264 phys_addr_t physicalAddress = _Allocate32BitPage(); 265 266 // put the page into the page table 267 page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i; 268 X86PagingMethod32Bit::PutPageTableEntryInTable(entry, 269 physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, 270 true); 271 272 // Write the page's physical address into the page itself, so we 273 // don't need to look it up later. 274 *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress; 275 } 276 277 fAllocatedPages = (uint8*)virtualBase; 278 fAllocatedPagesCount = pagesNeeded; 279 fUsedPagesCount = 0; 280 fFreeVirtualSlot 281 = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE); 282 } 283 284 phys_addr_t _Allocate32BitPage() 285 { 286 phys_addr_t physicalAddress 287 = (phys_addr_t)vm_allocate_early_physical_page_etc(fKernelArgs, 0xffffffff) 288 * B_PAGE_SIZE; 289 if (physicalAddress == 0 || physicalAddress > 0xffffffff) { 290 panic("Failed to allocate 32-bit-addressable page for the switch " 291 "to PAE!"); 292 return 0; 293 } 294 return physicalAddress; 295 } 296 297 void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress) 298 { 299 if (fUsedPagesCount >= fAllocatedPagesCount) { 300 panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more " 301 "allocated pages!"); 302 return NULL; 303 } 304 305 void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE; 306 _physicalAddress = *((phys_addr_t*)page); 307 308 if (clearPage) 309 memset(page, 0, B_PAGE_SIZE); 310 311 return page; 312 } 313 314 bool _IsVirtualAddressAllocated(addr_t address) const 315 { 316 for (uint32 i = 0; i < fKernelArgs->num_virtual_allocated_ranges; i++) { 317 addr_t start = fKernelArgs->virtual_allocated_range[i].start; 318 addr_t end = start + fKernelArgs->virtual_allocated_range[i].size; 319 if (address < start) 320 return false; 321 if (address <= end - 1) 322 return true; 323 } 324 325 return false; 326 } 327 328 private: 329 kernel_args* fKernelArgs; 330 page_table_entry* fPageHole; 331 page_directory_entry* fPageHolePageDir; 332 phys_addr_t fPhysicalPageDir; 333 uint8* fAllocatedPages; 334 uint32 fAllocatedPagesCount; 335 uint32 fUsedPagesCount; 336 addr_t fFreeVirtualSlot; 337 pae_page_table_entry* fFreeVirtualSlotPTE; 338 pae_page_directory_entry* fPageDirs[4]; 339 phys_addr_t fPhysicalPageDirs[4]; 340 }; 341 342 343 // #pragma mark - PhysicalPageSlotPool 344 345 346 struct X86PagingMethodPAE::PhysicalPageSlotPool final 347 : X86LargePhysicalPageMapper::PhysicalPageSlotPool { 348 public: 349 virtual ~PhysicalPageSlotPool(); 350 351 status_t InitInitial(X86PagingMethodPAE* method, 352 kernel_args* args); 353 status_t InitInitialPostArea(kernel_args* args); 354 355 void Init(area_id dataArea, 356 pae_page_table_entry* pageTable, 357 area_id virtualArea, addr_t virtualBase); 358 359 virtual status_t AllocatePool( 360 X86LargePhysicalPageMapper 361 ::PhysicalPageSlotPool*& _pool); 362 virtual void Map(phys_addr_t physicalAddress, 363 addr_t virtualAddress); 364 365 public: 366 static PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS]; 367 368 private: 369 area_id fDataArea; 370 area_id fVirtualArea; 371 addr_t fVirtualBase; 372 pae_page_table_entry* fPageTable; 373 }; 374 375 376 X86PagingMethodPAE::PhysicalPageSlotPool 377 X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[ 378 MAX_INITIAL_POOLS]; 379 380 381 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool() 382 { 383 } 384 385 386 status_t 387 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial( 388 X86PagingMethodPAE* method, kernel_args* args) 389 { 390 // allocate a virtual address range for the pages to be mapped into 391 addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0, 392 kPAEPageTableRange); 393 if (virtualBase == 0) { 394 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve " 395 "physical page pool space in virtual address space!"); 396 return B_ERROR; 397 } 398 399 // allocate memory for the page table and data 400 size_t areaSize = B_PAGE_SIZE 401 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 402 pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early( 403 args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0); 404 if (pageTable == 0) { 405 panic("X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(): Failed " 406 "to allocate memory for page table!"); 407 return B_ERROR; 408 } 409 410 // clear the page table and put it in the page dir 411 memset(pageTable, 0, B_PAGE_SIZE); 412 413 phys_addr_t physicalTable = 0; 414 method->_EarlyQuery((addr_t)pageTable, &physicalTable); 415 416 pae_page_directory_entry* entry = PageDirEntryForAddress( 417 method->KernelVirtualPageDirs(), virtualBase); 418 PutPageTableInPageDir(entry, physicalTable, 419 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 420 421 // init the pool structure and add the initial pool 422 Init(-1, pageTable, -1, (addr_t)virtualBase); 423 424 return B_OK; 425 } 426 427 428 status_t 429 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea( 430 kernel_args* args) 431 { 432 // create an area for the (already allocated) data 433 size_t areaSize = B_PAGE_SIZE 434 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 435 void* temp = fPageTable; 436 area_id area = create_area("physical page pool", &temp, 437 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED, 438 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 439 if (area < B_OK) { 440 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 441 "create area for physical page pool."); 442 return area; 443 } 444 fDataArea = area; 445 446 // create an area for the virtual address space 447 temp = (void*)fVirtualBase; 448 area = vm_create_null_area(VMAddressSpace::KernelID(), 449 "physical page pool space", &temp, B_EXACT_ADDRESS, 450 kPAEPageTableRange, 0); 451 if (area < B_OK) { 452 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 453 "create area for physical page pool space."); 454 return area; 455 } 456 fVirtualArea = area; 457 458 return B_OK; 459 } 460 461 462 void 463 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea, 464 pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase) 465 { 466 fDataArea = dataArea; 467 fVirtualArea = virtualArea; 468 fVirtualBase = virtualBase; 469 fPageTable = pageTable; 470 471 // init slot list 472 fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount); 473 addr_t slotAddress = virtualBase; 474 for (uint32 i = 0; i < kPAEPageTableEntryCount; 475 i++, slotAddress += B_PAGE_SIZE) { 476 PhysicalPageSlot* slot = &fSlots[i]; 477 slot->next = slot + 1; 478 slot->pool = this; 479 slot->address = slotAddress; 480 } 481 482 fSlots[kPAEPageTableEntryCount - 1].next = NULL; 483 // terminate list 484 } 485 486 487 void 488 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress, 489 addr_t virtualAddress) 490 { 491 pae_page_table_entry& pte = fPageTable[ 492 (virtualAddress - fVirtualBase) / B_PAGE_SIZE]; 493 pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 494 | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT; 495 496 invalidate_TLB(virtualAddress); 497 } 498 499 500 status_t 501 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool( 502 X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool) 503 { 504 // create the pool structure 505 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool; 506 if (pool == NULL) 507 return B_NO_MEMORY; 508 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool); 509 510 // create an area that can contain the page table and the slot 511 // structures 512 size_t areaSize = B_PAGE_SIZE 513 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]); 514 void* data; 515 virtual_address_restrictions virtualRestrictions = {}; 516 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 517 physical_address_restrictions physicalRestrictions = {}; 518 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool", 519 PAGE_ALIGN(areaSize), B_FULL_LOCK, 520 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 521 &virtualRestrictions, &physicalRestrictions, &data); 522 if (dataArea < 0) 523 return dataArea; 524 525 // create the null area for the virtual address space 526 void* virtualBase; 527 area_id virtualArea = vm_create_null_area( 528 VMAddressSpace::KernelID(), "physical page pool space", 529 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange, 530 CREATE_AREA_PRIORITY_VIP); 531 if (virtualArea < 0) { 532 delete_area(dataArea); 533 return virtualArea; 534 } 535 536 // prepare the page table 537 memset(data, 0, B_PAGE_SIZE); 538 539 // get the page table's physical address 540 phys_addr_t physicalTable; 541 X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>( 542 VMAddressSpace::Kernel()->TranslationMap()); 543 uint32 dummyFlags; 544 cpu_status state = disable_interrupts(); 545 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags); 546 restore_interrupts(state); 547 548 // put the page table into the page directory 549 pae_page_directory_entry* pageDirEntry 550 = X86PagingMethodPAE::PageDirEntryForAddress( 551 map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase); 552 PutPageTableInPageDir(pageDirEntry, physicalTable, 553 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 554 555 // init the pool structure 556 pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea, 557 (addr_t)virtualBase); 558 poolDeleter.Detach(); 559 _pool = pool; 560 return B_OK; 561 } 562 563 564 // #pragma mark - X86PagingMethodPAE 565 566 567 X86PagingMethodPAE::X86PagingMethodPAE() 568 : 569 fPhysicalPageMapper(NULL), 570 fKernelPhysicalPageMapper(NULL), 571 fFreePages(NULL), 572 fFreePagesCount(0) 573 { 574 mutex_init(&fFreePagesLock, "x86 PAE free pages"); 575 } 576 577 578 X86PagingMethodPAE::~X86PagingMethodPAE() 579 { 580 } 581 582 583 status_t 584 X86PagingMethodPAE::Init(kernel_args* args, 585 VMPhysicalPageMapper** _physicalPageMapper) 586 { 587 // Ignore all memory beyond the maximum PAE address. 588 static const phys_addr_t kLimit = 1ULL << 36; 589 for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) { 590 addr_range& range = args->physical_memory_range[i]; 591 if (range.start >= kLimit) 592 range.size = 0; 593 else if ((range.start + range.size) > kLimit) 594 range.size = kLimit - range.start; 595 } 596 597 // switch to PAE 598 ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable, 599 fKernelPhysicalPageDirPointerTable, fEarlyPageStructures, 600 fEarlyPageStructuresSize, fKernelVirtualPageDirs, 601 fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE); 602 603 // create the initial pools for the physical page mapper 604 int32 poolCount = _GetInitialPoolCount(); 605 PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool; 606 607 for (int32 i = 0; i < poolCount; i++) { 608 new(&pool[i]) PhysicalPageSlotPool; 609 status_t error = pool[i].InitInitial(this, args); 610 if (error != B_OK) { 611 panic("X86PagingMethodPAE::Init(): Failed to create initial pool " 612 "for physical page mapper!"); 613 return error; 614 } 615 } 616 617 // create physical page mapper 618 large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool), 619 fPhysicalPageMapper, fKernelPhysicalPageMapper); 620 621 *_physicalPageMapper = fPhysicalPageMapper; 622 return B_OK; 623 } 624 625 626 status_t 627 X86PagingMethodPAE::InitPostArea(kernel_args* args) 628 { 629 // wrap the kernel paging structures in an area 630 area_id area = create_area("kernel paging structs", &fEarlyPageStructures, 631 B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED, 632 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 633 if (area < B_OK) 634 return area; 635 636 // let the initial page pools create areas for its structures 637 int32 poolCount = _GetInitialPoolCount(); 638 for (int32 i = 0; i < poolCount; i++) { 639 status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i] 640 .InitInitialPostArea(args); 641 if (error != B_OK) 642 return error; 643 } 644 645 // The early physical page mapping mechanism is no longer needed. Unmap the 646 // slot. 647 *fFreeVirtualSlotPTE = 0; 648 invalidate_TLB(fFreeVirtualSlot); 649 650 fFreeVirtualSlotPTE = NULL; 651 fFreeVirtualSlot = 0; 652 653 return B_OK; 654 } 655 656 657 status_t 658 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 659 { 660 X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE; 661 if (map == NULL) 662 return B_NO_MEMORY; 663 664 status_t error = map->Init(kernel); 665 if (error != B_OK) { 666 delete map; 667 return error; 668 } 669 670 *_map = map; 671 return B_OK; 672 } 673 674 675 status_t 676 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress, 677 phys_addr_t physicalAddress, uint8 attributes, 678 page_num_t (*get_free_page)(kernel_args*)) 679 { 680 // check to see if a page table exists for this range 681 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 682 fKernelVirtualPageDirs, virtualAddress); 683 pae_page_table_entry* pageTable; 684 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 685 // we need to allocate a page table 686 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE; 687 688 TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for " 689 "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable); 690 691 // put it in the page dir 692 PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes); 693 694 // zero it out 695 pageTable = _EarlyGetPageTable(physicalPageTable); 696 memset(pageTable, 0, B_PAGE_SIZE); 697 } else { 698 // table already exists -- map it 699 pageTable = _EarlyGetPageTable( 700 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); 701 } 702 703 pae_page_table_entry* entry = pageTable 704 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 705 706 ASSERT_PRINT( 707 (*entry & X86_PAE_PTE_PRESENT) == 0, 708 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64 709 ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry); 710 711 // now, fill in the pentry 712 PutPageTableEntryInTable(entry, physicalAddress, attributes, 0, 713 IS_KERNEL_ADDRESS(virtualAddress)); 714 715 return B_OK; 716 } 717 718 719 bool 720 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress, 721 uint32 protection) 722 { 723 // we can't check much without the physical page mapper 724 if (fPhysicalPageMapper == NULL) 725 return false; 726 727 // We only trust the kernel team's page directories. So switch to the 728 // kernel PDPT first. Always set it to make sure the TLBs don't contain 729 // obsolete data. 730 uint32 physicalPDPT = x86_read_cr3(); 731 x86_write_cr3(fKernelPhysicalPageDirPointerTable); 732 733 // get the PDPT entry for the address 734 pae_page_directory_pointer_table_entry pdptEntry = 0; 735 if (physicalPDPT == fKernelPhysicalPageDirPointerTable) { 736 pdptEntry = fKernelVirtualPageDirPointerTable[ 737 virtualAddress / kPAEPageDirRange]; 738 } else { 739 // map the original PDPT and get the entry 740 void* handle; 741 addr_t virtualPDPT; 742 status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT, 743 &virtualPDPT, &handle); 744 if (error == B_OK) { 745 pdptEntry = ((pae_page_directory_pointer_table_entry*) 746 virtualPDPT)[virtualAddress / kPAEPageDirRange]; 747 fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle); 748 } 749 } 750 751 // map the page dir and get the entry 752 pae_page_directory_entry pageDirEntry = 0; 753 if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) { 754 void* handle; 755 addr_t virtualPageDir; 756 status_t error = fPhysicalPageMapper->GetPageDebug( 757 pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle); 758 if (error == B_OK) { 759 pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[ 760 virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount]; 761 fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle); 762 } 763 } 764 765 // map the page table and get the entry 766 pae_page_table_entry pageTableEntry = 0; 767 if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) { 768 void* handle; 769 addr_t virtualPageTable; 770 status_t error = fPhysicalPageMapper->GetPageDebug( 771 pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable, 772 &handle); 773 if (error == B_OK) { 774 pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[ 775 virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount]; 776 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); 777 } 778 } 779 780 // switch back to the original page directory 781 if (physicalPDPT != fKernelPhysicalPageDirPointerTable) 782 x86_write_cr3(physicalPDPT); 783 784 if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0) 785 return false; 786 787 // present means kernel-readable, so check for writable 788 return (protection & B_KERNEL_WRITE_AREA) == 0 789 || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0; 790 } 791 792 793 /*static*/ void 794 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry, 795 phys_addr_t physicalTable, uint32 attributes) 796 { 797 SetTableEntry(entry, (physicalTable & X86_PAE_PDE_ADDRESS_MASK) 798 | X86_PAE_PDE_PRESENT 799 | X86_PAE_PDE_WRITABLE 800 | X86_PAE_PDE_USER); 801 // TODO: We ignore the attributes of the page table -- for compatibility 802 // with BeOS we allow having user accessible areas in the kernel address 803 // space. This is currently being used by some drivers, mainly for the 804 // frame buffer. Our current real time data implementation makes use of 805 // this fact, too. 806 // We might want to get rid of this possibility one day, especially if 807 // we intend to port it to a platform that does not support this. 808 } 809 810 811 /*static*/ void 812 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry, 813 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 814 bool globalPage) 815 { 816 pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK) 817 | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0) 818 | MemoryTypeToPageTableEntryFlags(memoryType); 819 820 // if the page is user accessible, it's automatically 821 // accessible in kernel space, too (but with the same 822 // protection) 823 if ((attributes & B_USER_PROTECTION) != 0) { 824 page |= X86_PAE_PTE_USER; 825 if ((attributes & B_WRITE_AREA) != 0) 826 page |= X86_PAE_PTE_WRITABLE; 827 if ((attributes & B_EXECUTE_AREA) == 0 828 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 829 page |= X86_PAE_PTE_NOT_EXECUTABLE; 830 } 831 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 832 page |= X86_PAE_PTE_WRITABLE; 833 834 // put it in the page table 835 SetTableEntry(entry, page); 836 } 837 838 839 void* 840 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress, 841 void*& _handle) 842 { 843 // get a free page 844 MutexLocker locker(fFreePagesLock); 845 vm_page* page; 846 if (fFreePages != NULL) { 847 page = fFreePages; 848 fFreePages = page->cache_next; 849 fFreePagesCount--; 850 locker.Unlock(); 851 } else { 852 // no pages -- allocate one 853 locker.Unlock(); 854 855 physical_address_restrictions restrictions = {}; 856 restrictions.high_address = 0x100000000LL; 857 page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions, 858 VM_PRIORITY_SYSTEM); 859 if (page == NULL) 860 return NULL; 861 862 DEBUG_PAGE_ACCESS_END(page); 863 } 864 865 // map the page 866 phys_addr_t physicalAddress 867 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 868 addr_t virtualAddress; 869 if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle) 870 != B_OK) { 871 // mapping failed -- free page 872 locker.Lock(); 873 page->cache_next = fFreePages; 874 fFreePages = page; 875 fFreePagesCount++; 876 return NULL; 877 } 878 879 _physicalAddress = physicalAddress; 880 return (void*)virtualAddress; 881 } 882 883 884 void 885 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress, 886 void* handle) 887 { 888 // unmap the page 889 fPhysicalPageMapper->PutPage((addr_t)address, handle); 890 891 // free it 892 vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); 893 MutexLocker locker(fFreePagesLock); 894 if (fFreePagesCount < kMaxFree32BitPagesCount) { 895 // cache not full yet -- cache it 896 page->cache_next = fFreePages; 897 fFreePages = page; 898 fFreePagesCount++; 899 } else { 900 // cache full -- free it 901 locker.Unlock(); 902 DEBUG_PAGE_ACCESS_START(page); 903 vm_page_free(NULL, page); 904 } 905 } 906 907 908 inline int32 909 X86PagingMethodPAE::_GetInitialPoolCount() 910 { 911 int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU 912 + EXTRA_SLOTS; 913 return (requiredSlots + kPAEPageTableEntryCount - 1) 914 / kPAEPageTableEntryCount; 915 } 916 917 918 bool 919 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress, 920 phys_addr_t* _physicalAddress) 921 { 922 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress( 923 fKernelVirtualPageDirs, virtualAddress); 924 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { 925 // no pagetable here 926 return false; 927 } 928 929 pae_page_table_entry* entry = _EarlyGetPageTable( 930 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK) 931 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount; 932 if ((*entry & X86_PAE_PTE_PRESENT) == 0) { 933 // page mapping not valid 934 return false; 935 } 936 937 *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK; 938 return true; 939 } 940 941 942 pae_page_table_entry* 943 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address) 944 { 945 *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK) 946 | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL; 947 948 invalidate_TLB(fFreeVirtualSlot); 949 950 return (pae_page_table_entry*)fFreeVirtualSlot; 951 } 952 953 954 #endif // B_HAIKU_PHYSICAL_BITS == 64 955