1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "paging/32bit/X86PagingMethod32Bit.h" 12 13 #include <stdlib.h> 14 #include <string.h> 15 16 #include <AutoDeleter.h> 17 18 #include <arch_system_info.h> 19 #include <boot/kernel_args.h> 20 #include <int.h> 21 #include <thread.h> 22 #include <vm/vm.h> 23 #include <vm/VMAddressSpace.h> 24 25 #include "paging/32bit/X86PagingStructures32Bit.h" 26 #include "paging/32bit/X86VMTranslationMap32Bit.h" 27 #include "paging/x86_physical_page_mapper.h" 28 #include "paging/x86_physical_page_mapper_large_memory.h" 29 30 31 //#define TRACE_X86_PAGING_METHOD_32_BIT 32 #ifdef TRACE_X86_PAGING_METHOD_32_BIT 33 # define TRACE(x...) dprintf(x) 34 #else 35 # define TRACE(x...) ; 36 #endif 37 38 39 using X86LargePhysicalPageMapper::PhysicalPageSlot; 40 41 42 // #pragma mark - X86PagingMethod32Bit::PhysicalPageSlotPool 43 44 45 struct X86PagingMethod32Bit::PhysicalPageSlotPool 46 : X86LargePhysicalPageMapper::PhysicalPageSlotPool { 47 public: 48 virtual ~PhysicalPageSlotPool(); 49 50 status_t InitInitial(kernel_args* args); 51 status_t InitInitialPostArea(kernel_args* args); 52 53 void Init(area_id dataArea, void* data, 54 area_id virtualArea, addr_t virtualBase); 55 56 virtual status_t AllocatePool( 57 X86LargePhysicalPageMapper 58 ::PhysicalPageSlotPool*& _pool); 59 virtual void Map(phys_addr_t physicalAddress, 60 addr_t virtualAddress); 61 62 public: 63 static PhysicalPageSlotPool sInitialPhysicalPagePool; 64 65 private: 66 area_id fDataArea; 67 area_id fVirtualArea; 68 addr_t fVirtualBase; 69 page_table_entry* fPageTable; 70 }; 71 72 73 X86PagingMethod32Bit::PhysicalPageSlotPool 74 X86PagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool; 75 76 77 X86PagingMethod32Bit::PhysicalPageSlotPool::~PhysicalPageSlotPool() 78 { 79 } 80 81 82 status_t 83 X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitial(kernel_args* args) 84 { 85 // allocate a virtual address range for the pages to be mapped into 86 addr_t virtualBase = vm_allocate_early(args, 1024 * B_PAGE_SIZE, 0, 0, 87 kPageTableAlignment); 88 if (virtualBase == 0) { 89 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve " 90 "physical page pool space in virtual address space!"); 91 return B_ERROR; 92 } 93 94 // allocate memory for the page table and data 95 size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]); 96 page_table_entry* pageTable = (page_table_entry*)vm_allocate_early(args, 97 areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0); 98 99 // prepare the page table 100 _EarlyPreparePageTables(pageTable, virtualBase, 1024 * B_PAGE_SIZE); 101 102 // init the pool structure and add the initial pool 103 Init(-1, pageTable, -1, (addr_t)virtualBase); 104 105 return B_OK; 106 } 107 108 109 status_t 110 X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitialPostArea( 111 kernel_args* args) 112 { 113 // create an area for the (already allocated) data 114 size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]); 115 void* temp = fPageTable; 116 area_id area = create_area("physical page pool", &temp, 117 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED, 118 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 119 if (area < B_OK) { 120 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 121 "create area for physical page pool."); 122 return area; 123 } 124 fDataArea = area; 125 126 // create an area for the virtual address space 127 temp = (void*)fVirtualBase; 128 area = vm_create_null_area(VMAddressSpace::KernelID(), 129 "physical page pool space", &temp, B_EXACT_ADDRESS, 130 1024 * B_PAGE_SIZE, 0); 131 if (area < B_OK) { 132 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 133 "create area for physical page pool space."); 134 return area; 135 } 136 fVirtualArea = area; 137 138 return B_OK; 139 } 140 141 142 void 143 X86PagingMethod32Bit::PhysicalPageSlotPool::Init(area_id dataArea, void* data, 144 area_id virtualArea, addr_t virtualBase) 145 { 146 fDataArea = dataArea; 147 fVirtualArea = virtualArea; 148 fVirtualBase = virtualBase; 149 fPageTable = (page_table_entry*)data; 150 151 // init slot list 152 fSlots = (PhysicalPageSlot*)(fPageTable + 1024); 153 addr_t slotAddress = virtualBase; 154 for (int32 i = 0; i < 1024; i++, slotAddress += B_PAGE_SIZE) { 155 PhysicalPageSlot* slot = &fSlots[i]; 156 slot->next = slot + 1; 157 slot->pool = this; 158 slot->address = slotAddress; 159 } 160 161 fSlots[1023].next = NULL; 162 // terminate list 163 } 164 165 166 void 167 X86PagingMethod32Bit::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress, 168 addr_t virtualAddress) 169 { 170 page_table_entry& pte = fPageTable[ 171 (virtualAddress - fVirtualBase) / B_PAGE_SIZE]; 172 pte = (physicalAddress & X86_PTE_ADDRESS_MASK) 173 | X86_PTE_WRITABLE | X86_PTE_GLOBAL | X86_PTE_PRESENT; 174 175 invalidate_TLB(virtualAddress); 176 } 177 178 179 status_t 180 X86PagingMethod32Bit::PhysicalPageSlotPool::AllocatePool( 181 X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool) 182 { 183 // create the pool structure 184 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool; 185 if (pool == NULL) 186 return B_NO_MEMORY; 187 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool); 188 189 // create an area that can contain the page table and the slot 190 // structures 191 size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]); 192 void* data; 193 virtual_address_restrictions virtualRestrictions = {}; 194 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 195 physical_address_restrictions physicalRestrictions = {}; 196 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool", 197 PAGE_ALIGN(areaSize), B_FULL_LOCK, 198 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 199 &virtualRestrictions, &physicalRestrictions, &data); 200 if (dataArea < 0) 201 return dataArea; 202 203 // create the null area for the virtual address space 204 void* virtualBase; 205 area_id virtualArea = vm_create_null_area( 206 VMAddressSpace::KernelID(), "physical page pool space", 207 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, 1024 * B_PAGE_SIZE, 208 CREATE_AREA_PRIORITY_VIP); 209 if (virtualArea < 0) { 210 delete_area(dataArea); 211 return virtualArea; 212 } 213 214 // prepare the page table 215 memset(data, 0, B_PAGE_SIZE); 216 217 // get the page table's physical address 218 phys_addr_t physicalTable; 219 X86VMTranslationMap32Bit* map = static_cast<X86VMTranslationMap32Bit*>( 220 VMAddressSpace::Kernel()->TranslationMap()); 221 uint32 dummyFlags; 222 cpu_status state = disable_interrupts(); 223 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags); 224 restore_interrupts(state); 225 226 // put the page table into the page directory 227 int32 index = (addr_t)virtualBase / (B_PAGE_SIZE * 1024); 228 page_directory_entry* entry 229 = &map->PagingStructures32Bit()->pgdir_virt[index]; 230 PutPageTableInPageDir(entry, physicalTable, 231 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 232 X86PagingStructures32Bit::UpdateAllPageDirs(index, *entry); 233 234 // init the pool structure 235 pool->Init(dataArea, data, virtualArea, (addr_t)virtualBase); 236 poolDeleter.Detach(); 237 _pool = pool; 238 return B_OK; 239 } 240 241 242 // #pragma mark - X86PagingMethod32Bit 243 244 245 X86PagingMethod32Bit::X86PagingMethod32Bit() 246 : 247 fPageHole(NULL), 248 fPageHolePageDir(NULL), 249 fKernelPhysicalPageDirectory(0), 250 fKernelVirtualPageDirectory(NULL), 251 fPhysicalPageMapper(NULL), 252 fKernelPhysicalPageMapper(NULL) 253 { 254 } 255 256 257 X86PagingMethod32Bit::~X86PagingMethod32Bit() 258 { 259 } 260 261 262 status_t 263 X86PagingMethod32Bit::Init(kernel_args* args, 264 VMPhysicalPageMapper** _physicalPageMapper) 265 { 266 TRACE("X86PagingMethod32Bit::Init(): entry\n"); 267 268 // page hole set up in stage2 269 fPageHole = (page_table_entry*)(addr_t)args->arch_args.page_hole; 270 // calculate where the pgdir would be 271 fPageHolePageDir = (page_directory_entry*) 272 (((addr_t)args->arch_args.page_hole) 273 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); 274 // clear out the bottom 2 GB, unmap everything 275 memset(fPageHolePageDir + FIRST_USER_PGDIR_ENT, 0, 276 sizeof(page_directory_entry) * NUM_USER_PGDIR_ENTS); 277 278 fKernelPhysicalPageDirectory = args->arch_args.phys_pgdir; 279 fKernelVirtualPageDirectory = (page_directory_entry*)(addr_t) 280 args->arch_args.vir_pgdir; 281 282 #ifdef TRACE_X86_PAGING_METHOD_32_BIT 283 TRACE("page hole: %p, page dir: %p\n", fPageHole, fPageHolePageDir); 284 TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n", 285 fKernelVirtualPageDirectory, fKernelPhysicalPageDirectory); 286 #endif 287 288 X86PagingStructures32Bit::StaticInit(); 289 290 // create the initial pool for the physical page mapper 291 PhysicalPageSlotPool* pool 292 = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) 293 PhysicalPageSlotPool; 294 status_t error = pool->InitInitial(args); 295 if (error != B_OK) { 296 panic("X86PagingMethod32Bit::Init(): Failed to create initial pool " 297 "for physical page mapper!"); 298 return error; 299 } 300 301 // create physical page mapper 302 large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper, 303 fKernelPhysicalPageMapper); 304 // TODO: Select the best page mapper! 305 306 // enable global page feature if available 307 if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) { 308 // this prevents kernel pages from being flushed from TLB on 309 // context-switch 310 x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES); 311 } 312 313 TRACE("X86PagingMethod32Bit::Init(): done\n"); 314 315 *_physicalPageMapper = fPhysicalPageMapper; 316 return B_OK; 317 } 318 319 320 status_t 321 X86PagingMethod32Bit::InitPostArea(kernel_args* args) 322 { 323 // now that the vm is initialized, create an area that represents 324 // the page hole 325 void *temp; 326 status_t error; 327 area_id area; 328 329 // unmap the page hole hack we were using before 330 fKernelVirtualPageDirectory[1023] = 0; 331 fPageHolePageDir = NULL; 332 fPageHole = NULL; 333 334 temp = (void*)fKernelVirtualPageDirectory; 335 area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, B_PAGE_SIZE, 336 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 337 if (area < B_OK) 338 return area; 339 340 error = PhysicalPageSlotPool::sInitialPhysicalPagePool 341 .InitInitialPostArea(args); 342 if (error != B_OK) 343 return error; 344 345 return B_OK; 346 } 347 348 349 status_t 350 X86PagingMethod32Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 351 { 352 X86VMTranslationMap32Bit* map = new(std::nothrow) X86VMTranslationMap32Bit; 353 if (map == NULL) 354 return B_NO_MEMORY; 355 356 status_t error = map->Init(kernel); 357 if (error != B_OK) { 358 delete map; 359 return error; 360 } 361 362 *_map = map; 363 return B_OK; 364 } 365 366 367 status_t 368 X86PagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress, 369 phys_addr_t physicalAddress, uint8 attributes, 370 page_num_t (*get_free_page)(kernel_args*)) 371 { 372 // XXX horrible back door to map a page quickly regardless of translation 373 // map object, etc. used only during VM setup. 374 // uses a 'page hole' set up in the stage 2 bootloader. The page hole is 375 // created by pointing one of the pgdir entries back at itself, effectively 376 // mapping the contents of all of the 4MB of pagetables into a 4 MB region. 377 // It's only used here, and is later unmapped. 378 379 // check to see if a page table exists for this range 380 int index = VADDR_TO_PDENT(virtualAddress); 381 if ((fPageHolePageDir[index] & X86_PDE_PRESENT) == 0) { 382 phys_addr_t pgtable; 383 page_directory_entry *e; 384 // we need to allocate a pgtable 385 pgtable = get_free_page(args); 386 // pgtable is in pages, convert to physical address 387 pgtable *= B_PAGE_SIZE; 388 389 TRACE("X86PagingMethod32Bit::MapEarly(): asked for free page for " 390 "pgtable. %#" B_PRIxPHYSADDR "\n", pgtable); 391 392 // put it in the pgdir 393 e = &fPageHolePageDir[index]; 394 PutPageTableInPageDir(e, pgtable, attributes); 395 396 // zero it out in it's new mapping 397 memset((unsigned int*)((addr_t)fPageHole 398 + (virtualAddress / B_PAGE_SIZE / 1024) * B_PAGE_SIZE), 399 0, B_PAGE_SIZE); 400 } 401 402 ASSERT_PRINT( 403 (fPageHole[virtualAddress / B_PAGE_SIZE] & X86_PTE_PRESENT) == 0, 404 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32 405 ", existing pte: %#" B_PRIx32, virtualAddress, fPageHolePageDir[index], 406 fPageHole[virtualAddress / B_PAGE_SIZE]); 407 408 // now, fill in the pentry 409 PutPageTableEntryInTable(fPageHole + virtualAddress / B_PAGE_SIZE, 410 physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress)); 411 412 return B_OK; 413 } 414 415 416 bool 417 X86PagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress, 418 uint32 protection) 419 { 420 // We only trust the kernel team's page directory. So switch to it first. 421 // Always set it to make sure the TLBs don't contain obsolete data. 422 uint32 physicalPageDirectory = x86_read_cr3(); 423 x86_write_cr3(fKernelPhysicalPageDirectory); 424 425 // get the page directory entry for the address 426 page_directory_entry pageDirectoryEntry; 427 uint32 index = VADDR_TO_PDENT(virtualAddress); 428 429 if (physicalPageDirectory == fKernelPhysicalPageDirectory) { 430 pageDirectoryEntry = fKernelVirtualPageDirectory[index]; 431 } else if (fPhysicalPageMapper != NULL) { 432 // map the original page directory and get the entry 433 void* handle; 434 addr_t virtualPageDirectory; 435 status_t error = fPhysicalPageMapper->GetPageDebug( 436 physicalPageDirectory, &virtualPageDirectory, &handle); 437 if (error == B_OK) { 438 pageDirectoryEntry 439 = ((page_directory_entry*)virtualPageDirectory)[index]; 440 fPhysicalPageMapper->PutPageDebug(virtualPageDirectory, handle); 441 } else 442 pageDirectoryEntry = 0; 443 } else 444 pageDirectoryEntry = 0; 445 446 // map the page table and get the entry 447 page_table_entry pageTableEntry; 448 index = VADDR_TO_PTENT(virtualAddress); 449 450 if ((pageDirectoryEntry & X86_PDE_PRESENT) != 0 451 && fPhysicalPageMapper != NULL) { 452 void* handle; 453 addr_t virtualPageTable; 454 status_t error = fPhysicalPageMapper->GetPageDebug( 455 pageDirectoryEntry & X86_PDE_ADDRESS_MASK, &virtualPageTable, 456 &handle); 457 if (error == B_OK) { 458 pageTableEntry = ((page_table_entry*)virtualPageTable)[index]; 459 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); 460 } else 461 pageTableEntry = 0; 462 } else 463 pageTableEntry = 0; 464 465 // switch back to the original page directory 466 if (physicalPageDirectory != fKernelPhysicalPageDirectory) 467 x86_write_cr3(physicalPageDirectory); 468 469 if ((pageTableEntry & X86_PTE_PRESENT) == 0) 470 return false; 471 472 // present means kernel-readable, so check for writable 473 return (protection & B_KERNEL_WRITE_AREA) == 0 474 || (pageTableEntry & X86_PTE_WRITABLE) != 0; 475 } 476 477 478 /*static*/ void 479 X86PagingMethod32Bit::PutPageTableInPageDir(page_directory_entry* entry, 480 phys_addr_t pgtablePhysical, uint32 attributes) 481 { 482 *entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK) 483 | X86_PDE_PRESENT 484 | X86_PDE_WRITABLE 485 | X86_PDE_USER; 486 // TODO: we ignore the attributes of the page table - for compatibility 487 // with BeOS we allow having user accessible areas in the kernel address 488 // space. This is currently being used by some drivers, mainly for the 489 // frame buffer. Our current real time data implementation makes use of 490 // this fact, too. 491 // We might want to get rid of this possibility one day, especially if 492 // we intend to port it to a platform that does not support this. 493 } 494 495 496 /*static*/ void 497 X86PagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry, 498 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 499 bool globalPage) 500 { 501 page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK) 502 | X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0) 503 | MemoryTypeToPageTableEntryFlags(memoryType); 504 505 // if the page is user accessible, it's automatically 506 // accessible in kernel space, too (but with the same 507 // protection) 508 if ((attributes & B_USER_PROTECTION) != 0) { 509 page |= X86_PTE_USER; 510 if ((attributes & B_WRITE_AREA) != 0) 511 page |= X86_PTE_WRITABLE; 512 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 513 page |= X86_PTE_WRITABLE; 514 515 // put it in the page table 516 *(volatile page_table_entry*)entry = page; 517 } 518 519 520 /*static*/ void 521 X86PagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables, 522 addr_t address, size_t size) 523 { 524 memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024))); 525 526 // put the array of pgtables directly into the kernel pagedir 527 // these will be wired and kept mapped into virtual space to be easy to get 528 // to 529 { 530 addr_t virtualTable = (addr_t)pageTables; 531 532 page_directory_entry* pageHolePageDir 533 = X86PagingMethod32Bit::Method()->PageHolePageDir(); 534 535 for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024)); 536 i++, virtualTable += B_PAGE_SIZE) { 537 phys_addr_t physicalTable = 0; 538 _EarlyQuery(virtualTable, &physicalTable); 539 page_directory_entry* entry = &pageHolePageDir[ 540 (address / (B_PAGE_SIZE * 1024)) + i]; 541 PutPageTableInPageDir(entry, physicalTable, 542 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 543 } 544 } 545 } 546 547 548 //! TODO: currently assumes this translation map is active 549 /*static*/ status_t 550 X86PagingMethod32Bit::_EarlyQuery(addr_t virtualAddress, 551 phys_addr_t *_physicalAddress) 552 { 553 X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method(); 554 int index = VADDR_TO_PDENT(virtualAddress); 555 if ((method->PageHolePageDir()[index] & X86_PDE_PRESENT) == 0) { 556 // no pagetable here 557 return B_ERROR; 558 } 559 560 page_table_entry* entry = method->PageHole() + virtualAddress / B_PAGE_SIZE; 561 if ((*entry & X86_PTE_PRESENT) == 0) { 562 // page mapping not valid 563 return B_ERROR; 564 } 565 566 *_physicalAddress = *entry & X86_PTE_ADDRESS_MASK; 567 return B_OK; 568 } 569