1 /* 2 * Copyright 2010, Ithamar R. Adema, ithamar.adema@team-embedded.nl 3 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 4 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12 #include "paging/32bit/ARMPagingMethod32Bit.h" 13 14 #include <stdlib.h> 15 #include <string.h> 16 17 #include <AutoDeleter.h> 18 19 #include <arch_system_info.h> 20 #include <boot/kernel_args.h> 21 #include <int.h> 22 #include <thread.h> 23 #include <vm/vm.h> 24 #include <vm/VMAddressSpace.h> 25 #include <arm_mmu.h> 26 27 #include "paging/32bit/ARMPagingStructures32Bit.h" 28 #include "paging/32bit/ARMVMTranslationMap32Bit.h" 29 #include "paging/arm_physical_page_mapper.h" 30 #include "paging/arm_physical_page_mapper_large_memory.h" 31 32 33 #define TRACE_ARM_PAGING_METHOD_32_BIT 34 #ifdef TRACE_ARM_PAGING_METHOD_32_BIT 35 # define TRACE(x...) dprintf(x) 36 #else 37 # define TRACE(x...) ; 38 #endif 39 40 41 using ARMLargePhysicalPageMapper::PhysicalPageSlot; 42 43 44 // #pragma mark - ARMPagingMethod32Bit::PhysicalPageSlotPool 45 46 47 struct ARMPagingMethod32Bit::PhysicalPageSlotPool 48 : ARMLargePhysicalPageMapper::PhysicalPageSlotPool { 49 public: 50 virtual ~PhysicalPageSlotPool(); 51 52 status_t InitInitial(kernel_args* args); 53 status_t InitInitialPostArea(kernel_args* args); 54 55 void Init(area_id dataArea, void* data, 56 area_id virtualArea, addr_t virtualBase); 57 58 virtual status_t AllocatePool( 59 ARMLargePhysicalPageMapper 60 ::PhysicalPageSlotPool*& _pool); 61 virtual void Map(phys_addr_t physicalAddress, 62 addr_t virtualAddress); 63 64 public: 65 static PhysicalPageSlotPool sInitialPhysicalPagePool; 66 67 private: 68 area_id fDataArea; 69 area_id fVirtualArea; 70 addr_t fVirtualBase; 71 page_table_entry* fPageTable; 72 }; 73 74 75 ARMPagingMethod32Bit::PhysicalPageSlotPool 76 ARMPagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool; 77 78 79 ARMPagingMethod32Bit::PhysicalPageSlotPool::~PhysicalPageSlotPool() 80 { 81 } 82 83 84 status_t 85 ARMPagingMethod32Bit::PhysicalPageSlotPool::InitInitial(kernel_args* args) 86 { 87 // allocate a virtual address range for the pages to be mapped into 88 addr_t virtualBase = vm_allocate_early(args, 1024 * B_PAGE_SIZE, 0, 0, 89 kPageTableAlignment); 90 if (virtualBase == 0) { 91 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve " 92 "physical page pool space in virtual address space!"); 93 return B_ERROR; 94 } 95 96 // allocate memory for the page table and data 97 size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]); 98 page_table_entry* pageTable = (page_table_entry*)vm_allocate_early(args, 99 areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0); 100 101 // prepare the page table 102 _EarlyPreparePageTables(pageTable, virtualBase, 1024 * B_PAGE_SIZE); 103 104 // init the pool structure and add the initial pool 105 Init(-1, pageTable, -1, (addr_t)virtualBase); 106 107 return B_OK; 108 } 109 110 111 status_t 112 ARMPagingMethod32Bit::PhysicalPageSlotPool::InitInitialPostArea( 113 kernel_args* args) 114 { 115 // create an area for the (already allocated) data 116 size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]); 117 void* temp = fPageTable; 118 area_id area = create_area("physical page pool", &temp, 119 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED, 120 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 121 if (area < B_OK) { 122 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 123 "create area for physical page pool."); 124 return area; 125 } 126 fDataArea = area; 127 128 // create an area for the virtual address space 129 temp = (void*)fVirtualBase; 130 area = vm_create_null_area(VMAddressSpace::KernelID(), 131 "physical page pool space", &temp, B_EXACT_ADDRESS, 132 1024 * B_PAGE_SIZE, 0); 133 if (area < B_OK) { 134 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " 135 "create area for physical page pool space."); 136 return area; 137 } 138 fVirtualArea = area; 139 140 return B_OK; 141 } 142 143 144 void 145 ARMPagingMethod32Bit::PhysicalPageSlotPool::Init(area_id dataArea, void* data, 146 area_id virtualArea, addr_t virtualBase) 147 { 148 fDataArea = dataArea; 149 fVirtualArea = virtualArea; 150 fVirtualBase = virtualBase; 151 fPageTable = (page_table_entry*)data; 152 153 // init slot list 154 fSlots = (PhysicalPageSlot*)(fPageTable + 1024); 155 addr_t slotAddress = virtualBase; 156 for (int32 i = 0; i < 1024; i++, slotAddress += B_PAGE_SIZE) { 157 PhysicalPageSlot* slot = &fSlots[i]; 158 slot->next = slot + 1; 159 slot->pool = this; 160 slot->address = slotAddress; 161 } 162 163 fSlots[1023].next = NULL; 164 // terminate list 165 } 166 167 168 void 169 ARMPagingMethod32Bit::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress, 170 addr_t virtualAddress) 171 { 172 page_table_entry& pte = fPageTable[(virtualAddress - fVirtualBase) / B_PAGE_SIZE]; 173 pte = (physicalAddress & ARM_PTE_ADDRESS_MASK) 174 | ARM_PTE_TYPE_SMALL_PAGE; 175 176 arch_cpu_invalidate_TLB_range(virtualAddress, virtualAddress + B_PAGE_SIZE); 177 // invalidate_TLB(virtualAddress); 178 } 179 180 181 status_t 182 ARMPagingMethod32Bit::PhysicalPageSlotPool::AllocatePool( 183 ARMLargePhysicalPageMapper::PhysicalPageSlotPool*& _pool) 184 { 185 // create the pool structure 186 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool; 187 if (pool == NULL) 188 return B_NO_MEMORY; 189 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool); 190 191 // create an area that can contain the page table and the slot 192 // structures 193 size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]); 194 void* data; 195 virtual_address_restrictions virtualRestrictions = {}; 196 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 197 physical_address_restrictions physicalRestrictions = {}; 198 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool", 199 PAGE_ALIGN(areaSize), B_FULL_LOCK, 200 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 201 &virtualRestrictions, &physicalRestrictions, &data); 202 if (dataArea < 0) 203 return dataArea; 204 205 // create the null area for the virtual address space 206 void* virtualBase; 207 area_id virtualArea = vm_create_null_area( 208 VMAddressSpace::KernelID(), "physical page pool space", 209 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, 1024 * B_PAGE_SIZE, 210 CREATE_AREA_PRIORITY_VIP); 211 if (virtualArea < 0) { 212 delete_area(dataArea); 213 return virtualArea; 214 } 215 216 // prepare the page table 217 memset(data, 0, B_PAGE_SIZE); 218 219 // get the page table's physical address 220 phys_addr_t physicalTable; 221 ARMVMTranslationMap32Bit* map = static_cast<ARMVMTranslationMap32Bit*>( 222 VMAddressSpace::Kernel()->TranslationMap()); 223 uint32 dummyFlags; 224 cpu_status state = disable_interrupts(); 225 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags); 226 restore_interrupts(state); 227 228 // put the page table into the page directory 229 int32 index = VADDR_TO_PDENT((addr_t)virtualBase); 230 page_directory_entry* entry 231 = &map->PagingStructures32Bit()->pgdir_virt[index]; 232 PutPageTableInPageDir(entry, physicalTable, 233 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 234 ARMPagingStructures32Bit::UpdateAllPageDirs(index, *entry); 235 236 // init the pool structure 237 pool->Init(dataArea, data, virtualArea, (addr_t)virtualBase); 238 poolDeleter.Detach(); 239 _pool = pool; 240 return B_OK; 241 } 242 243 244 // #pragma mark - ARMPagingMethod32Bit 245 246 247 ARMPagingMethod32Bit::ARMPagingMethod32Bit() 248 : 249 fKernelPhysicalPageDirectory(0), 250 fKernelVirtualPageDirectory(NULL), 251 fPhysicalPageMapper(NULL), 252 fKernelPhysicalPageMapper(NULL) 253 { 254 } 255 256 257 ARMPagingMethod32Bit::~ARMPagingMethod32Bit() 258 { 259 } 260 261 262 status_t 263 ARMPagingMethod32Bit::Init(kernel_args* args, 264 VMPhysicalPageMapper** _physicalPageMapper) 265 { 266 TRACE("vm_translation_map_init: entry\n"); 267 268 fKernelPhysicalPageDirectory = args->arch_args.phys_pgdir; 269 fKernelVirtualPageDirectory = (page_directory_entry*) 270 args->arch_args.vir_pgdir; 271 272 TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n", 273 fKernelVirtualPageDirectory, fKernelPhysicalPageDirectory); 274 275 ARMPagingStructures32Bit::StaticInit(); 276 277 // create the initial pool for the physical page mapper 278 PhysicalPageSlotPool* pool 279 = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) 280 PhysicalPageSlotPool; 281 status_t error = pool->InitInitial(args); 282 if (error != B_OK) { 283 panic("ARMPagingMethod32Bit::Init(): Failed to create initial pool " 284 "for physical page mapper!"); 285 return error; 286 } 287 288 // create physical page mapper 289 large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper, 290 fKernelPhysicalPageMapper); 291 // TODO: Select the best page mapper! 292 293 // enable global page feature if available 294 #if 0 //IRA: check for ARMv6!! 295 if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) { 296 // this prevents kernel pages from being flushed from TLB on 297 // context-switch 298 x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES); 299 } 300 #endif 301 TRACE("vm_translation_map_init: done\n"); 302 303 *_physicalPageMapper = fPhysicalPageMapper; 304 return B_OK; 305 } 306 307 308 status_t 309 ARMPagingMethod32Bit::InitPostArea(kernel_args* args) 310 { 311 void *temp; 312 status_t error; 313 area_id area; 314 315 temp = (void*)fKernelVirtualPageDirectory; 316 area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, MMU_L1_TABLE_SIZE, 317 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 318 if (area < B_OK) 319 return area; 320 321 error = PhysicalPageSlotPool::sInitialPhysicalPagePool 322 .InitInitialPostArea(args); 323 if (error != B_OK) 324 return error; 325 326 return B_OK; 327 } 328 329 330 status_t 331 ARMPagingMethod32Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 332 { 333 ARMVMTranslationMap32Bit* map = new(std::nothrow) ARMVMTranslationMap32Bit; 334 if (map == NULL) 335 return B_NO_MEMORY; 336 337 status_t error = map->Init(kernel); 338 if (error != B_OK) { 339 delete map; 340 return error; 341 } 342 343 *_map = map; 344 return B_OK; 345 } 346 347 348 status_t 349 ARMPagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress, 350 phys_addr_t physicalAddress, uint8 attributes, 351 phys_addr_t (*get_free_page)(kernel_args*)) 352 { 353 // check to see if a page table exists for this range 354 int index = VADDR_TO_PDENT(virtualAddress); 355 if ((fKernelVirtualPageDirectory[index] & ARM_PDE_TYPE_MASK) == 0) { 356 phys_addr_t pgtable; 357 page_directory_entry *e; 358 // we need to allocate a pgtable 359 pgtable = get_free_page(args); 360 // pgtable is in pages, convert to physical address 361 pgtable *= B_PAGE_SIZE; 362 363 TRACE("ARMPagingMethod32Bit::MapEarly(): asked for free page for " 364 "pgtable. %#" B_PRIxPHYSADDR "\n", pgtable); 365 366 // put it in the pgdir 367 e = &fKernelVirtualPageDirectory[index]; 368 PutPageTableInPageDir(e, pgtable, attributes); 369 370 // zero it out in it's new mapping 371 memset((void*)pgtable, 0, B_PAGE_SIZE); 372 } 373 374 page_table_entry *ptEntry = (page_table_entry*) 375 (fKernelVirtualPageDirectory[index] & ARM_PDE_ADDRESS_MASK); 376 ptEntry += VADDR_TO_PTENT(virtualAddress); 377 378 ASSERT_PRINT( 379 (*ptEntry & ARM_PTE_TYPE_MASK) == 0, 380 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32 381 ", existing pte: %#" B_PRIx32, virtualAddress, fKernelVirtualPageDirectory[index], 382 *ptEntry); 383 384 // now, fill in the pentry 385 PutPageTableEntryInTable(ptEntry, 386 physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress)); 387 388 return B_OK; 389 } 390 391 392 bool 393 ARMPagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress, 394 uint32 protection) 395 { 396 #if 0 397 // We only trust the kernel team's page directory. So switch to it first. 398 // Always set it to make sure the TLBs don't contain obsolete data. 399 uint32 physicalPageDirectory; 400 read_cr3(physicalPageDirectory); 401 write_cr3(fKernelPhysicalPageDirectory); 402 403 // get the page directory entry for the address 404 page_directory_entry pageDirectoryEntry; 405 uint32 index = VADDR_TO_PDENT(virtualAddress); 406 407 if (physicalPageDirectory == fKernelPhysicalPageDirectory) { 408 pageDirectoryEntry = fKernelVirtualPageDirectory[index]; 409 } else if (fPhysicalPageMapper != NULL) { 410 // map the original page directory and get the entry 411 void* handle; 412 addr_t virtualPageDirectory; 413 status_t error = fPhysicalPageMapper->GetPageDebug( 414 physicalPageDirectory, &virtualPageDirectory, &handle); 415 if (error == B_OK) { 416 pageDirectoryEntry 417 = ((page_directory_entry*)virtualPageDirectory)[index]; 418 fPhysicalPageMapper->PutPageDebug(virtualPageDirectory, handle); 419 } else 420 pageDirectoryEntry = 0; 421 } else 422 pageDirectoryEntry = 0; 423 424 // map the page table and get the entry 425 page_table_entry pageTableEntry; 426 index = VADDR_TO_PTENT(virtualAddress); 427 428 if ((pageDirectoryEntry & ARM_PDE_PRESENT) != 0 429 && fPhysicalPageMapper != NULL) { 430 void* handle; 431 addr_t virtualPageTable; 432 status_t error = fPhysicalPageMapper->GetPageDebug( 433 pageDirectoryEntry & ARM_PDE_ADDRESS_MASK, &virtualPageTable, 434 &handle); 435 if (error == B_OK) { 436 pageTableEntry = ((page_table_entry*)virtualPageTable)[index]; 437 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); 438 } else 439 pageTableEntry = 0; 440 } else 441 pageTableEntry = 0; 442 443 // switch back to the original page directory 444 if (physicalPageDirectory != fKernelPhysicalPageDirectory) 445 write_cr3(physicalPageDirectory); 446 447 if ((pageTableEntry & ARM_PTE_PRESENT) == 0) 448 return false; 449 450 // present means kernel-readable, so check for writable 451 return (protection & B_KERNEL_WRITE_AREA) == 0 452 || (pageTableEntry & ARM_PTE_WRITABLE) != 0; 453 #endif 454 //IRA: fix the above! 455 return true; 456 } 457 458 459 /*static*/ void 460 ARMPagingMethod32Bit::PutPageTableInPageDir(page_directory_entry* entry, 461 phys_addr_t pgtablePhysical, uint32 attributes) 462 { 463 *entry = (pgtablePhysical & ARM_PDE_ADDRESS_MASK) 464 | ARM_PDE_TYPE_COARSE_L2_PAGE_TABLE; 465 // TODO: we ignore the attributes of the page table - for compatibility 466 // with BeOS we allow having user accessible areas in the kernel address 467 // space. This is currently being used by some drivers, mainly for the 468 // frame buffer. Our current real time data implementation makes use of 469 // this fact, too. 470 // We might want to get rid of this possibility one day, especially if 471 // we intend to port it to a platform that does not support this. 472 } 473 474 475 /*static*/ void 476 ARMPagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry, 477 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 478 bool globalPage) 479 { 480 page_table_entry page = (physicalAddress & ARM_PTE_ADDRESS_MASK) 481 | ARM_PTE_TYPE_SMALL_PAGE; 482 #if 0 //IRA 483 | ARM_PTE_PRESENT | (globalPage ? ARM_PTE_GLOBAL : 0) 484 | MemoryTypeToPageTableEntryFlags(memoryType); 485 486 // if the page is user accessible, it's automatically 487 // accessible in kernel space, too (but with the same 488 // protection) 489 if ((attributes & B_USER_PROTECTION) != 0) { 490 page |= ARM_PTE_USER; 491 if ((attributes & B_WRITE_AREA) != 0) 492 page |= ARM_PTE_WRITABLE; 493 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 494 page |= ARM_PTE_WRITABLE; 495 #endif 496 // put it in the page table 497 *(volatile page_table_entry*)entry = page; 498 } 499 500 501 /*static*/ void 502 ARMPagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables, 503 addr_t address, size_t size) 504 { 505 ARMPagingMethod32Bit* method = ARMPagingMethod32Bit::Method(); 506 memset(pageTables, 0, 256 * (size / (B_PAGE_SIZE * 256))); 507 508 // put the array of pgtables directly into the kernel pagedir 509 // these will be wired and kept mapped into virtual space to be easy to get 510 // to 511 { 512 addr_t virtualTable = (addr_t)pageTables; 513 514 for (size_t i = 0; i < (size / (B_PAGE_SIZE * 256)); 515 i++, virtualTable += 256*sizeof(page_directory_entry)) { 516 phys_addr_t physicalTable = 0; 517 _EarlyQuery(virtualTable, &physicalTable); 518 page_directory_entry* entry = method->KernelVirtualPageDirectory() 519 + VADDR_TO_PDENT(address) + i; 520 PutPageTableInPageDir(entry, physicalTable, 521 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 522 } 523 } 524 } 525 526 527 //! TODO: currently assumes this translation map is active 528 /*static*/ status_t 529 ARMPagingMethod32Bit::_EarlyQuery(addr_t virtualAddress, 530 phys_addr_t *_physicalAddress) 531 { 532 ARMPagingMethod32Bit* method = ARMPagingMethod32Bit::Method(); 533 int index = VADDR_TO_PDENT(virtualAddress); 534 if ((method->KernelVirtualPageDirectory()[index] & ARM_PDE_TYPE_MASK) == 0) { 535 // no pagetable here 536 return B_ERROR; 537 } 538 539 page_table_entry* entry = (page_table_entry*) 540 (method->KernelVirtualPageDirectory()[index] & ARM_PDE_ADDRESS_MASK); 541 entry += VADDR_TO_PTENT(virtualAddress); 542 543 if ((*entry & ARM_PTE_TYPE_MASK) == 0) { 544 // page mapping not valid 545 return B_ERROR; 546 } 547 548 *_physicalAddress = (*entry & ARM_PTE_ADDRESS_MASK) 549 | VADDR_TO_PGOFF(virtualAddress); 550 551 return B_OK; 552 } 553