1 /* 2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 3 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 4 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12 #include "paging/64bit/X86PagingMethod64Bit.h" 13 14 #include <stdlib.h> 15 #include <string.h> 16 17 #include <boot/kernel_args.h> 18 #include <util/AutoLock.h> 19 #include <vm/vm.h> 20 #include <vm/vm_page.h> 21 #include <vm/VMAddressSpace.h> 22 23 #include "paging/64bit/X86PagingStructures64Bit.h" 24 #include "paging/64bit/X86VMTranslationMap64Bit.h" 25 #include "paging/x86_physical_page_mapper.h" 26 #include "paging/x86_physical_page_mapper_mapped.h" 27 28 29 //#define TRACE_X86_PAGING_METHOD_64BIT 30 #ifdef TRACE_X86_PAGING_METHOD_64BIT 31 # define TRACE(x...) dprintf(x) 32 #else 33 # define TRACE(x...) ; 34 #endif 35 36 37 // #pragma mark - X86PagingMethod64Bit 38 39 40 X86PagingMethod64Bit::X86PagingMethod64Bit() 41 : 42 fKernelPhysicalPML4(0), 43 fKernelVirtualPML4(NULL), 44 fPhysicalPageMapper(NULL), 45 fKernelPhysicalPageMapper(NULL) 46 { 47 } 48 49 50 X86PagingMethod64Bit::~X86PagingMethod64Bit() 51 { 52 } 53 54 55 status_t 56 X86PagingMethod64Bit::Init(kernel_args* args, 57 VMPhysicalPageMapper** _physicalPageMapper) 58 { 59 fKernelPhysicalPML4 = args->arch_args.phys_pgdir; 60 fKernelVirtualPML4 = (uint64*)(addr_t)args->arch_args.vir_pgdir; 61 62 // if available enable NX-bit (No eXecute) 63 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) 64 call_all_cpus_sync(&_EnableExecutionDisable, NULL); 65 66 // Ensure that the user half of the address space is clear. This removes 67 // the temporary identity mapping made by the boot loader. 68 memset(fKernelVirtualPML4, 0, sizeof(uint64) * 256); 69 arch_cpu_global_TLB_invalidate(); 70 71 // Create the physical page mapper. 72 mapped_physical_page_ops_init(args, fPhysicalPageMapper, 73 fKernelPhysicalPageMapper); 74 75 *_physicalPageMapper = fPhysicalPageMapper; 76 return B_ERROR; 77 } 78 79 80 status_t 81 X86PagingMethod64Bit::InitPostArea(kernel_args* args) 82 { 83 // Create an area covering the physical map area. 84 void* address = (void*)KERNEL_PMAP_BASE; 85 area_id area = vm_create_null_area(VMAddressSpace::KernelID(), 86 "physical map area", &address, B_EXACT_ADDRESS, 87 KERNEL_PMAP_SIZE, 0); 88 if (area < B_OK) 89 return area; 90 91 // Create an area to represent the kernel PML4. 92 area = create_area("kernel pml4", (void**)&fKernelVirtualPML4, 93 B_EXACT_ADDRESS, B_PAGE_SIZE, B_ALREADY_WIRED, 94 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 95 if (area < B_OK) 96 return area; 97 98 return B_OK; 99 } 100 101 102 status_t 103 X86PagingMethod64Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map) 104 { 105 X86VMTranslationMap64Bit* map = new(std::nothrow) X86VMTranslationMap64Bit; 106 if (map == NULL) 107 return B_NO_MEMORY; 108 109 status_t error = map->Init(kernel); 110 if (error != B_OK) { 111 delete map; 112 return error; 113 } 114 115 *_map = map; 116 return B_OK; 117 } 118 119 120 status_t 121 X86PagingMethod64Bit::MapEarly(kernel_args* args, addr_t virtualAddress, 122 phys_addr_t physicalAddress, uint8 attributes, 123 page_num_t (*get_free_page)(kernel_args*)) 124 { 125 TRACE("X86PagingMethod64Bit::MapEarly(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR 126 ", %#" B_PRIx8 ")\n", virtualAddress, physicalAddress, attributes); 127 128 // Get the PDPT. We should be mapping on an existing PDPT at this stage. 129 uint64* pml4e = &fKernelVirtualPML4[VADDR_TO_PML4E(virtualAddress)]; 130 ASSERT((*pml4e & X86_64_PML4E_PRESENT) != 0); 131 uint64* virtualPDPT = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt( 132 *pml4e & X86_64_PML4E_ADDRESS_MASK); 133 134 // Get the page directory. 135 uint64* pdpte = &virtualPDPT[VADDR_TO_PDPTE(virtualAddress)]; 136 uint64* virtualPageDir; 137 if ((*pdpte & X86_64_PDPTE_PRESENT) == 0) { 138 phys_addr_t physicalPageDir = get_free_page(args) * B_PAGE_SIZE; 139 140 TRACE("X86PagingMethod64Bit::MapEarly(): creating page directory for va" 141 " %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress, 142 physicalPageDir); 143 144 SetTableEntry(pdpte, (physicalPageDir & X86_64_PDPTE_ADDRESS_MASK) 145 | X86_64_PDPTE_PRESENT 146 | X86_64_PDPTE_WRITABLE 147 | X86_64_PDPTE_USER); 148 149 // Map it and zero it. 150 virtualPageDir = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt( 151 physicalPageDir); 152 memset(virtualPageDir, 0, B_PAGE_SIZE); 153 } else { 154 virtualPageDir = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt( 155 *pdpte & X86_64_PDPTE_ADDRESS_MASK); 156 } 157 158 // Get the page table. 159 uint64* pde = &virtualPageDir[VADDR_TO_PDE(virtualAddress)]; 160 uint64* virtualPageTable; 161 if ((*pde & X86_64_PDE_PRESENT) == 0) { 162 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE; 163 164 TRACE("X86PagingMethod64Bit::MapEarly(): creating page table for va" 165 " %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress, 166 physicalPageTable); 167 168 SetTableEntry(pde, (physicalPageTable & X86_64_PDE_ADDRESS_MASK) 169 | X86_64_PDE_PRESENT 170 | X86_64_PDE_WRITABLE 171 | X86_64_PDE_USER); 172 173 // Map it and zero it. 174 virtualPageTable = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt( 175 physicalPageTable); 176 memset(virtualPageTable, 0, B_PAGE_SIZE); 177 } else { 178 virtualPageTable = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt( 179 *pde & X86_64_PDE_ADDRESS_MASK); 180 } 181 182 // The page table entry must not already be mapped. 183 uint64* pte = &virtualPageTable[VADDR_TO_PTE(virtualAddress)]; 184 ASSERT_PRINT( 185 (*pte & X86_64_PTE_PRESENT) == 0, 186 "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64, 187 virtualAddress, *pte); 188 189 // Fill in the table entry. 190 PutPageTableEntryInTable(pte, physicalAddress, attributes, 0, 191 IS_KERNEL_ADDRESS(virtualAddress)); 192 193 return B_OK; 194 } 195 196 197 bool 198 X86PagingMethod64Bit::IsKernelPageAccessible(addr_t virtualAddress, 199 uint32 protection) 200 { 201 return true; 202 } 203 204 205 /*! Traverses down the paging structure hierarchy to find the page directory 206 for a virtual address, allocating new tables if required. 207 */ 208 /*static*/ uint64* 209 X86PagingMethod64Bit::PageDirectoryForAddress(uint64* virtualPML4, 210 addr_t virtualAddress, bool isKernel, bool allocateTables, 211 vm_page_reservation* reservation, 212 TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount) 213 { 214 // Get the PDPT. 215 uint64* pml4e = &virtualPML4[VADDR_TO_PML4E(virtualAddress)]; 216 if ((*pml4e & X86_64_PML4E_PRESENT) == 0) { 217 if (!allocateTables) 218 return NULL; 219 220 // Allocate a new PDPT. 221 vm_page* page = vm_page_allocate_page(reservation, 222 PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); 223 224 DEBUG_PAGE_ACCESS_END(page); 225 226 phys_addr_t physicalPDPT 227 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 228 229 TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating PDPT " 230 "for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress, 231 physicalPDPT); 232 233 SetTableEntry(pml4e, (physicalPDPT & X86_64_PML4E_ADDRESS_MASK) 234 | X86_64_PML4E_PRESENT 235 | X86_64_PML4E_WRITABLE 236 | X86_64_PML4E_USER); 237 238 mapCount++; 239 } 240 241 uint64* virtualPDPT = (uint64*)pageMapper->GetPageTableAt( 242 *pml4e & X86_64_PML4E_ADDRESS_MASK); 243 244 // Get the page directory. 245 uint64* pdpte = &virtualPDPT[VADDR_TO_PDPTE(virtualAddress)]; 246 if ((*pdpte & X86_64_PDPTE_PRESENT) == 0) { 247 if (!allocateTables) 248 return NULL; 249 250 // Allocate a new page directory. 251 vm_page* page = vm_page_allocate_page(reservation, 252 PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); 253 254 DEBUG_PAGE_ACCESS_END(page); 255 256 phys_addr_t physicalPageDir 257 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 258 259 TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating page " 260 "directory for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", 261 virtualAddress, physicalPageDir); 262 263 SetTableEntry(pdpte, (physicalPageDir & X86_64_PDPTE_ADDRESS_MASK) 264 | X86_64_PDPTE_PRESENT 265 | X86_64_PDPTE_WRITABLE 266 | X86_64_PDPTE_USER); 267 268 mapCount++; 269 } 270 271 return (uint64*)pageMapper->GetPageTableAt( 272 *pdpte & X86_64_PDPTE_ADDRESS_MASK); 273 } 274 275 276 /*static*/ uint64* 277 X86PagingMethod64Bit::PageDirectoryEntryForAddress(uint64* virtualPML4, 278 addr_t virtualAddress, bool isKernel, bool allocateTables, 279 vm_page_reservation* reservation, 280 TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount) 281 { 282 uint64* virtualPageDirectory = PageDirectoryForAddress(virtualPML4, 283 virtualAddress, isKernel, allocateTables, reservation, pageMapper, 284 mapCount); 285 if (virtualPageDirectory == NULL) 286 return NULL; 287 288 return &virtualPageDirectory[VADDR_TO_PDE(virtualAddress)]; 289 } 290 291 292 /*! Traverses down the paging structure hierarchy to find the page table for a 293 virtual address, allocating new tables if required. 294 */ 295 /*static*/ uint64* 296 X86PagingMethod64Bit::PageTableForAddress(uint64* virtualPML4, 297 addr_t virtualAddress, bool isKernel, bool allocateTables, 298 vm_page_reservation* reservation, 299 TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount) 300 { 301 TRACE("X86PagingMethod64Bit::PageTableForAddress(%#" B_PRIxADDR ", " 302 "%d)\n", virtualAddress, allocateTables); 303 304 uint64* pde = PageDirectoryEntryForAddress(virtualPML4, virtualAddress, 305 isKernel, allocateTables, reservation, pageMapper, mapCount); 306 if (pde == NULL) 307 return NULL; 308 309 if ((*pde & X86_64_PDE_PRESENT) == 0) { 310 if (!allocateTables) 311 return NULL; 312 313 // Allocate a new page table. 314 vm_page* page = vm_page_allocate_page(reservation, 315 PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); 316 317 DEBUG_PAGE_ACCESS_END(page); 318 319 phys_addr_t physicalPageTable 320 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; 321 322 TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating page " 323 "table for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", 324 virtualAddress, physicalPageTable); 325 326 SetTableEntry(pde, (physicalPageTable & X86_64_PDE_ADDRESS_MASK) 327 | X86_64_PDE_PRESENT 328 | X86_64_PDE_WRITABLE 329 | X86_64_PDE_USER); 330 331 mapCount++; 332 } 333 334 // No proper large page support at the moment, but they are used for the 335 // physical map area. Ensure that nothing tries to treat that as normal 336 // address space. 337 ASSERT(!(*pde & X86_64_PDE_LARGE_PAGE)); 338 339 return (uint64*)pageMapper->GetPageTableAt(*pde & X86_64_PDE_ADDRESS_MASK); 340 } 341 342 343 /*static*/ uint64* 344 X86PagingMethod64Bit::PageTableEntryForAddress(uint64* virtualPML4, 345 addr_t virtualAddress, bool isKernel, bool allocateTables, 346 vm_page_reservation* reservation, 347 TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount) 348 { 349 uint64* virtualPageTable = PageTableForAddress(virtualPML4, virtualAddress, 350 isKernel, allocateTables, reservation, pageMapper, mapCount); 351 if (virtualPageTable == NULL) 352 return NULL; 353 354 return &virtualPageTable[VADDR_TO_PTE(virtualAddress)]; 355 } 356 357 358 /*static*/ void 359 X86PagingMethod64Bit::PutPageTableEntryInTable(uint64* entry, 360 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, 361 bool globalPage) 362 { 363 uint64 page = (physicalAddress & X86_64_PTE_ADDRESS_MASK) 364 | X86_64_PTE_PRESENT | (globalPage ? X86_64_PTE_GLOBAL : 0) 365 | MemoryTypeToPageTableEntryFlags(memoryType); 366 367 // if the page is user accessible, it's automatically 368 // accessible in kernel space, too (but with the same 369 // protection) 370 if ((attributes & B_USER_PROTECTION) != 0) { 371 page |= X86_64_PTE_USER; 372 if ((attributes & B_WRITE_AREA) != 0) 373 page |= X86_64_PTE_WRITABLE; 374 if ((attributes & B_EXECUTE_AREA) == 0 375 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 376 page |= X86_64_PTE_NOT_EXECUTABLE; 377 } 378 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) 379 page |= X86_64_PTE_WRITABLE; 380 381 // put it in the page table 382 SetTableEntry(entry, page); 383 } 384 385 386 /*static*/ void 387 X86PagingMethod64Bit::_EnableExecutionDisable(void* dummy, int cpu) 388 { 389 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER) 390 | IA32_MSR_EFER_NX); 391 } 392 393