1 /*
2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
4 * Distributed under the terms of the MIT License.
5 *
6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
7 * Distributed under the terms of the NewOS License.
8 */
9
10
11 #include "paging/pae/X86PagingMethodPAE.h"
12
13 #include <stdlib.h>
14 #include <string.h>
15
16 #include <AutoDeleter.h>
17
18 #include <arch/smp.h>
19 #include <boot/kernel_args.h>
20 #include <util/AutoLock.h>
21 #include <vm/vm.h>
22 #include <vm/vm_page.h>
23 #include <vm/VMAddressSpace.h>
24
25 #include "paging/32bit/paging.h"
26 #include "paging/32bit/X86PagingMethod32Bit.h"
27 #include "paging/pae/X86PagingStructuresPAE.h"
28 #include "paging/pae/X86VMTranslationMapPAE.h"
29 #include "paging/x86_physical_page_mapper.h"
30 #include "paging/x86_physical_page_mapper_large_memory.h"
31
32
33 //#define TRACE_X86_PAGING_METHOD_PAE
34 #ifdef TRACE_X86_PAGING_METHOD_PAE
35 # define TRACE(x...) dprintf(x)
36 #else
37 # define TRACE(x...) ;
38 #endif
39
40
41 #if B_HAIKU_PHYSICAL_BITS == 64
42
43
44 #define MAX_INITIAL_POOLS \
45 (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, \
46 kPAEPageTableEntryCount) \
47 / kPAEPageTableEntryCount)
48
49
50 using X86LargePhysicalPageMapper::PhysicalPageSlot;
51
52
53 // number of 32 bit pages that will be cached
54 static const page_num_t kMaxFree32BitPagesCount = 32;
55
56
57 // #pragma mark - ToPAESwitcher
58
59
60 struct X86PagingMethodPAE::ToPAESwitcher {
ToPAESwitcherX86PagingMethodPAE::ToPAESwitcher61 ToPAESwitcher(kernel_args* args)
62 :
63 fKernelArgs(args)
64 {
65 // page hole set up in the boot loader
66 fPageHole = (page_table_entry*)
67 (addr_t)fKernelArgs->arch_args.page_hole;
68
69 // calculate where the page dir would be
70 fPageHolePageDir = (page_directory_entry*)
71 (((addr_t)fKernelArgs->arch_args.page_hole)
72 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE));
73
74 fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir;
75
76 TRACE("page hole: %p\n", fPageHole);
77 TRACE("page dir: %p (physical: %#" B_PRIxPHYSADDR ")\n",
78 fPageHolePageDir, fPhysicalPageDir);
79 }
80
SwitchX86PagingMethodPAE::ToPAESwitcher81 void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT,
82 phys_addr_t& _physicalPDPT, void*& _pageStructures,
83 size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs,
84 phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot,
85 pae_page_table_entry*& _freeVirtualSlotPTE)
86 {
87 // count the page tables we have to translate
88 uint32 pageTableCount = 0;
89 for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
90 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
91 page_directory_entry entry = fPageHolePageDir[i];
92 if ((entry & X86_PDE_PRESENT) != 0)
93 pageTableCount++;
94 }
95
96 TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount);
97
98 // The pages we need to allocate to do our job:
99 // + 1 page dir pointer table
100 // + 4 page dirs
101 // + 2 * page tables (each has 512 instead of 1024 entries)
102 // + 1 page for the free virtual slot (no physical page needed)
103 uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1;
104
105 // We need additional PAE page tables for the new pages we're going to
106 // allocate: Two tables for every 1024 pages to map, i.e. 2 additional
107 // pages for every 1022 pages we want to allocate. We also need 32 bit
108 // page tables, but we don't need additional virtual space for them,
109 // since we can access then via the page hole.
110 pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2;
111
112 TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded);
113
114 // allocate the pages we need
115 _AllocateNeededPages(pagesNeeded);
116
117 // prepare the page directory pointer table
118 phys_addr_t physicalPDPT = 0;
119 pae_page_directory_pointer_table_entry* pdpt
120 = (pae_page_directory_pointer_table_entry*)_NextPage(true,
121 physicalPDPT);
122
123 for (int32 i = 0; i < 4; i++) {
124 fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true,
125 fPhysicalPageDirs[i]);
126
127 pdpt[i] = X86_PAE_PDPTE_PRESENT
128 | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK);
129 }
130
131 // Since we have to enable PAE in two steps -- setting cr3 to the PDPT
132 // and setting the cr4 PAE bit -- we copy the kernel page dir entries to
133 // the PDPT page, so after setting cr3, we continue to have working
134 // kernel mappings. This requires that the PDPTE registers and the
135 // page dir entries don't interect, obviously.
136 ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry)
137 <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry));
138
139 // translate the page tables
140 for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
141 i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
142 if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) {
143 // two PAE page tables per 32 bit page table
144 _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE);
145 _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE);
146
147 // copy the page directory entry to the PDPT page
148 ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i];
149 }
150 }
151
152 TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n",
153 fFreeVirtualSlot, fFreeVirtualSlotPTE);
154
155 // enable PAE on all CPUs
156 call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT);
157
158 // if available enable NX-bit (No eXecute)
159 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD))
160 call_all_cpus_sync(&_EnableExecutionDisable, NULL);
161
162 // set return values
163 _virtualPDPT = pdpt;
164 _physicalPDPT = physicalPDPT;
165 _pageStructures = fAllocatedPages;
166 _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE;
167 memcpy(pageDirs, fPageDirs, sizeof(fPageDirs));
168 memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs));
169
170 _freeVirtualSlot = fFreeVirtualSlot;
171 _freeVirtualSlotPTE = fFreeVirtualSlotPTE;
172 }
173
174 private:
_EnablePAEX86PagingMethodPAE::ToPAESwitcher175 static void _EnablePAE(void* physicalPDPT, int cpu)
176 {
177 x86_write_cr3((addr_t)physicalPDPT);
178 x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES);
179 }
180
_EnableExecutionDisableX86PagingMethodPAE::ToPAESwitcher181 static void _EnableExecutionDisable(void* dummy, int cpu)
182 {
183 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER)
184 | IA32_MSR_EFER_NX);
185 }
186
_TranslatePageTableX86PagingMethodPAE::ToPAESwitcher187 void _TranslatePageTable(addr_t virtualBase)
188 {
189 page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE];
190
191 // allocate a PAE page table
192 phys_addr_t physicalTable = 0;
193 pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false,
194 physicalTable);
195
196 // enter it into the page dir
197 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
198 fPageDirs, virtualBase);
199 PutPageTableInPageDir(pageDirEntry, physicalTable,
200 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
201
202 pae_page_table_entry* paeEntry = paeTable;
203 for (uint32 i = 0; i < kPAEPageTableEntryCount;
204 i++, entry++, paeEntry++) {
205 if ((*entry & X86_PTE_PRESENT) != 0
206 && _IsVirtualAddressAllocated(virtualBase + i * B_PAGE_SIZE)) {
207 // Note, we use the fact that the PAE flags are defined to the
208 // same values.
209 *paeEntry = *entry & (X86_PTE_PRESENT
210 | X86_PTE_WRITABLE
211 | X86_PTE_USER
212 | X86_PTE_WRITE_THROUGH
213 | X86_PTE_CACHING_DISABLED
214 | X86_PTE_GLOBAL
215 | X86_PTE_ADDRESS_MASK);
216 } else
217 *paeEntry = 0;
218 }
219
220 if (fFreeVirtualSlot / kPAEPageTableRange
221 == virtualBase / kPAEPageTableRange) {
222 fFreeVirtualSlotPTE = paeTable
223 + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount;
224 }
225 }
226
_AllocateNeededPagesX86PagingMethodPAE::ToPAESwitcher227 void _AllocateNeededPages(uint32 pagesNeeded)
228 {
229 size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE;
230 addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0,
231 kPageTableAlignment);
232 if (virtualBase == 0) {
233 panic("Failed to reserve virtual address space for the switch to "
234 "PAE!");
235 }
236
237 TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n",
238 virtualBase, virtualSize);
239
240 // allocate pages for the 32 bit page tables and prepare the tables
241 uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024;
242 for (uint32 i = 0; i < oldPageTableCount; i++) {
243 // allocate a page
244 phys_addr_t physicalTable = _Allocate32BitPage();
245
246 // put the page into the page dir
247 page_directory_entry* entry = &fPageHolePageDir[
248 virtualBase / B_PAGE_SIZE / 1024 + i];
249 X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable,
250 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
251
252 // clear the table
253 memset((void*)((addr_t)fPageHole
254 + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE),
255 0, B_PAGE_SIZE);
256 }
257
258 // We don't need a physical page for the free virtual slot.
259 pagesNeeded--;
260
261 // allocate and map the pages we need
262 for (uint32 i = 0; i < pagesNeeded; i++) {
263 // allocate a page
264 phys_addr_t physicalAddress = _Allocate32BitPage();
265
266 // put the page into the page table
267 page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i;
268 X86PagingMethod32Bit::PutPageTableEntryInTable(entry,
269 physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0,
270 true);
271
272 // Write the page's physical address into the page itself, so we
273 // don't need to look it up later.
274 *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress;
275 }
276
277 fAllocatedPages = (uint8*)virtualBase;
278 fAllocatedPagesCount = pagesNeeded;
279 fUsedPagesCount = 0;
280 fFreeVirtualSlot
281 = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE);
282 }
283
_Allocate32BitPageX86PagingMethodPAE::ToPAESwitcher284 phys_addr_t _Allocate32BitPage()
285 {
286 phys_addr_t physicalAddress
287 = (phys_addr_t)vm_allocate_early_physical_page_etc(fKernelArgs, 0xffffffff)
288 * B_PAGE_SIZE;
289 if (physicalAddress == 0 || physicalAddress > 0xffffffff) {
290 panic("Failed to allocate 32-bit-addressable page for the switch "
291 "to PAE!");
292 return 0;
293 }
294 return physicalAddress;
295 }
296
_NextPageX86PagingMethodPAE::ToPAESwitcher297 void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress)
298 {
299 if (fUsedPagesCount >= fAllocatedPagesCount) {
300 panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more "
301 "allocated pages!");
302 return NULL;
303 }
304
305 void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE;
306 _physicalAddress = *((phys_addr_t*)page);
307
308 if (clearPage)
309 memset(page, 0, B_PAGE_SIZE);
310
311 return page;
312 }
313
_IsVirtualAddressAllocatedX86PagingMethodPAE::ToPAESwitcher314 bool _IsVirtualAddressAllocated(addr_t address) const
315 {
316 for (uint32 i = 0; i < fKernelArgs->num_virtual_allocated_ranges; i++) {
317 addr_t start = fKernelArgs->virtual_allocated_range[i].start;
318 addr_t end = start + fKernelArgs->virtual_allocated_range[i].size;
319 if (address < start)
320 return false;
321 if (address <= end - 1)
322 return true;
323 }
324
325 return false;
326 }
327
328 private:
329 kernel_args* fKernelArgs;
330 page_table_entry* fPageHole;
331 page_directory_entry* fPageHolePageDir;
332 phys_addr_t fPhysicalPageDir;
333 uint8* fAllocatedPages;
334 uint32 fAllocatedPagesCount;
335 uint32 fUsedPagesCount;
336 addr_t fFreeVirtualSlot;
337 pae_page_table_entry* fFreeVirtualSlotPTE;
338 pae_page_directory_entry* fPageDirs[4];
339 phys_addr_t fPhysicalPageDirs[4];
340 };
341
342
343 // #pragma mark - PhysicalPageSlotPool
344
345
346 struct X86PagingMethodPAE::PhysicalPageSlotPool final
347 : X86LargePhysicalPageMapper::PhysicalPageSlotPool {
348 public:
349 virtual ~PhysicalPageSlotPool();
350
351 status_t InitInitial(X86PagingMethodPAE* method,
352 kernel_args* args);
353 status_t InitInitialPostArea(kernel_args* args);
354
355 void Init(area_id dataArea,
356 pae_page_table_entry* pageTable,
357 area_id virtualArea, addr_t virtualBase);
358
359 virtual status_t AllocatePool(
360 X86LargePhysicalPageMapper
361 ::PhysicalPageSlotPool*& _pool);
362 virtual void Map(phys_addr_t physicalAddress,
363 addr_t virtualAddress);
364
365 public:
366 static PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS];
367
368 private:
369 area_id fDataArea;
370 area_id fVirtualArea;
371 addr_t fVirtualBase;
372 pae_page_table_entry* fPageTable;
373 };
374
375
376 X86PagingMethodPAE::PhysicalPageSlotPool
377 X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[
378 MAX_INITIAL_POOLS];
379
380
~PhysicalPageSlotPool()381 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool()
382 {
383 }
384
385
386 status_t
InitInitial(X86PagingMethodPAE * method,kernel_args * args)387 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(
388 X86PagingMethodPAE* method, kernel_args* args)
389 {
390 // allocate a virtual address range for the pages to be mapped into
391 addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0,
392 kPAEPageTableRange);
393 if (virtualBase == 0) {
394 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
395 "physical page pool space in virtual address space!");
396 return B_ERROR;
397 }
398
399 // allocate memory for the page table and data
400 size_t areaSize = B_PAGE_SIZE
401 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
402 pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early(
403 args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
404 if (pageTable == 0) {
405 panic("X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(): Failed "
406 "to allocate memory for page table!");
407 return B_ERROR;
408 }
409
410 // clear the page table and put it in the page dir
411 memset(pageTable, 0, B_PAGE_SIZE);
412
413 phys_addr_t physicalTable = 0;
414 method->_EarlyQuery((addr_t)pageTable, &physicalTable);
415
416 pae_page_directory_entry* entry = PageDirEntryForAddress(
417 method->KernelVirtualPageDirs(), virtualBase);
418 PutPageTableInPageDir(entry, physicalTable,
419 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
420
421 // init the pool structure and add the initial pool
422 Init(-1, pageTable, -1, (addr_t)virtualBase);
423
424 return B_OK;
425 }
426
427
428 status_t
InitInitialPostArea(kernel_args * args)429 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea(
430 kernel_args* args)
431 {
432 // create an area for the (already allocated) data
433 size_t areaSize = B_PAGE_SIZE
434 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
435 void* temp = fPageTable;
436 area_id area = create_area("physical page pool", &temp,
437 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
438 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
439 if (area < B_OK) {
440 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
441 "create area for physical page pool.");
442 return area;
443 }
444 fDataArea = area;
445
446 // create an area for the virtual address space
447 temp = (void*)fVirtualBase;
448 area = vm_create_null_area(VMAddressSpace::KernelID(),
449 "physical page pool space", &temp, B_EXACT_ADDRESS,
450 kPAEPageTableRange, 0);
451 if (area < B_OK) {
452 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
453 "create area for physical page pool space.");
454 return area;
455 }
456 fVirtualArea = area;
457
458 return B_OK;
459 }
460
461
462 void
Init(area_id dataArea,pae_page_table_entry * pageTable,area_id virtualArea,addr_t virtualBase)463 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea,
464 pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase)
465 {
466 fDataArea = dataArea;
467 fVirtualArea = virtualArea;
468 fVirtualBase = virtualBase;
469 fPageTable = pageTable;
470
471 // init slot list
472 fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount);
473 addr_t slotAddress = virtualBase;
474 for (uint32 i = 0; i < kPAEPageTableEntryCount;
475 i++, slotAddress += B_PAGE_SIZE) {
476 PhysicalPageSlot* slot = &fSlots[i];
477 slot->next = slot + 1;
478 slot->pool = this;
479 slot->address = slotAddress;
480 }
481
482 fSlots[kPAEPageTableEntryCount - 1].next = NULL;
483 // terminate list
484 }
485
486
487 void
Map(phys_addr_t physicalAddress,addr_t virtualAddress)488 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
489 addr_t virtualAddress)
490 {
491 pae_page_table_entry& pte = fPageTable[
492 (virtualAddress - fVirtualBase) / B_PAGE_SIZE];
493 pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
494 | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT;
495
496 invalidate_TLB(virtualAddress);
497 }
498
499
500 status_t
AllocatePool(X86LargePhysicalPageMapper::PhysicalPageSlotPool * & _pool)501 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool(
502 X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
503 {
504 // create the pool structure
505 PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
506 if (pool == NULL)
507 return B_NO_MEMORY;
508 ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);
509
510 // create an area that can contain the page table and the slot
511 // structures
512 size_t areaSize = B_PAGE_SIZE
513 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
514 void* data;
515 virtual_address_restrictions virtualRestrictions = {};
516 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
517 physical_address_restrictions physicalRestrictions = {};
518 area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
519 PAGE_ALIGN(areaSize), B_FULL_LOCK,
520 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
521 &virtualRestrictions, &physicalRestrictions, &data);
522 if (dataArea < 0)
523 return dataArea;
524
525 // create the null area for the virtual address space
526 void* virtualBase;
527 area_id virtualArea = vm_create_null_area(
528 VMAddressSpace::KernelID(), "physical page pool space",
529 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange,
530 CREATE_AREA_PRIORITY_VIP);
531 if (virtualArea < 0) {
532 delete_area(dataArea);
533 return virtualArea;
534 }
535
536 // prepare the page table
537 memset(data, 0, B_PAGE_SIZE);
538
539 // get the page table's physical address
540 phys_addr_t physicalTable;
541 X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>(
542 VMAddressSpace::Kernel()->TranslationMap());
543 uint32 dummyFlags;
544 cpu_status state = disable_interrupts();
545 map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
546 restore_interrupts(state);
547
548 // put the page table into the page directory
549 pae_page_directory_entry* pageDirEntry
550 = X86PagingMethodPAE::PageDirEntryForAddress(
551 map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase);
552 PutPageTableInPageDir(pageDirEntry, physicalTable,
553 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
554
555 // init the pool structure
556 pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea,
557 (addr_t)virtualBase);
558 poolDeleter.Detach();
559 _pool = pool;
560 return B_OK;
561 }
562
563
564 // #pragma mark - X86PagingMethodPAE
565
566
X86PagingMethodPAE()567 X86PagingMethodPAE::X86PagingMethodPAE()
568 :
569 fPhysicalPageMapper(NULL),
570 fKernelPhysicalPageMapper(NULL),
571 fFreePages(NULL),
572 fFreePagesCount(0)
573 {
574 mutex_init(&fFreePagesLock, "x86 PAE free pages");
575 }
576
577
~X86PagingMethodPAE()578 X86PagingMethodPAE::~X86PagingMethodPAE()
579 {
580 }
581
582
583 status_t
Init(kernel_args * args,VMPhysicalPageMapper ** _physicalPageMapper)584 X86PagingMethodPAE::Init(kernel_args* args,
585 VMPhysicalPageMapper** _physicalPageMapper)
586 {
587 // Ignore all memory beyond the maximum PAE address.
588 static const phys_addr_t kLimit = 1ULL << 36;
589 for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) {
590 addr_range& range = args->physical_memory_range[i];
591 if (range.start >= kLimit)
592 range.size = 0;
593 else if ((range.start + range.size) > kLimit)
594 range.size = kLimit - range.start;
595 }
596
597 // switch to PAE
598 ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable,
599 fKernelPhysicalPageDirPointerTable, fEarlyPageStructures,
600 fEarlyPageStructuresSize, fKernelVirtualPageDirs,
601 fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE);
602
603 // create the initial pools for the physical page mapper
604 int32 poolCount = _GetInitialPoolCount();
605 PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool;
606
607 for (int32 i = 0; i < poolCount; i++) {
608 new(&pool[i]) PhysicalPageSlotPool;
609 status_t error = pool[i].InitInitial(this, args);
610 if (error != B_OK) {
611 panic("X86PagingMethodPAE::Init(): Failed to create initial pool "
612 "for physical page mapper!");
613 return error;
614 }
615 }
616
617 // create physical page mapper
618 large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool),
619 fPhysicalPageMapper, fKernelPhysicalPageMapper);
620
621 *_physicalPageMapper = fPhysicalPageMapper;
622 return B_OK;
623 }
624
625
626 status_t
InitPostArea(kernel_args * args)627 X86PagingMethodPAE::InitPostArea(kernel_args* args)
628 {
629 // wrap the kernel paging structures in an area
630 area_id area = create_area("kernel paging structs", &fEarlyPageStructures,
631 B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED,
632 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
633 if (area < B_OK)
634 return area;
635
636 // let the initial page pools create areas for its structures
637 int32 poolCount = _GetInitialPoolCount();
638 for (int32 i = 0; i < poolCount; i++) {
639 status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i]
640 .InitInitialPostArea(args);
641 if (error != B_OK)
642 return error;
643 }
644
645 // The early physical page mapping mechanism is no longer needed. Unmap the
646 // slot.
647 *fFreeVirtualSlotPTE = 0;
648 invalidate_TLB(fFreeVirtualSlot);
649
650 fFreeVirtualSlotPTE = NULL;
651 fFreeVirtualSlot = 0;
652
653 return B_OK;
654 }
655
656
657 status_t
CreateTranslationMap(bool kernel,VMTranslationMap ** _map)658 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
659 {
660 X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE;
661 if (map == NULL)
662 return B_NO_MEMORY;
663
664 status_t error = map->Init(kernel);
665 if (error != B_OK) {
666 delete map;
667 return error;
668 }
669
670 *_map = map;
671 return B_OK;
672 }
673
674
675 status_t
MapEarly(kernel_args * args,addr_t virtualAddress,phys_addr_t physicalAddress,uint8 attributes,page_num_t (* get_free_page)(kernel_args *))676 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress,
677 phys_addr_t physicalAddress, uint8 attributes,
678 page_num_t (*get_free_page)(kernel_args*))
679 {
680 // check to see if a page table exists for this range
681 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
682 fKernelVirtualPageDirs, virtualAddress);
683 pae_page_table_entry* pageTable;
684 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
685 // we need to allocate a page table
686 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE;
687
688 TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for "
689 "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable);
690
691 // put it in the page dir
692 PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes);
693
694 // zero it out
695 pageTable = _EarlyGetPageTable(physicalPageTable);
696 memset(pageTable, 0, B_PAGE_SIZE);
697 } else {
698 // table already exists -- map it
699 pageTable = _EarlyGetPageTable(
700 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK);
701 }
702
703 pae_page_table_entry* entry = pageTable
704 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
705
706 ASSERT_PRINT(
707 (*entry & X86_PAE_PTE_PRESENT) == 0,
708 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64
709 ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry);
710
711 // now, fill in the pentry
712 PutPageTableEntryInTable(entry, physicalAddress, attributes, 0,
713 IS_KERNEL_ADDRESS(virtualAddress));
714
715 return B_OK;
716 }
717
718
719 bool
IsKernelPageAccessible(addr_t virtualAddress,uint32 protection)720 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress,
721 uint32 protection)
722 {
723 // we can't check much without the physical page mapper
724 if (fPhysicalPageMapper == NULL)
725 return false;
726
727 // We only trust the kernel team's page directories. So switch to the
728 // kernel PDPT first. Always set it to make sure the TLBs don't contain
729 // obsolete data.
730 uint32 physicalPDPT = x86_read_cr3();
731 x86_write_cr3(fKernelPhysicalPageDirPointerTable);
732
733 // get the PDPT entry for the address
734 pae_page_directory_pointer_table_entry pdptEntry = 0;
735 if (physicalPDPT == fKernelPhysicalPageDirPointerTable) {
736 pdptEntry = fKernelVirtualPageDirPointerTable[
737 virtualAddress / kPAEPageDirRange];
738 } else {
739 // map the original PDPT and get the entry
740 void* handle;
741 addr_t virtualPDPT;
742 status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT,
743 &virtualPDPT, &handle);
744 if (error == B_OK) {
745 pdptEntry = ((pae_page_directory_pointer_table_entry*)
746 virtualPDPT)[virtualAddress / kPAEPageDirRange];
747 fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle);
748 }
749 }
750
751 // map the page dir and get the entry
752 pae_page_directory_entry pageDirEntry = 0;
753 if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) {
754 void* handle;
755 addr_t virtualPageDir;
756 status_t error = fPhysicalPageMapper->GetPageDebug(
757 pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle);
758 if (error == B_OK) {
759 pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[
760 virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount];
761 fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle);
762 }
763 }
764
765 // map the page table and get the entry
766 pae_page_table_entry pageTableEntry = 0;
767 if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) {
768 void* handle;
769 addr_t virtualPageTable;
770 status_t error = fPhysicalPageMapper->GetPageDebug(
771 pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable,
772 &handle);
773 if (error == B_OK) {
774 pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[
775 virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount];
776 fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
777 }
778 }
779
780 // switch back to the original page directory
781 if (physicalPDPT != fKernelPhysicalPageDirPointerTable)
782 x86_write_cr3(physicalPDPT);
783
784 if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0)
785 return false;
786
787 // present means kernel-readable, so check for writable
788 return (protection & B_KERNEL_WRITE_AREA) == 0
789 || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0;
790 }
791
792
793 /*static*/ void
PutPageTableInPageDir(pae_page_directory_entry * entry,phys_addr_t physicalTable,uint32 attributes)794 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry,
795 phys_addr_t physicalTable, uint32 attributes)
796 {
797 SetTableEntry(entry, (physicalTable & X86_PAE_PDE_ADDRESS_MASK)
798 | X86_PAE_PDE_PRESENT
799 | X86_PAE_PDE_WRITABLE
800 | X86_PAE_PDE_USER);
801 // TODO: We ignore the attributes of the page table -- for compatibility
802 // with BeOS we allow having user accessible areas in the kernel address
803 // space. This is currently being used by some drivers, mainly for the
804 // frame buffer. Our current real time data implementation makes use of
805 // this fact, too.
806 // We might want to get rid of this possibility one day, especially if
807 // we intend to port it to a platform that does not support this.
808 }
809
810
811 /*static*/ void
PutPageTableEntryInTable(pae_page_table_entry * entry,phys_addr_t physicalAddress,uint32 attributes,uint32 memoryType,bool globalPage)812 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry,
813 phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
814 bool globalPage)
815 {
816 pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
817 | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0)
818 | MemoryTypeToPageTableEntryFlags(memoryType);
819
820 // if the page is user accessible, it's automatically
821 // accessible in kernel space, too (but with the same
822 // protection)
823 if ((attributes & B_USER_PROTECTION) != 0) {
824 page |= X86_PAE_PTE_USER;
825 if ((attributes & B_WRITE_AREA) != 0)
826 page |= X86_PAE_PTE_WRITABLE;
827 if ((attributes & B_EXECUTE_AREA) == 0
828 && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
829 page |= X86_PAE_PTE_NOT_EXECUTABLE;
830 }
831 } else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
832 page |= X86_PAE_PTE_WRITABLE;
833
834 // put it in the page table
835 SetTableEntry(entry, page);
836 }
837
838
839 void*
Allocate32BitPage(phys_addr_t & _physicalAddress,void * & _handle)840 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress,
841 void*& _handle)
842 {
843 // get a free page
844 MutexLocker locker(fFreePagesLock);
845 vm_page* page;
846 if (fFreePages != NULL) {
847 page = fFreePages;
848 fFreePages = page->cache_next;
849 fFreePagesCount--;
850 locker.Unlock();
851 } else {
852 // no pages -- allocate one
853 locker.Unlock();
854
855 physical_address_restrictions restrictions = {};
856 restrictions.high_address = 0x100000000LL;
857 page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions,
858 VM_PRIORITY_SYSTEM);
859 if (page == NULL)
860 return NULL;
861
862 DEBUG_PAGE_ACCESS_END(page);
863 }
864
865 // map the page
866 phys_addr_t physicalAddress
867 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
868 addr_t virtualAddress;
869 if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle)
870 != B_OK) {
871 // mapping failed -- free page
872 locker.Lock();
873 page->cache_next = fFreePages;
874 fFreePages = page;
875 fFreePagesCount++;
876 return NULL;
877 }
878
879 _physicalAddress = physicalAddress;
880 return (void*)virtualAddress;
881 }
882
883
884 void
Free32BitPage(void * address,phys_addr_t physicalAddress,void * handle)885 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress,
886 void* handle)
887 {
888 // unmap the page
889 fPhysicalPageMapper->PutPage((addr_t)address, handle);
890
891 // free it
892 vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
893 MutexLocker locker(fFreePagesLock);
894 if (fFreePagesCount < kMaxFree32BitPagesCount) {
895 // cache not full yet -- cache it
896 page->cache_next = fFreePages;
897 fFreePages = page;
898 fFreePagesCount++;
899 } else {
900 // cache full -- free it
901 locker.Unlock();
902 DEBUG_PAGE_ACCESS_START(page);
903 vm_page_free(NULL, page);
904 }
905 }
906
907
908 inline int32
_GetInitialPoolCount()909 X86PagingMethodPAE::_GetInitialPoolCount()
910 {
911 int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU
912 + EXTRA_SLOTS;
913 return (requiredSlots + kPAEPageTableEntryCount - 1)
914 / kPAEPageTableEntryCount;
915 }
916
917
918 bool
_EarlyQuery(addr_t virtualAddress,phys_addr_t * _physicalAddress)919 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress,
920 phys_addr_t* _physicalAddress)
921 {
922 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
923 fKernelVirtualPageDirs, virtualAddress);
924 if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
925 // no pagetable here
926 return false;
927 }
928
929 pae_page_table_entry* entry = _EarlyGetPageTable(
930 *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK)
931 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
932 if ((*entry & X86_PAE_PTE_PRESENT) == 0) {
933 // page mapping not valid
934 return false;
935 }
936
937 *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK;
938 return true;
939 }
940
941
942 pae_page_table_entry*
_EarlyGetPageTable(phys_addr_t address)943 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address)
944 {
945 *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK)
946 | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL;
947
948 invalidate_TLB(fFreeVirtualSlot);
949
950 return (pae_page_table_entry*)fFreeVirtualSlot;
951 }
952
953
954 #endif // B_HAIKU_PHYSICAL_BITS == 64
955