xref: /haiku/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp (revision c04087a9afe0b29fdc27b2383b2593abd6c84fe9)
1 /*
2  * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
3  * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
4  * Distributed under the terms of the MIT License.
5  *
6  * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
7  * Distributed under the terms of the NewOS License.
8  */
9 
10 
11 #include "paging/pae/X86PagingMethodPAE.h"
12 
13 #include <stdlib.h>
14 #include <string.h>
15 
16 #include <AutoDeleter.h>
17 
18 #include <arch/smp.h>
19 #include <boot/kernel_args.h>
20 #include <util/AutoLock.h>
21 #include <vm/vm.h>
22 #include <vm/vm_page.h>
23 #include <vm/VMAddressSpace.h>
24 
25 #include "paging/32bit/paging.h"
26 #include "paging/32bit/X86PagingMethod32Bit.h"
27 #include "paging/pae/X86PagingStructuresPAE.h"
28 #include "paging/pae/X86VMTranslationMapPAE.h"
29 #include "paging/x86_physical_page_mapper.h"
30 #include "paging/x86_physical_page_mapper_large_memory.h"
31 
32 
33 //#define TRACE_X86_PAGING_METHOD_PAE
34 #ifdef TRACE_X86_PAGING_METHOD_PAE
35 #	define TRACE(x...) dprintf(x)
36 #else
37 #	define TRACE(x...) ;
38 #endif
39 
40 
41 #if B_HAIKU_PHYSICAL_BITS == 64
42 
43 
44 #define MAX_INITIAL_POOLS	\
45 	(ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS,	\
46 			kPAEPageTableEntryCount)	\
47 		/ kPAEPageTableEntryCount)
48 
49 
50 using X86LargePhysicalPageMapper::PhysicalPageSlot;
51 
52 
53 // number of 32 bit pages that will be cached
54 static const page_num_t kMaxFree32BitPagesCount = 32;
55 
56 
57 // #pragma mark - ToPAESwitcher
58 
59 
60 struct X86PagingMethodPAE::ToPAESwitcher {
ToPAESwitcherX86PagingMethodPAE::ToPAESwitcher61 	ToPAESwitcher(kernel_args* args)
62 		:
63 		fKernelArgs(args)
64 	{
65 		// page hole set up in the boot loader
66 		fPageHole = (page_table_entry*)
67 			(addr_t)fKernelArgs->arch_args.page_hole;
68 
69 		// calculate where the page dir would be
70 		fPageHolePageDir = (page_directory_entry*)
71 			(((addr_t)fKernelArgs->arch_args.page_hole)
72 				+ (B_PAGE_SIZE * 1024 - B_PAGE_SIZE));
73 
74 		fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir;
75 
76 		TRACE("page hole: %p\n", fPageHole);
77 		TRACE("page dir:  %p (physical: %#" B_PRIxPHYSADDR ")\n",
78 			fPageHolePageDir, fPhysicalPageDir);
79 	}
80 
SwitchX86PagingMethodPAE::ToPAESwitcher81 	void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT,
82 		phys_addr_t& _physicalPDPT, void*& _pageStructures,
83 		size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs,
84 		phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot,
85 		pae_page_table_entry*& _freeVirtualSlotPTE)
86 	{
87 		// count the page tables we have to translate
88 		uint32 pageTableCount = 0;
89 		for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
90 			i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
91 			page_directory_entry entry = fPageHolePageDir[i];
92 			if ((entry & X86_PDE_PRESENT) != 0)
93 				pageTableCount++;
94 		}
95 
96 		TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount);
97 
98 		// The pages we need to allocate to do our job:
99 		// + 1 page dir pointer table
100 		// + 4 page dirs
101 		// + 2 * page tables (each has 512 instead of 1024 entries)
102 		// + 1 page for the free virtual slot (no physical page needed)
103 		uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1;
104 
105 		// We need additional PAE page tables for the new pages we're going to
106 		// allocate: Two tables for every 1024 pages to map, i.e. 2 additional
107 		// pages for every 1022 pages we want to allocate. We also need 32 bit
108 		// page tables, but we don't need additional virtual space for them,
109 		// since we can access then via the page hole.
110 		pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2;
111 
112 		TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded);
113 
114 		// allocate the pages we need
115 		_AllocateNeededPages(pagesNeeded);
116 
117 		// prepare the page directory pointer table
118 		phys_addr_t physicalPDPT = 0;
119 		pae_page_directory_pointer_table_entry* pdpt
120 			= (pae_page_directory_pointer_table_entry*)_NextPage(true,
121 				physicalPDPT);
122 
123 		for (int32 i = 0; i < 4; i++) {
124 			fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true,
125 				fPhysicalPageDirs[i]);
126 
127 			pdpt[i] = X86_PAE_PDPTE_PRESENT
128 				| (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK);
129 		}
130 
131 		// Since we have to enable PAE in two steps -- setting cr3 to the PDPT
132 		// and setting the cr4 PAE bit -- we copy the kernel page dir entries to
133 		// the PDPT page, so after setting cr3, we continue to have working
134 		// kernel mappings. This requires that the PDPTE registers and the
135 		// page dir entries don't interect, obviously.
136 		ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry)
137 			<= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry));
138 
139 		// translate the page tables
140 		for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
141 			i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
142 			if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) {
143 				// two PAE page tables per 32 bit page table
144 				_TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE);
145 				_TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE);
146 
147 				// copy the page directory entry to the PDPT page
148 				((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i];
149 			}
150 		}
151 
152 		TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n",
153 			fFreeVirtualSlot, fFreeVirtualSlotPTE);
154 
155 		// enable PAE on all CPUs
156 		call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT);
157 
158 		// if available enable NX-bit (No eXecute)
159 		if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD))
160 			call_all_cpus_sync(&_EnableExecutionDisable, NULL);
161 
162 		// set return values
163 		_virtualPDPT = pdpt;
164 		_physicalPDPT = physicalPDPT;
165 		_pageStructures = fAllocatedPages;
166 		_pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE;
167 		memcpy(pageDirs, fPageDirs, sizeof(fPageDirs));
168 		memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs));
169 
170 		_freeVirtualSlot = fFreeVirtualSlot;
171 		_freeVirtualSlotPTE = fFreeVirtualSlotPTE;
172 	}
173 
174 private:
_EnablePAEX86PagingMethodPAE::ToPAESwitcher175 	static void _EnablePAE(void* physicalPDPT, int cpu)
176 	{
177 		x86_write_cr3((addr_t)physicalPDPT);
178 		x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES);
179 	}
180 
_EnableExecutionDisableX86PagingMethodPAE::ToPAESwitcher181 	static void _EnableExecutionDisable(void* dummy, int cpu)
182 	{
183 		x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER)
184 			| IA32_MSR_EFER_NX);
185 	}
186 
_TranslatePageTableX86PagingMethodPAE::ToPAESwitcher187 	void _TranslatePageTable(addr_t virtualBase)
188 	{
189 		page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE];
190 
191 		// allocate a PAE page table
192 		phys_addr_t physicalTable = 0;
193 		pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false,
194 			physicalTable);
195 
196 		// enter it into the page dir
197 		pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
198 			fPageDirs, virtualBase);
199 		PutPageTableInPageDir(pageDirEntry, physicalTable,
200 			B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
201 
202 		pae_page_table_entry* paeEntry = paeTable;
203 		for (uint32 i = 0; i < kPAEPageTableEntryCount;
204 				i++, entry++, paeEntry++) {
205 			if ((*entry & X86_PTE_PRESENT) != 0
206 				&& _IsVirtualAddressAllocated(virtualBase + i * B_PAGE_SIZE)) {
207 				// Note, we use the fact that the PAE flags are defined to the
208 				// same values.
209 				*paeEntry = *entry & (X86_PTE_PRESENT
210 					| X86_PTE_WRITABLE
211 					| X86_PTE_USER
212 					| X86_PTE_WRITE_THROUGH
213 					| X86_PTE_CACHING_DISABLED
214 					| X86_PTE_GLOBAL
215 					| X86_PTE_ADDRESS_MASK);
216 			} else
217 				*paeEntry = 0;
218 		}
219 
220 		if (fFreeVirtualSlot / kPAEPageTableRange
221 				== virtualBase / kPAEPageTableRange) {
222 			fFreeVirtualSlotPTE = paeTable
223 				+ fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount;
224 		}
225 	}
226 
_AllocateNeededPagesX86PagingMethodPAE::ToPAESwitcher227 	void _AllocateNeededPages(uint32 pagesNeeded)
228 	{
229 		size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE;
230 		addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0,
231 			kPageTableAlignment);
232 		if (virtualBase == 0) {
233 			panic("Failed to reserve virtual address space for the switch to "
234 				"PAE!");
235 		}
236 
237 		TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n",
238 			virtualBase, virtualSize);
239 
240 		// allocate pages for the 32 bit page tables and prepare the tables
241 		uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024;
242 		for (uint32 i = 0; i < oldPageTableCount; i++) {
243 			// allocate a page
244 			phys_addr_t physicalTable = _Allocate32BitPage();
245 
246 			// put the page into the page dir
247 			page_directory_entry* entry = &fPageHolePageDir[
248 				virtualBase / B_PAGE_SIZE / 1024 + i];
249 			X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable,
250 				B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
251 
252 			// clear the table
253 			memset((void*)((addr_t)fPageHole
254 					+ (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE),
255 				0, B_PAGE_SIZE);
256 		}
257 
258 		// We don't need a physical page for the free virtual slot.
259 		pagesNeeded--;
260 
261 		// allocate and map the pages we need
262 		for (uint32 i = 0; i < pagesNeeded; i++) {
263 			// allocate a page
264 			phys_addr_t physicalAddress = _Allocate32BitPage();
265 
266 			// put the page into the page table
267 			page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i;
268 			X86PagingMethod32Bit::PutPageTableEntryInTable(entry,
269 				physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0,
270 				true);
271 
272 			// Write the page's physical address into the page itself, so we
273 			// don't need to look it up later.
274 			*(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress;
275 		}
276 
277 		fAllocatedPages = (uint8*)virtualBase;
278 		fAllocatedPagesCount = pagesNeeded;
279 		fUsedPagesCount = 0;
280 		fFreeVirtualSlot
281 			= (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE);
282 	}
283 
_Allocate32BitPageX86PagingMethodPAE::ToPAESwitcher284 	phys_addr_t _Allocate32BitPage()
285 	{
286 		phys_addr_t physicalAddress
287 			= (phys_addr_t)vm_allocate_early_physical_page_etc(fKernelArgs, 0xffffffff)
288 				* B_PAGE_SIZE;
289 		if (physicalAddress == 0 || physicalAddress > 0xffffffff) {
290 			panic("Failed to allocate 32-bit-addressable page for the switch "
291 				"to PAE!");
292 			return 0;
293 		}
294 		return physicalAddress;
295 	}
296 
_NextPageX86PagingMethodPAE::ToPAESwitcher297 	void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress)
298 	{
299 		if (fUsedPagesCount >= fAllocatedPagesCount) {
300 			panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more "
301 				"allocated pages!");
302 			return NULL;
303 		}
304 
305 		void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE;
306 		_physicalAddress = *((phys_addr_t*)page);
307 
308 		if (clearPage)
309 			memset(page, 0, B_PAGE_SIZE);
310 
311 		return page;
312 	}
313 
_IsVirtualAddressAllocatedX86PagingMethodPAE::ToPAESwitcher314 	bool _IsVirtualAddressAllocated(addr_t address) const
315 	{
316 		for (uint32 i = 0; i < fKernelArgs->num_virtual_allocated_ranges; i++) {
317 			addr_t start = fKernelArgs->virtual_allocated_range[i].start;
318 			addr_t end = start + fKernelArgs->virtual_allocated_range[i].size;
319 			if (address < start)
320 				return false;
321 			if (address <= end - 1)
322 				return true;
323 		}
324 
325 		return false;
326 	}
327 
328 private:
329 	kernel_args*				fKernelArgs;
330 	page_table_entry*			fPageHole;
331 	page_directory_entry*		fPageHolePageDir;
332 	phys_addr_t					fPhysicalPageDir;
333 	uint8*						fAllocatedPages;
334 	uint32						fAllocatedPagesCount;
335 	uint32						fUsedPagesCount;
336 	addr_t						fFreeVirtualSlot;
337 	pae_page_table_entry*		fFreeVirtualSlotPTE;
338 	pae_page_directory_entry*	fPageDirs[4];
339 	phys_addr_t					fPhysicalPageDirs[4];
340 };
341 
342 
343 // #pragma mark - PhysicalPageSlotPool
344 
345 
346 struct X86PagingMethodPAE::PhysicalPageSlotPool final
347 	: X86LargePhysicalPageMapper::PhysicalPageSlotPool {
348 public:
349 	virtual						~PhysicalPageSlotPool();
350 
351 			status_t			InitInitial(X86PagingMethodPAE* method,
352 									kernel_args* args);
353 			status_t			InitInitialPostArea(kernel_args* args);
354 
355 			void				Init(area_id dataArea,
356 									pae_page_table_entry* pageTable,
357 									area_id virtualArea, addr_t virtualBase);
358 
359 	virtual	status_t			AllocatePool(
360 									X86LargePhysicalPageMapper
361 										::PhysicalPageSlotPool*& _pool);
362 	virtual	void				Map(phys_addr_t physicalAddress,
363 									addr_t virtualAddress);
364 
365 public:
366 	static	PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS];
367 
368 private:
369 			area_id				fDataArea;
370 			area_id				fVirtualArea;
371 			addr_t				fVirtualBase;
372 			pae_page_table_entry* fPageTable;
373 };
374 
375 
376 X86PagingMethodPAE::PhysicalPageSlotPool
377 	X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[
378 		MAX_INITIAL_POOLS];
379 
380 
~PhysicalPageSlotPool()381 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool()
382 {
383 }
384 
385 
386 status_t
InitInitial(X86PagingMethodPAE * method,kernel_args * args)387 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(
388 	X86PagingMethodPAE* method, kernel_args* args)
389 {
390 	// allocate a virtual address range for the pages to be mapped into
391 	addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0,
392 		kPAEPageTableRange);
393 	if (virtualBase == 0) {
394 		panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
395 			"physical page pool space in virtual address space!");
396 		return B_ERROR;
397 	}
398 
399 	// allocate memory for the page table and data
400 	size_t areaSize = B_PAGE_SIZE
401 		+ sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
402 	pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early(
403 		args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
404 	if (pageTable == 0) {
405 		panic("X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(): Failed "
406 			"to allocate memory for page table!");
407 		return B_ERROR;
408 	}
409 
410 	// clear the page table and put it in the page dir
411 	memset(pageTable, 0, B_PAGE_SIZE);
412 
413 	phys_addr_t physicalTable = 0;
414 	method->_EarlyQuery((addr_t)pageTable, &physicalTable);
415 
416 	pae_page_directory_entry* entry = PageDirEntryForAddress(
417 		method->KernelVirtualPageDirs(), virtualBase);
418 	PutPageTableInPageDir(entry, physicalTable,
419 		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
420 
421 	// init the pool structure and add the initial pool
422 	Init(-1, pageTable, -1, (addr_t)virtualBase);
423 
424 	return B_OK;
425 }
426 
427 
428 status_t
InitInitialPostArea(kernel_args * args)429 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea(
430 	kernel_args* args)
431 {
432 	// create an area for the (already allocated) data
433 	size_t areaSize = B_PAGE_SIZE
434 		+ sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
435 	void* temp = fPageTable;
436 	area_id area = create_area("physical page pool", &temp,
437 		B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
438 		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
439 	if (area < B_OK) {
440 		panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
441 			"create area for physical page pool.");
442 		return area;
443 	}
444 	fDataArea = area;
445 
446 	// create an area for the virtual address space
447 	temp = (void*)fVirtualBase;
448 	area = vm_create_null_area(VMAddressSpace::KernelID(),
449 		"physical page pool space", &temp, B_EXACT_ADDRESS,
450 		kPAEPageTableRange, 0);
451 	if (area < B_OK) {
452 		panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
453 			"create area for physical page pool space.");
454 		return area;
455 	}
456 	fVirtualArea = area;
457 
458 	return B_OK;
459 }
460 
461 
462 void
Init(area_id dataArea,pae_page_table_entry * pageTable,area_id virtualArea,addr_t virtualBase)463 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea,
464 	pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase)
465 {
466 	fDataArea = dataArea;
467 	fVirtualArea = virtualArea;
468 	fVirtualBase = virtualBase;
469 	fPageTable = pageTable;
470 
471 	// init slot list
472 	fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount);
473 	addr_t slotAddress = virtualBase;
474 	for (uint32 i = 0; i < kPAEPageTableEntryCount;
475 			i++, slotAddress += B_PAGE_SIZE) {
476 		PhysicalPageSlot* slot = &fSlots[i];
477 		slot->next = slot + 1;
478 		slot->pool = this;
479 		slot->address = slotAddress;
480 	}
481 
482 	fSlots[kPAEPageTableEntryCount - 1].next = NULL;
483 		// terminate list
484 }
485 
486 
487 void
Map(phys_addr_t physicalAddress,addr_t virtualAddress)488 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
489 	addr_t virtualAddress)
490 {
491 	pae_page_table_entry& pte = fPageTable[
492 		(virtualAddress - fVirtualBase) / B_PAGE_SIZE];
493 	pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
494 		| X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT;
495 
496 	invalidate_TLB(virtualAddress);
497 }
498 
499 
500 status_t
AllocatePool(X86LargePhysicalPageMapper::PhysicalPageSlotPool * & _pool)501 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool(
502 	X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
503 {
504 	// create the pool structure
505 	PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
506 	if (pool == NULL)
507 		return B_NO_MEMORY;
508 	ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);
509 
510 	// create an area that can contain the page table and the slot
511 	// structures
512 	size_t areaSize = B_PAGE_SIZE
513 		+ sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
514 	void* data;
515 	virtual_address_restrictions virtualRestrictions = {};
516 	virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
517 	physical_address_restrictions physicalRestrictions = {};
518 	area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
519 		PAGE_ALIGN(areaSize), B_FULL_LOCK,
520 		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
521 		&virtualRestrictions, &physicalRestrictions, &data);
522 	if (dataArea < 0)
523 		return dataArea;
524 
525 	// create the null area for the virtual address space
526 	void* virtualBase;
527 	area_id virtualArea = vm_create_null_area(
528 		VMAddressSpace::KernelID(), "physical page pool space",
529 		&virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange,
530 		CREATE_AREA_PRIORITY_VIP);
531 	if (virtualArea < 0) {
532 		delete_area(dataArea);
533 		return virtualArea;
534 	}
535 
536 	// prepare the page table
537 	memset(data, 0, B_PAGE_SIZE);
538 
539 	// get the page table's physical address
540 	phys_addr_t physicalTable;
541 	X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>(
542 		VMAddressSpace::Kernel()->TranslationMap());
543 	uint32 dummyFlags;
544 	cpu_status state = disable_interrupts();
545 	map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
546 	restore_interrupts(state);
547 
548 	// put the page table into the page directory
549 	pae_page_directory_entry* pageDirEntry
550 		= X86PagingMethodPAE::PageDirEntryForAddress(
551 			map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase);
552 	PutPageTableInPageDir(pageDirEntry, physicalTable,
553 		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
554 
555 	// init the pool structure
556 	pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea,
557 		(addr_t)virtualBase);
558 	poolDeleter.Detach();
559 	_pool = pool;
560 	return B_OK;
561 }
562 
563 
564 // #pragma mark - X86PagingMethodPAE
565 
566 
X86PagingMethodPAE()567 X86PagingMethodPAE::X86PagingMethodPAE()
568 	:
569 	fPhysicalPageMapper(NULL),
570 	fKernelPhysicalPageMapper(NULL),
571 	fFreePages(NULL),
572 	fFreePagesCount(0)
573 {
574 	mutex_init(&fFreePagesLock, "x86 PAE free pages");
575 }
576 
577 
~X86PagingMethodPAE()578 X86PagingMethodPAE::~X86PagingMethodPAE()
579 {
580 }
581 
582 
583 status_t
Init(kernel_args * args,VMPhysicalPageMapper ** _physicalPageMapper)584 X86PagingMethodPAE::Init(kernel_args* args,
585 	VMPhysicalPageMapper** _physicalPageMapper)
586 {
587 	// Ignore all memory beyond the maximum PAE address.
588 	static const phys_addr_t kLimit = 1ULL << 36;
589 	for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) {
590 		addr_range& range = args->physical_memory_range[i];
591 		if (range.start >= kLimit)
592 			range.size = 0;
593 		else if ((range.start + range.size) > kLimit)
594 			range.size = kLimit - range.start;
595 	}
596 
597 	// switch to PAE
598 	ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable,
599 		fKernelPhysicalPageDirPointerTable, fEarlyPageStructures,
600 		fEarlyPageStructuresSize, fKernelVirtualPageDirs,
601 		fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE);
602 
603 	// create the initial pools for the physical page mapper
604 	int32 poolCount = _GetInitialPoolCount();
605 	PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool;
606 
607 	for (int32 i = 0; i < poolCount; i++) {
608 		new(&pool[i]) PhysicalPageSlotPool;
609 		status_t error = pool[i].InitInitial(this, args);
610 		if (error != B_OK) {
611 			panic("X86PagingMethodPAE::Init(): Failed to create initial pool "
612 				"for physical page mapper!");
613 			return error;
614 		}
615 	}
616 
617 	// create physical page mapper
618 	large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool),
619 		fPhysicalPageMapper, fKernelPhysicalPageMapper);
620 
621 	*_physicalPageMapper = fPhysicalPageMapper;
622 	return B_OK;
623 }
624 
625 
626 status_t
InitPostArea(kernel_args * args)627 X86PagingMethodPAE::InitPostArea(kernel_args* args)
628 {
629 	// wrap the kernel paging structures in an area
630 	area_id area = create_area("kernel paging structs", &fEarlyPageStructures,
631 		B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED,
632 		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
633 	if (area < B_OK)
634 		return area;
635 
636 	// let the initial page pools create areas for its structures
637 	int32 poolCount = _GetInitialPoolCount();
638 	for (int32 i = 0; i < poolCount; i++) {
639 		status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i]
640 			.InitInitialPostArea(args);
641 		if (error != B_OK)
642 			return error;
643 	}
644 
645 	// The early physical page mapping mechanism is no longer needed. Unmap the
646 	// slot.
647 	*fFreeVirtualSlotPTE = 0;
648 	invalidate_TLB(fFreeVirtualSlot);
649 
650 	fFreeVirtualSlotPTE = NULL;
651 	fFreeVirtualSlot = 0;
652 
653 	return B_OK;
654 }
655 
656 
657 status_t
CreateTranslationMap(bool kernel,VMTranslationMap ** _map)658 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
659 {
660 	X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE;
661 	if (map == NULL)
662 		return B_NO_MEMORY;
663 
664 	status_t error = map->Init(kernel);
665 	if (error != B_OK) {
666 		delete map;
667 		return error;
668 	}
669 
670 	*_map = map;
671 	return B_OK;
672 }
673 
674 
675 status_t
MapEarly(kernel_args * args,addr_t virtualAddress,phys_addr_t physicalAddress,uint8 attributes,page_num_t (* get_free_page)(kernel_args *))676 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress,
677 	phys_addr_t physicalAddress, uint8 attributes,
678 	page_num_t (*get_free_page)(kernel_args*))
679 {
680 	// check to see if a page table exists for this range
681 	pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
682 		fKernelVirtualPageDirs, virtualAddress);
683 	pae_page_table_entry* pageTable;
684 	if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
685 		// we need to allocate a page table
686 		phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE;
687 
688 		TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for "
689 			"page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable);
690 
691 		// put it in the page dir
692 		PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes);
693 
694 		// zero it out
695 		pageTable = _EarlyGetPageTable(physicalPageTable);
696 		memset(pageTable, 0, B_PAGE_SIZE);
697 	} else {
698 		// table already exists -- map it
699 		pageTable = _EarlyGetPageTable(
700 			*pageDirEntry & X86_PAE_PDE_ADDRESS_MASK);
701 	}
702 
703 	pae_page_table_entry* entry = pageTable
704 		+ virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
705 
706 	ASSERT_PRINT(
707 		(*entry & X86_PAE_PTE_PRESENT) == 0,
708 		"virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64
709 		", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry);
710 
711 	// now, fill in the pentry
712 	PutPageTableEntryInTable(entry, physicalAddress, attributes, 0,
713 		IS_KERNEL_ADDRESS(virtualAddress));
714 
715 	return B_OK;
716 }
717 
718 
719 bool
IsKernelPageAccessible(addr_t virtualAddress,uint32 protection)720 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress,
721 	uint32 protection)
722 {
723 	// we can't check much without the physical page mapper
724 	if (fPhysicalPageMapper == NULL)
725 		return false;
726 
727 	// We only trust the kernel team's page directories. So switch to the
728 	// kernel PDPT first. Always set it to make sure the TLBs don't contain
729 	// obsolete data.
730 	uint32 physicalPDPT = x86_read_cr3();
731 	x86_write_cr3(fKernelPhysicalPageDirPointerTable);
732 
733 	// get the PDPT entry for the address
734 	pae_page_directory_pointer_table_entry pdptEntry = 0;
735 	if (physicalPDPT == fKernelPhysicalPageDirPointerTable) {
736 		pdptEntry = fKernelVirtualPageDirPointerTable[
737 			virtualAddress / kPAEPageDirRange];
738 	} else {
739 		// map the original PDPT and get the entry
740 		void* handle;
741 		addr_t virtualPDPT;
742 		status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT,
743 			&virtualPDPT, &handle);
744 		if (error == B_OK) {
745 			pdptEntry = ((pae_page_directory_pointer_table_entry*)
746 				virtualPDPT)[virtualAddress / kPAEPageDirRange];
747 			fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle);
748 		}
749 	}
750 
751 	// map the page dir and get the entry
752 	pae_page_directory_entry pageDirEntry = 0;
753 	if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) {
754 		void* handle;
755 		addr_t virtualPageDir;
756 		status_t error = fPhysicalPageMapper->GetPageDebug(
757 			pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle);
758 		if (error == B_OK) {
759 			pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[
760 				virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount];
761 			fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle);
762 		}
763 	}
764 
765 	// map the page table and get the entry
766 	pae_page_table_entry pageTableEntry = 0;
767 	if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) {
768 		void* handle;
769 		addr_t virtualPageTable;
770 		status_t error = fPhysicalPageMapper->GetPageDebug(
771 			pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable,
772 			&handle);
773 		if (error == B_OK) {
774 			pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[
775 				virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount];
776 			fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
777 		}
778 	}
779 
780 	// switch back to the original page directory
781 	if (physicalPDPT != fKernelPhysicalPageDirPointerTable)
782 		x86_write_cr3(physicalPDPT);
783 
784 	if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0)
785 		return false;
786 
787 	// present means kernel-readable, so check for writable
788 	return (protection & B_KERNEL_WRITE_AREA) == 0
789 		|| (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0;
790 }
791 
792 
793 /*static*/ void
PutPageTableInPageDir(pae_page_directory_entry * entry,phys_addr_t physicalTable,uint32 attributes)794 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry,
795 	phys_addr_t physicalTable, uint32 attributes)
796 {
797 	SetTableEntry(entry, (physicalTable & X86_PAE_PDE_ADDRESS_MASK)
798 		| X86_PAE_PDE_PRESENT
799 		| X86_PAE_PDE_WRITABLE
800 		| X86_PAE_PDE_USER);
801 		// TODO: We ignore the attributes of the page table -- for compatibility
802 		// with BeOS we allow having user accessible areas in the kernel address
803 		// space. This is currently being used by some drivers, mainly for the
804 		// frame buffer. Our current real time data implementation makes use of
805 		// this fact, too.
806 		// We might want to get rid of this possibility one day, especially if
807 		// we intend to port it to a platform that does not support this.
808 }
809 
810 
811 /*static*/ void
PutPageTableEntryInTable(pae_page_table_entry * entry,phys_addr_t physicalAddress,uint32 attributes,uint32 memoryType,bool globalPage)812 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry,
813 	phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
814 	bool globalPage)
815 {
816 	pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
817 		| X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0)
818 		| MemoryTypeToPageTableEntryFlags(memoryType);
819 
820 	// if the page is user accessible, it's automatically
821 	// accessible in kernel space, too (but with the same
822 	// protection)
823 	if ((attributes & B_USER_PROTECTION) != 0) {
824 		page |= X86_PAE_PTE_USER;
825 		if ((attributes & B_WRITE_AREA) != 0)
826 			page |= X86_PAE_PTE_WRITABLE;
827 		if ((attributes & B_EXECUTE_AREA) == 0
828 			&& x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
829 			page |= X86_PAE_PTE_NOT_EXECUTABLE;
830 		}
831 	} else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
832 		page |= X86_PAE_PTE_WRITABLE;
833 
834 	// put it in the page table
835 	SetTableEntry(entry, page);
836 }
837 
838 
839 void*
Allocate32BitPage(phys_addr_t & _physicalAddress,void * & _handle)840 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress,
841 	void*& _handle)
842 {
843 	// get a free page
844 	MutexLocker locker(fFreePagesLock);
845 	vm_page* page;
846 	if (fFreePages != NULL) {
847 		page = fFreePages;
848 		fFreePages = page->cache_next;
849 		fFreePagesCount--;
850 		locker.Unlock();
851 	} else {
852 		// no pages -- allocate one
853 		locker.Unlock();
854 
855 		physical_address_restrictions restrictions = {};
856 		restrictions.high_address = 0x100000000LL;
857 		page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions,
858 			VM_PRIORITY_SYSTEM);
859 		if (page == NULL)
860 			return NULL;
861 
862 		DEBUG_PAGE_ACCESS_END(page);
863 	}
864 
865 	// map the page
866 	phys_addr_t physicalAddress
867 		= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
868 	addr_t virtualAddress;
869 	if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle)
870 			!= B_OK) {
871 		// mapping failed -- free page
872 		locker.Lock();
873 		page->cache_next = fFreePages;
874 		fFreePages = page;
875 		fFreePagesCount++;
876 		return NULL;
877 	}
878 
879 	_physicalAddress = physicalAddress;
880 	return (void*)virtualAddress;
881 }
882 
883 
884 void
Free32BitPage(void * address,phys_addr_t physicalAddress,void * handle)885 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress,
886 	void* handle)
887 {
888 	// unmap the page
889 	fPhysicalPageMapper->PutPage((addr_t)address, handle);
890 
891 	// free it
892 	vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
893 	MutexLocker locker(fFreePagesLock);
894 	if (fFreePagesCount < kMaxFree32BitPagesCount) {
895 		// cache not full yet -- cache it
896 		page->cache_next = fFreePages;
897 		fFreePages = page;
898 		fFreePagesCount++;
899 	} else {
900 		// cache full -- free it
901 		locker.Unlock();
902 		DEBUG_PAGE_ACCESS_START(page);
903 		vm_page_free(NULL, page);
904 	}
905 }
906 
907 
908 inline int32
_GetInitialPoolCount()909 X86PagingMethodPAE::_GetInitialPoolCount()
910 {
911 	int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU
912 			+ EXTRA_SLOTS;
913 	return (requiredSlots + kPAEPageTableEntryCount - 1)
914 		/ kPAEPageTableEntryCount;
915 }
916 
917 
918 bool
_EarlyQuery(addr_t virtualAddress,phys_addr_t * _physicalAddress)919 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress,
920 	phys_addr_t* _physicalAddress)
921 {
922 	pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
923 		fKernelVirtualPageDirs, virtualAddress);
924 	if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
925 		// no pagetable here
926 		return false;
927 	}
928 
929 	pae_page_table_entry* entry = _EarlyGetPageTable(
930 			*pageDirEntry & X86_PAE_PDE_ADDRESS_MASK)
931 		+ virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
932 	if ((*entry & X86_PAE_PTE_PRESENT) == 0) {
933 		// page mapping not valid
934 		return false;
935 	}
936 
937 	*_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK;
938 	return true;
939 }
940 
941 
942 pae_page_table_entry*
_EarlyGetPageTable(phys_addr_t address)943 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address)
944 {
945 	*fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK)
946 		| X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL;
947 
948 	invalidate_TLB(fFreeVirtualSlot);
949 
950 	return (pae_page_table_entry*)fFreeVirtualSlot;
951 }
952 
953 
954 #endif	// B_HAIKU_PHYSICAL_BITS == 64
955