xref: /haiku/src/system/kernel/device_manager/IOCache.cpp (revision f967e2c0c73efb25581afdc95a838f7633e8c03b)
1 /*
2  * Copyright 2010-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
3  * Distributed under the terms of the MIT License.
4  */
5 
6 
7 #include "IOCache.h"
8 
9 #include <algorithm>
10 
11 #include <condition_variable.h>
12 #include <heap.h>
13 #include <low_resource_manager.h>
14 #include <util/AutoLock.h>
15 #include <vm/vm.h>
16 #include <vm/VMAddressSpace.h>
17 #include <vm/VMCache.h>
18 #include <vm/VMTranslationMap.h>
19 
20 
21 //#define TRACE_IO_CACHE 1
22 #ifdef TRACE_IO_CACHE
23 #	define TRACE(format...)	dprintf(format)
24 #else
25 #	define TRACE(format...)	do {} while (false)
26 #endif
27 
28 
29 static inline bool
30 page_physical_number_less(const vm_page* a, const vm_page* b)
31 {
32 	return a->physical_page_number < b->physical_page_number;
33 }
34 
35 
36 struct IOCache::Operation : IOOperation {
37 	ConditionVariable	finishedCondition;
38 };
39 
40 
41 IOCache::IOCache(DMAResource* resource, size_t cacheLineSize)
42 	:
43 	IOScheduler(resource),
44 	fDeviceCapacity(0),
45 	fLineSize(cacheLineSize),
46 	fPagesPerLine(cacheLineSize / B_PAGE_SIZE),
47 	fArea(-1),
48 	fCache(NULL),
49 	fPages(NULL),
50 	fVecs(NULL)
51 {
52 	ASSERT(resource != NULL);
53 	TRACE("%p->IOCache::IOCache(%p, %" B_PRIuSIZE ")\n", this, resource,
54 		cacheLineSize);
55 
56 	if (cacheLineSize < B_PAGE_SIZE
57 		|| (cacheLineSize & (cacheLineSize - 1)) != 0) {
58 		panic("Invalid cache line size (%" B_PRIuSIZE "). Must be a power of 2 "
59 			"multiple of the page size.", cacheLineSize);
60 	}
61 
62 	mutex_init(&fSerializationLock, "I/O cache request serialization");
63 
64 	fLineSizeShift = 0;
65 	while (cacheLineSize != 1) {
66 		fLineSizeShift++;
67 		cacheLineSize >>= 1;
68 	}
69 }
70 
71 
72 IOCache::~IOCache()
73 {
74 	if (fArea >= 0) {
75 		vm_page_unreserve_pages(&fMappingReservation);
76 		delete_area(fArea);
77 	}
78 
79 	delete[] fPages;
80 	delete[] fVecs;
81 
82 	mutex_destroy(&fSerializationLock);
83 }
84 
85 
86 status_t
87 IOCache::Init(const char* name)
88 {
89 	TRACE("%p->IOCache::Init(\"%s\")\n", this, name);
90 
91 	status_t error = IOScheduler::Init(name);
92 	if (error != B_OK)
93 		return error;
94 
95 	// create the area for mapping cache lines
96 	fArea = vm_create_null_area(B_SYSTEM_TEAM, "I/O cache line", &fAreaBase,
97 		B_ANY_KERNEL_ADDRESS, fLineSize, 0);
98 	if (fArea < 0)
99 		return fArea;
100 
101 	// reserve pages for mapping a complete cache line
102 	VMAddressSpace* addressSpace = VMAddressSpace::Kernel();
103 	VMTranslationMap* translationMap = addressSpace->TranslationMap();
104 	size_t pagesNeeded = translationMap->MaxPagesNeededToMap((addr_t)fAreaBase,
105 		(addr_t)fAreaBase + fLineSize - 1);
106 	vm_page_reserve_pages(&fMappingReservation, pagesNeeded,
107 		VM_PRIORITY_SYSTEM);
108 
109 	// get the area's cache
110 	VMArea* area = VMAreas::Lookup(fArea);
111 	if (area == NULL) {
112 		panic("IOCache::Init(): Where's our area (id: %" B_PRId32 ")?!", fArea);
113 		return B_ERROR;
114 	}
115 	fCache = area->cache;
116 
117 	// allocate arrays for pages and io vecs
118 	fPages = new(std::nothrow) vm_page*[fPagesPerLine];
119 	fVecs = new(std::nothrow) generic_io_vec[fPagesPerLine];
120 	if (fPages == NULL || fVecs == NULL)
121 		return B_NO_MEMORY;
122 
123 	return B_OK;
124 }
125 
126 
127 void
128 IOCache::SetDeviceCapacity(off_t deviceCapacity)
129 {
130 	TRACE("%p->IOCache::SetDeviceCapacity(%" B_PRIdOFF ")\n", this,
131 		deviceCapacity);
132 
133 	MutexLocker serializationLocker(fSerializationLock);
134 	AutoLocker<VMCache> cacheLocker(fCache);
135 
136 	fDeviceCapacity = deviceCapacity;
137 }
138 
139 
140 void
141 IOCache::MediaChanged()
142 {
143 	TRACE("%p->IOCache::MediaChanged()\n", this);
144 
145 	MutexLocker serializationLocker(fSerializationLock);
146 	AutoLocker<VMCache> cacheLocker(fCache);
147 
148 	// new media -- burn all cached data
149 	while (vm_page* page = fCache->pages.Root()) {
150 		DEBUG_PAGE_ACCESS_START(page);
151 		fCache->RemovePage(page);
152 		vm_page_free(NULL, page);
153 	}
154 }
155 
156 
157 status_t
158 IOCache::ScheduleRequest(IORequest* request)
159 {
160 	TRACE("%p->IOCache::ScheduleRequest(%p)\n", this, request);
161 
162 	// lock the request's memory
163 	status_t error;
164 	IOBuffer* buffer = request->Buffer();
165 	if (buffer->IsVirtual()) {
166 		error = buffer->LockMemory(request->TeamID(), request->IsWrite());
167 		if (error != B_OK) {
168 			request->SetStatusAndNotify(error);
169 			return error;
170 		}
171 	}
172 
173 	// we completely serialize all I/O in FIFO order
174 	MutexLocker serializationLocker(fSerializationLock);
175 	generic_size_t bytesTransferred = 0;
176 	error = _DoRequest(request, bytesTransferred);
177 	serializationLocker.Unlock();
178 
179 	// unlock memory
180 	if (buffer->IsVirtual())
181 		buffer->UnlockMemory(request->TeamID(), request->IsWrite());
182 
183 	// set status and notify
184 	if (error == B_OK) {
185 		request->SetTransferredBytes(bytesTransferred < request->Length(),
186 			bytesTransferred);
187 		request->SetStatusAndNotify(B_OK);
188 	} else
189 		request->SetStatusAndNotify(error);
190 
191 	return error;
192 }
193 
194 
195 void
196 IOCache::AbortRequest(IORequest* request, status_t status)
197 {
198 	// TODO:...
199 }
200 
201 
202 void
203 IOCache::OperationCompleted(IOOperation* operation, status_t status,
204 	generic_size_t transferredBytes)
205 {
206 	operation->SetStatus(status, transferredBytes);
207 
208 	if (status == B_OK) {
209 		// always fail in case of partial transfers
210 		((Operation*)operation)->finishedCondition.NotifyAll(
211 			transferredBytes == operation->Length() ? B_OK : B_ERROR);
212 	} else
213 		((Operation*)operation)->finishedCondition.NotifyAll(status);
214 }
215 
216 
217 void
218 IOCache::Dump() const
219 {
220 	kprintf("IOCache at %p\n", this);
221 	kprintf("  DMA resource:   %p\n", fDMAResource);
222 }
223 
224 
225 status_t
226 IOCache::_DoRequest(IORequest* request, generic_size_t& _bytesTransferred)
227 {
228 	off_t offset = request->Offset();
229 	generic_size_t length = request->Length();
230 
231 	TRACE("%p->IOCache::ScheduleRequest(%p): offset: %" B_PRIdOFF
232 		", length: %" B_PRIuSIZE "\n", this, request, offset, length);
233 
234 	if (offset < 0 || offset > fDeviceCapacity)
235 		return B_BAD_VALUE;
236 
237 	// truncate the request to the device capacity
238 	if (fDeviceCapacity - offset < (off_t)length)
239 		length = fDeviceCapacity - offset;
240 
241 	_bytesTransferred = 0;
242 
243 	while (length > 0) {
244 		// the start of the current cache line
245 		off_t lineOffset = (offset >> fLineSizeShift) << fLineSizeShift;
246 
247 		// intersection of request and cache line
248 		off_t cacheLineEnd = std::min(lineOffset + (off_t)fLineSize, fDeviceCapacity);
249 		size_t requestLineLength
250 			= std::min(cacheLineEnd - offset, (off_t)length);
251 
252 		// transfer the data of the cache line
253 		status_t error = _TransferRequestLine(request, lineOffset,
254 			cacheLineEnd - lineOffset, offset, requestLineLength);
255 		if (error != B_OK)
256 			return error;
257 
258 		offset = cacheLineEnd;
259 		length -= requestLineLength;
260 		_bytesTransferred += requestLineLength;
261 	}
262 
263 	return B_OK;
264 }
265 
266 
267 status_t
268 IOCache::_TransferRequestLine(IORequest* request, off_t lineOffset,
269 	size_t lineSize, off_t requestOffset, size_t requestLength)
270 {
271 	TRACE("%p->IOCache::_TransferRequestLine(%p, %" B_PRIdOFF
272 		", %" B_PRIdOFF  ", %" B_PRIuSIZE ")\n", this, request, lineOffset,
273 		requestOffset, requestLength);
274 
275 	// check whether there are pages of the cache line and the mark them used
276 	page_num_t firstPageOffset = lineOffset / B_PAGE_SIZE;
277 	page_num_t linePageCount = (lineSize + B_PAGE_SIZE - 1) / B_PAGE_SIZE;
278 
279 	AutoLocker<VMCache> cacheLocker(fCache);
280 
281 	page_num_t firstMissing = 0;
282 	page_num_t lastMissing = 0;
283 	page_num_t missingPages = 0;
284 	page_num_t pageOffset = firstPageOffset;
285 
286 	VMCachePagesTree::Iterator it = fCache->pages.GetIterator(pageOffset, true,
287 		true);
288 	while (pageOffset < firstPageOffset + linePageCount) {
289 		vm_page* page = it.Next();
290 		page_num_t currentPageOffset;
291 		if (page == NULL
292 			|| page->cache_offset >= firstPageOffset + linePageCount) {
293 			page = NULL;
294 			currentPageOffset = firstPageOffset + linePageCount;
295 		} else
296 			currentPageOffset = page->cache_offset;
297 
298 		if (pageOffset < currentPageOffset) {
299 			// pages are missing
300 			if (missingPages == 0)
301 				firstMissing = pageOffset;
302 			lastMissing = currentPageOffset - 1;
303 			missingPages += currentPageOffset - pageOffset;
304 
305 			for (; pageOffset < currentPageOffset; pageOffset++)
306 				fPages[pageOffset - firstPageOffset] = NULL;
307 		}
308 
309 		if (page != NULL) {
310 			fPages[pageOffset++ - firstPageOffset] = page;
311 			DEBUG_PAGE_ACCESS_START(page);
312 			vm_page_set_state(page, PAGE_STATE_UNUSED);
313 			DEBUG_PAGE_ACCESS_END(page);
314 		}
315 	}
316 
317 	cacheLocker.Unlock();
318 
319 	bool isVIP = (request->Flags() & B_VIP_IO_REQUEST) != 0;
320 
321 	if (missingPages > 0) {
322 // TODO: If this is a read request and the missing pages range doesn't intersect
323 // with the request, just satisfy the request and don't read anything at all.
324 		// There are pages of the cache line missing. We have to allocate fresh
325 		// ones.
326 
327 		// reserve
328 		vm_page_reservation reservation;
329 		if (!vm_page_try_reserve_pages(&reservation, missingPages,
330 				VM_PRIORITY_SYSTEM)) {
331 			_DiscardPages(firstMissing - firstPageOffset, missingPages);
332 
333 			// fall back to uncached transfer
334 			return _TransferRequestLineUncached(request, lineOffset,
335 				requestOffset, requestLength);
336 		}
337 
338 		// Allocate the missing pages and remove the already existing pages in
339 		// the range from the cache. We're going to read/write the whole range
340 		// anyway and this way we can sort it, possibly improving the physical
341 		// vecs.
342 // TODO: When memory is low, we should consider cannibalizing ourselves or
343 // simply transferring past the cache!
344 		for (pageOffset = firstMissing; pageOffset <= lastMissing;
345 				pageOffset++) {
346 			page_num_t index = pageOffset - firstPageOffset;
347 			if (fPages[index] == NULL) {
348 				fPages[index] = vm_page_allocate_page(&reservation,
349 					PAGE_STATE_UNUSED);
350 				DEBUG_PAGE_ACCESS_END(fPages[index]);
351 			} else {
352 				cacheLocker.Lock();
353 				fCache->RemovePage(fPages[index]);
354 				cacheLocker.Unlock();
355 			}
356 		}
357 
358 		missingPages = lastMissing - firstMissing + 1;
359 
360 		// sort the page array by physical page number
361 		std::sort(fPages + firstMissing - firstPageOffset,
362 			fPages + lastMissing - firstPageOffset + 1,
363 			page_physical_number_less);
364 
365 		// add the pages to the cache
366 		cacheLocker.Lock();
367 
368 		for (pageOffset = firstMissing; pageOffset <= lastMissing;
369 				pageOffset++) {
370 			page_num_t index = pageOffset - firstPageOffset;
371 			fCache->InsertPage(fPages[index], (off_t)pageOffset * B_PAGE_SIZE);
372 		}
373 
374 		cacheLocker.Unlock();
375 
376 		// Read in the missing pages, if this is a read request or a write
377 		// request that doesn't cover the complete missing range.
378 		if (request->IsRead()
379 			|| requestOffset < (off_t)firstMissing * B_PAGE_SIZE
380 			|| requestOffset + (off_t)requestLength
381 				> (off_t)(lastMissing + 1) * B_PAGE_SIZE) {
382 			status_t error = _TransferPages(firstMissing - firstPageOffset,
383 				missingPages, false, isVIP);
384 			if (error != B_OK) {
385 				dprintf("IOCache::_TransferRequestLine(): Failed to read into "
386 					"cache (offset: %" B_PRIdOFF ", length: %" B_PRIuSIZE "), "
387 					"trying uncached read (offset: %" B_PRIdOFF ", length: %"
388 					B_PRIuSIZE ")\n", (off_t)firstMissing * B_PAGE_SIZE,
389 					(size_t)missingPages * B_PAGE_SIZE, requestOffset,
390 					requestLength);
391 
392 				_DiscardPages(firstMissing - firstPageOffset, missingPages);
393 
394 				// Try again using an uncached transfer
395 				return _TransferRequestLineUncached(request, lineOffset,
396 					requestOffset, requestLength);
397 			}
398 		}
399 	}
400 
401 	if (request->IsRead()) {
402 		// copy data to request
403 		status_t error = _CopyPages(request, requestOffset - lineOffset,
404 			requestOffset, requestLength, true);
405 		_CachePages(0, linePageCount);
406 		return error;
407 	}
408 
409 	// copy data from request
410 	status_t error = _CopyPages(request, requestOffset - lineOffset,
411 		requestOffset, requestLength, false);
412 	if (error != B_OK) {
413 		_DiscardPages(0, linePageCount);
414 		return error;
415 	}
416 
417 	// write the pages to disk
418 	page_num_t firstPage = (requestOffset - lineOffset) / B_PAGE_SIZE;
419 	page_num_t endPage = (requestOffset + requestLength - lineOffset
420 		+ B_PAGE_SIZE - 1) / B_PAGE_SIZE;
421 	error = _TransferPages(firstPage, endPage - firstPage, true, isVIP);
422 
423 	if (error != B_OK) {
424 		_DiscardPages(firstPage, endPage - firstPage);
425 		return error;
426 	}
427 
428 	_CachePages(0, linePageCount);
429 	return error;
430 }
431 
432 
433 status_t
434 IOCache::_TransferRequestLineUncached(IORequest* request, off_t lineOffset,
435 	off_t requestOffset, size_t requestLength)
436 {
437 	TRACE("%p->IOCache::_TransferRequestLineUncached(%p, %" B_PRIdOFF
438 		", %" B_PRIdOFF  ", %" B_PRIuSIZE ")\n", this, request, lineOffset,
439 		requestOffset, requestLength);
440 
441 	// Advance the request to the interesting offset, so the DMAResource can
442 	// provide us with fitting operations.
443 	off_t actualRequestOffset
444 		= request->Offset() + request->Length() - request->RemainingBytes();
445 	if (actualRequestOffset > requestOffset) {
446 		dprintf("IOCache::_TransferRequestLineUncached(): Request %p advanced "
447 			"beyond current cache line (%" B_PRIdOFF " vs. %" B_PRIdOFF ")\n",
448 			request, actualRequestOffset, requestOffset);
449 		return B_BAD_VALUE;
450 	}
451 
452 	if (actualRequestOffset < requestOffset)
453 		request->Advance(requestOffset - actualRequestOffset);
454 
455 	generic_size_t requestRemaining = request->RemainingBytes() - requestLength;
456 
457 	// Process single operations until the specified part of the request is
458 	// finished or until an error occurs.
459 	Operation operation;
460 	operation.finishedCondition.Init(this, "I/O cache operation finished");
461 
462 	while (request->RemainingBytes() > requestRemaining
463 		&& request->Status() > 0) {
464 		status_t error = fDMAResource->TranslateNext(request, &operation,
465 			request->RemainingBytes() - requestRemaining);
466 		if (error != B_OK)
467 			return error;
468 
469 		error = _DoOperation(operation);
470 
471 		request->OperationFinished(&operation);
472 		request->SetUnfinished();
473 			// Keep the request in unfinished state. ScheduleRequest() will set
474 			// the final status and notify.
475 
476 		fDMAResource->RecycleBuffer(operation.Buffer());
477 
478 		if (error != B_OK) {
479 			TRACE("%p->IOCache::_TransferRequestLineUncached(): operation at "
480 				"%" B_PRIdOFF " failed: %s\n", this, operation.Offset(),
481 				strerror(error));
482 			return error;
483 		}
484 	}
485 
486 	return B_OK;
487 }
488 
489 
490 status_t
491 IOCache::_DoOperation(Operation& operation)
492 {
493 	TRACE("%p->IOCache::_DoOperation(%" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this,
494 		operation.Offset(), operation.Length());
495 
496 	while (true) {
497 		ConditionVariableEntry waitEntry;
498 		operation.finishedCondition.Add(&waitEntry);
499 
500 		status_t error = fIOCallback(fIOCallbackData, &operation);
501 		if (error != B_OK) {
502 			operation.finishedCondition.NotifyAll(error);
503 				// removes the entry from the variable
504 			return error;
505 		}
506 
507 		// wait for the operation to finish
508 		error = waitEntry.Wait();
509 		if (error != B_OK)
510 			return error;
511 
512 		if (operation.Finish())
513 			return B_OK;
514 	}
515 }
516 
517 
518 status_t
519 IOCache::_TransferPages(size_t firstPage, size_t pageCount, bool isWrite,
520 	bool isVIP)
521 {
522 	TRACE("%p->IOCache::_TransferPages(%" B_PRIuSIZE ", %" B_PRIuSIZE
523 		", write: %d, vip: %d)\n", this, firstPage, pageCount, isWrite, isVIP);
524 
525 	off_t firstPageOffset = (off_t)fPages[firstPage]->cache_offset
526 		* B_PAGE_SIZE;
527 	generic_size_t requestLength = std::min(
528 			firstPageOffset + (off_t)pageCount * B_PAGE_SIZE, fDeviceCapacity)
529 		- firstPageOffset;
530 
531 	// prepare the I/O vecs
532 	size_t vecCount = 0;
533 	size_t endPage = firstPage + pageCount;
534 	phys_addr_t vecsEndAddress = 0;
535 	for (size_t i = firstPage; i < endPage; i++) {
536 		phys_addr_t pageAddress
537 			= (phys_addr_t)fPages[i]->physical_page_number * B_PAGE_SIZE;
538 		if (vecCount == 0 || pageAddress != vecsEndAddress) {
539 			fVecs[vecCount].base = pageAddress;
540 			fVecs[vecCount++].length = B_PAGE_SIZE;
541 			vecsEndAddress = pageAddress + B_PAGE_SIZE;
542 		} else {
543 			// extend the previous vec
544 			fVecs[vecCount - 1].length += B_PAGE_SIZE;
545 			vecsEndAddress += B_PAGE_SIZE;
546 		}
547 	}
548 
549 	// Don't try to read past the end of the device just to fill a page;
550 	// this makes sure that sum(fVecs[].length) == requestLength
551 	generic_size_t padLength = B_PAGE_SIZE - requestLength % B_PAGE_SIZE;
552 	if (vecCount > 0 && padLength != B_PAGE_SIZE)
553 		fVecs[vecCount - 1].length -= padLength;
554 
555 	// create a request for the transfer
556 	IORequest request;
557 	status_t error = request.Init(firstPageOffset, fVecs, vecCount,
558 		requestLength, isWrite,
559 		B_PHYSICAL_IO_REQUEST | (isVIP ? B_VIP_IO_REQUEST : 0));
560 	if (error != B_OK)
561 		return error;
562 
563 	// Process single operations until the complete request is finished or
564 	// until an error occurs.
565 	Operation operation;
566 	operation.finishedCondition.Init(this, "I/O cache operation finished");
567 
568 	while (request.RemainingBytes() > 0 && request.Status() > 0) {
569 		error = fDMAResource->TranslateNext(&request, &operation,
570 			requestLength);
571 		if (error != B_OK)
572 			return error;
573 
574 		error = _DoOperation(operation);
575 
576 		request.RemoveOperation(&operation);
577 
578 		fDMAResource->RecycleBuffer(operation.Buffer());
579 
580 		if (error != B_OK) {
581 			TRACE("%p->IOCache::_TransferLine(): operation at %" B_PRIdOFF
582 				" failed: %s\n", this, operation.Offset(), strerror(error));
583 			return error;
584 		}
585 	}
586 
587 	return B_OK;
588 }
589 
590 
591 /*!	Frees all pages in given range of the \c fPages array.
592 	\c NULL entries in the range are OK. All non \c NULL entries must refer
593 	to pages with \c PAGE_STATE_UNUSED. The pages may belong to \c fCache or
594 	may not have a cache.
595 	\c fCache must not be locked.
596 */
597 void
598 IOCache::_DiscardPages(size_t firstPage, size_t pageCount)
599 {
600 	TRACE("%p->IOCache::_DiscardPages(%" B_PRIuSIZE ", %" B_PRIuSIZE ")\n",
601 		this, firstPage, pageCount);
602 
603 	AutoLocker<VMCache> cacheLocker(fCache);
604 
605 	for (size_t i = firstPage; i < firstPage + pageCount; i++) {
606 		vm_page* page = fPages[i];
607 		if (page == NULL)
608 			continue;
609 
610 		DEBUG_PAGE_ACCESS_START(page);
611 
612 		ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED,
613 			"page: %p @! page -m %p", page, page);
614 
615 		if (page->Cache() != NULL)
616 			fCache->RemovePage(page);
617 
618 		vm_page_free(NULL, page);
619 	}
620 }
621 
622 
623 /*!	Marks all pages in the given range of the \c fPages array cached.
624 	There must not be any \c NULL entries in the given array range. All pages
625 	must belong to \c cache and have state \c PAGE_STATE_UNUSED.
626 	\c fCache must not be locked.
627 */
628 void
629 IOCache::_CachePages(size_t firstPage, size_t pageCount)
630 {
631 	TRACE("%p->IOCache::_CachePages(%" B_PRIuSIZE ", %" B_PRIuSIZE ")\n",
632 		this, firstPage, pageCount);
633 
634 	AutoLocker<VMCache> cacheLocker(fCache);
635 
636 	for (size_t i = firstPage; i < firstPage + pageCount; i++) {
637 		vm_page* page = fPages[i];
638 		ASSERT(page != NULL);
639 		ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED
640 				&& page->Cache() == fCache,
641 			"page: %p @! page -m %p", page, page);
642 
643 		DEBUG_PAGE_ACCESS_START(page);
644 		vm_page_set_state(page, PAGE_STATE_CACHED);
645 		DEBUG_PAGE_ACCESS_END(page);
646 	}
647 }
648 
649 
650 /*!	Copies the contents of pages in \c fPages to \a request, or vice versa.
651 	\param request The request.
652 	\param pagesRelativeOffset The offset relative to \c fPages[0] where to
653 		start copying.
654 	\param requestOffset The request offset where to start copying.
655 	\param requestLength The number of bytes to copy.
656 	\param toRequest If \c true the copy directory is from \c fPages to
657 		\a request, otherwise the other way around.
658 	\return \c B_OK, if copying went fine, another error code otherwise.
659 */
660 status_t
661 IOCache::_CopyPages(IORequest* request, size_t pagesRelativeOffset,
662 	off_t requestOffset, size_t requestLength, bool toRequest)
663 {
664 	TRACE("%p->IOCache::_CopyPages(%p, %" B_PRIuSIZE ", %" B_PRIdOFF
665 		", %" B_PRIuSIZE ", %d)\n", this, request, pagesRelativeOffset,
666 		requestOffset, requestLength, toRequest);
667 
668 	size_t firstPage = pagesRelativeOffset / B_PAGE_SIZE;
669 	size_t endPage = (pagesRelativeOffset + requestLength + B_PAGE_SIZE - 1)
670 		/ B_PAGE_SIZE;
671 
672 	// map the pages
673 	status_t error = _MapPages(firstPage, endPage);
674 // TODO: _MapPages() cannot fail, so the fallback is never needed. Test which
675 // method is faster (probably the active one)!
676 #if 0
677 	if (error != B_OK) {
678 		// fallback to copying individual pages
679 		size_t inPageOffset = pagesRelativeOffset % B_PAGE_SIZE;
680 		for (size_t i = firstPage; i < endPage; i++) {
681 			// map the page
682 			void* handle;
683 			addr_t address;
684 			error = vm_get_physical_page(
685 				fPages[i]->physical_page_number * B_PAGE_SIZE, &address,
686 				&handle);
687 			if (error != B_OK)
688 				return error;
689 
690 			// copy the page's data
691 			size_t toCopy = std::min(B_PAGE_SIZE - inPageOffset, requestLength);
692 
693 			if (toRequest) {
694 				error = request->CopyData((uint8*)(address + inPageOffset),
695 					requestOffset, toCopy);
696 			} else {
697 				error = request->CopyData(requestOffset,
698 					(uint8*)(address + inPageOffset), toCopy);
699 			}
700 
701 			// unmap the page
702 			vm_put_physical_page(address, handle);
703 
704 			if (error != B_OK)
705 				return error;
706 
707 			inPageOffset = 0;
708 			requestOffset += toCopy;
709 			requestLength -= toCopy;
710 		}
711 
712 		return B_OK;
713 	}
714 #endif	// 0
715 
716 	// copy
717 	if (toRequest) {
718 		error = request->CopyData((uint8*)fAreaBase + pagesRelativeOffset,
719 			requestOffset, requestLength);
720 	} else {
721 		error = request->CopyData(requestOffset,
722 			(uint8*)fAreaBase + pagesRelativeOffset, requestLength);
723 	}
724 
725 	// unmap the pages
726 	_UnmapPages(firstPage, endPage);
727 
728 	return error;
729 }
730 
731 
732 /*!	Maps a range of pages in \c fPages into fArea.
733 
734 	If successful, it must be balanced by a call to _UnmapPages().
735 
736 	\param firstPage The \c fPages relative index of the first page to map.
737 	\param endPage The \c fPages relative index of the page after the last page
738 		to map.
739 	\return \c B_OK, if mapping went fine, another error code otherwise.
740 */
741 status_t
742 IOCache::_MapPages(size_t firstPage, size_t endPage)
743 {
744 	VMTranslationMap* translationMap
745 		= VMAddressSpace::Kernel()->TranslationMap();
746 
747 	translationMap->Lock();
748 
749 	for (size_t i = firstPage; i < endPage; i++) {
750 		vm_page* page = fPages[i];
751 
752 		ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED,
753 			"page: %p @! page -m %p", page, page);
754 
755 		translationMap->Map((addr_t)fAreaBase + i * B_PAGE_SIZE,
756 			page->physical_page_number * B_PAGE_SIZE,
757 			B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, &fMappingReservation);
758 		// NOTE: We don't increment gMappedPagesCount. Our pages have state
759 		// PAGE_STATE_UNUSED anyway and we map them only for a short time.
760 	}
761 
762 	translationMap->Unlock();
763 
764 	return B_OK;
765 }
766 
767 
768 /*!	Unmaps a range of pages in \c fPages into fArea.
769 
770 	Must balance a call to _MapPages().
771 
772 	\param firstPage The \c fPages relative index of the first page to unmap.
773 	\param endPage The \c fPages relative index of the page after the last page
774 		to unmap.
775 */
776 void
777 IOCache::_UnmapPages(size_t firstPage, size_t endPage)
778 {
779 	VMTranslationMap* translationMap
780 		= VMAddressSpace::Kernel()->TranslationMap();
781 
782 	translationMap->Lock();
783 
784 	translationMap->Unmap((addr_t)fAreaBase + firstPage * B_PAGE_SIZE,
785 		(addr_t)fAreaBase + endPage * B_PAGE_SIZE - 1);
786 
787 	translationMap->Unlock();
788 }
789