1 /* 2 * Copyright 2010-2011, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "IOCache.h" 8 9 #include <algorithm> 10 11 #include <condition_variable.h> 12 #include <heap.h> 13 #include <low_resource_manager.h> 14 #include <util/AutoLock.h> 15 #include <vm/vm.h> 16 #include <vm/VMAddressSpace.h> 17 #include <vm/VMCache.h> 18 #include <vm/VMTranslationMap.h> 19 20 21 //#define TRACE_IO_CACHE 1 22 #ifdef TRACE_IO_CACHE 23 # define TRACE(format...) dprintf(format) 24 #else 25 # define TRACE(format...) do {} while (false) 26 #endif 27 28 29 static inline bool 30 page_physical_number_less(const vm_page* a, const vm_page* b) 31 { 32 return a->physical_page_number < b->physical_page_number; 33 } 34 35 36 struct IOCache::Operation : IOOperation { 37 ConditionVariable finishedCondition; 38 }; 39 40 41 IOCache::IOCache(DMAResource* resource, size_t cacheLineSize) 42 : 43 IOScheduler(resource), 44 fDeviceCapacity(0), 45 fLineSize(cacheLineSize), 46 fPagesPerLine(cacheLineSize / B_PAGE_SIZE), 47 fArea(-1), 48 fCache(NULL), 49 fPages(NULL), 50 fVecs(NULL) 51 { 52 ASSERT(resource != NULL); 53 TRACE("%p->IOCache::IOCache(%p, %" B_PRIuSIZE ")\n", this, resource, 54 cacheLineSize); 55 56 if (cacheLineSize < B_PAGE_SIZE 57 || (cacheLineSize & (cacheLineSize - 1)) != 0) { 58 panic("Invalid cache line size (%" B_PRIuSIZE "). Must be a power of 2 " 59 "multiple of the page size.", cacheLineSize); 60 } 61 62 mutex_init(&fSerializationLock, "I/O cache request serialization"); 63 64 fLineSizeShift = 0; 65 while (cacheLineSize != 1) { 66 fLineSizeShift++; 67 cacheLineSize >>= 1; 68 } 69 } 70 71 72 IOCache::~IOCache() 73 { 74 if (fArea >= 0) { 75 vm_page_unreserve_pages(&fMappingReservation); 76 delete_area(fArea); 77 } 78 79 delete[] fPages; 80 delete[] fVecs; 81 82 mutex_destroy(&fSerializationLock); 83 } 84 85 86 status_t 87 IOCache::Init(const char* name) 88 { 89 TRACE("%p->IOCache::Init(\"%s\")\n", this, name); 90 91 status_t error = IOScheduler::Init(name); 92 if (error != B_OK) 93 return error; 94 95 // create the area for mapping cache lines 96 fArea = vm_create_null_area(B_SYSTEM_TEAM, "I/O cache line", &fAreaBase, 97 B_ANY_KERNEL_ADDRESS, fLineSize, 0); 98 if (fArea < 0) 99 return fArea; 100 101 // reserve pages for mapping a complete cache line 102 VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); 103 VMTranslationMap* translationMap = addressSpace->TranslationMap(); 104 size_t pagesNeeded = translationMap->MaxPagesNeededToMap((addr_t)fAreaBase, 105 (addr_t)fAreaBase + fLineSize - 1); 106 vm_page_reserve_pages(&fMappingReservation, pagesNeeded, 107 VM_PRIORITY_SYSTEM); 108 109 // get the area's cache 110 VMArea* area = VMAreaHash::Lookup(fArea); 111 if (area == NULL) { 112 panic("IOCache::Init(): Where's our area (id: %" B_PRId32 ")?!", fArea); 113 return B_ERROR; 114 } 115 fCache = area->cache; 116 117 // allocate arrays for pages and io vecs 118 fPages = new(std::nothrow) vm_page*[fPagesPerLine]; 119 fVecs = new(std::nothrow) generic_io_vec[fPagesPerLine]; 120 if (fPages == NULL || fVecs == NULL) 121 return B_NO_MEMORY; 122 123 return B_OK; 124 } 125 126 127 void 128 IOCache::SetDeviceCapacity(off_t deviceCapacity) 129 { 130 TRACE("%p->IOCache::SetDeviceCapacity(%" B_PRIdOFF ")\n", this, 131 deviceCapacity); 132 133 MutexLocker serializationLocker(fSerializationLock); 134 AutoLocker<VMCache> cacheLocker(fCache); 135 136 fDeviceCapacity = deviceCapacity; 137 } 138 139 140 void 141 IOCache::MediaChanged() 142 { 143 TRACE("%p->IOCache::MediaChanged()\n", this); 144 145 MutexLocker serializationLocker(fSerializationLock); 146 AutoLocker<VMCache> cacheLocker(fCache); 147 148 // new media -- burn all cached data 149 while (vm_page* page = fCache->pages.Root()) { 150 DEBUG_PAGE_ACCESS_START(page); 151 fCache->RemovePage(page); 152 vm_page_free(NULL, page); 153 } 154 } 155 156 157 status_t 158 IOCache::ScheduleRequest(IORequest* request) 159 { 160 TRACE("%p->IOCache::ScheduleRequest(%p)\n", this, request); 161 162 // lock the request's memory 163 status_t error; 164 IOBuffer* buffer = request->Buffer(); 165 if (buffer->IsVirtual()) { 166 error = buffer->LockMemory(request->TeamID(), request->IsWrite()); 167 if (error != B_OK) { 168 request->SetStatusAndNotify(error); 169 return error; 170 } 171 } 172 173 // we completely serialize all I/O in FIFO order 174 MutexLocker serializationLocker(fSerializationLock); 175 generic_size_t bytesTransferred = 0; 176 error = _DoRequest(request, bytesTransferred); 177 serializationLocker.Unlock(); 178 179 // unlock memory 180 if (buffer->IsVirtual()) 181 buffer->UnlockMemory(request->TeamID(), request->IsWrite()); 182 183 // set status and notify 184 if (error == B_OK) { 185 request->SetTransferredBytes(bytesTransferred < request->Length(), 186 bytesTransferred); 187 request->SetStatusAndNotify(B_OK); 188 } else 189 request->SetStatusAndNotify(error); 190 191 return error; 192 } 193 194 195 void 196 IOCache::AbortRequest(IORequest* request, status_t status) 197 { 198 // TODO:... 199 } 200 201 202 void 203 IOCache::OperationCompleted(IOOperation* operation, status_t status, 204 generic_size_t transferredBytes) 205 { 206 if (status == B_OK) { 207 // always fail in case of partial transfers 208 ((Operation*)operation)->finishedCondition.NotifyAll(false, 209 transferredBytes == operation->Length() ? B_OK : B_ERROR); 210 } else 211 ((Operation*)operation)->finishedCondition.NotifyAll(false, status); 212 } 213 214 215 void 216 IOCache::Dump() const 217 { 218 kprintf("IOCache at %p\n", this); 219 kprintf(" DMA resource: %p\n", fDMAResource); 220 } 221 222 223 status_t 224 IOCache::_DoRequest(IORequest* request, generic_size_t& _bytesTransferred) 225 { 226 off_t offset = request->Offset(); 227 generic_size_t length = request->Length(); 228 229 TRACE("%p->IOCache::ScheduleRequest(%p): offset: %" B_PRIdOFF 230 ", length: %" B_PRIuSIZE "\n", this, request, offset, length); 231 232 if (offset < 0 || offset > fDeviceCapacity) 233 return B_BAD_VALUE; 234 235 // truncate the request to the device capacity 236 if (fDeviceCapacity - offset < (off_t)length) 237 length = fDeviceCapacity - offset; 238 239 _bytesTransferred = 0; 240 241 while (length > 0) { 242 // the start of the current cache line 243 off_t lineOffset = (offset >> fLineSizeShift) << fLineSizeShift; 244 245 // intersection of request and cache line 246 off_t cacheLineEnd = std::min(lineOffset + (off_t)fLineSize, fDeviceCapacity); 247 size_t requestLineLength 248 = std::min(cacheLineEnd - offset, (off_t)length); 249 250 // transfer the data of the cache line 251 status_t error = _TransferRequestLine(request, lineOffset, 252 cacheLineEnd - lineOffset, offset, requestLineLength); 253 if (error != B_OK) 254 return error; 255 256 offset = cacheLineEnd; 257 length -= requestLineLength; 258 _bytesTransferred += requestLineLength; 259 } 260 261 return B_OK; 262 } 263 264 265 status_t 266 IOCache::_TransferRequestLine(IORequest* request, off_t lineOffset, 267 size_t lineSize, off_t requestOffset, size_t requestLength) 268 { 269 TRACE("%p->IOCache::_TransferRequestLine(%p, %" B_PRIdOFF 270 ", %" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this, request, lineOffset, 271 requestOffset, requestLength); 272 273 // check whether there are pages of the cache line and the mark them used 274 page_num_t firstPageOffset = lineOffset / B_PAGE_SIZE; 275 page_num_t linePageCount = (lineSize + B_PAGE_SIZE - 1) / B_PAGE_SIZE; 276 277 AutoLocker<VMCache> cacheLocker(fCache); 278 279 page_num_t firstMissing = 0; 280 page_num_t lastMissing = 0; 281 page_num_t missingPages = 0; 282 page_num_t pageOffset = firstPageOffset; 283 284 VMCachePagesTree::Iterator it = fCache->pages.GetIterator(pageOffset, true, 285 true); 286 while (pageOffset < firstPageOffset + linePageCount) { 287 vm_page* page = it.Next(); 288 page_num_t currentPageOffset; 289 if (page == NULL 290 || page->cache_offset >= firstPageOffset + linePageCount) { 291 page = NULL; 292 currentPageOffset = firstPageOffset + linePageCount; 293 } else 294 currentPageOffset = page->cache_offset; 295 296 if (pageOffset < currentPageOffset) { 297 // pages are missing 298 if (missingPages == 0) 299 firstMissing = pageOffset; 300 lastMissing = currentPageOffset - 1; 301 missingPages += currentPageOffset - pageOffset; 302 303 for (; pageOffset < currentPageOffset; pageOffset++) 304 fPages[pageOffset - firstPageOffset] = NULL; 305 } 306 307 if (page != NULL) { 308 fPages[pageOffset++ - firstPageOffset] = page; 309 DEBUG_PAGE_ACCESS_START(page); 310 vm_page_set_state(page, PAGE_STATE_UNUSED); 311 DEBUG_PAGE_ACCESS_END(page); 312 } 313 } 314 315 cacheLocker.Unlock(); 316 317 bool isVIP = (request->Flags() & B_VIP_IO_REQUEST) != 0; 318 319 if (missingPages > 0) { 320 // TODO: If this is a read request and the missing pages range doesn't intersect 321 // with the request, just satisfy the request and don't read anything at all. 322 // There are pages of the cache line missing. We have to allocate fresh 323 // ones. 324 325 // reserve 326 vm_page_reservation reservation; 327 if (!vm_page_try_reserve_pages(&reservation, missingPages, 328 VM_PRIORITY_SYSTEM)) { 329 _DiscardPages(firstMissing - firstPageOffset, missingPages); 330 331 // fall back to uncached transfer 332 return _TransferRequestLineUncached(request, lineOffset, 333 requestOffset, requestLength); 334 } 335 336 // Allocate the missing pages and remove the already existing pages in 337 // the range from the cache. We're going to read/write the whole range 338 // anyway and this way we can sort it, possibly improving the physical 339 // vecs. 340 // TODO: When memory is low, we should consider cannibalizing ourselves or 341 // simply transferring past the cache! 342 for (pageOffset = firstMissing; pageOffset <= lastMissing; 343 pageOffset++) { 344 page_num_t index = pageOffset - firstPageOffset; 345 if (fPages[index] == NULL) { 346 fPages[index] = vm_page_allocate_page(&reservation, 347 PAGE_STATE_UNUSED); 348 DEBUG_PAGE_ACCESS_END(fPages[index]); 349 } else { 350 cacheLocker.Lock(); 351 fCache->RemovePage(fPages[index]); 352 cacheLocker.Unlock(); 353 } 354 } 355 356 missingPages = lastMissing - firstMissing + 1; 357 358 // sort the page array by physical page number 359 std::sort(fPages + firstMissing - firstPageOffset, 360 fPages + lastMissing - firstPageOffset + 1, 361 page_physical_number_less); 362 363 // add the pages to the cache 364 cacheLocker.Lock(); 365 366 for (pageOffset = firstMissing; pageOffset <= lastMissing; 367 pageOffset++) { 368 page_num_t index = pageOffset - firstPageOffset; 369 fCache->InsertPage(fPages[index], (off_t)pageOffset * B_PAGE_SIZE); 370 } 371 372 cacheLocker.Unlock(); 373 374 // Read in the missing pages, if this is a read request or a write 375 // request that doesn't cover the complete missing range. 376 if (request->IsRead() 377 || requestOffset < (off_t)firstMissing * B_PAGE_SIZE 378 || requestOffset + (off_t)requestLength 379 > (off_t)(lastMissing + 1) * B_PAGE_SIZE) { 380 status_t error = _TransferPages(firstMissing - firstPageOffset, 381 missingPages, false, isVIP); 382 if (error != B_OK) { 383 dprintf("IOCache::_TransferRequestLine(): Failed to read into " 384 "cache (offset: %" B_PRIdOFF ", length: %" B_PRIuSIZE "), " 385 "trying uncached read (offset: %" B_PRIdOFF ", length: %" 386 B_PRIuSIZE ")\n", (off_t)firstMissing * B_PAGE_SIZE, 387 (size_t)missingPages * B_PAGE_SIZE, requestOffset, 388 requestLength); 389 390 _DiscardPages(firstMissing - firstPageOffset, missingPages); 391 392 // Try again using an uncached transfer 393 return _TransferRequestLineUncached(request, lineOffset, 394 requestOffset, requestLength); 395 } 396 } 397 } 398 399 if (request->IsRead()) { 400 // copy data to request 401 status_t error = _CopyPages(request, requestOffset - lineOffset, 402 requestOffset, requestLength, true); 403 _CachePages(0, linePageCount); 404 return error; 405 } 406 407 // copy data from request 408 status_t error = _CopyPages(request, requestOffset - lineOffset, 409 requestOffset, requestLength, false); 410 if (error != B_OK) { 411 _DiscardPages(0, linePageCount); 412 return error; 413 } 414 415 // write the pages to disk 416 page_num_t firstPage = (requestOffset - lineOffset) / B_PAGE_SIZE; 417 page_num_t endPage = (requestOffset + requestLength - lineOffset 418 + B_PAGE_SIZE - 1) / B_PAGE_SIZE; 419 error = _TransferPages(firstPage, endPage - firstPage, true, isVIP); 420 421 if (error != B_OK) { 422 _DiscardPages(firstPage, endPage - firstPage); 423 return error; 424 } 425 426 _CachePages(0, linePageCount); 427 return error; 428 } 429 430 431 status_t 432 IOCache::_TransferRequestLineUncached(IORequest* request, off_t lineOffset, 433 off_t requestOffset, size_t requestLength) 434 { 435 TRACE("%p->IOCache::_TransferRequestLineUncached(%p, %" B_PRIdOFF 436 ", %" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this, request, lineOffset, 437 requestOffset, requestLength); 438 439 // Advance the request to the interesting offset, so the DMAResource can 440 // provide us with fitting operations. 441 off_t actualRequestOffset 442 = request->Offset() + request->Length() - request->RemainingBytes(); 443 if (actualRequestOffset > requestOffset) { 444 dprintf("IOCache::_TransferRequestLineUncached(): Request %p advanced " 445 "beyond current cache line (%" B_PRIdOFF " vs. %" B_PRIdOFF ")\n", 446 request, actualRequestOffset, requestOffset); 447 return B_BAD_VALUE; 448 } 449 450 if (actualRequestOffset < requestOffset) 451 request->Advance(requestOffset - actualRequestOffset); 452 453 generic_size_t requestRemaining = request->RemainingBytes() - requestLength; 454 455 // Process single operations until the specified part of the request is 456 // finished or until an error occurs. 457 Operation operation; 458 operation.finishedCondition.Init(this, "I/O cache operation finished"); 459 460 while (request->RemainingBytes() > requestRemaining 461 && request->Status() > 0) { 462 status_t error = fDMAResource->TranslateNext(request, &operation, 463 request->RemainingBytes() - requestRemaining); 464 if (error != B_OK) 465 return error; 466 467 error = _DoOperation(operation); 468 469 request->OperationFinished(&operation, error, false, 470 error == B_OK ? operation.OriginalLength() : 0); 471 request->SetUnfinished(); 472 // Keep the request in unfinished state. ScheduleRequest() will set 473 // the final status and notify. 474 475 fDMAResource->RecycleBuffer(operation.Buffer()); 476 477 if (error != B_OK) { 478 TRACE("%p->IOCache::_TransferRequestLineUncached(): operation at " 479 "%" B_PRIdOFF " failed: %s\n", this, operation.Offset(), 480 strerror(error)); 481 return error; 482 } 483 } 484 485 return B_OK; 486 } 487 488 489 status_t 490 IOCache::_DoOperation(Operation& operation) 491 { 492 TRACE("%p->IOCache::_DoOperation(%" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this, 493 operation.Offset(), operation.Length()); 494 495 while (true) { 496 ConditionVariableEntry waitEntry; 497 operation.finishedCondition.Add(&waitEntry); 498 499 status_t error = fIOCallback(fIOCallbackData, &operation); 500 if (error != B_OK) { 501 operation.finishedCondition.NotifyAll(false, error); 502 // removes the entry from the variable 503 return error; 504 } 505 506 // wait for the operation to finish 507 error = waitEntry.Wait(); 508 if (error != B_OK) 509 return error; 510 511 if (operation.Finish()) 512 return B_OK; 513 } 514 } 515 516 517 status_t 518 IOCache::_TransferPages(size_t firstPage, size_t pageCount, bool isWrite, 519 bool isVIP) 520 { 521 TRACE("%p->IOCache::_TransferPages(%" B_PRIuSIZE ", %" B_PRIuSIZE 522 ", write: %d, vip: %d)\n", this, firstPage, pageCount, isWrite, isVIP); 523 524 off_t firstPageOffset = (off_t)fPages[firstPage]->cache_offset 525 * B_PAGE_SIZE; 526 generic_size_t requestLength = std::min( 527 firstPageOffset + (off_t)pageCount * B_PAGE_SIZE, fDeviceCapacity) 528 - firstPageOffset; 529 530 // prepare the I/O vecs 531 size_t vecCount = 0; 532 size_t endPage = firstPage + pageCount; 533 phys_addr_t vecsEndAddress = 0; 534 for (size_t i = firstPage; i < endPage; i++) { 535 phys_addr_t pageAddress 536 = (phys_addr_t)fPages[i]->physical_page_number * B_PAGE_SIZE; 537 if (vecCount == 0 || pageAddress != vecsEndAddress) { 538 fVecs[vecCount].base = pageAddress; 539 fVecs[vecCount++].length = B_PAGE_SIZE; 540 vecsEndAddress = pageAddress + B_PAGE_SIZE; 541 } else { 542 // extend the previous vec 543 fVecs[vecCount - 1].length += B_PAGE_SIZE; 544 vecsEndAddress += B_PAGE_SIZE; 545 } 546 } 547 548 // create a request for the transfer 549 IORequest request; 550 status_t error = request.Init(firstPageOffset, fVecs, vecCount, 551 requestLength, isWrite, 552 B_PHYSICAL_IO_REQUEST | (isVIP ? B_VIP_IO_REQUEST : 0)); 553 if (error != B_OK) 554 return error; 555 556 // Process single operations until the complete request is finished or 557 // until an error occurs. 558 Operation operation; 559 operation.finishedCondition.Init(this, "I/O cache operation finished"); 560 561 while (request.RemainingBytes() > 0 && request.Status() > 0) { 562 error = fDMAResource->TranslateNext(&request, &operation, 563 requestLength); 564 if (error != B_OK) 565 return error; 566 567 error = _DoOperation(operation); 568 569 request.RemoveOperation(&operation); 570 571 fDMAResource->RecycleBuffer(operation.Buffer()); 572 573 if (error != B_OK) { 574 TRACE("%p->IOCache::_TransferLine(): operation at %" B_PRIdOFF 575 " failed: %s\n", this, operation.Offset(), strerror(error)); 576 return error; 577 } 578 } 579 580 return B_OK; 581 } 582 583 584 /*! Frees all pages in given range of the \c fPages array. 585 \c NULL entries in the range are OK. All non \c NULL entries must refer 586 to pages with \c PAGE_STATE_UNUSED. The pages may belong to \c fCache or 587 may not have a cache. 588 \c fCache must not be locked. 589 */ 590 void 591 IOCache::_DiscardPages(size_t firstPage, size_t pageCount) 592 { 593 TRACE("%p->IOCache::_DiscardPages(%" B_PRIuSIZE ", %" B_PRIuSIZE ")\n", 594 this, firstPage, pageCount); 595 596 AutoLocker<VMCache> cacheLocker(fCache); 597 598 for (size_t i = firstPage; i < firstPage + pageCount; i++) { 599 vm_page* page = fPages[i]; 600 if (page == NULL) 601 continue; 602 603 DEBUG_PAGE_ACCESS_START(page); 604 605 ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED, 606 "page: %p @! page -m %p", page, page); 607 608 if (page->Cache() != NULL) 609 fCache->RemovePage(page); 610 611 vm_page_free(NULL, page); 612 } 613 } 614 615 616 /*! Marks all pages in the given range of the \c fPages array cached. 617 There must not be any \c NULL entries in the given array range. All pages 618 must belong to \c cache and have state \c PAGE_STATE_UNUSED. 619 \c fCache must not be locked. 620 */ 621 void 622 IOCache::_CachePages(size_t firstPage, size_t pageCount) 623 { 624 TRACE("%p->IOCache::_CachePages(%" B_PRIuSIZE ", %" B_PRIuSIZE ")\n", 625 this, firstPage, pageCount); 626 627 AutoLocker<VMCache> cacheLocker(fCache); 628 629 for (size_t i = firstPage; i < firstPage + pageCount; i++) { 630 vm_page* page = fPages[i]; 631 ASSERT(page != NULL); 632 ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED 633 && page->Cache() == fCache, 634 "page: %p @! page -m %p", page, page); 635 636 DEBUG_PAGE_ACCESS_START(page); 637 vm_page_set_state(page, PAGE_STATE_CACHED); 638 DEBUG_PAGE_ACCESS_END(page); 639 } 640 } 641 642 643 /*! Copies the contents of pages in \c fPages to \a request, or vice versa. 644 \param request The request. 645 \param pagesRelativeOffset The offset relative to \c fPages[0] where to 646 start copying. 647 \param requestOffset The request offset where to start copying. 648 \param requestLength The number of bytes to copy. 649 \param toRequest If \c true the copy directory is from \c fPages to 650 \a request, otherwise the other way around. 651 \return \c B_OK, if copying went fine, another error code otherwise. 652 */ 653 status_t 654 IOCache::_CopyPages(IORequest* request, size_t pagesRelativeOffset, 655 off_t requestOffset, size_t requestLength, bool toRequest) 656 { 657 TRACE("%p->IOCache::_CopyPages(%p, %" B_PRIuSIZE ", %" B_PRIdOFF 658 ", %" B_PRIuSIZE ", %d)\n", this, request, pagesRelativeOffset, 659 requestOffset, requestLength, toRequest); 660 661 size_t firstPage = pagesRelativeOffset / B_PAGE_SIZE; 662 size_t endPage = (pagesRelativeOffset + requestLength + B_PAGE_SIZE - 1) 663 / B_PAGE_SIZE; 664 665 // map the pages 666 status_t error = _MapPages(firstPage, endPage); 667 // TODO: _MapPages() cannot fail, so the fallback is never needed. Test which 668 // method is faster (probably the active one)! 669 #if 0 670 if (error != B_OK) { 671 // fallback to copying individual pages 672 size_t inPageOffset = pagesRelativeOffset % B_PAGE_SIZE; 673 for (size_t i = firstPage; i < endPage; i++) { 674 // map the page 675 void* handle; 676 addr_t address; 677 error = vm_get_physical_page( 678 fPages[i]->physical_page_number * B_PAGE_SIZE, &address, 679 &handle); 680 if (error != B_OK) 681 return error; 682 683 // copy the page's data 684 size_t toCopy = std::min(B_PAGE_SIZE - inPageOffset, requestLength); 685 686 if (toRequest) { 687 error = request->CopyData((uint8*)(address + inPageOffset), 688 requestOffset, toCopy); 689 } else { 690 error = request->CopyData(requestOffset, 691 (uint8*)(address + inPageOffset), toCopy); 692 } 693 694 // unmap the page 695 vm_put_physical_page(address, handle); 696 697 if (error != B_OK) 698 return error; 699 700 inPageOffset = 0; 701 requestOffset += toCopy; 702 requestLength -= toCopy; 703 } 704 705 return B_OK; 706 } 707 #endif // 0 708 709 // copy 710 if (toRequest) { 711 error = request->CopyData((uint8*)fAreaBase + pagesRelativeOffset, 712 requestOffset, requestLength); 713 } else { 714 error = request->CopyData(requestOffset, 715 (uint8*)fAreaBase + pagesRelativeOffset, requestLength); 716 } 717 718 // unmap the pages 719 _UnmapPages(firstPage, endPage); 720 721 return error; 722 } 723 724 725 /*! Maps a range of pages in \c fPages into fArea. 726 727 If successful, it must be balanced by a call to _UnmapPages(). 728 729 \param firstPage The \c fPages relative index of the first page to map. 730 \param endPage The \c fPages relative index of the page after the last page 731 to map. 732 \return \c B_OK, if mapping went fine, another error code otherwise. 733 */ 734 status_t 735 IOCache::_MapPages(size_t firstPage, size_t endPage) 736 { 737 VMTranslationMap* translationMap 738 = VMAddressSpace::Kernel()->TranslationMap(); 739 740 translationMap->Lock(); 741 742 for (size_t i = firstPage; i < endPage; i++) { 743 vm_page* page = fPages[i]; 744 745 ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED, 746 "page: %p @! page -m %p", page, page); 747 748 translationMap->Map((addr_t)fAreaBase + i * B_PAGE_SIZE, 749 page->physical_page_number * B_PAGE_SIZE, 750 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, &fMappingReservation); 751 // NOTE: We don't increment gMappedPagesCount. Our pages have state 752 // PAGE_STATE_UNUSED anyway and we map them only for a short time. 753 } 754 755 translationMap->Unlock(); 756 757 return B_OK; 758 } 759 760 761 /*! Unmaps a range of pages in \c fPages into fArea. 762 763 Must balance a call to _MapPages(). 764 765 \param firstPage The \c fPages relative index of the first page to unmap. 766 \param endPage The \c fPages relative index of the page after the last page 767 to unmap. 768 */ 769 void 770 IOCache::_UnmapPages(size_t firstPage, size_t endPage) 771 { 772 VMTranslationMap* translationMap 773 = VMAddressSpace::Kernel()->TranslationMap(); 774 775 translationMap->Lock(); 776 777 translationMap->Unmap((addr_t)fAreaBase + firstPage * B_PAGE_SIZE, 778 (addr_t)fAreaBase + endPage * B_PAGE_SIZE - 1); 779 780 translationMap->Unlock(); 781 } 782