1 /* 2 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "IORequest.h" 9 10 #include <string.h> 11 12 #include <arch/debug.h> 13 #include <debug.h> 14 #include <heap.h> 15 #include <kernel.h> 16 #include <thread.h> 17 #include <util/AutoLock.h> 18 #include <vm/vm.h> 19 #include <vm/VMAddressSpace.h> 20 21 #include "dma_resources.h" 22 23 24 //#define TRACE_IO_REQUEST 25 #ifdef TRACE_IO_REQUEST 26 # define TRACE(x...) dprintf(x) 27 #else 28 # define TRACE(x...) ; 29 #endif 30 31 32 // partial I/O operation phases 33 enum { 34 PHASE_READ_BEGIN = 0, 35 PHASE_READ_END = 1, 36 PHASE_DO_ALL = 2 37 }; 38 39 40 // #pragma mark - 41 42 43 IORequestChunk::IORequestChunk() 44 : 45 fParent(NULL), 46 fStatus(1) 47 { 48 } 49 50 51 IORequestChunk::~IORequestChunk() 52 { 53 } 54 55 56 // #pragma mark - 57 58 59 struct virtual_vec_cookie { 60 uint32 vec_index; 61 generic_size_t vec_offset; 62 area_id mapped_area; 63 void* physical_page_handle; 64 addr_t virtual_address; 65 }; 66 67 68 IOBuffer* 69 IOBuffer::Create(uint32 count, bool vip) 70 { 71 size_t size = sizeof(IOBuffer) + sizeof(generic_io_vec) * (count - 1); 72 IOBuffer* buffer 73 = (IOBuffer*)(malloc_etc(size, vip ? HEAP_PRIORITY_VIP : 0)); 74 if (buffer == NULL) 75 return NULL; 76 77 buffer->fCapacity = count; 78 buffer->fVecCount = 0; 79 buffer->fUser = false; 80 buffer->fPhysical = false; 81 buffer->fVIP = vip; 82 buffer->fMemoryLocked = false; 83 84 return buffer; 85 } 86 87 88 void 89 IOBuffer::Delete() 90 { 91 if (this == NULL) 92 return; 93 94 free_etc(this, fVIP ? HEAP_PRIORITY_VIP : 0); 95 } 96 97 98 void 99 IOBuffer::SetVecs(generic_size_t firstVecOffset, const generic_io_vec* vecs, 100 uint32 count, generic_size_t length, uint32 flags) 101 { 102 memcpy(fVecs, vecs, sizeof(generic_io_vec) * count); 103 104 if (count > 0 && firstVecOffset > 0) { 105 fVecs[0].base += firstVecOffset; 106 fVecs[0].length -= firstVecOffset; 107 } 108 109 fVecCount = count; 110 fLength = length; 111 fPhysical = (flags & B_PHYSICAL_IO_REQUEST) != 0; 112 fUser = !fPhysical && IS_USER_ADDRESS(vecs[0].base); 113 } 114 115 116 status_t 117 IOBuffer::GetNextVirtualVec(void*& _cookie, iovec& vector) 118 { 119 virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie; 120 if (cookie == NULL) { 121 cookie = new(malloc_flags(fVIP ? HEAP_PRIORITY_VIP : 0)) 122 virtual_vec_cookie; 123 if (cookie == NULL) 124 return B_NO_MEMORY; 125 126 cookie->vec_index = 0; 127 cookie->vec_offset = 0; 128 cookie->mapped_area = -1; 129 cookie->physical_page_handle = NULL; 130 cookie->virtual_address = 0; 131 _cookie = cookie; 132 } 133 134 // recycle a potential previously mapped page 135 if (cookie->physical_page_handle != NULL) { 136 // TODO: This check is invalid! The physical page mapper is not required to 137 // return a non-NULL handle (the generic implementation does not)! 138 vm_put_physical_page(cookie->virtual_address, 139 cookie->physical_page_handle); 140 } 141 142 if (cookie->vec_index >= fVecCount) 143 return B_BAD_INDEX; 144 145 if (!fPhysical) { 146 vector.iov_base = (void*)(addr_t)fVecs[cookie->vec_index].base; 147 vector.iov_len = fVecs[cookie->vec_index++].length; 148 return B_OK; 149 } 150 151 if (cookie->vec_index == 0 152 && (fVecCount > 1 || fVecs[0].length > B_PAGE_SIZE)) { 153 void* mappedAddress; 154 addr_t mappedSize; 155 156 // TODO: This is a potential violation of the VIP requirement, since 157 // vm_map_physical_memory_vecs() allocates memory without special flags! 158 cookie->mapped_area = vm_map_physical_memory_vecs( 159 VMAddressSpace::KernelID(), "io buffer mapped physical vecs", 160 &mappedAddress, B_ANY_KERNEL_ADDRESS, &mappedSize, 161 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, fVecs, fVecCount); 162 163 if (cookie->mapped_area >= 0) { 164 vector.iov_base = mappedAddress; 165 vector.iov_len = mappedSize; 166 return B_OK; 167 } else 168 ktrace_printf("failed to map area: %s\n", strerror(cookie->mapped_area)); 169 } 170 171 // fallback to page wise mapping 172 generic_io_vec& currentVec = fVecs[cookie->vec_index]; 173 generic_addr_t address = currentVec.base + cookie->vec_offset; 174 size_t pageOffset = address % B_PAGE_SIZE; 175 176 // TODO: This is a potential violation of the VIP requirement, since 177 // vm_get_physical_page() may allocate memory without special flags! 178 status_t result = vm_get_physical_page(address - pageOffset, 179 &cookie->virtual_address, &cookie->physical_page_handle); 180 if (result != B_OK) 181 return result; 182 183 generic_size_t length = min_c(currentVec.length - cookie->vec_offset, 184 B_PAGE_SIZE - pageOffset); 185 186 vector.iov_base = (void*)(cookie->virtual_address + pageOffset); 187 vector.iov_len = length; 188 189 cookie->vec_offset += length; 190 if (cookie->vec_offset >= currentVec.length) { 191 cookie->vec_index++; 192 cookie->vec_offset = 0; 193 } 194 195 return B_OK; 196 } 197 198 199 void 200 IOBuffer::FreeVirtualVecCookie(void* _cookie) 201 { 202 virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie; 203 if (cookie->mapped_area >= 0) 204 delete_area(cookie->mapped_area); 205 // TODO: A vm_get_physical_page() may still be unmatched! 206 207 free_etc(cookie, fVIP ? HEAP_PRIORITY_VIP : 0); 208 } 209 210 211 status_t 212 IOBuffer::LockMemory(team_id team, bool isWrite) 213 { 214 if (fMemoryLocked) { 215 panic("memory already locked!"); 216 return B_BAD_VALUE; 217 } 218 219 for (uint32 i = 0; i < fVecCount; i++) { 220 status_t status = lock_memory_etc(team, (void*)(addr_t)fVecs[i].base, 221 fVecs[i].length, isWrite ? 0 : B_READ_DEVICE); 222 if (status != B_OK) { 223 _UnlockMemory(team, i, isWrite); 224 return status; 225 } 226 } 227 228 fMemoryLocked = true; 229 return B_OK; 230 } 231 232 233 void 234 IOBuffer::_UnlockMemory(team_id team, size_t count, bool isWrite) 235 { 236 for (uint32 i = 0; i < count; i++) { 237 unlock_memory_etc(team, (void*)(addr_t)fVecs[i].base, fVecs[i].length, 238 isWrite ? 0 : B_READ_DEVICE); 239 } 240 } 241 242 243 void 244 IOBuffer::UnlockMemory(team_id team, bool isWrite) 245 { 246 if (!fMemoryLocked) { 247 panic("memory not locked"); 248 return; 249 } 250 251 _UnlockMemory(team, fVecCount, isWrite); 252 fMemoryLocked = false; 253 } 254 255 256 void 257 IOBuffer::Dump() const 258 { 259 kprintf("IOBuffer at %p\n", this); 260 261 kprintf(" origin: %s\n", fUser ? "user" : "kernel"); 262 kprintf(" kind: %s\n", fPhysical ? "physical" : "virtual"); 263 kprintf(" length: %" B_PRIuGENADDR "\n", fLength); 264 kprintf(" capacity: %" B_PRIuSIZE "\n", fCapacity); 265 kprintf(" vecs: %" B_PRIuSIZE "\n", fVecCount); 266 267 for (uint32 i = 0; i < fVecCount; i++) { 268 kprintf(" [%" B_PRIu32 "] %#" B_PRIxGENADDR ", %" B_PRIuGENADDR "\n", 269 i, fVecs[i].base, fVecs[i].length); 270 } 271 } 272 273 274 // #pragma mark - 275 276 277 bool 278 IOOperation::Finish() 279 { 280 TRACE("IOOperation::Finish()\n"); 281 if (fStatus == B_OK) { 282 if (fParent->IsWrite()) { 283 TRACE(" is write\n"); 284 if (fPhase == PHASE_READ_BEGIN) { 285 TRACE(" phase read begin\n"); 286 // repair phase adjusted vec 287 fDMABuffer->VecAt(fSavedVecIndex).length = fSavedVecLength; 288 289 // partial write: copy partial begin to bounce buffer 290 bool skipReadEndPhase; 291 status_t error = _CopyPartialBegin(true, skipReadEndPhase); 292 if (error == B_OK) { 293 // We're done with the first phase only (read in begin). 294 // Get ready for next phase... 295 fPhase = HasPartialEnd() && !skipReadEndPhase 296 ? PHASE_READ_END : PHASE_DO_ALL; 297 _PrepareVecs(); 298 ResetStatus(); 299 // TODO: Is there a race condition, if the request is 300 // aborted at the same time? 301 return false; 302 } 303 304 SetStatus(error); 305 } else if (fPhase == PHASE_READ_END) { 306 TRACE(" phase read end\n"); 307 // repair phase adjusted vec 308 generic_io_vec& vec = fDMABuffer->VecAt(fSavedVecIndex); 309 vec.base += vec.length - fSavedVecLength; 310 vec.length = fSavedVecLength; 311 312 // partial write: copy partial end to bounce buffer 313 status_t error = _CopyPartialEnd(true); 314 if (error == B_OK) { 315 // We're done with the second phase only (read in end). 316 // Get ready for next phase... 317 fPhase = PHASE_DO_ALL; 318 ResetStatus(); 319 // TODO: Is there a race condition, if the request is 320 // aborted at the same time? 321 return false; 322 } 323 324 SetStatus(error); 325 } 326 } 327 } 328 329 if (fParent->IsRead() && UsesBounceBuffer()) { 330 TRACE(" read with bounce buffer\n"); 331 // copy the bounce buffer segments to the final location 332 uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); 333 phys_addr_t bounceBufferStart 334 = fDMABuffer->PhysicalBounceBufferAddress(); 335 phys_addr_t bounceBufferEnd = bounceBufferStart 336 + fDMABuffer->BounceBufferSize(); 337 338 const generic_io_vec* vecs = fDMABuffer->Vecs(); 339 uint32 vecCount = fDMABuffer->VecCount(); 340 341 status_t error = B_OK; 342 343 // We iterate through the vecs we have read, moving offset (the device 344 // offset) as we go. If [offset, offset + vec.length) intersects with 345 // [startOffset, endOffset) we copy to the final location. 346 off_t offset = fOffset; 347 const off_t startOffset = fOriginalOffset; 348 const off_t endOffset = fOriginalOffset + fOriginalLength; 349 350 for (uint32 i = 0; error == B_OK && i < vecCount; i++) { 351 const generic_io_vec& vec = vecs[i]; 352 generic_addr_t base = vec.base; 353 generic_size_t length = vec.length; 354 355 if (offset < startOffset) { 356 // If the complete vector is before the start offset, skip it. 357 if (offset + (off_t)length <= startOffset) { 358 offset += length; 359 continue; 360 } 361 362 // The vector starts before the start offset, but intersects 363 // with it. Skip the part we aren't interested in. 364 generic_size_t diff = startOffset - offset; 365 offset += diff; 366 base += diff; 367 length -= diff; 368 } 369 370 if (offset + (off_t)length > endOffset) { 371 // If we're already beyond the end offset, we're done. 372 if (offset >= endOffset) 373 break; 374 375 // The vector extends beyond the end offset -- cut it. 376 length = endOffset - offset; 377 } 378 379 if (base >= bounceBufferStart && base < bounceBufferEnd) { 380 error = fParent->CopyData( 381 bounceBuffer + (base - bounceBufferStart), offset, length); 382 } 383 384 offset += length; 385 } 386 387 if (error != B_OK) 388 SetStatus(error); 389 } 390 391 return true; 392 } 393 394 395 /*! Note: SetPartial() must be called first! 396 */ 397 status_t 398 IOOperation::Prepare(IORequest* request) 399 { 400 if (fParent != NULL) 401 fParent->RemoveOperation(this); 402 403 fParent = request; 404 405 fTransferredBytes = 0; 406 407 // set initial phase 408 fPhase = PHASE_DO_ALL; 409 if (fParent->IsWrite()) { 410 // Copy data to bounce buffer segments, save the partial begin/end vec, 411 // which will be copied after their respective read phase. 412 if (UsesBounceBuffer()) { 413 TRACE(" write with bounce buffer\n"); 414 uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); 415 phys_addr_t bounceBufferStart 416 = fDMABuffer->PhysicalBounceBufferAddress(); 417 phys_addr_t bounceBufferEnd = bounceBufferStart 418 + fDMABuffer->BounceBufferSize(); 419 420 const generic_io_vec* vecs = fDMABuffer->Vecs(); 421 uint32 vecCount = fDMABuffer->VecCount(); 422 generic_size_t vecOffset = 0; 423 uint32 i = 0; 424 425 off_t offset = fOffset; 426 off_t endOffset = fOffset + fLength; 427 428 if (HasPartialBegin()) { 429 // skip first block 430 generic_size_t toSkip = fBlockSize; 431 while (toSkip > 0) { 432 if (vecs[i].length <= toSkip) { 433 toSkip -= vecs[i].length; 434 i++; 435 } else { 436 vecOffset = toSkip; 437 break; 438 } 439 } 440 441 offset += fBlockSize; 442 } 443 444 if (HasPartialEnd()) { 445 // skip last block 446 generic_size_t toSkip = fBlockSize; 447 while (toSkip > 0) { 448 if (vecs[vecCount - 1].length <= toSkip) { 449 toSkip -= vecs[vecCount - 1].length; 450 vecCount--; 451 } else 452 break; 453 } 454 455 endOffset -= fBlockSize; 456 } 457 458 for (; i < vecCount; i++) { 459 const generic_io_vec& vec = vecs[i]; 460 generic_addr_t base = vec.base + vecOffset; 461 generic_size_t length = vec.length - vecOffset; 462 vecOffset = 0; 463 464 if (base >= bounceBufferStart && base < bounceBufferEnd) { 465 if (offset + (off_t)length > endOffset) 466 length = endOffset - offset; 467 status_t error = fParent->CopyData(offset, 468 bounceBuffer + (base - bounceBufferStart), length); 469 if (error != B_OK) 470 return error; 471 } 472 473 offset += length; 474 } 475 } 476 477 if (HasPartialBegin()) 478 fPhase = PHASE_READ_BEGIN; 479 else if (HasPartialEnd()) 480 fPhase = PHASE_READ_END; 481 482 _PrepareVecs(); 483 } 484 485 ResetStatus(); 486 487 if (fParent != NULL) 488 fParent->AddOperation(this); 489 490 return B_OK; 491 } 492 493 494 void 495 IOOperation::SetOriginalRange(off_t offset, generic_size_t length) 496 { 497 fOriginalOffset = fOffset = offset; 498 fOriginalLength = fLength = length; 499 } 500 501 502 void 503 IOOperation::SetRange(off_t offset, generic_size_t length) 504 { 505 fOffset = offset; 506 fLength = length; 507 } 508 509 510 off_t 511 IOOperation::Offset() const 512 { 513 return fPhase == PHASE_READ_END ? fOffset + fLength - fBlockSize : fOffset; 514 } 515 516 517 generic_size_t 518 IOOperation::Length() const 519 { 520 return fPhase == PHASE_DO_ALL ? fLength : fBlockSize; 521 } 522 523 524 generic_io_vec* 525 IOOperation::Vecs() const 526 { 527 switch (fPhase) { 528 case PHASE_READ_END: 529 return fDMABuffer->Vecs() + fSavedVecIndex; 530 case PHASE_READ_BEGIN: 531 case PHASE_DO_ALL: 532 default: 533 return fDMABuffer->Vecs(); 534 } 535 } 536 537 538 uint32 539 IOOperation::VecCount() const 540 { 541 switch (fPhase) { 542 case PHASE_READ_BEGIN: 543 return fSavedVecIndex + 1; 544 case PHASE_READ_END: 545 return fDMABuffer->VecCount() - fSavedVecIndex; 546 case PHASE_DO_ALL: 547 default: 548 return fDMABuffer->VecCount(); 549 } 550 } 551 552 553 void 554 IOOperation::SetPartial(bool partialBegin, bool partialEnd) 555 { 556 TRACE("partial begin %d, end %d\n", partialBegin, partialEnd); 557 fPartialBegin = partialBegin; 558 fPartialEnd = partialEnd; 559 } 560 561 562 bool 563 IOOperation::IsWrite() const 564 { 565 return fParent->IsWrite() && fPhase == PHASE_DO_ALL; 566 } 567 568 569 bool 570 IOOperation::IsRead() const 571 { 572 return fParent->IsRead(); 573 } 574 575 576 void 577 IOOperation::_PrepareVecs() 578 { 579 // we need to prepare the vecs for consumption by the drivers 580 if (fPhase == PHASE_READ_BEGIN) { 581 generic_io_vec* vecs = fDMABuffer->Vecs(); 582 uint32 vecCount = fDMABuffer->VecCount(); 583 generic_size_t vecLength = fBlockSize; 584 for (uint32 i = 0; i < vecCount; i++) { 585 generic_io_vec& vec = vecs[i]; 586 if (vec.length >= vecLength) { 587 fSavedVecIndex = i; 588 fSavedVecLength = vec.length; 589 vec.length = vecLength; 590 break; 591 } 592 vecLength -= vec.length; 593 } 594 } else if (fPhase == PHASE_READ_END) { 595 generic_io_vec* vecs = fDMABuffer->Vecs(); 596 uint32 vecCount = fDMABuffer->VecCount(); 597 generic_size_t vecLength = fBlockSize; 598 for (int32 i = vecCount - 1; i >= 0; i--) { 599 generic_io_vec& vec = vecs[i]; 600 if (vec.length >= vecLength) { 601 fSavedVecIndex = i; 602 fSavedVecLength = vec.length; 603 vec.base += vec.length - vecLength; 604 vec.length = vecLength; 605 break; 606 } 607 vecLength -= vec.length; 608 } 609 } 610 } 611 612 613 status_t 614 IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly) 615 { 616 generic_size_t relativeOffset = OriginalOffset() - fOffset; 617 generic_size_t length = fBlockSize - relativeOffset; 618 619 singleBlockOnly = length >= OriginalLength(); 620 if (singleBlockOnly) 621 length = OriginalLength(); 622 623 TRACE("_CopyPartialBegin(%s, single only %d)\n", 624 isWrite ? "write" : "read", singleBlockOnly); 625 626 if (isWrite) { 627 return fParent->CopyData(OriginalOffset(), 628 (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length); 629 } else { 630 return fParent->CopyData( 631 (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, 632 OriginalOffset(), length); 633 } 634 } 635 636 637 status_t 638 IOOperation::_CopyPartialEnd(bool isWrite) 639 { 640 TRACE("_CopyPartialEnd(%s)\n", isWrite ? "write" : "read"); 641 642 const generic_io_vec& lastVec 643 = fDMABuffer->VecAt(fDMABuffer->VecCount() - 1); 644 off_t lastVecPos = fOffset + fLength - fBlockSize; 645 uint8* base = (uint8*)fDMABuffer->BounceBufferAddress() 646 + (lastVec.base + lastVec.length - fBlockSize 647 - fDMABuffer->PhysicalBounceBufferAddress()); 648 // NOTE: this won't work if we don't use the bounce buffer contiguously 649 // (because of boundary alignments). 650 generic_size_t length = OriginalOffset() + OriginalLength() - lastVecPos; 651 652 if (isWrite) 653 return fParent->CopyData(lastVecPos, base, length); 654 655 return fParent->CopyData(base, lastVecPos, length); 656 } 657 658 659 void 660 IOOperation::Dump() const 661 { 662 kprintf("io_operation at %p\n", this); 663 664 kprintf(" parent: %p\n", fParent); 665 kprintf(" status: %s\n", strerror(fStatus)); 666 kprintf(" dma buffer: %p\n", fDMABuffer); 667 kprintf(" offset: %-8Ld (original: %Ld)\n", fOffset, 668 fOriginalOffset); 669 kprintf(" length: %-8" B_PRIuGENADDR " (original: %" 670 B_PRIuGENADDR ")\n", fLength, fOriginalLength); 671 kprintf(" transferred: %" B_PRIuGENADDR "\n", fTransferredBytes); 672 kprintf(" block size: %" B_PRIuGENADDR "\n", fBlockSize); 673 kprintf(" saved vec index: %u\n", fSavedVecIndex); 674 kprintf(" saved vec length: %u\n", fSavedVecLength); 675 kprintf(" r/w: %s\n", IsWrite() ? "write" : "read"); 676 kprintf(" phase: %s\n", fPhase == PHASE_READ_BEGIN 677 ? "read begin" : fPhase == PHASE_READ_END ? "read end" 678 : fPhase == PHASE_DO_ALL ? "do all" : "unknown"); 679 kprintf(" partial begin: %s\n", fPartialBegin ? "yes" : "no"); 680 kprintf(" partial end: %s\n", fPartialEnd ? "yes" : "no"); 681 kprintf(" bounce buffer: %s\n", fUsesBounceBuffer ? "yes" : "no"); 682 683 set_debug_variable("_parent", (addr_t)fParent); 684 set_debug_variable("_buffer", (addr_t)fDMABuffer); 685 } 686 687 688 // #pragma mark - 689 690 691 IORequest::IORequest() 692 : 693 fIsNotified(false), 694 fFinishedCallback(NULL), 695 fFinishedCookie(NULL), 696 fIterationCallback(NULL), 697 fIterationCookie(NULL) 698 { 699 mutex_init(&fLock, "I/O request lock"); 700 fFinishedCondition.Init(this, "I/O request finished"); 701 } 702 703 704 IORequest::~IORequest() 705 { 706 mutex_lock(&fLock); 707 DeleteSubRequests(); 708 fBuffer->Delete(); 709 mutex_destroy(&fLock); 710 } 711 712 713 /* static */ IORequest* 714 IORequest::Create(bool vip) 715 { 716 return vip 717 ? new(malloc_flags(HEAP_PRIORITY_VIP)) IORequest 718 : new(std::nothrow) IORequest; 719 } 720 721 722 status_t 723 IORequest::Init(off_t offset, generic_addr_t buffer, generic_size_t length, 724 bool write, uint32 flags) 725 { 726 generic_io_vec vec; 727 vec.base = buffer; 728 vec.length = length; 729 return Init(offset, &vec, 1, length, write, flags); 730 } 731 732 733 status_t 734 IORequest::Init(off_t offset, generic_size_t firstVecOffset, 735 const generic_io_vec* vecs, size_t count, generic_size_t length, bool write, 736 uint32 flags) 737 { 738 fBuffer = IOBuffer::Create(count, (flags & B_VIP_IO_REQUEST) != 0); 739 if (fBuffer == NULL) 740 return B_NO_MEMORY; 741 742 fBuffer->SetVecs(firstVecOffset, vecs, count, length, flags); 743 744 fOwner = NULL; 745 fOffset = offset; 746 fLength = length; 747 fRelativeParentOffset = 0; 748 fTransferSize = 0; 749 fFlags = flags; 750 Thread* thread = thread_get_current_thread(); 751 fTeam = thread->team->id; 752 fThread = thread->id; 753 fIsWrite = write; 754 fPartialTransfer = false; 755 fSuppressChildNotifications = false; 756 757 // these are for iteration 758 fVecIndex = 0; 759 fVecOffset = 0; 760 fRemainingBytes = length; 761 762 fPendingChildren = 0; 763 764 fStatus = 1; 765 766 return B_OK; 767 } 768 769 770 status_t 771 IORequest::CreateSubRequest(off_t parentOffset, off_t offset, 772 generic_size_t length, IORequest*& _subRequest) 773 { 774 ASSERT(parentOffset >= fOffset && length <= fLength 775 && parentOffset - fOffset <= (off_t)(fLength - length)); 776 777 // find start vec 778 generic_size_t vecOffset = parentOffset - fOffset; 779 generic_io_vec* vecs = fBuffer->Vecs(); 780 int32 vecCount = fBuffer->VecCount(); 781 int32 startVec = 0; 782 for (; startVec < vecCount; startVec++) { 783 const generic_io_vec& vec = vecs[startVec]; 784 if (vecOffset < vec.length) 785 break; 786 787 vecOffset -= vec.length; 788 } 789 790 // count vecs 791 generic_size_t currentVecOffset = vecOffset; 792 int32 endVec = startVec; 793 generic_size_t remainingLength = length; 794 for (; endVec < vecCount; endVec++) { 795 const generic_io_vec& vec = vecs[endVec]; 796 if (vec.length - currentVecOffset >= remainingLength) 797 break; 798 799 remainingLength -= vec.length - currentVecOffset; 800 currentVecOffset = 0; 801 } 802 803 // create subrequest 804 IORequest* subRequest = Create((fFlags & B_VIP_IO_REQUEST) != 0); 805 if (subRequest == NULL) 806 return B_NO_MEMORY; 807 808 status_t error = subRequest->Init(offset, vecOffset, vecs + startVec, 809 endVec - startVec + 1, length, fIsWrite, fFlags & ~B_DELETE_IO_REQUEST); 810 if (error != B_OK) { 811 delete subRequest; 812 return error; 813 } 814 815 subRequest->fRelativeParentOffset = parentOffset - fOffset; 816 subRequest->fTeam = fTeam; 817 subRequest->fThread = fThread; 818 819 _subRequest = subRequest; 820 subRequest->SetParent(this); 821 822 MutexLocker _(fLock); 823 824 fChildren.Add(subRequest); 825 fPendingChildren++; 826 TRACE("IORequest::CreateSubRequest(): request: %p, subrequest: %p\n", this, 827 subRequest); 828 829 return B_OK; 830 } 831 832 833 void 834 IORequest::DeleteSubRequests() 835 { 836 while (IORequestChunk* chunk = fChildren.RemoveHead()) 837 delete chunk; 838 fPendingChildren = 0; 839 } 840 841 842 void 843 IORequest::SetFinishedCallback(io_request_finished_callback callback, 844 void* cookie) 845 { 846 fFinishedCallback = callback; 847 fFinishedCookie = cookie; 848 } 849 850 851 void 852 IORequest::SetIterationCallback(io_request_iterate_callback callback, 853 void* cookie) 854 { 855 fIterationCallback = callback; 856 fIterationCookie = cookie; 857 } 858 859 860 io_request_finished_callback 861 IORequest::FinishedCallback(void** _cookie) const 862 { 863 if (_cookie != NULL) 864 *_cookie = fFinishedCookie; 865 return fFinishedCallback; 866 } 867 868 869 status_t 870 IORequest::Wait(uint32 flags, bigtime_t timeout) 871 { 872 MutexLocker locker(fLock); 873 874 if (IsFinished() && fIsNotified) 875 return Status(); 876 877 ConditionVariableEntry entry; 878 fFinishedCondition.Add(&entry); 879 880 locker.Unlock(); 881 882 status_t error = entry.Wait(flags, timeout); 883 if (error != B_OK) 884 return error; 885 886 return Status(); 887 } 888 889 890 void 891 IORequest::NotifyFinished() 892 { 893 TRACE("IORequest::NotifyFinished(): request: %p\n", this); 894 895 MutexLocker locker(fLock); 896 897 if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) { 898 // The request is not really done yet. If it has an iteration callback, 899 // call it. 900 if (fIterationCallback != NULL) { 901 ResetStatus(); 902 locker.Unlock(); 903 bool partialTransfer = false; 904 status_t error = fIterationCallback(fIterationCookie, this, 905 &partialTransfer); 906 if (error == B_OK && !partialTransfer) 907 return; 908 909 // Iteration failed, which means we're responsible for notifying the 910 // requests finished. 911 locker.Lock(); 912 fStatus = error; 913 fPartialTransfer = true; 914 } 915 } 916 917 ASSERT(!fIsNotified); 918 ASSERT(fPendingChildren == 0); 919 ASSERT(fChildren.IsEmpty() 920 || dynamic_cast<IOOperation*>(fChildren.Head()) == NULL); 921 922 // unlock the memory 923 if (fBuffer->IsMemoryLocked()) 924 fBuffer->UnlockMemory(fTeam, fIsWrite); 925 926 // Cache the callbacks before we unblock waiters and unlock. Any of the 927 // following could delete this request, so we don't want to touch it 928 // once we have started telling others that it is done. 929 IORequest* parent = fParent; 930 io_request_finished_callback finishedCallback = fFinishedCallback; 931 void* finishedCookie = fFinishedCookie; 932 status_t status = fStatus; 933 generic_size_t lastTransferredOffset 934 = fRelativeParentOffset + fTransferSize; 935 bool partialTransfer = status != B_OK || fPartialTransfer; 936 bool deleteRequest = (fFlags & B_DELETE_IO_REQUEST) != 0; 937 938 // unblock waiters 939 fIsNotified = true; 940 fFinishedCondition.NotifyAll(); 941 942 locker.Unlock(); 943 944 // notify callback 945 if (finishedCallback != NULL) { 946 finishedCallback(finishedCookie, this, status, partialTransfer, 947 lastTransferredOffset); 948 } 949 950 // notify parent 951 if (parent != NULL) { 952 parent->SubRequestFinished(this, status, partialTransfer, 953 lastTransferredOffset); 954 } 955 956 if (deleteRequest) 957 delete this; 958 } 959 960 961 /*! Returns whether this request or any of it's ancestors has a finished or 962 notification callback. Used to decide whether NotifyFinished() can be called 963 synchronously. 964 */ 965 bool 966 IORequest::HasCallbacks() const 967 { 968 if (fFinishedCallback != NULL || fIterationCallback != NULL) 969 return true; 970 971 return fParent != NULL && fParent->HasCallbacks(); 972 } 973 974 975 void 976 IORequest::SetStatusAndNotify(status_t status) 977 { 978 MutexLocker locker(fLock); 979 980 if (fStatus != 1) 981 return; 982 983 fStatus = status; 984 985 locker.Unlock(); 986 987 NotifyFinished(); 988 } 989 990 991 void 992 IORequest::OperationFinished(IOOperation* operation, status_t status, 993 bool partialTransfer, generic_size_t transferEndOffset) 994 { 995 TRACE("IORequest::OperationFinished(%p, %#lx): request: %p\n", operation, 996 status, this); 997 998 MutexLocker locker(fLock); 999 1000 fChildren.Remove(operation); 1001 operation->SetParent(NULL); 1002 1003 if (status != B_OK || partialTransfer) { 1004 if (fTransferSize > transferEndOffset) 1005 fTransferSize = transferEndOffset; 1006 fPartialTransfer = true; 1007 } 1008 1009 if (status != B_OK && fStatus == 1) 1010 fStatus = status; 1011 1012 if (--fPendingChildren > 0) 1013 return; 1014 1015 // last child finished 1016 1017 // set status, if not done yet 1018 if (fStatus == 1) 1019 fStatus = B_OK; 1020 } 1021 1022 1023 void 1024 IORequest::SubRequestFinished(IORequest* request, status_t status, 1025 bool partialTransfer, generic_size_t transferEndOffset) 1026 { 1027 TRACE("IORequest::SubrequestFinished(%p, %#" B_PRIx32 ", %d, %" 1028 B_PRIuGENADDR "): request: %p\n", request, status, partialTransfer, transferEndOffset, this); 1029 1030 MutexLocker locker(fLock); 1031 1032 if (status != B_OK || partialTransfer) { 1033 if (fTransferSize > transferEndOffset) 1034 fTransferSize = transferEndOffset; 1035 fPartialTransfer = true; 1036 } 1037 1038 if (status != B_OK && fStatus == 1) 1039 fStatus = status; 1040 1041 if (--fPendingChildren > 0 || fSuppressChildNotifications) 1042 return; 1043 1044 // last child finished 1045 1046 // set status, if not done yet 1047 if (fStatus == 1) 1048 fStatus = B_OK; 1049 1050 locker.Unlock(); 1051 1052 NotifyFinished(); 1053 } 1054 1055 1056 void 1057 IORequest::SetUnfinished() 1058 { 1059 MutexLocker _(fLock); 1060 ResetStatus(); 1061 } 1062 1063 1064 void 1065 IORequest::SetTransferredBytes(bool partialTransfer, 1066 generic_size_t transferredBytes) 1067 { 1068 TRACE("%p->IORequest::SetTransferredBytes(%d, %" B_PRIuGENADDR ")\n", this, 1069 partialTransfer, transferredBytes); 1070 1071 MutexLocker _(fLock); 1072 1073 fPartialTransfer = partialTransfer; 1074 fTransferSize = transferredBytes; 1075 } 1076 1077 1078 void 1079 IORequest::SetSuppressChildNotifications(bool suppress) 1080 { 1081 fSuppressChildNotifications = suppress; 1082 } 1083 1084 1085 void 1086 IORequest::Advance(generic_size_t bySize) 1087 { 1088 TRACE("IORequest::Advance(%" B_PRIuGENADDR "): remaining: %" B_PRIuGENADDR 1089 " -> %" B_PRIuGENADDR "\n", bySize, fRemainingBytes, 1090 fRemainingBytes - bySize); 1091 fRemainingBytes -= bySize; 1092 fTransferSize += bySize; 1093 1094 generic_io_vec* vecs = fBuffer->Vecs(); 1095 uint32 vecCount = fBuffer->VecCount(); 1096 while (fVecIndex < vecCount 1097 && vecs[fVecIndex].length - fVecOffset <= bySize) { 1098 bySize -= vecs[fVecIndex].length - fVecOffset; 1099 fVecOffset = 0; 1100 fVecIndex++; 1101 } 1102 1103 fVecOffset += bySize; 1104 } 1105 1106 1107 IORequest* 1108 IORequest::FirstSubRequest() 1109 { 1110 return dynamic_cast<IORequest*>(fChildren.Head()); 1111 } 1112 1113 1114 IORequest* 1115 IORequest::NextSubRequest(IORequest* previous) 1116 { 1117 if (previous == NULL) 1118 return NULL; 1119 return dynamic_cast<IORequest*>(fChildren.GetNext(previous)); 1120 } 1121 1122 1123 void 1124 IORequest::AddOperation(IOOperation* operation) 1125 { 1126 MutexLocker locker(fLock); 1127 TRACE("IORequest::AddOperation(%p): request: %p\n", operation, this); 1128 fChildren.Add(operation); 1129 fPendingChildren++; 1130 } 1131 1132 1133 void 1134 IORequest::RemoveOperation(IOOperation* operation) 1135 { 1136 MutexLocker locker(fLock); 1137 fChildren.Remove(operation); 1138 operation->SetParent(NULL); 1139 } 1140 1141 1142 status_t 1143 IORequest::CopyData(off_t offset, void* buffer, size_t size) 1144 { 1145 return _CopyData(buffer, offset, size, true); 1146 } 1147 1148 1149 status_t 1150 IORequest::CopyData(const void* buffer, off_t offset, size_t size) 1151 { 1152 return _CopyData((void*)buffer, offset, size, false); 1153 } 1154 1155 1156 status_t 1157 IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn) 1158 { 1159 if (size == 0) 1160 return B_OK; 1161 1162 uint8* buffer = (uint8*)_buffer; 1163 1164 if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) { 1165 panic("IORequest::_CopyData(): invalid range: (%lld, %lu)", offset, 1166 size); 1167 return B_BAD_VALUE; 1168 } 1169 1170 // If we can, we directly copy from/to the virtual buffer. The memory is 1171 // locked in this case. 1172 status_t (*copyFunction)(void*, generic_addr_t, size_t, team_id, bool); 1173 if (fBuffer->IsPhysical()) { 1174 copyFunction = &IORequest::_CopyPhysical; 1175 } else { 1176 copyFunction = fBuffer->IsUser() 1177 ? &IORequest::_CopyUser : &IORequest::_CopySimple; 1178 } 1179 1180 // skip bytes if requested 1181 generic_io_vec* vecs = fBuffer->Vecs(); 1182 generic_size_t skipBytes = offset - fOffset; 1183 generic_size_t vecOffset = 0; 1184 while (skipBytes > 0) { 1185 if (vecs[0].length > skipBytes) { 1186 vecOffset = skipBytes; 1187 break; 1188 } 1189 1190 skipBytes -= vecs[0].length; 1191 vecs++; 1192 } 1193 1194 // copy vector-wise 1195 while (size > 0) { 1196 generic_size_t toCopy = min_c(size, vecs[0].length - vecOffset); 1197 status_t error = copyFunction(buffer, vecs[0].base + vecOffset, toCopy, 1198 fTeam, copyIn); 1199 if (error != B_OK) 1200 return error; 1201 1202 buffer += toCopy; 1203 size -= toCopy; 1204 vecs++; 1205 vecOffset = 0; 1206 } 1207 1208 return B_OK; 1209 } 1210 1211 1212 /* static */ status_t 1213 IORequest::_CopySimple(void* bounceBuffer, generic_addr_t external, size_t size, 1214 team_id team, bool copyIn) 1215 { 1216 TRACE(" IORequest::_CopySimple(%p, %#" B_PRIxGENADDR ", %lu, %d)\n", 1217 bounceBuffer, external, size, copyIn); 1218 if (copyIn) 1219 memcpy(bounceBuffer, (void*)(addr_t)external, size); 1220 else 1221 memcpy((void*)(addr_t)external, bounceBuffer, size); 1222 return B_OK; 1223 } 1224 1225 1226 /* static */ status_t 1227 IORequest::_CopyPhysical(void* bounceBuffer, generic_addr_t external, 1228 size_t size, team_id team, bool copyIn) 1229 { 1230 if (copyIn) 1231 return vm_memcpy_from_physical(bounceBuffer, external, size, false); 1232 1233 return vm_memcpy_to_physical(external, bounceBuffer, size, false); 1234 } 1235 1236 1237 /* static */ status_t 1238 IORequest::_CopyUser(void* _bounceBuffer, generic_addr_t _external, size_t size, 1239 team_id team, bool copyIn) 1240 { 1241 uint8* bounceBuffer = (uint8*)_bounceBuffer; 1242 uint8* external = (uint8*)(addr_t)_external; 1243 1244 while (size > 0) { 1245 static const int32 kEntryCount = 8; 1246 physical_entry entries[kEntryCount]; 1247 1248 uint32 count = kEntryCount; 1249 status_t error = get_memory_map_etc(team, external, size, entries, 1250 &count); 1251 if (error != B_OK && error != B_BUFFER_OVERFLOW) { 1252 panic("IORequest::_CopyUser(): Failed to get physical memory for " 1253 "user memory %p\n", external); 1254 return B_BAD_ADDRESS; 1255 } 1256 1257 for (uint32 i = 0; i < count; i++) { 1258 const physical_entry& entry = entries[i]; 1259 error = _CopyPhysical(bounceBuffer, entry.address, entry.size, team, 1260 copyIn); 1261 if (error != B_OK) 1262 return error; 1263 1264 size -= entry.size; 1265 bounceBuffer += entry.size; 1266 external += entry.size; 1267 } 1268 } 1269 1270 return B_OK; 1271 } 1272 1273 1274 void 1275 IORequest::Dump() const 1276 { 1277 kprintf("io_request at %p\n", this); 1278 1279 kprintf(" owner: %p\n", fOwner); 1280 kprintf(" parent: %p\n", fParent); 1281 kprintf(" status: %s\n", strerror(fStatus)); 1282 kprintf(" mutex: %p\n", &fLock); 1283 kprintf(" IOBuffer: %p\n", fBuffer); 1284 kprintf(" offset: %Ld\n", fOffset); 1285 kprintf(" length: %" B_PRIuGENADDR "\n", fLength); 1286 kprintf(" transfer size: %" B_PRIuGENADDR "\n", fTransferSize); 1287 kprintf(" relative offset: %" B_PRIuGENADDR "\n", fRelativeParentOffset); 1288 kprintf(" pending children: %ld\n", fPendingChildren); 1289 kprintf(" flags: %#lx\n", fFlags); 1290 kprintf(" team: %ld\n", fTeam); 1291 kprintf(" thread: %ld\n", fThread); 1292 kprintf(" r/w: %s\n", fIsWrite ? "write" : "read"); 1293 kprintf(" partial transfer: %s\n", fPartialTransfer ? "yes" : "no"); 1294 kprintf(" finished cvar: %p\n", &fFinishedCondition); 1295 kprintf(" iteration:\n"); 1296 kprintf(" vec index: %lu\n", fVecIndex); 1297 kprintf(" vec offset: %" B_PRIuGENADDR "\n", fVecOffset); 1298 kprintf(" remaining bytes: %" B_PRIuGENADDR "\n", fRemainingBytes); 1299 kprintf(" callbacks:\n"); 1300 kprintf(" finished %p, cookie %p\n", fFinishedCallback, fFinishedCookie); 1301 kprintf(" iteration %p, cookie %p\n", fIterationCallback, 1302 fIterationCookie); 1303 kprintf(" children:\n"); 1304 1305 IORequestChunkList::ConstIterator iterator = fChildren.GetIterator(); 1306 while (iterator.HasNext()) { 1307 kprintf(" %p\n", iterator.Next()); 1308 } 1309 1310 set_debug_variable("_parent", (addr_t)fParent); 1311 set_debug_variable("_mutex", (addr_t)&fLock); 1312 set_debug_variable("_buffer", (addr_t)fBuffer); 1313 set_debug_variable("_cvar", (addr_t)&fFinishedCondition); 1314 } 1315