1 /* 2 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2008-2017, Axel Dörfler, axeld@pinc-software.de. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "IORequest.h" 9 10 #include <string.h> 11 12 #include <arch/debug.h> 13 #include <debug.h> 14 #include <heap.h> 15 #include <kernel.h> 16 #include <team.h> 17 #include <thread.h> 18 #include <util/AutoLock.h> 19 #include <vm/vm.h> 20 #include <vm/VMAddressSpace.h> 21 22 #include "dma_resources.h" 23 24 25 //#define TRACE_IO_REQUEST 26 #ifdef TRACE_IO_REQUEST 27 # define TRACE(x...) dprintf(x) 28 #else 29 # define TRACE(x...) ; 30 #endif 31 32 33 // partial I/O operation phases 34 enum { 35 PHASE_READ_BEGIN = 0, 36 PHASE_READ_END = 1, 37 PHASE_DO_ALL = 2 38 }; 39 40 41 struct virtual_vec_cookie { 42 uint32 vec_index; 43 generic_size_t vec_offset; 44 area_id mapped_area; 45 void* physical_page_handle; 46 addr_t virtual_address; 47 48 virtual_vec_cookie() 49 : 50 vec_index(0), 51 vec_offset(0), 52 mapped_area(-1), 53 physical_page_handle(NULL), 54 virtual_address((addr_t)-1) 55 { 56 } 57 58 void PutPhysicalPageIfNeeded() 59 { 60 if (virtual_address != (addr_t)-1) { 61 vm_put_physical_page(virtual_address, physical_page_handle); 62 virtual_address = (addr_t)-1; 63 } 64 } 65 }; 66 67 68 // #pragma mark - 69 70 71 IORequestChunk::IORequestChunk() 72 : 73 fParent(NULL), 74 fStatus(1) 75 { 76 } 77 78 79 IORequestChunk::~IORequestChunk() 80 { 81 } 82 83 84 // #pragma mark - 85 86 87 IOBuffer* 88 IOBuffer::Create(uint32 count, bool vip) 89 { 90 size_t size = sizeof(IOBuffer) + sizeof(generic_io_vec) * (count - 1); 91 IOBuffer* buffer 92 = (IOBuffer*)(malloc_etc(size, vip ? HEAP_PRIORITY_VIP : 0)); 93 if (buffer == NULL) 94 return NULL; 95 96 buffer->fCapacity = count; 97 buffer->fVecCount = 0; 98 buffer->fUser = false; 99 buffer->fPhysical = false; 100 buffer->fVIP = vip; 101 buffer->fMemoryLocked = false; 102 103 return buffer; 104 } 105 106 107 void 108 IOBuffer::Delete() 109 { 110 free_etc(this, fVIP ? HEAP_PRIORITY_VIP : 0); 111 } 112 113 114 void 115 IOBuffer::SetVecs(generic_size_t firstVecOffset, generic_size_t lastVecSize, 116 const generic_io_vec* vecs, uint32 count, generic_size_t length, uint32 flags) 117 { 118 memcpy(fVecs, vecs, sizeof(generic_io_vec) * count); 119 120 if (count > 0 && firstVecOffset > 0) { 121 fVecs[0].base += firstVecOffset; 122 fVecs[0].length -= firstVecOffset; 123 } 124 if (lastVecSize > 0) 125 fVecs[count - 1].length = lastVecSize; 126 127 fVecCount = count; 128 fLength = length; 129 fPhysical = (flags & B_PHYSICAL_IO_REQUEST) != 0; 130 fUser = !fPhysical && IS_USER_ADDRESS(vecs[0].base); 131 132 #if KDEBUG 133 generic_size_t actualLength = 0; 134 for (size_t i = 0; i < fVecCount; i++) 135 actualLength += fVecs[i].length; 136 137 ASSERT(actualLength == fLength); 138 #endif 139 } 140 141 142 status_t 143 IOBuffer::GetNextVirtualVec(void*& _cookie, iovec& vector) 144 { 145 virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie; 146 if (cookie == NULL) { 147 cookie = new(malloc_flags(fVIP ? HEAP_PRIORITY_VIP : 0)) 148 virtual_vec_cookie; 149 if (cookie == NULL) 150 return B_NO_MEMORY; 151 152 _cookie = cookie; 153 } 154 155 // recycle a potential previously mapped page 156 cookie->PutPhysicalPageIfNeeded(); 157 158 if (cookie->vec_index >= fVecCount) 159 return B_BAD_INDEX; 160 161 if (!fPhysical) { 162 vector.iov_base = (void*)(addr_t)fVecs[cookie->vec_index].base; 163 vector.iov_len = fVecs[cookie->vec_index++].length; 164 return B_OK; 165 } 166 167 if (cookie->vec_index == 0 168 && (fVecCount > 1 || fVecs[0].length > B_PAGE_SIZE)) { 169 void* mappedAddress; 170 addr_t mappedSize; 171 172 // TODO: This is a potential violation of the VIP requirement, since 173 // vm_map_physical_memory_vecs() allocates memory without special flags! 174 cookie->mapped_area = vm_map_physical_memory_vecs( 175 VMAddressSpace::KernelID(), "io buffer mapped physical vecs", 176 &mappedAddress, B_ANY_KERNEL_ADDRESS, &mappedSize, 177 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, fVecs, fVecCount); 178 179 if (cookie->mapped_area >= 0) { 180 vector.iov_base = mappedAddress; 181 vector.iov_len = mappedSize; 182 return B_OK; 183 } else 184 ktrace_printf("failed to map area: %s\n", strerror(cookie->mapped_area)); 185 } 186 187 // fallback to page wise mapping 188 generic_io_vec& currentVec = fVecs[cookie->vec_index]; 189 generic_addr_t address = currentVec.base + cookie->vec_offset; 190 size_t pageOffset = address % B_PAGE_SIZE; 191 192 // TODO: This is a potential violation of the VIP requirement, since 193 // vm_get_physical_page() may allocate memory without special flags! 194 status_t result = vm_get_physical_page(address - pageOffset, 195 &cookie->virtual_address, &cookie->physical_page_handle); 196 if (result != B_OK) 197 return result; 198 199 generic_size_t length = min_c(currentVec.length - cookie->vec_offset, 200 B_PAGE_SIZE - pageOffset); 201 202 vector.iov_base = (void*)(cookie->virtual_address + pageOffset); 203 vector.iov_len = length; 204 205 cookie->vec_offset += length; 206 if (cookie->vec_offset >= currentVec.length) { 207 cookie->vec_index++; 208 cookie->vec_offset = 0; 209 } 210 211 return B_OK; 212 } 213 214 215 void 216 IOBuffer::FreeVirtualVecCookie(void* _cookie) 217 { 218 virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie; 219 if (cookie->mapped_area >= 0) 220 delete_area(cookie->mapped_area); 221 222 cookie->PutPhysicalPageIfNeeded(); 223 224 free_etc(cookie, fVIP ? HEAP_PRIORITY_VIP : 0); 225 } 226 227 228 status_t 229 IOBuffer::LockMemory(team_id team, bool isWrite) 230 { 231 if (fMemoryLocked) { 232 panic("memory already locked!"); 233 return B_BAD_VALUE; 234 } 235 236 for (uint32 i = 0; i < fVecCount; i++) { 237 status_t status = lock_memory_etc(team, (void*)(addr_t)fVecs[i].base, 238 fVecs[i].length, isWrite ? 0 : B_READ_DEVICE); 239 if (status != B_OK) { 240 _UnlockMemory(team, i, isWrite); 241 return status; 242 } 243 } 244 245 fMemoryLocked = true; 246 return B_OK; 247 } 248 249 250 void 251 IOBuffer::_UnlockMemory(team_id team, size_t count, bool isWrite) 252 { 253 for (uint32 i = 0; i < count; i++) { 254 unlock_memory_etc(team, (void*)(addr_t)fVecs[i].base, fVecs[i].length, 255 isWrite ? 0 : B_READ_DEVICE); 256 } 257 } 258 259 260 void 261 IOBuffer::UnlockMemory(team_id team, bool isWrite) 262 { 263 if (!fMemoryLocked) { 264 panic("memory not locked"); 265 return; 266 } 267 268 _UnlockMemory(team, fVecCount, isWrite); 269 fMemoryLocked = false; 270 } 271 272 273 void 274 IOBuffer::Dump() const 275 { 276 kprintf("IOBuffer at %p\n", this); 277 278 kprintf(" origin: %s\n", fUser ? "user" : "kernel"); 279 kprintf(" kind: %s\n", fPhysical ? "physical" : "virtual"); 280 kprintf(" length: %" B_PRIuGENADDR "\n", fLength); 281 kprintf(" capacity: %" B_PRIuSIZE "\n", fCapacity); 282 kprintf(" vecs: %" B_PRIuSIZE "\n", fVecCount); 283 284 for (uint32 i = 0; i < fVecCount; i++) { 285 kprintf(" [%" B_PRIu32 "] %#" B_PRIxGENADDR ", %" B_PRIuGENADDR "\n", 286 i, fVecs[i].base, fVecs[i].length); 287 } 288 } 289 290 291 // #pragma mark - 292 293 294 void 295 IOOperation::SetStatus(status_t status, generic_size_t completedLength) 296 { 297 IORequestChunk::SetStatus(status); 298 if (IsWrite() == fParent->IsWrite()) { 299 // Determine how many bytes we actually read or wrote, 300 // relative to the original range, not the translated range. 301 const generic_size_t partialBegin = (fOriginalOffset - fOffset); 302 generic_size_t originalTransferredBytes = completedLength; 303 if (originalTransferredBytes < partialBegin) 304 originalTransferredBytes = 0; 305 else 306 originalTransferredBytes -= partialBegin; 307 308 if (originalTransferredBytes > fOriginalLength) 309 originalTransferredBytes = fOriginalLength; 310 311 fTransferredBytes += originalTransferredBytes; 312 } 313 } 314 315 316 bool 317 IOOperation::Finish() 318 { 319 TRACE("IOOperation::Finish()\n"); 320 321 if (fStatus == B_OK) { 322 if (fParent->IsWrite()) { 323 TRACE(" is write\n"); 324 if (fPhase == PHASE_READ_BEGIN) { 325 TRACE(" phase read begin\n"); 326 // repair phase adjusted vec 327 fDMABuffer->VecAt(fSavedVecIndex).length = fSavedVecLength; 328 329 // partial write: copy partial begin to bounce buffer 330 bool skipReadEndPhase; 331 status_t error = _CopyPartialBegin(true, skipReadEndPhase); 332 if (error == B_OK) { 333 // We're done with the first phase only (read in begin). 334 // Get ready for next phase... 335 fPhase = HasPartialEnd() && !skipReadEndPhase 336 ? PHASE_READ_END : PHASE_DO_ALL; 337 _PrepareVecs(); 338 ResetStatus(); 339 // TODO: Is there a race condition, if the request is 340 // aborted at the same time? 341 return false; 342 } 343 344 IORequestChunk::SetStatus(error); 345 } else if (fPhase == PHASE_READ_END) { 346 TRACE(" phase read end\n"); 347 // repair phase adjusted vec 348 generic_io_vec& vec = fDMABuffer->VecAt(fSavedVecIndex); 349 vec.base += vec.length - fSavedVecLength; 350 vec.length = fSavedVecLength; 351 352 // partial write: copy partial end to bounce buffer 353 status_t error = _CopyPartialEnd(true); 354 if (error == B_OK) { 355 // We're done with the second phase only (read in end). 356 // Get ready for next phase... 357 fPhase = PHASE_DO_ALL; 358 ResetStatus(); 359 // TODO: Is there a race condition, if the request is 360 // aborted at the same time? 361 return false; 362 } 363 364 IORequestChunk::SetStatus(error); 365 } 366 } 367 } 368 369 if (fParent->IsRead() && UsesBounceBuffer()) { 370 TRACE(" read with bounce buffer\n"); 371 // copy the bounce buffer segments to the final location 372 uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); 373 phys_addr_t bounceBufferStart 374 = fDMABuffer->PhysicalBounceBufferAddress(); 375 phys_addr_t bounceBufferEnd = bounceBufferStart 376 + fDMABuffer->BounceBufferSize(); 377 378 const generic_io_vec* vecs = fDMABuffer->Vecs(); 379 uint32 vecCount = fDMABuffer->VecCount(); 380 381 status_t error = B_OK; 382 383 // We iterate through the vecs we have read, moving offset (the device 384 // offset) as we go. If [offset, offset + vec.length) intersects with 385 // [startOffset, endOffset) we copy to the final location. 386 off_t offset = fOffset; 387 const off_t startOffset = fOriginalOffset; 388 const off_t endOffset = fOriginalOffset + fOriginalLength; 389 390 for (uint32 i = 0; error == B_OK && i < vecCount; i++) { 391 const generic_io_vec& vec = vecs[i]; 392 generic_addr_t base = vec.base; 393 generic_size_t length = vec.length; 394 395 if (offset < startOffset) { 396 // If the complete vector is before the start offset, skip it. 397 if (offset + (off_t)length <= startOffset) { 398 offset += length; 399 continue; 400 } 401 402 // The vector starts before the start offset, but intersects 403 // with it. Skip the part we aren't interested in. 404 generic_size_t diff = startOffset - offset; 405 offset += diff; 406 base += diff; 407 length -= diff; 408 } 409 410 if (offset + (off_t)length > endOffset) { 411 // If we're already beyond the end offset, we're done. 412 if (offset >= endOffset) 413 break; 414 415 // The vector extends beyond the end offset -- cut it. 416 length = endOffset - offset; 417 } 418 419 if (base >= bounceBufferStart && base < bounceBufferEnd) { 420 error = fParent->CopyData( 421 bounceBuffer + (base - bounceBufferStart), offset, length); 422 } 423 424 offset += length; 425 } 426 427 if (error != B_OK) 428 IORequestChunk::SetStatus(error); 429 } 430 431 return true; 432 } 433 434 435 /*! Note: SetPartial() must be called first! 436 */ 437 status_t 438 IOOperation::Prepare(IORequest* request) 439 { 440 if (fParent != NULL) 441 fParent->RemoveOperation(this); 442 443 fParent = request; 444 445 fTransferredBytes = 0; 446 447 // set initial phase 448 fPhase = PHASE_DO_ALL; 449 if (fParent->IsWrite()) { 450 // Copy data to bounce buffer segments, save the partial begin/end vec, 451 // which will be copied after their respective read phase. 452 if (UsesBounceBuffer()) { 453 TRACE(" write with bounce buffer\n"); 454 uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); 455 phys_addr_t bounceBufferStart 456 = fDMABuffer->PhysicalBounceBufferAddress(); 457 phys_addr_t bounceBufferEnd = bounceBufferStart 458 + fDMABuffer->BounceBufferSize(); 459 460 const generic_io_vec* vecs = fDMABuffer->Vecs(); 461 uint32 vecCount = fDMABuffer->VecCount(); 462 generic_size_t vecOffset = 0; 463 uint32 i = 0; 464 465 off_t offset = fOffset; 466 off_t endOffset = fOffset + fLength; 467 468 if (HasPartialBegin()) { 469 // skip first block 470 generic_size_t toSkip = fBlockSize; 471 while (toSkip > 0) { 472 if (vecs[i].length <= toSkip) { 473 toSkip -= vecs[i].length; 474 i++; 475 } else { 476 vecOffset = toSkip; 477 break; 478 } 479 } 480 481 offset += fBlockSize; 482 } 483 484 if (HasPartialEnd()) { 485 // skip last block 486 generic_size_t toSkip = fBlockSize; 487 while (toSkip > 0) { 488 if (vecs[vecCount - 1].length <= toSkip) { 489 toSkip -= vecs[vecCount - 1].length; 490 vecCount--; 491 } else 492 break; 493 } 494 495 endOffset -= fBlockSize; 496 } 497 498 for (; i < vecCount; i++) { 499 const generic_io_vec& vec = vecs[i]; 500 generic_addr_t base = vec.base + vecOffset; 501 generic_size_t length = vec.length - vecOffset; 502 vecOffset = 0; 503 504 if (base >= bounceBufferStart && base < bounceBufferEnd) { 505 if (offset + (off_t)length > endOffset) 506 length = endOffset - offset; 507 status_t error = fParent->CopyData(offset, 508 bounceBuffer + (base - bounceBufferStart), length); 509 if (error != B_OK) 510 return error; 511 } 512 513 offset += length; 514 } 515 } 516 517 if (HasPartialBegin()) 518 fPhase = PHASE_READ_BEGIN; 519 else if (HasPartialEnd()) 520 fPhase = PHASE_READ_END; 521 522 _PrepareVecs(); 523 } 524 525 ResetStatus(); 526 527 if (fParent != NULL) 528 fParent->AddOperation(this); 529 530 return B_OK; 531 } 532 533 534 void 535 IOOperation::SetOriginalRange(off_t offset, generic_size_t length) 536 { 537 fOriginalOffset = fOffset = offset; 538 fOriginalLength = fLength = length; 539 } 540 541 542 void 543 IOOperation::SetRange(off_t offset, generic_size_t length) 544 { 545 fOffset = offset; 546 fLength = length; 547 } 548 549 550 off_t 551 IOOperation::Offset() const 552 { 553 return fPhase == PHASE_READ_END ? fOffset + fLength - fBlockSize : fOffset; 554 } 555 556 557 generic_size_t 558 IOOperation::Length() const 559 { 560 return fPhase == PHASE_DO_ALL ? fLength : fBlockSize; 561 } 562 563 564 generic_io_vec* 565 IOOperation::Vecs() const 566 { 567 switch (fPhase) { 568 case PHASE_READ_END: 569 return fDMABuffer->Vecs() + fSavedVecIndex; 570 case PHASE_READ_BEGIN: 571 case PHASE_DO_ALL: 572 default: 573 return fDMABuffer->Vecs(); 574 } 575 } 576 577 578 uint32 579 IOOperation::VecCount() const 580 { 581 switch (fPhase) { 582 case PHASE_READ_BEGIN: 583 return fSavedVecIndex + 1; 584 case PHASE_READ_END: 585 return fDMABuffer->VecCount() - fSavedVecIndex; 586 case PHASE_DO_ALL: 587 default: 588 return fDMABuffer->VecCount(); 589 } 590 } 591 592 593 void 594 IOOperation::SetPartial(bool partialBegin, bool partialEnd) 595 { 596 TRACE("partial begin %d, end %d\n", partialBegin, partialEnd); 597 fPartialBegin = partialBegin; 598 fPartialEnd = partialEnd; 599 } 600 601 602 bool 603 IOOperation::IsWrite() const 604 { 605 return fParent->IsWrite() && fPhase == PHASE_DO_ALL; 606 } 607 608 609 bool 610 IOOperation::IsRead() const 611 { 612 return fParent->IsRead(); 613 } 614 615 616 void 617 IOOperation::_PrepareVecs() 618 { 619 // we need to prepare the vecs for consumption by the drivers 620 if (fPhase == PHASE_READ_BEGIN) { 621 generic_io_vec* vecs = fDMABuffer->Vecs(); 622 uint32 vecCount = fDMABuffer->VecCount(); 623 generic_size_t vecLength = fBlockSize; 624 for (uint32 i = 0; i < vecCount; i++) { 625 generic_io_vec& vec = vecs[i]; 626 if (vec.length >= vecLength) { 627 fSavedVecIndex = i; 628 fSavedVecLength = vec.length; 629 vec.length = vecLength; 630 break; 631 } 632 vecLength -= vec.length; 633 } 634 } else if (fPhase == PHASE_READ_END) { 635 generic_io_vec* vecs = fDMABuffer->Vecs(); 636 uint32 vecCount = fDMABuffer->VecCount(); 637 generic_size_t vecLength = fBlockSize; 638 for (int32 i = vecCount - 1; i >= 0; i--) { 639 generic_io_vec& vec = vecs[i]; 640 if (vec.length >= vecLength) { 641 fSavedVecIndex = i; 642 fSavedVecLength = vec.length; 643 vec.base += vec.length - vecLength; 644 vec.length = vecLength; 645 break; 646 } 647 vecLength -= vec.length; 648 } 649 } 650 } 651 652 653 status_t 654 IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly) 655 { 656 generic_size_t relativeOffset = OriginalOffset() - fOffset; 657 generic_size_t length = fBlockSize - relativeOffset; 658 659 singleBlockOnly = length >= OriginalLength(); 660 if (singleBlockOnly) 661 length = OriginalLength(); 662 663 TRACE("_CopyPartialBegin(%s, single only %d)\n", 664 isWrite ? "write" : "read", singleBlockOnly); 665 666 if (isWrite) { 667 return fParent->CopyData(OriginalOffset(), 668 (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length); 669 } else { 670 return fParent->CopyData( 671 (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, 672 OriginalOffset(), length); 673 } 674 } 675 676 677 status_t 678 IOOperation::_CopyPartialEnd(bool isWrite) 679 { 680 TRACE("_CopyPartialEnd(%s)\n", isWrite ? "write" : "read"); 681 682 const generic_io_vec& lastVec 683 = fDMABuffer->VecAt(fDMABuffer->VecCount() - 1); 684 off_t lastVecPos = fOffset + fLength - fBlockSize; 685 uint8* base = (uint8*)fDMABuffer->BounceBufferAddress() 686 + (lastVec.base + lastVec.length - fBlockSize 687 - fDMABuffer->PhysicalBounceBufferAddress()); 688 // NOTE: this won't work if we don't use the bounce buffer contiguously 689 // (because of boundary alignments). 690 generic_size_t length = OriginalOffset() + OriginalLength() - lastVecPos; 691 692 if (isWrite) 693 return fParent->CopyData(lastVecPos, base, length); 694 695 return fParent->CopyData(base, lastVecPos, length); 696 } 697 698 699 void 700 IOOperation::Dump() const 701 { 702 kprintf("io_operation at %p\n", this); 703 704 kprintf(" parent: %p\n", fParent); 705 kprintf(" status: %s\n", strerror(fStatus)); 706 kprintf(" dma buffer: %p\n", fDMABuffer); 707 kprintf(" offset: %-8" B_PRIdOFF " (original: %" B_PRIdOFF ")\n", 708 fOffset, fOriginalOffset); 709 kprintf(" length: %-8" B_PRIuGENADDR " (original: %" 710 B_PRIuGENADDR ")\n", fLength, fOriginalLength); 711 kprintf(" transferred: %" B_PRIuGENADDR "\n", fTransferredBytes); 712 kprintf(" block size: %" B_PRIuGENADDR "\n", fBlockSize); 713 kprintf(" saved vec index: %u\n", fSavedVecIndex); 714 kprintf(" saved vec length: %u\n", fSavedVecLength); 715 kprintf(" r/w: %s\n", IsWrite() ? "write" : "read"); 716 kprintf(" phase: %s\n", fPhase == PHASE_READ_BEGIN 717 ? "read begin" : fPhase == PHASE_READ_END ? "read end" 718 : fPhase == PHASE_DO_ALL ? "do all" : "unknown"); 719 kprintf(" partial begin: %s\n", fPartialBegin ? "yes" : "no"); 720 kprintf(" partial end: %s\n", fPartialEnd ? "yes" : "no"); 721 kprintf(" bounce buffer: %s\n", fUsesBounceBuffer ? "yes" : "no"); 722 723 set_debug_variable("_parent", (addr_t)fParent); 724 set_debug_variable("_buffer", (addr_t)fDMABuffer); 725 } 726 727 728 // #pragma mark - 729 730 731 IORequest::IORequest() 732 : 733 fIsNotified(false), 734 fFinishedCallback(NULL), 735 fFinishedCookie(NULL), 736 fIterationCallback(NULL), 737 fIterationCookie(NULL) 738 { 739 mutex_init(&fLock, "I/O request lock"); 740 fFinishedCondition.Init(this, "I/O request finished"); 741 } 742 743 744 IORequest::~IORequest() 745 { 746 mutex_lock(&fLock); 747 DeleteSubRequests(); 748 if (fBuffer != NULL) 749 fBuffer->Delete(); 750 mutex_destroy(&fLock); 751 } 752 753 754 /* static */ IORequest* 755 IORequest::Create(bool vip) 756 { 757 return vip 758 ? new(malloc_flags(HEAP_PRIORITY_VIP)) IORequest 759 : new(std::nothrow) IORequest; 760 } 761 762 763 status_t 764 IORequest::Init(off_t offset, generic_addr_t buffer, generic_size_t length, 765 bool write, uint32 flags) 766 { 767 ASSERT(offset >= 0); 768 769 generic_io_vec vec; 770 vec.base = buffer; 771 vec.length = length; 772 return Init(offset, &vec, 1, length, write, flags); 773 } 774 775 776 status_t 777 IORequest::Init(off_t offset, generic_size_t firstVecOffset, 778 generic_size_t lastVecSize, const generic_io_vec* vecs, size_t count, 779 generic_size_t length, bool write, uint32 flags) 780 { 781 ASSERT(offset >= 0); 782 783 fBuffer = IOBuffer::Create(count, (flags & B_VIP_IO_REQUEST) != 0); 784 if (fBuffer == NULL) 785 return B_NO_MEMORY; 786 787 fBuffer->SetVecs(firstVecOffset, lastVecSize, vecs, count, length, flags); 788 789 fOwner = NULL; 790 fOffset = offset; 791 fLength = length; 792 fRelativeParentOffset = 0; 793 fTransferSize = 0; 794 fFlags = flags; 795 Thread* thread = thread_get_current_thread(); 796 fTeam = thread->team->id; 797 fThread = thread->id; 798 fIsWrite = write; 799 fPartialTransfer = false; 800 fSuppressChildNotifications = false; 801 802 // these are for iteration 803 fVecIndex = 0; 804 fVecOffset = 0; 805 fRemainingBytes = length; 806 807 fPendingChildren = 0; 808 809 fStatus = 1; 810 811 return B_OK; 812 } 813 814 815 status_t 816 IORequest::CreateSubRequest(off_t parentOffset, off_t offset, 817 generic_size_t length, IORequest*& _subRequest) 818 { 819 ASSERT(parentOffset >= fOffset && length <= fLength 820 && parentOffset - fOffset <= (off_t)(fLength - length)); 821 822 // find start vec 823 generic_size_t vecOffset = parentOffset - fOffset; 824 generic_io_vec* vecs = fBuffer->Vecs(); 825 int32 vecCount = fBuffer->VecCount(); 826 int32 startVec = 0; 827 for (; startVec < vecCount; startVec++) { 828 const generic_io_vec& vec = vecs[startVec]; 829 if (vecOffset < vec.length) 830 break; 831 832 vecOffset -= vec.length; 833 } 834 835 // count vecs 836 generic_size_t currentVecOffset = vecOffset; 837 int32 endVec = startVec; 838 generic_size_t remainingLength = length; 839 for (; endVec < vecCount; endVec++) { 840 const generic_io_vec& vec = vecs[endVec]; 841 if (vec.length - currentVecOffset >= remainingLength) 842 break; 843 844 remainingLength -= vec.length - currentVecOffset; 845 currentVecOffset = 0; 846 } 847 848 // create subrequest 849 IORequest* subRequest = Create((fFlags & B_VIP_IO_REQUEST) != 0); 850 if (subRequest == NULL) 851 return B_NO_MEMORY; 852 853 status_t error = subRequest->Init(offset, vecOffset, remainingLength, 854 vecs + startVec, endVec - startVec + 1, length, fIsWrite, 855 fFlags & ~B_DELETE_IO_REQUEST); 856 if (error != B_OK) { 857 delete subRequest; 858 return error; 859 } 860 861 subRequest->fRelativeParentOffset = parentOffset - fOffset; 862 subRequest->fTeam = fTeam; 863 subRequest->fThread = fThread; 864 865 _subRequest = subRequest; 866 subRequest->SetParent(this); 867 868 MutexLocker _(fLock); 869 870 fChildren.Add(subRequest); 871 fPendingChildren++; 872 TRACE("IORequest::CreateSubRequest(): request: %p, subrequest: %p\n", this, 873 subRequest); 874 875 return B_OK; 876 } 877 878 879 void 880 IORequest::DeleteSubRequests() 881 { 882 while (IORequestChunk* chunk = fChildren.RemoveHead()) 883 delete chunk; 884 fPendingChildren = 0; 885 } 886 887 888 void 889 IORequest::SetFinishedCallback(io_request_finished_callback callback, 890 void* cookie) 891 { 892 fFinishedCallback = callback; 893 fFinishedCookie = cookie; 894 } 895 896 897 void 898 IORequest::SetIterationCallback(io_request_iterate_callback callback, 899 void* cookie) 900 { 901 fIterationCallback = callback; 902 fIterationCookie = cookie; 903 } 904 905 906 io_request_finished_callback 907 IORequest::FinishedCallback(void** _cookie) const 908 { 909 if (_cookie != NULL) 910 *_cookie = fFinishedCookie; 911 return fFinishedCallback; 912 } 913 914 915 status_t 916 IORequest::Wait(uint32 flags, bigtime_t timeout) 917 { 918 MutexLocker locker(fLock); 919 920 if (IsFinished() && fIsNotified) 921 return Status(); 922 923 ConditionVariableEntry entry; 924 fFinishedCondition.Add(&entry); 925 926 locker.Unlock(); 927 928 status_t error = entry.Wait(flags, timeout); 929 if (error != B_OK) 930 return error; 931 932 return Status(); 933 } 934 935 936 void 937 IORequest::NotifyFinished() 938 { 939 TRACE("IORequest::NotifyFinished(): request: %p\n", this); 940 941 MutexLocker locker(fLock); 942 ASSERT(fStatus != 1); 943 944 if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) { 945 // The request is not really done yet. If it has an iteration callback, 946 // call it. 947 if (fIterationCallback != NULL) { 948 ResetStatus(); 949 locker.Unlock(); 950 bool partialTransfer = false; 951 status_t error = fIterationCallback(fIterationCookie, this, 952 &partialTransfer); 953 if (error == B_OK && !partialTransfer) 954 return; 955 956 // Iteration failed, which means we're responsible for notifying the 957 // requests finished. 958 locker.Lock(); 959 fStatus = error; 960 fPartialTransfer = true; 961 } 962 } 963 964 ASSERT(!fIsNotified); 965 ASSERT(fPendingChildren == 0); 966 ASSERT(fChildren.IsEmpty() 967 || dynamic_cast<IOOperation*>(fChildren.Head()) == NULL); 968 ASSERT(fTransferSize <= fLength); 969 970 // unlock the memory 971 if (fBuffer->IsMemoryLocked()) 972 fBuffer->UnlockMemory(fTeam, fIsWrite); 973 974 // Cache the callbacks before we unblock waiters and unlock. Any of the 975 // following could delete this request, so we don't want to touch it 976 // once we have started telling others that it is done. 977 IORequest* parent = fParent; 978 io_request_finished_callback finishedCallback = fFinishedCallback; 979 void* finishedCookie = fFinishedCookie; 980 status_t status = fStatus; 981 generic_size_t transferredBytes = fTransferSize; 982 generic_size_t lastTransferredOffset 983 = fRelativeParentOffset + transferredBytes; 984 bool partialTransfer = status != B_OK || fPartialTransfer; 985 bool deleteRequest = (fFlags & B_DELETE_IO_REQUEST) != 0; 986 987 // unblock waiters 988 fIsNotified = true; 989 fFinishedCondition.NotifyAll(); 990 991 locker.Unlock(); 992 993 // notify callback 994 if (finishedCallback != NULL) { 995 finishedCallback(finishedCookie, this, status, partialTransfer, 996 transferredBytes); 997 } 998 999 // notify parent 1000 if (parent != NULL) { 1001 parent->SubRequestFinished(this, status, partialTransfer, 1002 lastTransferredOffset); 1003 } 1004 1005 if (deleteRequest) 1006 delete this; 1007 } 1008 1009 1010 /*! Returns whether this request or any of it's ancestors has a finished or 1011 notification callback. Used to decide whether NotifyFinished() can be called 1012 synchronously. 1013 */ 1014 bool 1015 IORequest::HasCallbacks() const 1016 { 1017 if (fFinishedCallback != NULL || fIterationCallback != NULL) 1018 return true; 1019 1020 return fParent != NULL && fParent->HasCallbacks(); 1021 } 1022 1023 1024 void 1025 IORequest::SetStatusAndNotify(status_t status) 1026 { 1027 MutexLocker locker(fLock); 1028 1029 if (fStatus != 1) 1030 return; 1031 1032 fStatus = status; 1033 1034 locker.Unlock(); 1035 1036 NotifyFinished(); 1037 } 1038 1039 1040 void 1041 IORequest::OperationFinished(IOOperation* operation) 1042 { 1043 TRACE("IORequest::OperationFinished(%p, %#" B_PRIx32 "): request: %p\n", 1044 operation, status, this); 1045 1046 MutexLocker locker(fLock); 1047 1048 fChildren.Remove(operation); 1049 operation->SetParent(NULL); 1050 1051 const status_t status = operation->Status(); 1052 const bool partialTransfer = 1053 (operation->TransferredBytes() < operation->OriginalLength()); 1054 const generic_size_t transferEndOffset = 1055 (operation->OriginalOffset() + operation->TransferredBytes()); 1056 1057 if (status != B_OK || partialTransfer) { 1058 if (fTransferSize > transferEndOffset) 1059 fTransferSize = transferEndOffset; 1060 fPartialTransfer = true; 1061 } 1062 1063 if (status != B_OK && fStatus == 1) 1064 fStatus = status; 1065 1066 if (--fPendingChildren > 0) 1067 return; 1068 1069 // last child finished 1070 1071 // set status, if not done yet 1072 if (fStatus == 1) 1073 fStatus = B_OK; 1074 } 1075 1076 1077 void 1078 IORequest::SubRequestFinished(IORequest* request, status_t status, 1079 bool partialTransfer, generic_size_t transferEndOffset) 1080 { 1081 TRACE("IORequest::SubrequestFinished(%p, %#" B_PRIx32 ", %d, %" B_PRIuGENADDR 1082 "): request: %p\n", request, status, partialTransfer, transferEndOffset, this); 1083 1084 MutexLocker locker(fLock); 1085 1086 if (status != B_OK || partialTransfer) { 1087 if (fTransferSize > transferEndOffset) 1088 fTransferSize = transferEndOffset; 1089 fPartialTransfer = true; 1090 } 1091 1092 if (status != B_OK && fStatus == 1) 1093 fStatus = status; 1094 1095 if (--fPendingChildren > 0 || fSuppressChildNotifications) 1096 return; 1097 1098 // last child finished 1099 1100 // set status, if not done yet 1101 if (fStatus == 1) 1102 fStatus = B_OK; 1103 1104 locker.Unlock(); 1105 1106 NotifyFinished(); 1107 } 1108 1109 1110 void 1111 IORequest::SetUnfinished() 1112 { 1113 MutexLocker _(fLock); 1114 ResetStatus(); 1115 } 1116 1117 1118 void 1119 IORequest::SetTransferredBytes(bool partialTransfer, 1120 generic_size_t transferredBytes) 1121 { 1122 TRACE("%p->IORequest::SetTransferredBytes(%d, %" B_PRIuGENADDR ")\n", this, 1123 partialTransfer, transferredBytes); 1124 1125 MutexLocker _(fLock); 1126 1127 fPartialTransfer = partialTransfer; 1128 fTransferSize = transferredBytes; 1129 } 1130 1131 1132 void 1133 IORequest::SetSuppressChildNotifications(bool suppress) 1134 { 1135 fSuppressChildNotifications = suppress; 1136 } 1137 1138 1139 void 1140 IORequest::Advance(generic_size_t bySize) 1141 { 1142 TRACE("IORequest::Advance(%" B_PRIuGENADDR "): remaining: %" B_PRIuGENADDR 1143 " -> %" B_PRIuGENADDR "\n", bySize, fRemainingBytes, 1144 fRemainingBytes - bySize); 1145 fRemainingBytes -= bySize; 1146 fTransferSize += bySize; 1147 1148 generic_io_vec* vecs = fBuffer->Vecs(); 1149 uint32 vecCount = fBuffer->VecCount(); 1150 while (fVecIndex < vecCount 1151 && vecs[fVecIndex].length - fVecOffset <= bySize) { 1152 bySize -= vecs[fVecIndex].length - fVecOffset; 1153 fVecOffset = 0; 1154 fVecIndex++; 1155 } 1156 1157 fVecOffset += bySize; 1158 } 1159 1160 1161 IORequest* 1162 IORequest::FirstSubRequest() 1163 { 1164 return dynamic_cast<IORequest*>(fChildren.Head()); 1165 } 1166 1167 1168 IORequest* 1169 IORequest::NextSubRequest(IORequest* previous) 1170 { 1171 if (previous == NULL) 1172 return NULL; 1173 return dynamic_cast<IORequest*>(fChildren.GetNext(previous)); 1174 } 1175 1176 1177 void 1178 IORequest::AddOperation(IOOperation* operation) 1179 { 1180 MutexLocker locker(fLock); 1181 TRACE("IORequest::AddOperation(%p): request: %p\n", operation, this); 1182 fChildren.Add(operation); 1183 fPendingChildren++; 1184 } 1185 1186 1187 void 1188 IORequest::RemoveOperation(IOOperation* operation) 1189 { 1190 MutexLocker locker(fLock); 1191 fChildren.Remove(operation); 1192 operation->SetParent(NULL); 1193 } 1194 1195 1196 status_t 1197 IORequest::CopyData(off_t offset, void* buffer, size_t size) 1198 { 1199 return _CopyData(buffer, offset, size, true); 1200 } 1201 1202 1203 status_t 1204 IORequest::CopyData(const void* buffer, off_t offset, size_t size) 1205 { 1206 return _CopyData((void*)buffer, offset, size, false); 1207 } 1208 1209 1210 status_t 1211 IORequest::ClearData(off_t offset, generic_size_t size) 1212 { 1213 if (size == 0) 1214 return B_OK; 1215 1216 if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) { 1217 panic("IORequest::ClearData(): invalid range: (%" B_PRIdOFF 1218 ", %" B_PRIuGENADDR ")", offset, size); 1219 return B_BAD_VALUE; 1220 } 1221 1222 // If we can, we directly copy from/to the virtual buffer. The memory is 1223 // locked in this case. 1224 status_t (*clearFunction)(generic_addr_t, generic_size_t, team_id); 1225 if (fBuffer->IsPhysical()) { 1226 clearFunction = &IORequest::_ClearDataPhysical; 1227 } else { 1228 clearFunction = fBuffer->IsUser() && fTeam != team_get_current_team_id() 1229 ? &IORequest::_ClearDataUser : &IORequest::_ClearDataSimple; 1230 } 1231 1232 // skip bytes if requested 1233 generic_io_vec* vecs = fBuffer->Vecs(); 1234 generic_size_t skipBytes = offset - fOffset; 1235 generic_size_t vecOffset = 0; 1236 while (skipBytes > 0) { 1237 if (vecs[0].length > skipBytes) { 1238 vecOffset = skipBytes; 1239 break; 1240 } 1241 1242 skipBytes -= vecs[0].length; 1243 vecs++; 1244 } 1245 1246 // clear vector-wise 1247 while (size > 0) { 1248 generic_size_t toClear = min_c(size, vecs[0].length - vecOffset); 1249 status_t error = clearFunction(vecs[0].base + vecOffset, toClear, 1250 fTeam); 1251 if (error != B_OK) 1252 return error; 1253 1254 size -= toClear; 1255 vecs++; 1256 vecOffset = 0; 1257 } 1258 1259 return B_OK; 1260 1261 } 1262 1263 1264 status_t 1265 IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn) 1266 { 1267 if (size == 0) 1268 return B_OK; 1269 1270 uint8* buffer = (uint8*)_buffer; 1271 1272 if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) { 1273 panic("IORequest::_CopyData(): invalid range: (%" B_PRIdOFF ", %lu)", 1274 offset, size); 1275 return B_BAD_VALUE; 1276 } 1277 1278 // If we can, we directly copy from/to the virtual buffer. The memory is 1279 // locked in this case. 1280 status_t (*copyFunction)(void*, generic_addr_t, size_t, team_id, bool); 1281 if (fBuffer->IsPhysical()) { 1282 copyFunction = &IORequest::_CopyPhysical; 1283 } else { 1284 copyFunction = fBuffer->IsUser() && fTeam != team_get_current_team_id() 1285 ? &IORequest::_CopyUser : &IORequest::_CopySimple; 1286 } 1287 1288 // skip bytes if requested 1289 generic_io_vec* vecs = fBuffer->Vecs(); 1290 generic_size_t skipBytes = offset - fOffset; 1291 generic_size_t vecOffset = 0; 1292 while (skipBytes > 0) { 1293 if (vecs[0].length > skipBytes) { 1294 vecOffset = skipBytes; 1295 break; 1296 } 1297 1298 skipBytes -= vecs[0].length; 1299 vecs++; 1300 } 1301 1302 // copy vector-wise 1303 while (size > 0) { 1304 generic_size_t toCopy = min_c(size, vecs[0].length - vecOffset); 1305 status_t error = copyFunction(buffer, vecs[0].base + vecOffset, toCopy, 1306 fTeam, copyIn); 1307 if (error != B_OK) 1308 return error; 1309 1310 buffer += toCopy; 1311 size -= toCopy; 1312 vecs++; 1313 vecOffset = 0; 1314 } 1315 1316 return B_OK; 1317 } 1318 1319 1320 /* static */ status_t 1321 IORequest::_CopySimple(void* bounceBuffer, generic_addr_t external, size_t size, 1322 team_id team, bool copyIn) 1323 { 1324 TRACE(" IORequest::_CopySimple(%p, %#" B_PRIxGENADDR ", %lu, %d)\n", 1325 bounceBuffer, external, size, copyIn); 1326 if (IS_USER_ADDRESS(external)) { 1327 status_t status = B_OK; 1328 if (copyIn) 1329 status = user_memcpy(bounceBuffer, (void*)(addr_t)external, size); 1330 else 1331 status = user_memcpy((void*)(addr_t)external, bounceBuffer, size); 1332 if (status < B_OK) 1333 return status; 1334 return B_OK; 1335 } 1336 if (copyIn) 1337 memcpy(bounceBuffer, (void*)(addr_t)external, size); 1338 else 1339 memcpy((void*)(addr_t)external, bounceBuffer, size); 1340 return B_OK; 1341 } 1342 1343 1344 /* static */ status_t 1345 IORequest::_CopyPhysical(void* bounceBuffer, generic_addr_t external, 1346 size_t size, team_id team, bool copyIn) 1347 { 1348 if (copyIn) 1349 return vm_memcpy_from_physical(bounceBuffer, external, size, false); 1350 1351 return vm_memcpy_to_physical(external, bounceBuffer, size, false); 1352 } 1353 1354 1355 /* static */ status_t 1356 IORequest::_CopyUser(void* _bounceBuffer, generic_addr_t _external, size_t size, 1357 team_id team, bool copyIn) 1358 { 1359 uint8* bounceBuffer = (uint8*)_bounceBuffer; 1360 uint8* external = (uint8*)(addr_t)_external; 1361 1362 while (size > 0) { 1363 static const int32 kEntryCount = 8; 1364 physical_entry entries[kEntryCount]; 1365 1366 uint32 count = kEntryCount; 1367 status_t error = get_memory_map_etc(team, external, size, entries, 1368 &count); 1369 if (error != B_OK && error != B_BUFFER_OVERFLOW) { 1370 panic("IORequest::_CopyUser(): Failed to get physical memory for " 1371 "user memory %p\n", external); 1372 return B_BAD_ADDRESS; 1373 } 1374 1375 for (uint32 i = 0; i < count; i++) { 1376 const physical_entry& entry = entries[i]; 1377 error = _CopyPhysical(bounceBuffer, entry.address, entry.size, team, 1378 copyIn); 1379 if (error != B_OK) 1380 return error; 1381 1382 size -= entry.size; 1383 bounceBuffer += entry.size; 1384 external += entry.size; 1385 } 1386 } 1387 1388 return B_OK; 1389 } 1390 1391 1392 /*static*/ status_t 1393 IORequest::_ClearDataSimple(generic_addr_t external, generic_size_t size, 1394 team_id team) 1395 { 1396 memset((void*)(addr_t)external, 0, (size_t)size); 1397 return B_OK; 1398 } 1399 1400 1401 /*static*/ status_t 1402 IORequest::_ClearDataPhysical(generic_addr_t external, generic_size_t size, 1403 team_id team) 1404 { 1405 return vm_memset_physical((phys_addr_t)external, 0, (phys_size_t)size); 1406 } 1407 1408 1409 /*static*/ status_t 1410 IORequest::_ClearDataUser(generic_addr_t _external, generic_size_t size, 1411 team_id team) 1412 { 1413 uint8* external = (uint8*)(addr_t)_external; 1414 1415 while (size > 0) { 1416 static const int32 kEntryCount = 8; 1417 physical_entry entries[kEntryCount]; 1418 1419 uint32 count = kEntryCount; 1420 status_t error = get_memory_map_etc(team, external, size, entries, 1421 &count); 1422 if (error != B_OK && error != B_BUFFER_OVERFLOW) { 1423 panic("IORequest::_ClearDataUser(): Failed to get physical memory " 1424 "for user memory %p\n", external); 1425 return B_BAD_ADDRESS; 1426 } 1427 1428 for (uint32 i = 0; i < count; i++) { 1429 const physical_entry& entry = entries[i]; 1430 error = _ClearDataPhysical(entry.address, entry.size, team); 1431 if (error != B_OK) 1432 return error; 1433 1434 size -= entry.size; 1435 external += entry.size; 1436 } 1437 } 1438 1439 return B_OK; 1440 } 1441 1442 1443 void 1444 IORequest::Dump() const 1445 { 1446 kprintf("io_request at %p\n", this); 1447 1448 kprintf(" owner: %p\n", fOwner); 1449 kprintf(" parent: %p\n", fParent); 1450 kprintf(" status: %s\n", strerror(fStatus)); 1451 kprintf(" mutex: %p\n", &fLock); 1452 kprintf(" IOBuffer: %p\n", fBuffer); 1453 kprintf(" offset: %" B_PRIdOFF "\n", fOffset); 1454 kprintf(" length: %" B_PRIuGENADDR "\n", fLength); 1455 kprintf(" transfer size: %" B_PRIuGENADDR "\n", fTransferSize); 1456 kprintf(" relative offset: %" B_PRIuGENADDR "\n", fRelativeParentOffset); 1457 kprintf(" pending children: %" B_PRId32 "\n", fPendingChildren); 1458 kprintf(" flags: %#" B_PRIx32 "\n", fFlags); 1459 kprintf(" team: %" B_PRId32 "\n", fTeam); 1460 kprintf(" thread: %" B_PRId32 "\n", fThread); 1461 kprintf(" r/w: %s\n", fIsWrite ? "write" : "read"); 1462 kprintf(" partial transfer: %s\n", fPartialTransfer ? "yes" : "no"); 1463 kprintf(" finished cvar: %p\n", &fFinishedCondition); 1464 kprintf(" iteration:\n"); 1465 kprintf(" vec index: %" B_PRIu32 "\n", fVecIndex); 1466 kprintf(" vec offset: %" B_PRIuGENADDR "\n", fVecOffset); 1467 kprintf(" remaining bytes: %" B_PRIuGENADDR "\n", fRemainingBytes); 1468 kprintf(" callbacks:\n"); 1469 kprintf(" finished %p, cookie %p\n", fFinishedCallback, fFinishedCookie); 1470 kprintf(" iteration %p, cookie %p\n", fIterationCallback, 1471 fIterationCookie); 1472 kprintf(" children:\n"); 1473 1474 IORequestChunkList::ConstIterator iterator = fChildren.GetIterator(); 1475 while (iterator.HasNext()) { 1476 kprintf(" %p\n", iterator.Next()); 1477 } 1478 1479 set_debug_variable("_parent", (addr_t)fParent); 1480 set_debug_variable("_mutex", (addr_t)&fLock); 1481 set_debug_variable("_buffer", (addr_t)fBuffer); 1482 set_debug_variable("_cvar", (addr_t)&fFinishedCondition); 1483 } 1484