1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 // included by vfs.cpp 7 8 9 //#define TRACE_VFS_REQUEST_IO 10 #ifdef TRACE_VFS_REQUEST_IO 11 # define TRACE_RIO(x...) dprintf(x) 12 #else 13 # define TRACE_RIO(x...) do {} while (false) 14 #endif 15 16 17 #include <heap.h> 18 #include <AutoDeleterDrivers.h> 19 20 21 // #pragma mark - AsyncIOCallback 22 23 24 AsyncIOCallback::~AsyncIOCallback() 25 { 26 } 27 28 29 /* static */ void 30 AsyncIOCallback::IORequestCallback(void* data, io_request* request, 31 status_t status, bool partialTransfer, generic_size_t bytesTransferred) 32 { 33 ((AsyncIOCallback*)data)->IOFinished(status, partialTransfer, 34 bytesTransferred); 35 } 36 37 38 // #pragma mark - StackableAsyncIOCallback 39 40 41 StackableAsyncIOCallback::StackableAsyncIOCallback(AsyncIOCallback* next) 42 : 43 fNextCallback(next) 44 { 45 } 46 47 48 // #pragma mark - 49 50 51 struct iterative_io_cookie { 52 struct vnode* vnode; 53 file_descriptor* descriptor; 54 iterative_io_get_vecs get_vecs; 55 iterative_io_finished finished; 56 void* cookie; 57 off_t request_offset; 58 io_request_finished_callback next_finished_callback; 59 void* next_finished_cookie; 60 }; 61 62 63 class DoIO { 64 public: 65 DoIO(bool write) 66 : 67 fWrite(write) 68 { 69 } 70 71 virtual ~DoIO() 72 { 73 } 74 75 virtual status_t IO(off_t offset, void* buffer, size_t* length) = 0; 76 77 protected: 78 bool fWrite; 79 }; 80 81 82 class CallbackIO : public DoIO { 83 public: 84 CallbackIO(bool write, 85 status_t (*doIO)(void* cookie, off_t offset, void* buffer, 86 size_t* length), 87 void* cookie) 88 : 89 DoIO(write), 90 fDoIO(doIO), 91 fCookie(cookie) 92 { 93 } 94 95 virtual status_t IO(off_t offset, void* buffer, size_t* length) 96 { 97 return fDoIO(fCookie, offset, buffer, length); 98 } 99 100 private: 101 status_t (*fDoIO)(void*, off_t, void*, size_t*); 102 void* fCookie; 103 }; 104 105 106 class VnodeIO : public DoIO { 107 public: 108 VnodeIO(bool write, struct vnode* vnode, void* cookie) 109 : 110 DoIO(write), 111 fVnode(vnode), 112 fCookie(cookie) 113 { 114 } 115 116 virtual status_t IO(off_t offset, void* buffer, size_t* length) 117 { 118 iovec vec; 119 vec.iov_base = buffer; 120 vec.iov_len = *length; 121 122 if (fWrite) { 123 return FS_CALL(fVnode, write_pages, fCookie, offset, &vec, 1, 124 length); 125 } 126 127 return FS_CALL(fVnode, read_pages, fCookie, offset, &vec, 1, length); 128 } 129 130 private: 131 struct vnode* fVnode; 132 void* fCookie; 133 }; 134 135 136 static status_t 137 do_iterative_fd_io_iterate(void* _cookie, io_request* request, 138 bool* _partialTransfer) 139 { 140 TRACE_RIO("[%ld] do_iterative_fd_io_iterate(request: %p)\n", 141 find_thread(NULL), request); 142 143 static const size_t kMaxSubRequests = 8; 144 145 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; 146 147 request->DeleteSubRequests(); 148 149 off_t requestOffset = cookie->request_offset; 150 size_t requestLength = request->Length() 151 - (requestOffset - request->Offset()); 152 153 // get the next file vecs 154 file_io_vec vecs[kMaxSubRequests]; 155 size_t vecCount = kMaxSubRequests; 156 status_t error = cookie->get_vecs(cookie->cookie, request, requestOffset, 157 requestLength, vecs, &vecCount); 158 if (error != B_OK && error != B_BUFFER_OVERFLOW) 159 return error; 160 if (vecCount == 0) { 161 *_partialTransfer = true; 162 return B_OK; 163 } 164 TRACE_RIO("[%ld] got %zu file vecs\n", find_thread(NULL), vecCount); 165 166 // Reset the error code for the loop below 167 error = B_OK; 168 169 // create subrequests for the file vecs we've got 170 size_t subRequestCount = 0; 171 for (size_t i = 0; 172 i < vecCount && subRequestCount < kMaxSubRequests && error == B_OK; 173 i++) { 174 off_t vecOffset = vecs[i].offset; 175 off_t vecLength = min_c(vecs[i].length, (off_t)requestLength); 176 TRACE_RIO("[%ld] vec %lu offset: %lld, length: %lld\n", 177 find_thread(NULL), i, vecOffset, vecLength); 178 179 // Special offset -1 means that this is part of sparse file that is 180 // zero. We fill it in right here. 181 if (vecOffset == -1) { 182 if (request->IsWrite()) { 183 panic("do_iterative_fd_io_iterate(): write to sparse file " 184 "vector"); 185 error = B_BAD_VALUE; 186 break; 187 } 188 189 error = request->ClearData(requestOffset, vecLength); 190 if (error != B_OK) 191 break; 192 193 requestOffset += vecLength; 194 requestLength -= vecLength; 195 continue; 196 } 197 198 while (vecLength > 0 && subRequestCount < kMaxSubRequests) { 199 TRACE_RIO("[%ld] creating subrequest: offset: %lld, length: " 200 "%lld\n", find_thread(NULL), vecOffset, vecLength); 201 IORequest* subRequest; 202 error = request->CreateSubRequest(requestOffset, vecOffset, 203 vecLength, subRequest); 204 if (error != B_OK) 205 break; 206 207 subRequestCount++; 208 209 size_t lengthProcessed = subRequest->Length(); 210 vecOffset += lengthProcessed; 211 vecLength -= lengthProcessed; 212 requestOffset += lengthProcessed; 213 requestLength -= lengthProcessed; 214 } 215 } 216 217 // Only if we couldn't create any subrequests, we fail. 218 if (error != B_OK && subRequestCount == 0) 219 return error; 220 221 // Reset the error code for the loop below 222 error = B_OK; 223 224 request->Advance(requestOffset - cookie->request_offset); 225 cookie->request_offset = requestOffset; 226 227 // If we don't have any sub requests at this point, that means all that 228 // remained were zeroed sparse file vectors. So the request is done now. 229 if (subRequestCount == 0) { 230 ASSERT(request->RemainingBytes() == 0); 231 request->SetStatusAndNotify(B_OK); 232 return B_OK; 233 } 234 235 // Schedule the subrequests. 236 IORequest* nextSubRequest = request->FirstSubRequest(); 237 while (nextSubRequest != NULL) { 238 IORequest* subRequest = nextSubRequest; 239 nextSubRequest = request->NextSubRequest(subRequest); 240 241 if (error == B_OK) { 242 TRACE_RIO("[%ld] scheduling subrequest: %p\n", find_thread(NULL), 243 subRequest); 244 error = vfs_vnode_io(cookie->vnode, cookie->descriptor->cookie, 245 subRequest); 246 } else { 247 // Once scheduling a subrequest failed, we cancel all subsequent 248 // subrequests. 249 subRequest->SetStatusAndNotify(B_CANCELED); 250 } 251 } 252 253 // TODO: Cancel the subrequests that were scheduled successfully. 254 255 return B_OK; 256 } 257 258 259 static void 260 do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status, 261 bool partialTransfer, generic_size_t bytesTransferred) 262 { 263 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; 264 265 if (cookie->finished != NULL) { 266 cookie->finished(cookie->cookie, request, status, partialTransfer, 267 bytesTransferred); 268 } 269 270 put_fd(cookie->descriptor); 271 272 if (cookie->next_finished_callback != NULL) { 273 cookie->next_finished_callback(cookie->next_finished_cookie, request, 274 status, partialTransfer, bytesTransferred); 275 } 276 277 delete cookie; 278 } 279 280 281 static status_t 282 do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie, 283 io_request* request, iterative_io_get_vecs getVecs, 284 iterative_io_finished finished, void* cookie) 285 { 286 IOBuffer* buffer = request->Buffer(); 287 VnodeIO io(request->IsWrite(), vnode, openCookie); 288 289 iovec vector; 290 void* virtualVecCookie = NULL; 291 off_t offset = request->Offset(); 292 generic_size_t length = request->Length(); 293 294 status_t error = B_OK; 295 bool partial = false; 296 297 for (; error == B_OK && length > 0 && !partial 298 && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) { 299 uint8* vecBase = (uint8*)vector.iov_base; 300 generic_size_t vecLength = min_c(vector.iov_len, length); 301 302 while (error == B_OK && vecLength > 0) { 303 file_io_vec fileVecs[8]; 304 size_t fileVecCount = 8; 305 if (getVecs != NULL) { 306 error = getVecs(cookie, request, offset, vecLength, fileVecs, 307 &fileVecCount); 308 } else { 309 fileVecs[0].offset = offset; 310 fileVecs[0].length = vecLength; 311 fileVecCount = 1; 312 } 313 if (error != B_OK) 314 break; 315 if (fileVecCount == 0) { 316 partial = true; 317 break; 318 } 319 320 for (size_t i = 0; i < fileVecCount; i++) { 321 const file_io_vec& fileVec = fileVecs[i]; 322 size_t toTransfer = min_c(fileVec.length, (off_t)length); 323 size_t transferred = toTransfer; 324 error = io.IO(fileVec.offset, vecBase, &transferred); 325 if (error != B_OK) 326 break; 327 328 offset += transferred; 329 length -= transferred; 330 vecBase += transferred; 331 vecLength -= transferred; 332 333 if (transferred != toTransfer) { 334 partial = true; 335 break; 336 } 337 } 338 } 339 } 340 341 buffer->FreeVirtualVecCookie(virtualVecCookie); 342 343 partial = (partial || length > 0); 344 size_t bytesTransferred = request->Length() - length; 345 request->SetTransferredBytes(partial, bytesTransferred); 346 if (finished != NULL) 347 finished(cookie, request, error, partial, bytesTransferred); 348 request->SetStatusAndNotify(error); 349 return error; 350 } 351 352 353 static status_t 354 synchronous_io(io_request* request, DoIO& io) 355 { 356 TRACE_RIO("[%" B_PRId32 "] synchronous_io(request: %p (offset: %" B_PRIdOFF 357 ", length: %" B_PRIuGENADDR "))\n", find_thread(NULL), request, 358 request->Offset(), request->Length()); 359 360 IOBuffer* buffer = request->Buffer(); 361 362 iovec vector; 363 void* virtualVecCookie = NULL; 364 off_t offset = request->Offset(); 365 generic_size_t length = request->Length(); 366 367 for (; length > 0 368 && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) { 369 void* vecBase = (void*)(addr_t)vector.iov_base; 370 size_t vecLength = min_c(vector.iov_len, length); 371 372 TRACE_RIO("[%ld] I/O: offset: %lld, vecBase: %p, length: %lu\n", 373 find_thread(NULL), offset, vecBase, vecLength); 374 375 size_t transferred = vecLength; 376 status_t error = io.IO(offset, vecBase, &transferred); 377 if (error != B_OK) { 378 TRACE_RIO("[%ld] I/O failed: %#lx\n", find_thread(NULL), error); 379 buffer->FreeVirtualVecCookie(virtualVecCookie); 380 request->SetStatusAndNotify(error); 381 return error; 382 } 383 384 offset += transferred; 385 length -= transferred; 386 387 if (transferred != vecLength) 388 break; 389 } 390 391 TRACE_RIO("[%ld] synchronous_io() succeeded\n", find_thread(NULL)); 392 393 buffer->FreeVirtualVecCookie(virtualVecCookie); 394 request->SetTransferredBytes(length > 0, request->Length() - length); 395 request->SetStatusAndNotify(B_OK); 396 return B_OK; 397 } 398 399 400 // #pragma mark - kernel private API 401 402 403 status_t 404 vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request) 405 { 406 status_t result = B_ERROR; 407 if (!HAS_FS_CALL(vnode, io) 408 || (result = FS_CALL(vnode, io, cookie, request)) == B_UNSUPPORTED) { 409 // no io() call -- fall back to synchronous I/O 410 VnodeIO io(request->IsWrite(), vnode, cookie); 411 return synchronous_io(request, io); 412 } 413 414 return result; 415 } 416 417 418 status_t 419 vfs_synchronous_io(io_request* request, 420 status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t* length), 421 void* cookie) 422 { 423 CallbackIO io(request->IsWrite(), doIO, cookie); 424 return synchronous_io(request, io); 425 } 426 427 428 status_t 429 vfs_asynchronous_read_pages(struct vnode* vnode, void* cookie, off_t pos, 430 const generic_io_vec* vecs, size_t count, generic_size_t numBytes, 431 uint32 flags, AsyncIOCallback* callback) 432 { 433 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0); 434 if (request == NULL) { 435 callback->IOFinished(B_NO_MEMORY, true, 0); 436 return B_NO_MEMORY; 437 } 438 439 status_t status = request->Init(pos, vecs, count, numBytes, false, 440 flags | B_DELETE_IO_REQUEST); 441 if (status != B_OK) { 442 delete request; 443 callback->IOFinished(status, true, 0); 444 return status; 445 } 446 447 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback, 448 callback); 449 450 return vfs_vnode_io(vnode, cookie, request); 451 } 452 453 454 status_t 455 vfs_asynchronous_write_pages(struct vnode* vnode, void* cookie, off_t pos, 456 const generic_io_vec* vecs, size_t count, generic_size_t numBytes, 457 uint32 flags, AsyncIOCallback* callback) 458 { 459 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0); 460 if (request == NULL) { 461 callback->IOFinished(B_NO_MEMORY, true, 0); 462 return B_NO_MEMORY; 463 } 464 465 status_t status = request->Init(pos, vecs, count, numBytes, true, 466 flags | B_DELETE_IO_REQUEST); 467 if (status != B_OK) { 468 delete request; 469 callback->IOFinished(status, true, 0); 470 return status; 471 } 472 473 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback, 474 callback); 475 476 return vfs_vnode_io(vnode, cookie, request); 477 } 478 479 480 // #pragma mark - public API 481 482 483 status_t 484 do_fd_io(int fd, io_request* request) 485 { 486 return do_iterative_fd_io(fd, request, NULL, NULL, NULL); 487 } 488 489 490 status_t 491 do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs, 492 iterative_io_finished finished, void* cookie) 493 { 494 TRACE_RIO("[%" B_PRId32 "] do_iterative_fd_io(fd: %d, request: %p " 495 "(offset: %" B_PRIdOFF ", length: %" B_PRIuGENADDR "))\n", 496 find_thread(NULL), fd, request, request->Offset(), request->Length()); 497 498 struct vnode* vnode; 499 file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true); 500 if (descriptor == NULL) { 501 if (finished != NULL) 502 finished(cookie, request, B_FILE_ERROR, true, 0); 503 request->SetStatusAndNotify(B_FILE_ERROR); 504 return B_FILE_ERROR; 505 } 506 507 FileDescriptorPutter descriptorPutter(descriptor); 508 509 if (!HAS_FS_CALL(vnode, io)) { 510 // no io() call -- fall back to synchronous I/O 511 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie, 512 request, getVecs, finished, cookie); 513 } 514 515 iterative_io_cookie* iterationCookie 516 = (request->Flags() & B_VIP_IO_REQUEST) != 0 517 ? new(malloc_flags(HEAP_PRIORITY_VIP)) iterative_io_cookie 518 : new(std::nothrow) iterative_io_cookie; 519 if (iterationCookie == NULL) { 520 // no memory -- fall back to synchronous I/O 521 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie, 522 request, getVecs, finished, cookie); 523 } 524 525 iterationCookie->vnode = vnode; 526 iterationCookie->descriptor = descriptor; 527 iterationCookie->get_vecs = getVecs; 528 iterationCookie->finished = finished; 529 iterationCookie->cookie = cookie; 530 iterationCookie->request_offset = request->Offset(); 531 iterationCookie->next_finished_callback = request->FinishedCallback( 532 &iterationCookie->next_finished_cookie); 533 534 request->SetFinishedCallback(&do_iterative_fd_io_finish, iterationCookie); 535 if (getVecs != NULL) 536 request->SetIterationCallback(&do_iterative_fd_io_iterate, iterationCookie); 537 538 descriptorPutter.Detach(); 539 // From now on the descriptor is put by our finish callback. 540 541 if (getVecs != NULL) { 542 bool partialTransfer = false; 543 status_t error = do_iterative_fd_io_iterate(iterationCookie, request, 544 &partialTransfer); 545 if (error != B_OK || partialTransfer) { 546 if (partialTransfer) { 547 request->SetTransferredBytes(partialTransfer, 548 request->TransferredBytes()); 549 } 550 551 request->SetStatusAndNotify(error); 552 return error; 553 } 554 } else { 555 return vfs_vnode_io(vnode, descriptor->cookie, request); 556 } 557 558 return B_OK; 559 } 560