1 /* 2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 // included by vfs.cpp 7 8 9 //#define TRACE_VFS_REQUEST_IO 10 #ifdef TRACE_VFS_REQUEST_IO 11 # define TRACE_RIO(x...) dprintf(x) 12 #else 13 # define TRACE_RIO(x...) do {} while (false) 14 #endif 15 16 17 #include <heap.h> 18 19 20 // #pragma mark - AsyncIOCallback 21 22 23 AsyncIOCallback::~AsyncIOCallback() 24 { 25 } 26 27 28 /* static */ status_t 29 AsyncIOCallback::IORequestCallback(void* data, io_request* request, 30 status_t status, bool partialTransfer, size_t transferEndOffset) 31 { 32 ((AsyncIOCallback*)data)->IOFinished(status, partialTransfer, 33 transferEndOffset); 34 return B_OK; 35 } 36 37 38 // #pragma mark - StackableAsyncIOCallback 39 40 41 StackableAsyncIOCallback::StackableAsyncIOCallback(AsyncIOCallback* next) 42 : 43 fNextCallback(next) 44 { 45 } 46 47 48 // #pragma mark - 49 50 51 struct iterative_io_cookie { 52 struct vnode* vnode; 53 file_descriptor* descriptor; 54 iterative_io_get_vecs get_vecs; 55 iterative_io_finished finished; 56 void* cookie; 57 off_t request_offset; 58 io_request_finished_callback next_finished_callback; 59 void* next_finished_cookie; 60 }; 61 62 63 class DoIO { 64 public: 65 DoIO(bool write) 66 : 67 fWrite(write) 68 { 69 } 70 71 virtual ~DoIO() 72 { 73 } 74 75 virtual status_t IO(off_t offset, void* buffer, size_t* length) = 0; 76 77 protected: 78 bool fWrite; 79 }; 80 81 82 class CallbackIO : public DoIO { 83 public: 84 CallbackIO(bool write, 85 status_t (*doIO)(void* cookie, off_t offset, void* buffer, 86 size_t* length), 87 void* cookie) 88 : 89 DoIO(write), 90 fDoIO(doIO), 91 fCookie(cookie) 92 { 93 } 94 95 virtual status_t IO(off_t offset, void* buffer, size_t* length) 96 { 97 return fDoIO(fCookie, offset, buffer, length); 98 } 99 100 private: 101 status_t (*fDoIO)(void*, off_t, void*, size_t*); 102 void* fCookie; 103 }; 104 105 106 class VnodeIO : public DoIO { 107 public: 108 VnodeIO(bool write, struct vnode* vnode, void* cookie) 109 : 110 DoIO(write), 111 fVnode(vnode), 112 fCookie(cookie) 113 { 114 } 115 116 virtual status_t IO(off_t offset, void* buffer, size_t* length) 117 { 118 iovec vec; 119 vec.iov_base = buffer; 120 vec.iov_len = *length; 121 122 if (fWrite) { 123 return FS_CALL(fVnode, write_pages, fCookie, offset, &vec, 1, 124 length); 125 } 126 127 return FS_CALL(fVnode, read_pages, fCookie, offset, &vec, 1, length); 128 } 129 130 private: 131 struct vnode* fVnode; 132 void* fCookie; 133 }; 134 135 136 static status_t 137 do_iterative_fd_io_iterate(void* _cookie, io_request* request, 138 bool* _partialTransfer) 139 { 140 TRACE_RIO("[%ld] do_iterative_fd_io_iterate(request: %p)\n", 141 find_thread(NULL), request); 142 143 static const int32 kMaxSubRequests = 8; 144 145 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; 146 147 request->DeleteSubRequests(); 148 149 off_t requestOffset = cookie->request_offset; 150 size_t requestLength = request->Length() 151 - (requestOffset - request->Offset()); 152 153 // get the next file vecs 154 file_io_vec vecs[kMaxSubRequests]; 155 uint32 vecCount = kMaxSubRequests; 156 status_t error = cookie->get_vecs(cookie->cookie, request, requestOffset, 157 requestLength, vecs, &vecCount); 158 if (error != B_OK && error != B_BUFFER_OVERFLOW) 159 return error; 160 if (vecCount == 0) { 161 *_partialTransfer = true; 162 return B_OK; 163 } 164 TRACE_RIO("[%ld] got %lu file vecs\n", find_thread(NULL), vecCount); 165 166 // create subrequests for the file vecs we've got 167 int32 subRequestCount = 0; 168 for (uint32 i = 0; i < vecCount && subRequestCount < kMaxSubRequests; i++) { 169 off_t vecOffset = vecs[i].offset; 170 off_t vecLength = min_c(vecs[i].length, requestLength); 171 TRACE_RIO("[%ld] vec %lu offset: %lld, length: %lld\n", 172 find_thread(NULL), i, vecOffset, vecLength); 173 174 while (vecLength > 0 && subRequestCount < kMaxSubRequests) { 175 TRACE_RIO("[%ld] creating subrequest: offset: %lld, length: " 176 "%lld\n", find_thread(NULL), vecOffset, vecLength); 177 IORequest* subRequest; 178 error = request->CreateSubRequest(requestOffset, vecOffset, 179 vecLength, subRequest); 180 if (error != B_OK) 181 break; 182 183 subRequestCount++; 184 185 size_t lengthProcessed = subRequest->Length(); 186 vecOffset += lengthProcessed; 187 vecLength -= lengthProcessed; 188 requestOffset += lengthProcessed; 189 requestLength -= lengthProcessed; 190 } 191 } 192 193 // Only if we couldn't create any subrequests, we fail. 194 if (error != B_OK && subRequestCount == 0) 195 return error; 196 197 // Reset the error code for the loop below 198 error = B_OK; 199 200 request->Advance(requestOffset - cookie->request_offset); 201 cookie->request_offset = requestOffset; 202 203 // Schedule the subrequests. 204 IORequest* nextSubRequest = request->FirstSubRequest(); 205 while (nextSubRequest != NULL) { 206 IORequest* subRequest = nextSubRequest; 207 nextSubRequest = request->NextSubRequest(subRequest); 208 209 if (error == B_OK) { 210 TRACE_RIO("[%ld] scheduling subrequest: %p\n", find_thread(NULL), 211 subRequest); 212 error = vfs_vnode_io(cookie->vnode, cookie->descriptor->cookie, 213 subRequest); 214 } else { 215 // Once scheduling a subrequest failed, we cancel all subsequent 216 // subrequests. 217 subRequest->SetStatusAndNotify(B_CANCELED); 218 } 219 } 220 221 // TODO: Cancel the subrequests that were scheduled successfully. 222 223 return B_OK; 224 } 225 226 227 static status_t 228 do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status, 229 bool partialTransfer, size_t transferEndOffset) 230 { 231 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; 232 233 if (cookie->finished != NULL) { 234 cookie->finished(cookie->cookie, request, status, partialTransfer, 235 transferEndOffset); 236 } 237 238 put_fd(cookie->descriptor); 239 240 if (cookie->next_finished_callback != NULL) { 241 cookie->next_finished_callback(cookie->next_finished_cookie, request, 242 status, partialTransfer, transferEndOffset); 243 } 244 245 delete cookie; 246 247 return B_OK; 248 } 249 250 251 static status_t 252 do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie, 253 io_request* request, iterative_io_get_vecs getVecs, 254 iterative_io_finished finished, void* cookie) 255 { 256 IOBuffer* buffer = request->Buffer(); 257 VnodeIO io(request->IsWrite(), vnode, openCookie); 258 259 iovec vector; 260 void* virtualVecCookie = NULL; 261 off_t offset = request->Offset(); 262 size_t length = request->Length(); 263 264 status_t error = B_OK; 265 266 for (; error == B_OK && length > 0 267 && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) { 268 uint8* vecBase = (uint8*)vector.iov_base; 269 size_t vecLength = min_c(vector.iov_len, length); 270 271 while (error == B_OK && vecLength > 0) { 272 file_io_vec fileVecs[8]; 273 uint32 fileVecCount = 8; 274 error = getVecs(cookie, request, offset, vecLength, fileVecs, 275 &fileVecCount); 276 if (error != B_OK || fileVecCount == 0) 277 break; 278 279 for (uint32 i = 0; i < fileVecCount; i++) { 280 const file_io_vec& fileVec = fileVecs[i]; 281 size_t toTransfer = min_c(fileVec.length, length); 282 size_t transferred = toTransfer; 283 error = io.IO(fileVec.offset, vecBase, &transferred); 284 if (error != B_OK) 285 break; 286 287 offset += transferred; 288 length -= transferred; 289 vecBase += transferred; 290 vecLength -= transferred; 291 292 if (transferred != toTransfer) 293 break; 294 } 295 } 296 } 297 298 buffer->FreeVirtualVecCookie(virtualVecCookie); 299 300 bool partial = length > 0; 301 size_t bytesTransferred = request->Length() - length; 302 request->SetTransferredBytes(partial, bytesTransferred); 303 finished(cookie, request, error, partial, bytesTransferred); 304 request->SetStatusAndNotify(error); 305 return error; 306 } 307 308 309 static status_t 310 synchronous_io(io_request* request, DoIO& io) 311 { 312 TRACE_RIO("[%ld] synchronous_io(request: %p (offset: %lld, length: %lu))\n", 313 find_thread(NULL), request, request->Offset(), request->Length()); 314 315 IOBuffer* buffer = request->Buffer(); 316 317 iovec vector; 318 void* virtualVecCookie = NULL; 319 off_t offset = request->Offset(); 320 size_t length = request->Length(); 321 322 for (; length > 0 323 && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) { 324 void* vecBase = vector.iov_base; 325 size_t vecLength = min_c(vector.iov_len, length); 326 327 TRACE_RIO("[%ld] I/O: offset: %lld, vecBase: %p, length: %lu\n", 328 find_thread(NULL), offset, vecBase, vecLength); 329 330 size_t transferred = vecLength; 331 status_t error = io.IO(offset, vecBase, &transferred); 332 if (error != B_OK) { 333 TRACE_RIO("[%ld] I/O failed: %#lx\n", find_thread(NULL), error); 334 buffer->FreeVirtualVecCookie(virtualVecCookie); 335 request->SetStatusAndNotify(error); 336 return error; 337 } 338 339 offset += transferred; 340 length -= transferred; 341 342 if (transferred != vecLength) 343 break; 344 } 345 346 TRACE_RIO("[%ld] synchronous_io() succeeded\n", find_thread(NULL)); 347 348 buffer->FreeVirtualVecCookie(virtualVecCookie); 349 request->SetTransferredBytes(length > 0, request->Length() - length); 350 request->SetStatusAndNotify(B_OK); 351 return B_OK; 352 } 353 354 355 // #pragma mark - kernel private API 356 357 358 status_t 359 vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request) 360 { 361 status_t result = B_ERROR; 362 if (!HAS_FS_CALL(vnode, io) 363 || (result = FS_CALL(vnode, io, cookie, request)) == B_UNSUPPORTED) { 364 // no io() call -- fall back to synchronous I/O 365 VnodeIO io(request->IsWrite(), vnode, cookie); 366 return synchronous_io(request, io); 367 } 368 369 return result; 370 } 371 372 373 status_t 374 vfs_synchronous_io(io_request* request, 375 status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t* length), 376 void* cookie) 377 { 378 CallbackIO io(request->IsWrite(), doIO, cookie); 379 return synchronous_io(request, io); 380 } 381 382 383 status_t 384 vfs_asynchronous_read_pages(struct vnode* vnode, void* cookie, off_t pos, 385 const iovec* vecs, size_t count, size_t numBytes, uint32 flags, 386 AsyncIOCallback* callback) 387 { 388 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0); 389 if (request == NULL) { 390 callback->IOFinished(B_NO_MEMORY, true, 0); 391 return B_NO_MEMORY; 392 } 393 394 status_t status = request->Init(pos, vecs, count, numBytes, false, 395 flags | B_DELETE_IO_REQUEST); 396 if (status != B_OK) { 397 delete request; 398 callback->IOFinished(status, true, 0); 399 return status; 400 } 401 402 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback, 403 callback); 404 405 return vfs_vnode_io(vnode, cookie, request); 406 } 407 408 409 status_t 410 vfs_asynchronous_write_pages(struct vnode* vnode, void* cookie, off_t pos, 411 const iovec* vecs, size_t count, size_t numBytes, uint32 flags, 412 AsyncIOCallback* callback) 413 { 414 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0); 415 if (request == NULL) { 416 callback->IOFinished(B_NO_MEMORY, true, 0); 417 return B_NO_MEMORY; 418 } 419 420 status_t status = request->Init(pos, vecs, count, numBytes, true, 421 flags | B_DELETE_IO_REQUEST); 422 if (status != B_OK) { 423 delete request; 424 callback->IOFinished(status, true, 0); 425 return status; 426 } 427 428 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback, 429 callback); 430 431 return vfs_vnode_io(vnode, cookie, request); 432 } 433 434 435 // #pragma mark - public API 436 437 438 status_t 439 do_fd_io(int fd, io_request* request) 440 { 441 struct vnode* vnode; 442 file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true); 443 if (descriptor == NULL) { 444 request->SetStatusAndNotify(B_FILE_ERROR); 445 return B_FILE_ERROR; 446 } 447 448 CObjectDeleter<file_descriptor> descriptorPutter(descriptor, put_fd); 449 450 return vfs_vnode_io(vnode, descriptor->cookie, request); 451 } 452 453 454 status_t 455 do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs, 456 iterative_io_finished finished, void* cookie) 457 { 458 TRACE_RIO("[%ld] do_iterative_fd_io(fd: %d, request: %p (offset: %lld, " 459 "length: %ld))\n", find_thread(NULL), fd, request, request->Offset(), 460 request->Length()); 461 462 struct vnode* vnode; 463 file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true); 464 if (descriptor == NULL) { 465 finished(cookie, request, B_FILE_ERROR, true, 0); 466 request->SetStatusAndNotify(B_FILE_ERROR); 467 return B_FILE_ERROR; 468 } 469 470 CObjectDeleter<file_descriptor> descriptorPutter(descriptor, put_fd); 471 472 if (!HAS_FS_CALL(vnode, io)) { 473 // no io() call -- fall back to synchronous I/O 474 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie, 475 request, getVecs, finished, cookie); 476 } 477 478 iterative_io_cookie* iterationCookie 479 = (request->Flags() & B_VIP_IO_REQUEST) != 0 480 ? new(malloc_flags(HEAP_PRIORITY_VIP)) iterative_io_cookie 481 : new(std::nothrow) iterative_io_cookie; 482 if (iterationCookie == NULL) { 483 // no memory -- fall back to synchronous I/O 484 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie, 485 request, getVecs, finished, cookie); 486 } 487 488 iterationCookie->vnode = vnode; 489 iterationCookie->descriptor = descriptor; 490 iterationCookie->get_vecs = getVecs; 491 iterationCookie->finished = finished; 492 iterationCookie->cookie = cookie; 493 iterationCookie->request_offset = request->Offset(); 494 iterationCookie->next_finished_callback = request->FinishedCallback( 495 &iterationCookie->next_finished_cookie); 496 497 request->SetFinishedCallback(&do_iterative_fd_io_finish, iterationCookie); 498 request->SetIterationCallback(&do_iterative_fd_io_iterate, iterationCookie); 499 500 descriptorPutter.Detach(); 501 // From now on the descriptor is put by our finish callback. 502 503 bool partialTransfer = false; 504 status_t error = do_iterative_fd_io_iterate(iterationCookie, request, 505 &partialTransfer); 506 if (error != B_OK || partialTransfer) { 507 if (partialTransfer) { 508 request->SetTransferredBytes(partialTransfer, 509 request->TransferredBytes()); 510 } 511 512 request->SetStatusAndNotify(error); 513 return error; 514 } 515 516 return B_OK; 517 } 518