1 /* 2 * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 // included by vfs.cpp 7 8 9 //#define TRACE_VFS_REQUEST_IO 10 #ifdef TRACE_VFS_REQUEST_IO 11 # define TRACE_RIO(x...) dprintf(x) 12 #else 13 # define TRACE_RIO(x...) do {} while (false) 14 #endif 15 16 17 // #pragma mark - AsyncIOCallback 18 19 20 AsyncIOCallback::~AsyncIOCallback() 21 { 22 } 23 24 25 void 26 AsyncIOCallback::operator delete(void* address, size_t size) 27 { 28 io_request_free(address); 29 } 30 31 32 /* static */ status_t 33 AsyncIOCallback::IORequestCallback(void* data, io_request* request, 34 status_t status, bool partialTransfer, size_t transferEndOffset) 35 { 36 ((AsyncIOCallback*)data)->IOFinished(status, partialTransfer, 37 transferEndOffset); 38 return B_OK; 39 } 40 41 42 // #pragma mark - StackableAsyncIOCallback 43 44 45 StackableAsyncIOCallback::StackableAsyncIOCallback(AsyncIOCallback* next) 46 : 47 fNextCallback(next) 48 { 49 } 50 51 52 // #pragma mark - 53 54 55 struct iterative_io_cookie { 56 struct vnode* vnode; 57 file_descriptor* descriptor; 58 iterative_io_get_vecs get_vecs; 59 iterative_io_finished finished; 60 void* cookie; 61 off_t request_offset; 62 io_request_finished_callback next_finished_callback; 63 void* next_finished_cookie; 64 65 void operator delete(void* address, size_t size) 66 { io_request_free(address); } 67 }; 68 69 70 class DoIO { 71 public: 72 DoIO(bool write, bool isPhysical) 73 : 74 fWrite(write), 75 fIsPhysical(isPhysical) 76 { 77 } 78 79 virtual ~DoIO() 80 { 81 } 82 83 status_t IO(off_t offset, void* _buffer, size_t* _length) 84 { 85 if (!fIsPhysical) 86 return InternalIO(offset, _buffer, _length); 87 88 // buffer points to physical address -- map pages 89 addr_t buffer = (addr_t)_buffer; 90 size_t length = *_length; 91 92 while (length > 0) { 93 addr_t pageOffset = buffer % B_PAGE_SIZE; 94 addr_t virtualAddress; 95 status_t error = vm_get_physical_page(buffer - pageOffset, 96 &virtualAddress, 0); 97 if (error != B_OK) 98 return error; 99 100 size_t toTransfer = min_c(length, B_PAGE_SIZE - pageOffset); 101 size_t transferred = toTransfer; 102 error = InternalIO(offset, (void*)(virtualAddress + pageOffset), 103 &transferred); 104 105 vm_put_physical_page(virtualAddress); 106 107 if (error != B_OK) 108 return error; 109 110 offset += transferred; 111 buffer += transferred; 112 length -= transferred; 113 114 if (transferred != toTransfer) 115 break; 116 } 117 118 *_length -= length; 119 return B_OK; 120 } 121 122 protected: 123 virtual status_t InternalIO(off_t offset, void* buffer, size_t* length) = 0; 124 125 protected: 126 bool fWrite; 127 bool fIsPhysical; 128 }; 129 130 131 class CallbackIO : public DoIO { 132 public: 133 CallbackIO(bool write, bool isPhysical, 134 status_t (*doIO)(void* cookie, off_t offset, void* buffer, 135 size_t* length), 136 void* cookie) 137 : 138 DoIO(write, isPhysical), 139 fDoIO(doIO), 140 fCookie(cookie) 141 { 142 } 143 144 protected: 145 virtual status_t InternalIO(off_t offset, void* buffer, size_t* length) 146 { 147 return fDoIO(fCookie, offset, buffer, length); 148 } 149 150 private: 151 status_t (*fDoIO)(void*, off_t, void*, size_t*); 152 void* fCookie; 153 }; 154 155 156 class VnodeIO : public DoIO { 157 public: 158 VnodeIO(bool write, bool isPhysical, struct vnode* vnode, void* cookie) 159 : 160 DoIO(write, isPhysical), 161 fVnode(vnode), 162 fCookie(cookie) 163 { 164 } 165 166 protected: 167 virtual status_t InternalIO(off_t offset, void* buffer, size_t* length) 168 { 169 iovec vec; 170 vec.iov_base = buffer; 171 vec.iov_len = *length; 172 173 if (fWrite) { 174 return FS_CALL(fVnode, write_pages, fCookie, offset, &vec, 1, 175 length); 176 } 177 178 return FS_CALL(fVnode, read_pages, fCookie, offset, &vec, 1, length); 179 } 180 181 private: 182 struct vnode* fVnode; 183 void* fCookie; 184 }; 185 186 187 static status_t 188 do_iterative_fd_io_iterate(void* _cookie, io_request* request, 189 bool* _partialTransfer) 190 { 191 TRACE_RIO("[%ld] do_iterative_fd_io_iterate(request: %p)\n", 192 find_thread(NULL), request); 193 194 static const int32 kMaxSubRequests = 8; 195 196 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; 197 198 request->DeleteSubRequests(); 199 200 off_t requestOffset = cookie->request_offset; 201 size_t requestLength = request->Length() 202 - (requestOffset - request->Offset()); 203 204 // get the next file vecs 205 file_io_vec vecs[kMaxSubRequests]; 206 uint32 vecCount = kMaxSubRequests; 207 status_t error = cookie->get_vecs(cookie->cookie, request, requestOffset, 208 requestLength, vecs, &vecCount); 209 if (error != B_OK) 210 return error; 211 if (vecCount == 0) { 212 *_partialTransfer = true; 213 return B_OK; 214 } 215 TRACE_RIO("[%ld] got %lu file vecs\n", find_thread(NULL), vecCount); 216 217 // create subrequests for the file vecs we've got 218 int32 subRequestCount = 0; 219 for (uint32 i = 0; i < vecCount && subRequestCount < kMaxSubRequests; i++) { 220 off_t vecOffset = vecs[i].offset; 221 off_t vecLength = min_c(vecs[i].length, requestLength); 222 TRACE_RIO("[%ld] vec %lu offset: %lld, length: %lld\n", 223 find_thread(NULL), i, vecOffset, vecLength); 224 225 while (vecLength > 0 && subRequestCount < kMaxSubRequests) { 226 TRACE_RIO("[%ld] creating subrequest: offset: %lld, length: " 227 "%lld\n", find_thread(NULL), vecOffset, vecLength); 228 IORequest* subRequest; 229 error = request->CreateSubRequest(requestOffset, vecOffset, 230 vecLength, subRequest); 231 if (error != B_OK) 232 break; 233 234 subRequestCount++; 235 236 size_t lengthProcessed = subRequest->Length(); 237 vecOffset += lengthProcessed; 238 vecLength -= lengthProcessed; 239 requestOffset += lengthProcessed; 240 requestLength -= lengthProcessed; 241 } 242 } 243 244 // Only if we couldn't create any subrequests, we fail. 245 if (error != B_OK && subRequestCount == 0) 246 return error; 247 248 request->Advance(requestOffset - cookie->request_offset); 249 cookie->request_offset = requestOffset; 250 251 // Schedule the subrequests. 252 IORequest* nextSubRequest = request->FirstSubRequest(); 253 while (nextSubRequest != NULL) { 254 IORequest* subRequest = nextSubRequest; 255 nextSubRequest = request->NextSubRequest(subRequest); 256 257 if (error == B_OK) { 258 TRACE_RIO("[%ld] scheduling subrequest: %p\n", find_thread(NULL), 259 subRequest); 260 error = vfs_vnode_io(cookie->vnode, cookie->descriptor->cookie, 261 subRequest); 262 } else { 263 // Once scheduling a subrequest failed, we cancel all subsequent 264 // subrequests. 265 subRequest->SetStatusAndNotify(B_CANCELED); 266 } 267 } 268 269 // TODO: Cancel the subrequests that were scheduled successfully. 270 271 return B_OK; 272 } 273 274 275 static status_t 276 do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status, 277 bool partialTransfer, size_t transferEndOffset) 278 { 279 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; 280 281 if (cookie->finished != NULL) { 282 cookie->finished(cookie->cookie, request, status, partialTransfer, 283 transferEndOffset); 284 } 285 286 put_fd(cookie->descriptor); 287 288 if (cookie->next_finished_callback != NULL) { 289 cookie->next_finished_callback(cookie->next_finished_cookie, request, 290 status, partialTransfer, transferEndOffset); 291 } 292 293 delete cookie; 294 295 return B_OK; 296 } 297 298 299 static status_t 300 do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie, 301 io_request* request, iterative_io_get_vecs getVecs, 302 iterative_io_finished finished, void* cookie) 303 { 304 IOBuffer* buffer = request->Buffer(); 305 VnodeIO io(request->IsWrite(), buffer->IsPhysical(), vnode, openCookie); 306 307 iovec* vecs = buffer->Vecs(); 308 int32 vecCount = buffer->VecCount(); 309 off_t offset = request->Offset(); 310 size_t length = request->Length(); 311 312 status_t error = B_OK; 313 314 for (int32 i = 0; error == B_OK && length > 0 && i < vecCount; i++) { 315 uint8* vecBase = (uint8*)vecs[i].iov_base; 316 size_t vecLength = min_c(vecs[i].iov_len, length); 317 318 while (error == B_OK && vecLength > 0) { 319 file_io_vec fileVecs[8]; 320 uint32 fileVecCount = 8; 321 error = getVecs(cookie, request, offset, vecLength, fileVecs, 322 &fileVecCount); 323 if (error != B_OK || fileVecCount == 0) 324 break; 325 326 for (uint32 k = 0; k < fileVecCount; k++) { 327 const file_io_vec& fileVec = fileVecs[i]; 328 size_t toTransfer = min_c(fileVec.length, length); 329 size_t transferred = toTransfer; 330 error = io.IO(fileVec.offset, vecBase, &transferred); 331 if (error != B_OK) 332 break; 333 334 offset += transferred; 335 length -= transferred; 336 vecBase += transferred; 337 vecLength -= transferred; 338 339 if (transferred != toTransfer) 340 break; 341 } 342 } 343 } 344 345 bool partial = length > 0; 346 size_t bytesTransferred = request->Length() - length; 347 request->SetTransferredBytes(partial, bytesTransferred); 348 finished(cookie, request, error, partial, bytesTransferred); 349 request->SetStatusAndNotify(error); 350 return error; 351 } 352 353 354 static status_t 355 synchronous_io(io_request* request, DoIO& io) 356 { 357 TRACE_RIO("[%ld] synchronous_io(request: %p (offset: %lld, length: %lu))\n", 358 find_thread(NULL), request, request->Offset(), request->Length()); 359 360 IOBuffer* buffer = request->Buffer(); 361 362 iovec* vecs = buffer->Vecs(); 363 int32 vecCount = buffer->VecCount(); 364 off_t offset = request->Offset(); 365 size_t length = request->Length(); 366 367 for (int32 i = 0; length > 0 && i < vecCount; i++) { 368 void* vecBase = vecs[i].iov_base; 369 size_t vecLength = min_c(vecs[i].iov_len, length); 370 371 TRACE_RIO("[%ld] I/O: offset: %lld, vecBase: %p, length: %lu\n", 372 find_thread(NULL), offset, vecBase, vecLength); 373 374 size_t transferred = vecLength; 375 status_t error = io.IO(offset, vecBase, &transferred); 376 if (error != B_OK) { 377 TRACE_RIO("[%ld] I/O failed: %#lx\n", find_thread(NULL), error); 378 request->SetStatusAndNotify(error); 379 return error; 380 } 381 382 offset += transferred; 383 length -= transferred; 384 385 if (transferred != vecLength) 386 break; 387 } 388 389 TRACE_RIO("[%ld] synchronous_io() succeeded\n", find_thread(NULL)); 390 391 request->SetTransferredBytes(length > 0, request->Length() - length); 392 request->SetStatusAndNotify(B_OK); 393 return B_OK; 394 } 395 396 397 // #pragma mark - kernel private API 398 399 400 status_t 401 vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request) 402 { 403 if (!HAS_FS_CALL(vnode, io)) { 404 // no io() call -- fall back to synchronous I/O 405 IOBuffer* buffer = request->Buffer(); 406 VnodeIO io(request->IsWrite(), buffer->IsPhysical(), vnode, cookie); 407 return synchronous_io(request, io); 408 } 409 410 return FS_CALL(vnode, io, cookie, request); 411 } 412 413 414 status_t 415 vfs_synchronous_io(io_request* request, 416 status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t* length), 417 void* cookie) 418 { 419 IOBuffer* buffer = request->Buffer(); 420 CallbackIO io(request->IsWrite(), buffer->IsPhysical(), doIO, cookie); 421 422 return synchronous_io(request, io); 423 } 424 425 426 status_t 427 vfs_asynchronous_write_pages(struct vnode* vnode, void* cookie, off_t pos, 428 const iovec* vecs, size_t count, size_t numBytes, uint32 flags, 429 AsyncIOCallback* callback) 430 { 431 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0); 432 if (request == NULL) { 433 callback->IOFinished(B_NO_MEMORY, true, 0); 434 return B_NO_MEMORY; 435 } 436 437 status_t status = request->Init(pos, vecs, count, numBytes, true, 438 flags | B_DELETE_IO_REQUEST); 439 if (status != B_OK) { 440 delete request; 441 callback->IOFinished(status, true, 0); 442 return status; 443 } 444 445 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback, 446 callback); 447 448 return vfs_vnode_io(vnode, cookie, request); 449 } 450 451 452 // #pragma mark - public API 453 454 455 status_t 456 do_fd_io(int fd, io_request* request) 457 { 458 struct vnode* vnode; 459 file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true); 460 if (descriptor == NULL) { 461 request->SetStatusAndNotify(B_FILE_ERROR); 462 return B_FILE_ERROR; 463 } 464 465 CObjectDeleter<file_descriptor> descriptorPutter(descriptor, put_fd); 466 467 return vfs_vnode_io(vnode, descriptor->cookie, request); 468 } 469 470 471 status_t 472 do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs, 473 iterative_io_finished finished, void* cookie) 474 { 475 TRACE_RIO("[%ld] do_iterative_fd_io(fd: %d, request: %p (offset: %lld, " 476 "length: %ld))\n", find_thread(NULL), fd, request, request->Offset(), 477 request->Length()); 478 479 struct vnode* vnode; 480 file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true); 481 if (descriptor == NULL) { 482 finished(cookie, request, B_FILE_ERROR, true, 0); 483 request->SetStatusAndNotify(B_FILE_ERROR); 484 return B_FILE_ERROR; 485 } 486 487 CObjectDeleter<file_descriptor> descriptorPutter(descriptor, put_fd); 488 489 if (!HAS_FS_CALL(vnode, io)) { 490 // no io() call -- fall back to synchronous I/O 491 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie, 492 request, getVecs, finished, cookie); 493 } 494 495 iterative_io_cookie* iterationCookie 496 = (request->Flags() & B_VIP_IO_REQUEST) != 0 497 ? new(vip_io_alloc) iterative_io_cookie 498 : new(std::nothrow) iterative_io_cookie; 499 if (iterationCookie == NULL) { 500 // no memory -- fall back to synchronous I/O 501 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie, 502 request, getVecs, finished, cookie); 503 } 504 505 iterationCookie->vnode = vnode; 506 iterationCookie->descriptor = descriptor; 507 iterationCookie->get_vecs = getVecs; 508 iterationCookie->finished = finished; 509 iterationCookie->cookie = cookie; 510 iterationCookie->request_offset = request->Offset(); 511 iterationCookie->next_finished_callback = request->FinishedCallback( 512 &iterationCookie->next_finished_cookie); 513 514 request->SetFinishedCallback(&do_iterative_fd_io_finish, iterationCookie); 515 request->SetIterationCallback(&do_iterative_fd_io_iterate, iterationCookie); 516 517 descriptorPutter.Detach(); 518 // From now on the descriptor is put by our finish callback. 519 520 bool partialTransfer = false; 521 status_t error = do_iterative_fd_io_iterate(iterationCookie, request, 522 &partialTransfer); 523 if (error != B_OK || partialTransfer) { 524 if (partialTransfer) { 525 request->SetTransferredBytes(partialTransfer, 526 request->TransferredBytes()); 527 } 528 529 request->SetStatusAndNotify(error); 530 return error; 531 } 532 533 return B_OK; 534 } 535