1 /*
2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
3 * Distributed under the terms of the MIT License.
4 */
5
6 // included by vfs.cpp
7
8
9 //#define TRACE_VFS_REQUEST_IO
10 #ifdef TRACE_VFS_REQUEST_IO
11 # define TRACE_RIO(x...) dprintf(x)
12 #else
13 # define TRACE_RIO(x...) do {} while (false)
14 #endif
15
16
17 #include <heap.h>
18 #include <AutoDeleterDrivers.h>
19
20
21 // #pragma mark - AsyncIOCallback
22
23
~AsyncIOCallback()24 AsyncIOCallback::~AsyncIOCallback()
25 {
26 }
27
28
29 /* static */ void
IORequestCallback(void * data,io_request * request,status_t status,bool partialTransfer,generic_size_t bytesTransferred)30 AsyncIOCallback::IORequestCallback(void* data, io_request* request,
31 status_t status, bool partialTransfer, generic_size_t bytesTransferred)
32 {
33 ((AsyncIOCallback*)data)->IOFinished(status, partialTransfer,
34 bytesTransferred);
35 }
36
37
38 // #pragma mark - StackableAsyncIOCallback
39
40
StackableAsyncIOCallback(AsyncIOCallback * next)41 StackableAsyncIOCallback::StackableAsyncIOCallback(AsyncIOCallback* next)
42 :
43 fNextCallback(next)
44 {
45 }
46
47
48 // #pragma mark -
49
50
51 struct iterative_io_cookie {
52 struct vnode* vnode;
53 file_descriptor* descriptor;
54 iterative_io_get_vecs get_vecs;
55 iterative_io_finished finished;
56 void* cookie;
57 off_t request_offset;
58 io_request_finished_callback next_finished_callback;
59 void* next_finished_cookie;
60 };
61
62
63 class DoIO {
64 public:
DoIO(bool write)65 DoIO(bool write)
66 :
67 fWrite(write)
68 {
69 }
70
~DoIO()71 virtual ~DoIO()
72 {
73 }
74
75 virtual status_t IO(off_t offset, void* buffer, size_t* length) = 0;
76
77 protected:
78 bool fWrite;
79 };
80
81
82 class CallbackIO : public DoIO {
83 public:
CallbackIO(bool write,status_t (* doIO)(void * cookie,off_t offset,void * buffer,size_t * length),void * cookie)84 CallbackIO(bool write,
85 status_t (*doIO)(void* cookie, off_t offset, void* buffer,
86 size_t* length),
87 void* cookie)
88 :
89 DoIO(write),
90 fDoIO(doIO),
91 fCookie(cookie)
92 {
93 }
94
IO(off_t offset,void * buffer,size_t * length)95 virtual status_t IO(off_t offset, void* buffer, size_t* length)
96 {
97 return fDoIO(fCookie, offset, buffer, length);
98 }
99
100 private:
101 status_t (*fDoIO)(void*, off_t, void*, size_t*);
102 void* fCookie;
103 };
104
105
106 class VnodeIO : public DoIO {
107 public:
VnodeIO(bool write,struct vnode * vnode,void * cookie)108 VnodeIO(bool write, struct vnode* vnode, void* cookie)
109 :
110 DoIO(write),
111 fVnode(vnode),
112 fCookie(cookie)
113 {
114 }
115
IO(off_t offset,void * buffer,size_t * length)116 virtual status_t IO(off_t offset, void* buffer, size_t* length)
117 {
118 iovec vec;
119 vec.iov_base = buffer;
120 vec.iov_len = *length;
121
122 if (fWrite) {
123 return FS_CALL(fVnode, write_pages, fCookie, offset, &vec, 1,
124 length);
125 }
126
127 return FS_CALL(fVnode, read_pages, fCookie, offset, &vec, 1, length);
128 }
129
130 private:
131 struct vnode* fVnode;
132 void* fCookie;
133 };
134
135
136 static status_t
do_iterative_fd_io_iterate(void * _cookie,io_request * request,bool * _partialTransfer)137 do_iterative_fd_io_iterate(void* _cookie, io_request* request,
138 bool* _partialTransfer)
139 {
140 TRACE_RIO("[%ld] do_iterative_fd_io_iterate(request: %p)\n",
141 find_thread(NULL), request);
142
143 static const size_t kMaxSubRequests = 8;
144
145 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie;
146
147 request->DeleteSubRequests();
148
149 off_t requestOffset = cookie->request_offset;
150 size_t requestLength = request->Length()
151 - (requestOffset - request->Offset());
152
153 // get the next file vecs
154 file_io_vec vecs[kMaxSubRequests];
155 size_t vecCount = kMaxSubRequests;
156 status_t error = cookie->get_vecs(cookie->cookie, request, requestOffset,
157 requestLength, vecs, &vecCount);
158 if (error != B_OK && error != B_BUFFER_OVERFLOW)
159 return error;
160 if (vecCount == 0) {
161 *_partialTransfer = true;
162 return B_OK;
163 }
164 TRACE_RIO("[%ld] got %zu file vecs\n", find_thread(NULL), vecCount);
165
166 // Reset the error code for the loop below
167 error = B_OK;
168
169 // create subrequests for the file vecs we've got
170 size_t subRequestCount = 0;
171 for (size_t i = 0;
172 i < vecCount && subRequestCount < kMaxSubRequests && error == B_OK;
173 i++) {
174 off_t vecOffset = vecs[i].offset;
175 off_t vecLength = min_c(vecs[i].length, (off_t)requestLength);
176 TRACE_RIO("[%ld] vec %lu offset: %lld, length: %lld\n",
177 find_thread(NULL), i, vecOffset, vecLength);
178
179 // Special offset -1 means that this is part of sparse file that is
180 // zero. We fill it in right here.
181 if (vecOffset == -1) {
182 if (request->IsWrite()) {
183 panic("do_iterative_fd_io_iterate(): write to sparse file "
184 "vector");
185 error = B_BAD_VALUE;
186 break;
187 }
188
189 error = request->ClearData(requestOffset, vecLength);
190 if (error != B_OK)
191 break;
192
193 requestOffset += vecLength;
194 requestLength -= vecLength;
195 continue;
196 }
197
198 while (vecLength > 0 && subRequestCount < kMaxSubRequests) {
199 TRACE_RIO("[%ld] creating subrequest: offset: %lld, length: "
200 "%lld\n", find_thread(NULL), vecOffset, vecLength);
201 IORequest* subRequest;
202 error = request->CreateSubRequest(requestOffset, vecOffset,
203 vecLength, subRequest);
204 if (error != B_OK)
205 break;
206
207 subRequestCount++;
208
209 size_t lengthProcessed = subRequest->Length();
210 vecOffset += lengthProcessed;
211 vecLength -= lengthProcessed;
212 requestOffset += lengthProcessed;
213 requestLength -= lengthProcessed;
214 }
215 }
216
217 // Only if we couldn't create any subrequests, we fail.
218 if (error != B_OK && subRequestCount == 0)
219 return error;
220
221 // Reset the error code for the loop below
222 error = B_OK;
223
224 request->Advance(requestOffset - cookie->request_offset);
225 cookie->request_offset = requestOffset;
226
227 // If we don't have any sub requests at this point, that means all that
228 // remained were zeroed sparse file vectors. So the request is done now.
229 if (subRequestCount == 0) {
230 ASSERT(request->RemainingBytes() == 0);
231 request->SetStatusAndNotify(B_OK);
232 return B_OK;
233 }
234
235 // Schedule the subrequests.
236 IORequest* nextSubRequest = request->FirstSubRequest();
237 while (nextSubRequest != NULL) {
238 IORequest* subRequest = nextSubRequest;
239 nextSubRequest = request->NextSubRequest(subRequest);
240
241 if (error == B_OK) {
242 TRACE_RIO("[%ld] scheduling subrequest: %p\n", find_thread(NULL),
243 subRequest);
244 error = vfs_vnode_io(cookie->vnode, cookie->descriptor->cookie,
245 subRequest);
246 } else {
247 // Once scheduling a subrequest failed, we cancel all subsequent
248 // subrequests.
249 subRequest->SetStatusAndNotify(B_CANCELED);
250 }
251 }
252
253 // TODO: Cancel the subrequests that were scheduled successfully.
254
255 return B_OK;
256 }
257
258
259 static void
do_iterative_fd_io_finish(void * _cookie,io_request * request,status_t status,bool partialTransfer,generic_size_t bytesTransferred)260 do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status,
261 bool partialTransfer, generic_size_t bytesTransferred)
262 {
263 iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie;
264
265 if (cookie->finished != NULL) {
266 cookie->finished(cookie->cookie, request, status, partialTransfer,
267 bytesTransferred);
268 }
269
270 put_fd(cookie->descriptor);
271
272 if (cookie->next_finished_callback != NULL) {
273 cookie->next_finished_callback(cookie->next_finished_cookie, request,
274 status, partialTransfer, bytesTransferred);
275 }
276
277 delete cookie;
278 }
279
280
281 static status_t
do_synchronous_iterative_vnode_io(struct vnode * vnode,void * openCookie,io_request * request,iterative_io_get_vecs getVecs,iterative_io_finished finished,void * cookie)282 do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie,
283 io_request* request, iterative_io_get_vecs getVecs,
284 iterative_io_finished finished, void* cookie)
285 {
286 IOBuffer* buffer = request->Buffer();
287 VnodeIO io(request->IsWrite(), vnode, openCookie);
288
289 iovec vector;
290 void* virtualVecCookie = NULL;
291 off_t offset = request->Offset();
292 generic_size_t length = request->Length();
293
294 status_t error = B_OK;
295 bool partial = false;
296
297 for (; error == B_OK && length > 0 && !partial
298 && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) {
299 uint8* vecBase = (uint8*)vector.iov_base;
300 generic_size_t vecLength = min_c(vector.iov_len, length);
301
302 while (error == B_OK && vecLength > 0) {
303 file_io_vec fileVecs[8];
304 size_t fileVecCount = 8;
305 if (getVecs != NULL) {
306 error = getVecs(cookie, request, offset, vecLength, fileVecs,
307 &fileVecCount);
308 } else {
309 fileVecs[0].offset = offset;
310 fileVecs[0].length = vecLength;
311 fileVecCount = 1;
312 }
313 if (error != B_OK)
314 break;
315 if (fileVecCount == 0) {
316 partial = true;
317 break;
318 }
319
320 for (size_t i = 0; i < fileVecCount; i++) {
321 const file_io_vec& fileVec = fileVecs[i];
322 size_t toTransfer = min_c(fileVec.length, (off_t)length);
323 size_t transferred = toTransfer;
324 error = io.IO(fileVec.offset, vecBase, &transferred);
325 if (error != B_OK)
326 break;
327
328 offset += transferred;
329 length -= transferred;
330 vecBase += transferred;
331 vecLength -= transferred;
332
333 if (transferred != toTransfer) {
334 partial = true;
335 break;
336 }
337 }
338 }
339 }
340
341 buffer->FreeVirtualVecCookie(virtualVecCookie);
342
343 partial = (partial || length > 0);
344 size_t bytesTransferred = request->Length() - length;
345 request->SetTransferredBytes(partial, bytesTransferred);
346 if (finished != NULL)
347 finished(cookie, request, error, partial, bytesTransferred);
348 request->SetStatusAndNotify(error);
349 return error;
350 }
351
352
353 static status_t
synchronous_io(io_request * request,DoIO & io)354 synchronous_io(io_request* request, DoIO& io)
355 {
356 TRACE_RIO("[%" B_PRId32 "] synchronous_io(request: %p (offset: %" B_PRIdOFF
357 ", length: %" B_PRIuGENADDR "))\n", find_thread(NULL), request,
358 request->Offset(), request->Length());
359
360 IOBuffer* buffer = request->Buffer();
361
362 iovec vector;
363 void* virtualVecCookie = NULL;
364 off_t offset = request->Offset();
365 generic_size_t length = request->Length();
366
367 for (; length > 0
368 && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) {
369 void* vecBase = (void*)(addr_t)vector.iov_base;
370 size_t vecLength = min_c(vector.iov_len, length);
371
372 TRACE_RIO("[%ld] I/O: offset: %lld, vecBase: %p, length: %lu\n",
373 find_thread(NULL), offset, vecBase, vecLength);
374
375 size_t transferred = vecLength;
376 status_t error = io.IO(offset, vecBase, &transferred);
377 if (error != B_OK) {
378 TRACE_RIO("[%ld] I/O failed: %#lx\n", find_thread(NULL), error);
379 buffer->FreeVirtualVecCookie(virtualVecCookie);
380 request->SetStatusAndNotify(error);
381 return error;
382 }
383
384 offset += transferred;
385 length -= transferred;
386
387 if (transferred != vecLength)
388 break;
389 }
390
391 TRACE_RIO("[%ld] synchronous_io() succeeded\n", find_thread(NULL));
392
393 buffer->FreeVirtualVecCookie(virtualVecCookie);
394 request->SetTransferredBytes(length > 0, request->Length() - length);
395 request->SetStatusAndNotify(B_OK);
396 return B_OK;
397 }
398
399
400 // #pragma mark - kernel private API
401
402
403 status_t
vfs_vnode_io(struct vnode * vnode,void * cookie,io_request * request)404 vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request)
405 {
406 status_t result = B_ERROR;
407 if (!HAS_FS_CALL(vnode, io)
408 || (result = FS_CALL(vnode, io, cookie, request)) == B_UNSUPPORTED) {
409 // no io() call -- fall back to synchronous I/O
410 VnodeIO io(request->IsWrite(), vnode, cookie);
411 return synchronous_io(request, io);
412 }
413
414 return result;
415 }
416
417
418 status_t
vfs_synchronous_io(io_request * request,status_t (* doIO)(void * cookie,off_t offset,void * buffer,size_t * length),void * cookie)419 vfs_synchronous_io(io_request* request,
420 status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t* length),
421 void* cookie)
422 {
423 CallbackIO io(request->IsWrite(), doIO, cookie);
424 return synchronous_io(request, io);
425 }
426
427
428 status_t
vfs_asynchronous_read_pages(struct vnode * vnode,void * cookie,off_t pos,const generic_io_vec * vecs,size_t count,generic_size_t numBytes,uint32 flags,AsyncIOCallback * callback)429 vfs_asynchronous_read_pages(struct vnode* vnode, void* cookie, off_t pos,
430 const generic_io_vec* vecs, size_t count, generic_size_t numBytes,
431 uint32 flags, AsyncIOCallback* callback)
432 {
433 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0);
434 if (request == NULL) {
435 callback->IOFinished(B_NO_MEMORY, true, 0);
436 return B_NO_MEMORY;
437 }
438
439 status_t status = request->Init(pos, vecs, count, numBytes, false,
440 flags | B_DELETE_IO_REQUEST);
441 if (status != B_OK) {
442 delete request;
443 callback->IOFinished(status, true, 0);
444 return status;
445 }
446
447 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback,
448 callback);
449
450 return vfs_vnode_io(vnode, cookie, request);
451 }
452
453
454 status_t
vfs_asynchronous_write_pages(struct vnode * vnode,void * cookie,off_t pos,const generic_io_vec * vecs,size_t count,generic_size_t numBytes,uint32 flags,AsyncIOCallback * callback)455 vfs_asynchronous_write_pages(struct vnode* vnode, void* cookie, off_t pos,
456 const generic_io_vec* vecs, size_t count, generic_size_t numBytes,
457 uint32 flags, AsyncIOCallback* callback)
458 {
459 IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0);
460 if (request == NULL) {
461 callback->IOFinished(B_NO_MEMORY, true, 0);
462 return B_NO_MEMORY;
463 }
464
465 status_t status = request->Init(pos, vecs, count, numBytes, true,
466 flags | B_DELETE_IO_REQUEST);
467 if (status != B_OK) {
468 delete request;
469 callback->IOFinished(status, true, 0);
470 return status;
471 }
472
473 request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback,
474 callback);
475
476 return vfs_vnode_io(vnode, cookie, request);
477 }
478
479
480 // #pragma mark - public API
481
482
483 status_t
do_fd_io(int fd,io_request * request)484 do_fd_io(int fd, io_request* request)
485 {
486 return do_iterative_fd_io(fd, request, NULL, NULL, NULL);
487 }
488
489
490 status_t
do_iterative_fd_io(int fd,io_request * request,iterative_io_get_vecs getVecs,iterative_io_finished finished,void * cookie)491 do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs,
492 iterative_io_finished finished, void* cookie)
493 {
494 TRACE_RIO("[%" B_PRId32 "] do_iterative_fd_io(fd: %d, request: %p "
495 "(offset: %" B_PRIdOFF ", length: %" B_PRIuGENADDR "))\n",
496 find_thread(NULL), fd, request, request->Offset(), request->Length());
497
498 struct vnode* vnode;
499 file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true);
500 if (descriptor == NULL) {
501 if (finished != NULL)
502 finished(cookie, request, B_FILE_ERROR, true, 0);
503 request->SetStatusAndNotify(B_FILE_ERROR);
504 return B_FILE_ERROR;
505 }
506
507 FileDescriptorPutter descriptorPutter(descriptor);
508
509 if (!HAS_FS_CALL(vnode, io)) {
510 // no io() call -- fall back to synchronous I/O
511 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie,
512 request, getVecs, finished, cookie);
513 }
514
515 iterative_io_cookie* iterationCookie
516 = (request->Flags() & B_VIP_IO_REQUEST) != 0
517 ? new(malloc_flags(HEAP_PRIORITY_VIP)) iterative_io_cookie
518 : new(std::nothrow) iterative_io_cookie;
519 if (iterationCookie == NULL) {
520 // no memory -- fall back to synchronous I/O
521 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie,
522 request, getVecs, finished, cookie);
523 }
524
525 iterationCookie->vnode = vnode;
526 iterationCookie->descriptor = descriptor;
527 iterationCookie->get_vecs = getVecs;
528 iterationCookie->finished = finished;
529 iterationCookie->cookie = cookie;
530 iterationCookie->request_offset = request->Offset();
531 iterationCookie->next_finished_callback = request->FinishedCallback(
532 &iterationCookie->next_finished_cookie);
533
534 request->SetFinishedCallback(&do_iterative_fd_io_finish, iterationCookie);
535 if (getVecs != NULL)
536 request->SetIterationCallback(&do_iterative_fd_io_iterate, iterationCookie);
537
538 descriptorPutter.Detach();
539 // From now on the descriptor is put by our finish callback.
540
541 if (getVecs != NULL) {
542 bool partialTransfer = false;
543 status_t error = do_iterative_fd_io_iterate(iterationCookie, request,
544 &partialTransfer);
545 if (error != B_OK || partialTransfer) {
546 if (partialTransfer) {
547 request->SetTransferredBytes(partialTransfer,
548 request->TransferredBytes());
549 }
550
551 request->SetStatusAndNotify(error);
552 return error;
553 }
554 } else {
555 return vfs_vnode_io(vnode, descriptor->cookie, request);
556 }
557
558 return B_OK;
559 }
560