1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * Copyright 2014, Axel Dörfler, axeld@pinc-software.de 4 * All rights reserved. Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "CopyEngine.h" 9 10 #include <new> 11 12 #include <math.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <sys/resource.h> 16 17 #include <Directory.h> 18 #include <fs_attr.h> 19 #include <NodeInfo.h> 20 #include <Path.h> 21 #include <SymLink.h> 22 23 #include "SemaphoreLocker.h" 24 #include "ProgressReporter.h" 25 26 27 using std::nothrow; 28 29 30 // #pragma mark - EntryFilter 31 32 33 CopyEngine::EntryFilter::~EntryFilter() 34 { 35 } 36 37 38 // #pragma mark - CopyEngine 39 40 41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 42 : 43 fBufferQueue(), 44 fWriterThread(-1), 45 fQuitting(false), 46 47 fAbsoluteSourcePath(), 48 49 fBytesRead(0), 50 fLastBytesRead(0), 51 fItemsCopied(0), 52 fLastItemsCopied(0), 53 fTimeRead(0), 54 55 fBytesWritten(0), 56 fTimeWritten(0), 57 58 fCurrentTargetFolder(NULL), 59 fCurrentItem(NULL), 60 61 fProgressReporter(reporter), 62 fEntryFilter(entryFilter) 63 { 64 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 65 B_NORMAL_PRIORITY, this); 66 67 if (fWriterThread >= B_OK) 68 resume_thread(fWriterThread); 69 70 // ask for a bunch more file descriptors so that nested copying works well 71 struct rlimit rl; 72 rl.rlim_cur = 512; 73 rl.rlim_max = RLIM_SAVED_MAX; 74 setrlimit(RLIMIT_NOFILE, &rl); 75 } 76 77 78 CopyEngine::~CopyEngine() 79 { 80 while (fBufferQueue.Size() > 0) 81 snooze(10000); 82 83 fQuitting = true; 84 if (fWriterThread >= B_OK) { 85 int32 exitValue; 86 wait_for_thread(fWriterThread, &exitValue); 87 } 88 } 89 90 91 void 92 CopyEngine::ResetTargets(const char* source) 93 { 94 // TODO: One could subtract the bytes/items which were added to the 95 // ProgressReporter before resetting them... 96 97 fAbsoluteSourcePath = source; 98 99 fBytesRead = 0; 100 fLastBytesRead = 0; 101 fItemsCopied = 0; 102 fLastItemsCopied = 0; 103 fTimeRead = 0; 104 105 fBytesWritten = 0; 106 fTimeWritten = 0; 107 108 fCurrentTargetFolder = NULL; 109 fCurrentItem = NULL; 110 } 111 112 113 status_t 114 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 115 { 116 int32 level = 0; 117 off_t bytesToCopy = 0; 118 uint64 itemsToCopy = 0; 119 status_t ret = _CollectCopyInfo(source, level, cancelSemaphore, bytesToCopy, 120 itemsToCopy); 121 if (ret == B_OK && fProgressReporter != NULL) 122 fProgressReporter->AddItems(itemsToCopy, bytesToCopy); 123 return ret; 124 } 125 126 127 status_t 128 CopyEngine::Copy(const char* _source, const char* _destination, 129 sem_id cancelSemaphore, bool copyAttributes) 130 { 131 int32 level = 0; 132 status_t ret; 133 134 BEntry source(_source); 135 ret = source.InitCheck(); 136 if (ret != B_OK) 137 return ret; 138 139 BEntry destination(_destination); 140 ret = destination.InitCheck(); 141 if (ret != B_OK) 142 return ret; 143 144 return _Copy(source, destination, level, cancelSemaphore, copyAttributes); 145 } 146 147 148 status_t 149 CopyEngine::_CopyData(const BEntry& _source, const BEntry& _destination, 150 sem_id cancelSemaphore) 151 { 152 SemaphoreLocker lock(cancelSemaphore); 153 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 154 // We are supposed to quit 155 return B_CANCELED; 156 } 157 158 BFile source(&_source, B_READ_ONLY); 159 status_t ret = source.InitCheck(); 160 if (ret < B_OK) 161 return ret; 162 163 BFile* destination = new (nothrow) BFile(&_destination, 164 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 165 ret = destination->InitCheck(); 166 if (ret < B_OK) { 167 delete destination; 168 return ret; 169 } 170 171 int32 loopIteration = 0; 172 173 while (true) { 174 if (fBufferQueue.Size() >= BUFFER_COUNT) { 175 // the queue is "full", just wait a bit, the 176 // write thread will empty it 177 snooze(1000); 178 continue; 179 } 180 181 // allocate buffer 182 Buffer* buffer = new (nothrow) Buffer(destination); 183 if (!buffer || !buffer->buffer) { 184 delete destination; 185 delete buffer; 186 fprintf(stderr, "reading loop: out of memory\n"); 187 return B_NO_MEMORY; 188 } 189 190 // fill buffer 191 ssize_t read = source.Read(buffer->buffer, buffer->size); 192 if (read < 0) { 193 ret = (status_t)read; 194 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 195 delete buffer; 196 delete destination; 197 break; 198 } 199 200 fBytesRead += read; 201 loopIteration += 1; 202 if (loopIteration % 2 == 0) 203 _UpdateProgress(); 204 205 buffer->deleteFile = read == 0; 206 if (read > 0) 207 buffer->validBytes = (size_t)read; 208 else 209 buffer->validBytes = 0; 210 211 // enqueue the buffer 212 ret = fBufferQueue.Push(buffer); 213 if (ret < B_OK) { 214 buffer->deleteFile = false; 215 delete buffer; 216 delete destination; 217 return ret; 218 } 219 220 // quit if done 221 if (read == 0) 222 break; 223 } 224 225 return ret; 226 } 227 228 229 // #pragma mark - 230 231 232 status_t 233 CopyEngine::_CollectCopyInfo(const char* _source, int32& level, 234 sem_id cancelSemaphore, off_t& bytesToCopy, uint64& itemsToCopy) 235 { 236 level++; 237 238 BEntry source(_source); 239 status_t ret = source.InitCheck(); 240 if (ret < B_OK) 241 return ret; 242 243 struct stat statInfo; 244 ret = source.GetStat(&statInfo); 245 if (ret < B_OK) 246 return ret; 247 248 SemaphoreLocker lock(cancelSemaphore); 249 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 250 // We are supposed to quit 251 return B_CANCELED; 252 } 253 254 if (fEntryFilter != NULL 255 && !fEntryFilter->ShouldCopyEntry(source, 256 _RelativeEntryPath(_source), statInfo, level)) { 257 // Skip this entry 258 return B_OK; 259 } 260 261 if (cancelSemaphore >= 0) 262 lock.Unlock(); 263 264 if (S_ISDIR(statInfo.st_mode)) { 265 BDirectory srcFolder(&source); 266 ret = srcFolder.InitCheck(); 267 if (ret < B_OK) 268 return ret; 269 270 BEntry entry; 271 while (srcFolder.GetNextEntry(&entry) == B_OK) { 272 BPath entryPath; 273 ret = entry.GetPath(&entryPath); 274 if (ret < B_OK) 275 return ret; 276 277 ret = _CollectCopyInfo(entryPath.Path(), level, 278 cancelSemaphore, bytesToCopy, itemsToCopy); 279 if (ret < B_OK) 280 return ret; 281 } 282 } else if (S_ISLNK(statInfo.st_mode)) { 283 // link, ignore size 284 } else { 285 bytesToCopy += statInfo.st_size; 286 } 287 288 itemsToCopy++; 289 level--; 290 return B_OK; 291 } 292 293 294 status_t 295 CopyEngine::_Copy(BEntry &source, BEntry &destination, 296 int32& level, sem_id cancelSemaphore, bool copyAttributes) 297 { 298 level++; 299 300 struct stat sourceInfo; 301 status_t ret = source.GetStat(&sourceInfo); 302 if (ret != B_OK) 303 return ret; 304 305 SemaphoreLocker lock(cancelSemaphore); 306 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 307 // We are supposed to quit 308 return B_CANCELED; 309 } 310 311 BPath sourcePath(&source); 312 ret = sourcePath.InitCheck(); 313 if (ret != B_OK) 314 return ret; 315 316 BPath destPath(&destination); 317 ret = destPath.InitCheck(); 318 if (ret != B_OK) 319 return ret; 320 321 const char *relativeSourcePath = _RelativeEntryPath(sourcePath.Path()); 322 if (fEntryFilter != NULL 323 && !fEntryFilter->ShouldCopyEntry(source, relativeSourcePath, 324 sourceInfo, level)) { 325 // Silently skip the filtered entry. 326 return B_OK; 327 } 328 329 if (cancelSemaphore >= 0) 330 lock.Unlock(); 331 332 bool copyAttributesToTarget = copyAttributes; 333 // attributes of the current source to the destination will be copied 334 // when copyAttributes is set to true, but there may be exceptions, so 335 // allow the recursively used copyAttribute parameter to be overridden 336 // for the current target. 337 if (S_ISDIR(sourceInfo.st_mode)) { 338 BDirectory sourceDirectory(&source); 339 ret = sourceDirectory.InitCheck(); 340 if (ret != B_OK) 341 return ret; 342 343 if (destination.Exists()) { 344 if (destination.IsDirectory()) { 345 if (fEntryFilter 346 && fEntryFilter->ShouldClobberFolder(source, 347 relativeSourcePath, sourceInfo, level)) { 348 ret = _RemoveFolder(destination); 349 } else { 350 // Do not overwrite attributes on folders that exist. 351 // This should work better when the install target 352 // already contains a Haiku installation. 353 copyAttributesToTarget = false; 354 } 355 } else { 356 ret = destination.Remove(); 357 } 358 359 if (ret != B_OK) { 360 fprintf(stderr, "Failed to make room for folder '%s': " 361 "%s\n", sourcePath.Path(), strerror(ret)); 362 return ret; 363 } 364 } else { 365 ret = create_directory(destPath.Path(), 0777); 366 if (ret != B_OK && ret != B_FILE_EXISTS) { 367 fprintf(stderr, "Could not create '%s': %s\n", destPath.Path(), 368 strerror(ret)); 369 return ret; 370 } 371 } 372 373 BDirectory destDirectory(&destination); 374 ret = destDirectory.InitCheck(); 375 if (ret != B_OK) 376 return ret; 377 378 BEntry entry; 379 while (sourceDirectory.GetNextEntry(&entry) == B_OK) { 380 BEntry dest(&destDirectory, entry.Name()); 381 ret = dest.InitCheck(); 382 if (ret != B_OK) 383 return ret; 384 ret = _Copy(entry, dest, level, 385 cancelSemaphore, copyAttributes); 386 if (ret != B_OK) 387 return ret; 388 } 389 } else { 390 if (destination.Exists()) { 391 if (destination.IsDirectory()) 392 ret = _RemoveFolder(destination); 393 else 394 ret = destination.Remove(); 395 if (ret != B_OK) { 396 fprintf(stderr, "Failed to make room for entry '%s': " 397 "%s\n", sourcePath.Path(), strerror(ret)); 398 return ret; 399 } 400 } 401 402 fItemsCopied++; 403 BPath destDirectory; 404 ret = destPath.GetParent(&destDirectory); 405 if (ret != B_OK) 406 return ret; 407 fCurrentTargetFolder = destDirectory.Path(); 408 fCurrentItem = sourcePath.Leaf(); 409 _UpdateProgress(); 410 411 if (S_ISLNK(sourceInfo.st_mode)) { 412 // copy symbolic links 413 BSymLink srcLink(&source); 414 ret = srcLink.InitCheck(); 415 if (ret != B_OK) 416 return ret; 417 418 char linkPath[B_PATH_NAME_LENGTH]; 419 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 420 if (read < 0) 421 return (status_t)read; 422 423 BDirectory dstFolder; 424 ret = destination.GetParent(&dstFolder); 425 if (ret != B_OK) 426 return ret; 427 ret = dstFolder.CreateSymLink(sourcePath.Leaf(), linkPath, NULL); 428 if (ret != B_OK) 429 return ret; 430 } else { 431 // copy file data 432 // NOTE: Do not pass the locker, we simply keep holding the lock! 433 ret = _CopyData(source, destination); 434 if (ret != B_OK) 435 return ret; 436 } 437 } 438 439 if (copyAttributesToTarget) { 440 // copy attributes to the current target 441 BNode sourceNode(&source); 442 BNode targetNode(&destination); 443 char attrName[B_ATTR_NAME_LENGTH]; 444 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 445 attr_info info; 446 if (sourceNode.GetAttrInfo(attrName, &info) != B_OK) 447 continue; 448 size_t size = 4096; 449 uint8 buffer[size]; 450 off_t offset = 0; 451 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 452 offset, buffer, std::min((off_t)size, info.size)); 453 // NOTE: It's important to still write the attribute even if 454 // we have read 0 bytes! 455 while (read >= 0) { 456 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 457 offset += read; 458 read = sourceNode.ReadAttr(attrName, info.type, 459 offset, buffer, std::min((off_t)size, info.size - offset)); 460 if (read == 0) 461 break; 462 } 463 } 464 465 // copy basic attributes 466 destination.SetPermissions(sourceInfo.st_mode); 467 destination.SetOwner(sourceInfo.st_uid); 468 destination.SetGroup(sourceInfo.st_gid); 469 destination.SetModificationTime(sourceInfo.st_mtime); 470 destination.SetCreationTime(sourceInfo.st_crtime); 471 } 472 473 level--; 474 return B_OK; 475 } 476 477 478 status_t 479 CopyEngine::_RemoveFolder(BEntry& entry) 480 { 481 BDirectory directory(&entry); 482 status_t ret = directory.InitCheck(); 483 if (ret != B_OK) 484 return ret; 485 486 BEntry subEntry; 487 while (directory.GetNextEntry(&subEntry) == B_OK) { 488 if (subEntry.IsDirectory()) { 489 ret = _RemoveFolder(subEntry); 490 if (ret != B_OK) 491 return ret; 492 } else { 493 ret = subEntry.Remove(); 494 if (ret != B_OK) 495 return ret; 496 } 497 } 498 return entry.Remove(); 499 } 500 501 502 const char* 503 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 504 { 505 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 506 fAbsoluteSourcePath.Length()) != 0) { 507 return absoluteSourcePath; 508 } 509 510 const char* relativePath 511 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 512 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 513 } 514 515 516 void 517 CopyEngine::_UpdateProgress() 518 { 519 if (fProgressReporter == NULL) 520 return; 521 522 uint64 items = 0; 523 if (fLastItemsCopied < fItemsCopied) { 524 items = fItemsCopied - fLastItemsCopied; 525 fLastItemsCopied = fItemsCopied; 526 } 527 528 off_t bytes = 0; 529 if (fLastBytesRead < fBytesRead) { 530 bytes = fBytesRead - fLastBytesRead; 531 fLastBytesRead = fBytesRead; 532 } 533 534 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 535 fCurrentTargetFolder); 536 } 537 538 539 int32 540 CopyEngine::_WriteThreadEntry(void* cookie) 541 { 542 CopyEngine* engine = (CopyEngine*)cookie; 543 engine->_WriteThread(); 544 return B_OK; 545 } 546 547 548 void 549 CopyEngine::_WriteThread() 550 { 551 bigtime_t bufferWaitTimeout = 100000; 552 553 while (!fQuitting) { 554 555 bigtime_t now = system_time(); 556 557 Buffer* buffer = NULL; 558 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 559 if (ret == B_TIMED_OUT) { 560 // no buffer, timeout 561 continue; 562 } else if (ret == B_NO_INIT) { 563 // real error 564 return; 565 } else if (ret != B_OK) { 566 // no buffer, queue error 567 snooze(10000); 568 continue; 569 } 570 571 if (!buffer->deleteFile) { 572 ssize_t written = buffer->file->Write(buffer->buffer, 573 buffer->validBytes); 574 if (written != (ssize_t)buffer->validBytes) { 575 // TODO: this should somehow be propagated back 576 // to the main thread! 577 fprintf(stderr, "Failed to write data: %s\n", 578 strerror((status_t)written)); 579 } 580 fBytesWritten += written; 581 } 582 583 delete buffer; 584 585 // measure performance 586 fTimeWritten += system_time() - now; 587 } 588 589 double megaBytes = (double)fBytesWritten / (1024 * 1024); 590 double seconds = (double)fTimeWritten / 1000000; 591 if (seconds > 0) { 592 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 593 megaBytes / seconds); 594 } 595 } 596