1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * Copyright 2014, Axel Dörfler, axeld@pinc-software.de 4 * All rights reserved. Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "CopyEngine.h" 9 10 #include <new> 11 12 #include <math.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <sys/resource.h> 16 17 #include <Directory.h> 18 #include <fs_attr.h> 19 #include <NodeInfo.h> 20 #include <Path.h> 21 #include <SymLink.h> 22 23 #include "SemaphoreLocker.h" 24 #include "ProgressReporter.h" 25 26 27 using std::nothrow; 28 29 30 // #pragma mark - EntryFilter 31 32 33 CopyEngine::EntryFilter::~EntryFilter() 34 { 35 } 36 37 38 // #pragma mark - CopyEngine 39 40 41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 42 : 43 fBufferQueue(), 44 fWriterThread(-1), 45 fQuitting(false), 46 47 fAbsoluteSourcePath(), 48 49 fBytesRead(0), 50 fLastBytesRead(0), 51 fItemsCopied(0), 52 fLastItemsCopied(0), 53 fTimeRead(0), 54 55 fBytesWritten(0), 56 fTimeWritten(0), 57 58 fBytesToCopy(0), 59 fItemsToCopy(0), 60 61 fCurrentTargetFolder(NULL), 62 fCurrentItem(NULL), 63 64 fProgressReporter(reporter), 65 fEntryFilter(entryFilter) 66 { 67 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 68 B_NORMAL_PRIORITY, this); 69 70 if (fWriterThread >= B_OK) 71 resume_thread(fWriterThread); 72 73 // ask for a bunch more file descriptors so that nested copying works well 74 struct rlimit rl; 75 rl.rlim_cur = 512; 76 rl.rlim_max = RLIM_SAVED_MAX; 77 setrlimit(RLIMIT_NOFILE, &rl); 78 } 79 80 81 CopyEngine::~CopyEngine() 82 { 83 while (fBufferQueue.Size() > 0) 84 snooze(10000); 85 86 fQuitting = true; 87 if (fWriterThread >= B_OK) { 88 int32 exitValue; 89 wait_for_thread(fWriterThread, &exitValue); 90 } 91 } 92 93 94 void 95 CopyEngine::ResetTargets(const char* source) 96 { 97 // TODO: One could subtract the bytes/items which were added to the 98 // ProgressReporter before resetting them... 99 100 fAbsoluteSourcePath = source; 101 102 fBytesRead = 0; 103 fLastBytesRead = 0; 104 fItemsCopied = 0; 105 fLastItemsCopied = 0; 106 fTimeRead = 0; 107 108 fBytesWritten = 0; 109 fTimeWritten = 0; 110 111 fBytesToCopy = 0; 112 fItemsToCopy = 0; 113 114 fCurrentTargetFolder = NULL; 115 fCurrentItem = NULL; 116 } 117 118 119 status_t 120 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 121 { 122 int32 level = 0; 123 status_t ret = _CollectCopyInfo(source, level, cancelSemaphore); 124 if (ret == B_OK && fProgressReporter != NULL) 125 fProgressReporter->AddItems(fItemsToCopy, fBytesToCopy); 126 return ret; 127 } 128 129 130 status_t 131 CopyEngine::Copy(const char* _source, const char* _destination, 132 sem_id cancelSemaphore, bool copyAttributes) 133 { 134 int32 level = 0; 135 status_t ret; 136 137 BEntry source(_source); 138 ret = source.InitCheck(); 139 if (ret != B_OK) 140 return ret; 141 142 BEntry destination(_destination); 143 ret = destination.InitCheck(); 144 if (ret != B_OK) 145 return ret; 146 147 return _Copy(source, destination, level, cancelSemaphore, copyAttributes); 148 } 149 150 151 status_t 152 CopyEngine::_CopyData(const BEntry& _source, const BEntry& _destination, 153 sem_id cancelSemaphore) 154 { 155 SemaphoreLocker lock(cancelSemaphore); 156 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 157 // We are supposed to quit 158 return B_CANCELED; 159 } 160 161 BFile source(&_source, B_READ_ONLY); 162 status_t ret = source.InitCheck(); 163 if (ret < B_OK) 164 return ret; 165 166 BFile* destination = new (nothrow) BFile(&_destination, 167 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 168 ret = destination->InitCheck(); 169 if (ret < B_OK) { 170 delete destination; 171 return ret; 172 } 173 174 int32 loopIteration = 0; 175 176 while (true) { 177 if (fBufferQueue.Size() >= BUFFER_COUNT) { 178 // the queue is "full", just wait a bit, the 179 // write thread will empty it 180 snooze(1000); 181 continue; 182 } 183 184 // allocate buffer 185 Buffer* buffer = new (nothrow) Buffer(destination); 186 if (!buffer || !buffer->buffer) { 187 delete destination; 188 delete buffer; 189 fprintf(stderr, "reading loop: out of memory\n"); 190 return B_NO_MEMORY; 191 } 192 193 // fill buffer 194 ssize_t read = source.Read(buffer->buffer, buffer->size); 195 if (read < 0) { 196 ret = (status_t)read; 197 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 198 delete buffer; 199 delete destination; 200 break; 201 } 202 203 fBytesRead += read; 204 loopIteration += 1; 205 if (loopIteration % 2 == 0) 206 _UpdateProgress(); 207 208 buffer->deleteFile = read == 0; 209 if (read > 0) 210 buffer->validBytes = (size_t)read; 211 else 212 buffer->validBytes = 0; 213 214 // enqueue the buffer 215 ret = fBufferQueue.Push(buffer); 216 if (ret < B_OK) { 217 buffer->deleteFile = false; 218 delete buffer; 219 delete destination; 220 return ret; 221 } 222 223 // quit if done 224 if (read == 0) 225 break; 226 } 227 228 return ret; 229 } 230 231 232 // #pragma mark - 233 234 235 status_t 236 CopyEngine::_CollectCopyInfo(const char* _source, int32& level, 237 sem_id cancelSemaphore) 238 { 239 level++; 240 241 BEntry source(_source); 242 status_t ret = source.InitCheck(); 243 if (ret < B_OK) 244 return ret; 245 246 struct stat statInfo; 247 ret = source.GetStat(&statInfo); 248 if (ret < B_OK) 249 return ret; 250 251 SemaphoreLocker lock(cancelSemaphore); 252 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 253 // We are supposed to quit 254 return B_CANCELED; 255 } 256 257 if (fEntryFilter != NULL 258 && !fEntryFilter->ShouldCopyEntry(source, 259 _RelativeEntryPath(_source), statInfo, level)) { 260 // Skip this entry 261 return B_OK; 262 } 263 264 if (cancelSemaphore >= 0) 265 lock.Unlock(); 266 267 if (S_ISDIR(statInfo.st_mode)) { 268 BDirectory srcFolder(&source); 269 ret = srcFolder.InitCheck(); 270 if (ret < B_OK) 271 return ret; 272 273 BEntry entry; 274 while (srcFolder.GetNextEntry(&entry) == B_OK) { 275 BPath entryPath; 276 ret = entry.GetPath(&entryPath); 277 if (ret < B_OK) 278 return ret; 279 280 ret = _CollectCopyInfo(entryPath.Path(), level, cancelSemaphore); 281 if (ret < B_OK) 282 return ret; 283 } 284 } else if (S_ISLNK(statInfo.st_mode)) { 285 // link, ignore size 286 } else { 287 fBytesToCopy += statInfo.st_size; 288 } 289 290 fItemsToCopy++; 291 level--; 292 return B_OK; 293 } 294 295 296 status_t 297 CopyEngine::_Copy(BEntry &source, BEntry &destination, 298 int32& level, sem_id cancelSemaphore, bool copyAttributes) 299 { 300 level++; 301 302 struct stat sourceInfo; 303 status_t ret = source.GetStat(&sourceInfo); 304 if (ret != B_OK) 305 return ret; 306 307 SemaphoreLocker lock(cancelSemaphore); 308 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 309 // We are supposed to quit 310 return B_CANCELED; 311 } 312 313 BPath sourcePath(&source); 314 ret = sourcePath.InitCheck(); 315 if (ret != B_OK) 316 return ret; 317 318 BPath destPath(&destination); 319 ret = destPath.InitCheck(); 320 if (ret != B_OK) 321 return ret; 322 323 const char *relativeSourcePath = _RelativeEntryPath(sourcePath.Path()); 324 if (fEntryFilter != NULL 325 && !fEntryFilter->ShouldCopyEntry(source, relativeSourcePath, 326 sourceInfo, level)) { 327 // Silently skip the filtered entry. 328 return B_OK; 329 } 330 331 if (cancelSemaphore >= 0) 332 lock.Unlock(); 333 334 if (S_ISDIR(sourceInfo.st_mode)) { 335 BDirectory sourceDirectory(&source); 336 ret = sourceDirectory.InitCheck(); 337 if (ret != B_OK) 338 return ret; 339 340 if (destination.Exists()) { 341 if (destination.IsDirectory()) { 342 if (fEntryFilter 343 && fEntryFilter->ShouldClobberFolder(source, 344 relativeSourcePath, sourceInfo, level)) { 345 ret = _RemoveFolder(destination); 346 } else { 347 // Do not overwrite attributes on folders that exist. 348 // This should work better when the install target 349 // already contains a Haiku installation. 350 copyAttributes = false; 351 } 352 } else { 353 ret = destination.Remove(); 354 } 355 356 if (ret != B_OK) { 357 fprintf(stderr, "Failed to make room for folder '%s': " 358 "%s\n", sourcePath.Path(), strerror(ret)); 359 return ret; 360 } 361 } else { 362 ret = create_directory(destPath.Path(), 0777); 363 if (ret != B_OK && ret != B_FILE_EXISTS) { 364 fprintf(stderr, "Could not create '%s': %s\n", destPath.Path(), 365 strerror(ret)); 366 return ret; 367 } 368 } 369 370 BDirectory destDirectory(&destination); 371 ret = destDirectory.InitCheck(); 372 if (ret != B_OK) 373 return ret; 374 375 BEntry entry; 376 while (sourceDirectory.GetNextEntry(&entry) == B_OK) { 377 BEntry dest(&destDirectory, entry.Name()); 378 ret = dest.InitCheck(); 379 if (ret != B_OK) 380 return ret; 381 ret = _Copy(entry, dest, level, 382 cancelSemaphore, copyAttributes); 383 if (ret != B_OK) 384 return ret; 385 } 386 } else { 387 if (destination.Exists()) { 388 if (destination.IsDirectory()) 389 ret = _RemoveFolder(destination); 390 else 391 ret = destination.Remove(); 392 if (ret != B_OK) { 393 fprintf(stderr, "Failed to make room for entry '%s': " 394 "%s\n", sourcePath.Path(), strerror(ret)); 395 return ret; 396 } 397 } 398 399 fItemsCopied++; 400 BPath destDirectory; 401 ret = destPath.GetParent(&destDirectory); 402 if (ret != B_OK) 403 return ret; 404 fCurrentTargetFolder = destDirectory.Path(); 405 fCurrentItem = sourcePath.Leaf(); 406 _UpdateProgress(); 407 408 if (S_ISLNK(sourceInfo.st_mode)) { 409 // copy symbolic links 410 BSymLink srcLink(&source); 411 ret = srcLink.InitCheck(); 412 if (ret != B_OK) 413 return ret; 414 415 char linkPath[B_PATH_NAME_LENGTH]; 416 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 417 if (read < 0) 418 return (status_t)read; 419 420 BDirectory dstFolder; 421 ret = destination.GetParent(&dstFolder); 422 if (ret != B_OK) 423 return ret; 424 ret = dstFolder.CreateSymLink(sourcePath.Leaf(), linkPath, NULL); 425 if (ret != B_OK) 426 return ret; 427 } else { 428 // copy file data 429 // NOTE: Do not pass the locker, we simply keep holding the lock! 430 ret = _CopyData(source, destination); 431 if (ret != B_OK) 432 return ret; 433 } 434 } 435 436 if (copyAttributes) { 437 // copy attributes 438 BNode sourceNode(&source); 439 BNode targetNode(&destination); 440 char attrName[B_ATTR_NAME_LENGTH]; 441 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 442 attr_info info; 443 if (sourceNode.GetAttrInfo(attrName, &info) != B_OK) 444 continue; 445 size_t size = 4096; 446 uint8 buffer[size]; 447 off_t offset = 0; 448 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 449 offset, buffer, std::min((off_t)size, info.size)); 450 // NOTE: It's important to still write the attribute even if 451 // we have read 0 bytes! 452 while (read >= 0) { 453 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 454 offset += read; 455 read = sourceNode.ReadAttr(attrName, info.type, 456 offset, buffer, std::min((off_t)size, info.size - offset)); 457 if (read == 0) 458 break; 459 } 460 } 461 462 // copy basic attributes 463 destination.SetPermissions(sourceInfo.st_mode); 464 destination.SetOwner(sourceInfo.st_uid); 465 destination.SetGroup(sourceInfo.st_gid); 466 destination.SetModificationTime(sourceInfo.st_mtime); 467 destination.SetCreationTime(sourceInfo.st_crtime); 468 } 469 470 level--; 471 return B_OK; 472 } 473 474 475 status_t 476 CopyEngine::_RemoveFolder(BEntry& entry) 477 { 478 BDirectory directory(&entry); 479 status_t ret = directory.InitCheck(); 480 if (ret != B_OK) 481 return ret; 482 483 BEntry subEntry; 484 while (directory.GetNextEntry(&subEntry) == B_OK) { 485 if (subEntry.IsDirectory()) { 486 ret = _RemoveFolder(subEntry); 487 if (ret != B_OK) 488 return ret; 489 } else { 490 ret = subEntry.Remove(); 491 if (ret != B_OK) 492 return ret; 493 } 494 } 495 return entry.Remove(); 496 } 497 498 499 const char* 500 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 501 { 502 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 503 fAbsoluteSourcePath.Length()) != 0) { 504 return absoluteSourcePath; 505 } 506 507 const char* relativePath 508 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 509 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 510 } 511 512 513 void 514 CopyEngine::_UpdateProgress() 515 { 516 if (fProgressReporter == NULL) 517 return; 518 519 uint64 items = 0; 520 if (fLastItemsCopied < fItemsCopied) { 521 items = fItemsCopied - fLastItemsCopied; 522 fLastItemsCopied = fItemsCopied; 523 } 524 525 off_t bytes = 0; 526 if (fLastBytesRead < fBytesRead) { 527 bytes = fBytesRead - fLastBytesRead; 528 fLastBytesRead = fBytesRead; 529 } 530 531 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 532 fCurrentTargetFolder); 533 } 534 535 536 int32 537 CopyEngine::_WriteThreadEntry(void* cookie) 538 { 539 CopyEngine* engine = (CopyEngine*)cookie; 540 engine->_WriteThread(); 541 return B_OK; 542 } 543 544 545 void 546 CopyEngine::_WriteThread() 547 { 548 bigtime_t bufferWaitTimeout = 100000; 549 550 while (!fQuitting) { 551 552 bigtime_t now = system_time(); 553 554 Buffer* buffer = NULL; 555 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 556 if (ret == B_TIMED_OUT) { 557 // no buffer, timeout 558 continue; 559 } else if (ret == B_NO_INIT) { 560 // real error 561 return; 562 } else if (ret != B_OK) { 563 // no buffer, queue error 564 snooze(10000); 565 continue; 566 } 567 568 if (!buffer->deleteFile) { 569 ssize_t written = buffer->file->Write(buffer->buffer, 570 buffer->validBytes); 571 if (written != (ssize_t)buffer->validBytes) { 572 // TODO: this should somehow be propagated back 573 // to the main thread! 574 fprintf(stderr, "Failed to write data: %s\n", 575 strerror((status_t)written)); 576 } 577 fBytesWritten += written; 578 } 579 580 delete buffer; 581 582 // measure performance 583 fTimeWritten += system_time() - now; 584 } 585 586 double megaBytes = (double)fBytesWritten / (1024 * 1024); 587 double seconds = (double)fTimeWritten / 1000000; 588 if (seconds > 0) { 589 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 590 megaBytes / seconds); 591 } 592 } 593