1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * Copyright 2014, Axel Dörfler, axeld@pinc-software.de 4 * All rights reserved. Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "CopyEngine.h" 9 10 #include <new> 11 12 #include <math.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <sys/resource.h> 16 17 #include <Directory.h> 18 #include <fs_attr.h> 19 #include <NodeInfo.h> 20 #include <Path.h> 21 #include <SymLink.h> 22 23 #include "SemaphoreLocker.h" 24 #include "ProgressReporter.h" 25 26 27 using std::nothrow; 28 29 30 // #pragma mark - EntryFilter 31 32 33 CopyEngine::EntryFilter::~EntryFilter() 34 { 35 } 36 37 38 // #pragma mark - CopyEngine 39 40 41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 42 : 43 fBufferQueue(), 44 fWriterThread(-1), 45 fQuitting(false), 46 47 fAbsoluteSourcePath(), 48 49 fBytesRead(0), 50 fLastBytesRead(0), 51 fItemsCopied(0), 52 fLastItemsCopied(0), 53 fTimeRead(0), 54 55 fBytesWritten(0), 56 fTimeWritten(0), 57 58 fCurrentTargetFolder(NULL), 59 fCurrentItem(NULL), 60 61 fProgressReporter(reporter), 62 fEntryFilter(entryFilter) 63 { 64 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 65 B_NORMAL_PRIORITY, this); 66 67 if (fWriterThread >= B_OK) 68 resume_thread(fWriterThread); 69 70 // ask for a bunch more file descriptors so that nested copying works well 71 struct rlimit rl; 72 rl.rlim_cur = 512; 73 rl.rlim_max = RLIM_SAVED_MAX; 74 setrlimit(RLIMIT_NOFILE, &rl); 75 } 76 77 78 CopyEngine::~CopyEngine() 79 { 80 while (fBufferQueue.Size() > 0) 81 snooze(10000); 82 83 fQuitting = true; 84 if (fWriterThread >= B_OK) { 85 int32 exitValue; 86 wait_for_thread(fWriterThread, &exitValue); 87 } 88 } 89 90 91 void 92 CopyEngine::ResetTargets(const char* source) 93 { 94 // TODO: One could subtract the bytes/items which were added to the 95 // ProgressReporter before resetting them... 96 97 fAbsoluteSourcePath = source; 98 99 fBytesRead = 0; 100 fLastBytesRead = 0; 101 fItemsCopied = 0; 102 fLastItemsCopied = 0; 103 fTimeRead = 0; 104 105 fBytesWritten = 0; 106 fTimeWritten = 0; 107 108 fCurrentTargetFolder = NULL; 109 fCurrentItem = NULL; 110 } 111 112 113 status_t 114 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 115 { 116 int32 level = 0; 117 off_t bytesToCopy = 0; 118 uint64 itemsToCopy = 0; 119 status_t ret = _CollectCopyInfo(source, level, cancelSemaphore, bytesToCopy, 120 itemsToCopy); 121 if (ret == B_OK && fProgressReporter != NULL) 122 fProgressReporter->AddItems(itemsToCopy, bytesToCopy); 123 return ret; 124 } 125 126 127 status_t 128 CopyEngine::Copy(const char* _source, const char* _destination, 129 sem_id cancelSemaphore, bool copyAttributes) 130 { 131 int32 level = 0; 132 status_t ret; 133 134 BEntry source(_source); 135 ret = source.InitCheck(); 136 if (ret != B_OK) 137 return ret; 138 139 BEntry destination(_destination); 140 ret = destination.InitCheck(); 141 if (ret != B_OK) 142 return ret; 143 144 return _Copy(source, destination, level, cancelSemaphore, copyAttributes); 145 } 146 147 148 status_t 149 CopyEngine::_CopyData(const BEntry& _source, const BEntry& _destination, 150 sem_id cancelSemaphore) 151 { 152 SemaphoreLocker lock(cancelSemaphore); 153 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 154 // We are supposed to quit 155 return B_CANCELED; 156 } 157 158 BFile source(&_source, B_READ_ONLY); 159 status_t ret = source.InitCheck(); 160 if (ret < B_OK) 161 return ret; 162 163 BFile* destination = new (nothrow) BFile(&_destination, 164 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 165 ret = destination->InitCheck(); 166 if (ret < B_OK) { 167 delete destination; 168 return ret; 169 } 170 171 int32 loopIteration = 0; 172 173 while (true) { 174 if (fBufferQueue.Size() >= BUFFER_COUNT) { 175 // the queue is "full", just wait a bit, the 176 // write thread will empty it 177 snooze(1000); 178 continue; 179 } 180 181 // allocate buffer 182 Buffer* buffer = new (nothrow) Buffer(destination); 183 if (!buffer || !buffer->buffer) { 184 delete destination; 185 delete buffer; 186 fprintf(stderr, "reading loop: out of memory\n"); 187 return B_NO_MEMORY; 188 } 189 190 // fill buffer 191 ssize_t read = source.Read(buffer->buffer, buffer->size); 192 if (read < 0) { 193 ret = (status_t)read; 194 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 195 delete buffer; 196 delete destination; 197 break; 198 } 199 200 fBytesRead += read; 201 loopIteration += 1; 202 if (loopIteration % 2 == 0) 203 _UpdateProgress(); 204 205 buffer->deleteFile = read == 0; 206 if (read > 0) 207 buffer->validBytes = (size_t)read; 208 else 209 buffer->validBytes = 0; 210 211 // enqueue the buffer 212 ret = fBufferQueue.Push(buffer); 213 if (ret < B_OK) { 214 buffer->deleteFile = false; 215 delete buffer; 216 delete destination; 217 return ret; 218 } 219 220 // quit if done 221 if (read == 0) 222 break; 223 } 224 225 return ret; 226 } 227 228 229 // #pragma mark - 230 231 232 status_t 233 CopyEngine::_CollectCopyInfo(const char* _source, int32& level, 234 sem_id cancelSemaphore, off_t& bytesToCopy, uint64& itemsToCopy) 235 { 236 level++; 237 238 BEntry source(_source); 239 status_t ret = source.InitCheck(); 240 if (ret < B_OK) 241 return ret; 242 243 struct stat statInfo; 244 ret = source.GetStat(&statInfo); 245 if (ret < B_OK) 246 return ret; 247 248 SemaphoreLocker lock(cancelSemaphore); 249 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 250 // We are supposed to quit 251 return B_CANCELED; 252 } 253 254 if (fEntryFilter != NULL 255 && !fEntryFilter->ShouldCopyEntry(source, 256 _RelativeEntryPath(_source), statInfo, level)) { 257 // Skip this entry 258 return B_OK; 259 } 260 261 if (cancelSemaphore >= 0) 262 lock.Unlock(); 263 264 if (S_ISDIR(statInfo.st_mode)) { 265 BDirectory srcFolder(&source); 266 ret = srcFolder.InitCheck(); 267 if (ret < B_OK) 268 return ret; 269 270 BEntry entry; 271 while (srcFolder.GetNextEntry(&entry) == B_OK) { 272 BPath entryPath; 273 ret = entry.GetPath(&entryPath); 274 if (ret < B_OK) 275 return ret; 276 277 ret = _CollectCopyInfo(entryPath.Path(), level, 278 cancelSemaphore, bytesToCopy, itemsToCopy); 279 if (ret < B_OK) 280 return ret; 281 } 282 } else if (S_ISLNK(statInfo.st_mode)) { 283 // link, ignore size 284 } else { 285 bytesToCopy += statInfo.st_size; 286 } 287 288 itemsToCopy++; 289 level--; 290 return B_OK; 291 } 292 293 294 status_t 295 CopyEngine::_Copy(BEntry &source, BEntry &destination, 296 int32& level, sem_id cancelSemaphore, bool copyAttributes) 297 { 298 level++; 299 300 struct stat sourceInfo; 301 status_t ret = source.GetStat(&sourceInfo); 302 if (ret != B_OK) 303 return ret; 304 305 SemaphoreLocker lock(cancelSemaphore); 306 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 307 // We are supposed to quit 308 return B_CANCELED; 309 } 310 311 BPath sourcePath(&source); 312 ret = sourcePath.InitCheck(); 313 if (ret != B_OK) 314 return ret; 315 316 BPath destPath(&destination); 317 ret = destPath.InitCheck(); 318 if (ret != B_OK) 319 return ret; 320 321 const char *relativeSourcePath = _RelativeEntryPath(sourcePath.Path()); 322 if (fEntryFilter != NULL 323 && !fEntryFilter->ShouldCopyEntry(source, relativeSourcePath, 324 sourceInfo, level)) { 325 // Silently skip the filtered entry. 326 return B_OK; 327 } 328 329 if (cancelSemaphore >= 0) 330 lock.Unlock(); 331 332 if (S_ISDIR(sourceInfo.st_mode)) { 333 BDirectory sourceDirectory(&source); 334 ret = sourceDirectory.InitCheck(); 335 if (ret != B_OK) 336 return ret; 337 338 if (destination.Exists()) { 339 if (destination.IsDirectory()) { 340 if (fEntryFilter 341 && fEntryFilter->ShouldClobberFolder(source, 342 relativeSourcePath, sourceInfo, level)) { 343 ret = _RemoveFolder(destination); 344 } else { 345 // Do not overwrite attributes on folders that exist. 346 // This should work better when the install target 347 // already contains a Haiku installation. 348 copyAttributes = false; 349 } 350 } else { 351 ret = destination.Remove(); 352 } 353 354 if (ret != B_OK) { 355 fprintf(stderr, "Failed to make room for folder '%s': " 356 "%s\n", sourcePath.Path(), strerror(ret)); 357 return ret; 358 } 359 } else { 360 ret = create_directory(destPath.Path(), 0777); 361 if (ret != B_OK && ret != B_FILE_EXISTS) { 362 fprintf(stderr, "Could not create '%s': %s\n", destPath.Path(), 363 strerror(ret)); 364 return ret; 365 } 366 } 367 368 BDirectory destDirectory(&destination); 369 ret = destDirectory.InitCheck(); 370 if (ret != B_OK) 371 return ret; 372 373 BEntry entry; 374 while (sourceDirectory.GetNextEntry(&entry) == B_OK) { 375 BEntry dest(&destDirectory, entry.Name()); 376 ret = dest.InitCheck(); 377 if (ret != B_OK) 378 return ret; 379 ret = _Copy(entry, dest, level, 380 cancelSemaphore, copyAttributes); 381 if (ret != B_OK) 382 return ret; 383 } 384 } else { 385 if (destination.Exists()) { 386 if (destination.IsDirectory()) 387 ret = _RemoveFolder(destination); 388 else 389 ret = destination.Remove(); 390 if (ret != B_OK) { 391 fprintf(stderr, "Failed to make room for entry '%s': " 392 "%s\n", sourcePath.Path(), strerror(ret)); 393 return ret; 394 } 395 } 396 397 fItemsCopied++; 398 BPath destDirectory; 399 ret = destPath.GetParent(&destDirectory); 400 if (ret != B_OK) 401 return ret; 402 fCurrentTargetFolder = destDirectory.Path(); 403 fCurrentItem = sourcePath.Leaf(); 404 _UpdateProgress(); 405 406 if (S_ISLNK(sourceInfo.st_mode)) { 407 // copy symbolic links 408 BSymLink srcLink(&source); 409 ret = srcLink.InitCheck(); 410 if (ret != B_OK) 411 return ret; 412 413 char linkPath[B_PATH_NAME_LENGTH]; 414 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 415 if (read < 0) 416 return (status_t)read; 417 418 BDirectory dstFolder; 419 ret = destination.GetParent(&dstFolder); 420 if (ret != B_OK) 421 return ret; 422 ret = dstFolder.CreateSymLink(sourcePath.Leaf(), linkPath, NULL); 423 if (ret != B_OK) 424 return ret; 425 } else { 426 // copy file data 427 // NOTE: Do not pass the locker, we simply keep holding the lock! 428 ret = _CopyData(source, destination); 429 if (ret != B_OK) 430 return ret; 431 } 432 } 433 434 if (copyAttributes) { 435 // copy attributes 436 BNode sourceNode(&source); 437 BNode targetNode(&destination); 438 char attrName[B_ATTR_NAME_LENGTH]; 439 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 440 attr_info info; 441 if (sourceNode.GetAttrInfo(attrName, &info) != B_OK) 442 continue; 443 size_t size = 4096; 444 uint8 buffer[size]; 445 off_t offset = 0; 446 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 447 offset, buffer, std::min((off_t)size, info.size)); 448 // NOTE: It's important to still write the attribute even if 449 // we have read 0 bytes! 450 while (read >= 0) { 451 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 452 offset += read; 453 read = sourceNode.ReadAttr(attrName, info.type, 454 offset, buffer, std::min((off_t)size, info.size - offset)); 455 if (read == 0) 456 break; 457 } 458 } 459 460 // copy basic attributes 461 destination.SetPermissions(sourceInfo.st_mode); 462 destination.SetOwner(sourceInfo.st_uid); 463 destination.SetGroup(sourceInfo.st_gid); 464 destination.SetModificationTime(sourceInfo.st_mtime); 465 destination.SetCreationTime(sourceInfo.st_crtime); 466 } 467 468 level--; 469 return B_OK; 470 } 471 472 473 status_t 474 CopyEngine::_RemoveFolder(BEntry& entry) 475 { 476 BDirectory directory(&entry); 477 status_t ret = directory.InitCheck(); 478 if (ret != B_OK) 479 return ret; 480 481 BEntry subEntry; 482 while (directory.GetNextEntry(&subEntry) == B_OK) { 483 if (subEntry.IsDirectory()) { 484 ret = _RemoveFolder(subEntry); 485 if (ret != B_OK) 486 return ret; 487 } else { 488 ret = subEntry.Remove(); 489 if (ret != B_OK) 490 return ret; 491 } 492 } 493 return entry.Remove(); 494 } 495 496 497 const char* 498 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 499 { 500 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 501 fAbsoluteSourcePath.Length()) != 0) { 502 return absoluteSourcePath; 503 } 504 505 const char* relativePath 506 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 507 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 508 } 509 510 511 void 512 CopyEngine::_UpdateProgress() 513 { 514 if (fProgressReporter == NULL) 515 return; 516 517 uint64 items = 0; 518 if (fLastItemsCopied < fItemsCopied) { 519 items = fItemsCopied - fLastItemsCopied; 520 fLastItemsCopied = fItemsCopied; 521 } 522 523 off_t bytes = 0; 524 if (fLastBytesRead < fBytesRead) { 525 bytes = fBytesRead - fLastBytesRead; 526 fLastBytesRead = fBytesRead; 527 } 528 529 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 530 fCurrentTargetFolder); 531 } 532 533 534 int32 535 CopyEngine::_WriteThreadEntry(void* cookie) 536 { 537 CopyEngine* engine = (CopyEngine*)cookie; 538 engine->_WriteThread(); 539 return B_OK; 540 } 541 542 543 void 544 CopyEngine::_WriteThread() 545 { 546 bigtime_t bufferWaitTimeout = 100000; 547 548 while (!fQuitting) { 549 550 bigtime_t now = system_time(); 551 552 Buffer* buffer = NULL; 553 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 554 if (ret == B_TIMED_OUT) { 555 // no buffer, timeout 556 continue; 557 } else if (ret == B_NO_INIT) { 558 // real error 559 return; 560 } else if (ret != B_OK) { 561 // no buffer, queue error 562 snooze(10000); 563 continue; 564 } 565 566 if (!buffer->deleteFile) { 567 ssize_t written = buffer->file->Write(buffer->buffer, 568 buffer->validBytes); 569 if (written != (ssize_t)buffer->validBytes) { 570 // TODO: this should somehow be propagated back 571 // to the main thread! 572 fprintf(stderr, "Failed to write data: %s\n", 573 strerror((status_t)written)); 574 } 575 fBytesWritten += written; 576 } 577 578 delete buffer; 579 580 // measure performance 581 fTimeWritten += system_time() - now; 582 } 583 584 double megaBytes = (double)fBytesWritten / (1024 * 1024); 585 double seconds = (double)fTimeWritten / 1000000; 586 if (seconds > 0) { 587 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 588 megaBytes / seconds); 589 } 590 } 591