1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * Copyright 2014, Axel Dörfler, axeld@pinc-software.de 4 * All rights reserved. Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "CopyEngine.h" 9 10 #include <new> 11 12 #include <math.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <sys/resource.h> 16 17 #include <Directory.h> 18 #include <fs_attr.h> 19 #include <NodeInfo.h> 20 #include <Path.h> 21 #include <SymLink.h> 22 23 #include "SemaphoreLocker.h" 24 #include "ProgressReporter.h" 25 26 27 using std::nothrow; 28 29 30 // #pragma mark - EntryFilter 31 32 33 CopyEngine::EntryFilter::~EntryFilter() 34 { 35 } 36 37 38 // #pragma mark - CopyEngine 39 40 41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 42 : 43 fBufferQueue(), 44 fWriterThread(-1), 45 fQuitting(false), 46 47 fAbsoluteSourcePath(), 48 49 fBytesRead(0), 50 fLastBytesRead(0), 51 fItemsCopied(0), 52 fLastItemsCopied(0), 53 fTimeRead(0), 54 55 fBytesWritten(0), 56 fTimeWritten(0), 57 58 fCurrentTargetFolder(NULL), 59 fCurrentItem(NULL), 60 61 fProgressReporter(reporter), 62 fEntryFilter(entryFilter) 63 { 64 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 65 B_NORMAL_PRIORITY, this); 66 67 if (fWriterThread >= B_OK) 68 resume_thread(fWriterThread); 69 70 // ask for a bunch more file descriptors so that nested copying works well 71 struct rlimit rl; 72 rl.rlim_cur = 512; 73 rl.rlim_max = RLIM_SAVED_MAX; 74 setrlimit(RLIMIT_NOFILE, &rl); 75 } 76 77 78 CopyEngine::~CopyEngine() 79 { 80 while (fBufferQueue.Size() > 0) 81 snooze(10000); 82 83 fQuitting = true; 84 if (fWriterThread >= B_OK) { 85 int32 exitValue; 86 wait_for_thread(fWriterThread, &exitValue); 87 } 88 } 89 90 91 void 92 CopyEngine::ResetTargets(const char* source) 93 { 94 // TODO: One could subtract the bytes/items which were added to the 95 // ProgressReporter before resetting them... 96 97 fAbsoluteSourcePath = source; 98 99 fBytesRead = 0; 100 fLastBytesRead = 0; 101 fItemsCopied = 0; 102 fLastItemsCopied = 0; 103 fTimeRead = 0; 104 105 fBytesWritten = 0; 106 fTimeWritten = 0; 107 108 fCurrentTargetFolder = NULL; 109 fCurrentItem = NULL; 110 } 111 112 113 status_t 114 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 115 { 116 off_t bytesToCopy = 0; 117 uint64 itemsToCopy = 0; 118 status_t ret = _CollectCopyInfo(source, cancelSemaphore, bytesToCopy, 119 itemsToCopy); 120 if (ret == B_OK && fProgressReporter != NULL) 121 fProgressReporter->AddItems(itemsToCopy, bytesToCopy); 122 return ret; 123 } 124 125 126 status_t 127 CopyEngine::Copy(const char* _source, const char* _destination, 128 sem_id cancelSemaphore, bool copyAttributes) 129 { 130 status_t ret; 131 132 BEntry source(_source); 133 ret = source.InitCheck(); 134 if (ret != B_OK) 135 return ret; 136 137 BEntry destination(_destination); 138 ret = destination.InitCheck(); 139 if (ret != B_OK) 140 return ret; 141 142 return _Copy(source, destination, cancelSemaphore, copyAttributes); 143 } 144 145 146 status_t 147 CopyEngine::RemoveFolder(BEntry& entry) 148 { 149 BDirectory directory(&entry); 150 status_t ret = directory.InitCheck(); 151 if (ret != B_OK) 152 return ret; 153 154 BEntry subEntry; 155 while (directory.GetNextEntry(&subEntry) == B_OK) { 156 if (subEntry.IsDirectory()) { 157 ret = CopyEngine::RemoveFolder(subEntry); 158 if (ret != B_OK) 159 return ret; 160 } else { 161 ret = subEntry.Remove(); 162 if (ret != B_OK) 163 return ret; 164 } 165 } 166 return entry.Remove(); 167 } 168 169 170 status_t 171 CopyEngine::_CopyData(const BEntry& _source, const BEntry& _destination, 172 sem_id cancelSemaphore) 173 { 174 SemaphoreLocker lock(cancelSemaphore); 175 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 176 // We are supposed to quit 177 return B_CANCELED; 178 } 179 180 BFile source(&_source, B_READ_ONLY); 181 status_t ret = source.InitCheck(); 182 if (ret < B_OK) 183 return ret; 184 185 BFile* destination = new (nothrow) BFile(&_destination, 186 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 187 ret = destination->InitCheck(); 188 if (ret < B_OK) { 189 delete destination; 190 return ret; 191 } 192 193 int32 loopIteration = 0; 194 195 while (true) { 196 if (fBufferQueue.Size() >= BUFFER_COUNT) { 197 // the queue is "full", just wait a bit, the 198 // write thread will empty it 199 snooze(1000); 200 continue; 201 } 202 203 // allocate buffer 204 Buffer* buffer = new (nothrow) Buffer(destination); 205 if (!buffer || !buffer->buffer) { 206 delete destination; 207 delete buffer; 208 fprintf(stderr, "reading loop: out of memory\n"); 209 return B_NO_MEMORY; 210 } 211 212 // fill buffer 213 ssize_t read = source.Read(buffer->buffer, buffer->size); 214 if (read < 0) { 215 ret = (status_t)read; 216 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 217 delete buffer; 218 delete destination; 219 break; 220 } 221 222 fBytesRead += read; 223 loopIteration += 1; 224 if (loopIteration % 2 == 0) 225 _UpdateProgress(); 226 227 buffer->deleteFile = read == 0; 228 if (read > 0) 229 buffer->validBytes = (size_t)read; 230 else 231 buffer->validBytes = 0; 232 233 // enqueue the buffer 234 ret = fBufferQueue.Push(buffer); 235 if (ret < B_OK) { 236 buffer->deleteFile = false; 237 delete buffer; 238 delete destination; 239 return ret; 240 } 241 242 // quit if done 243 if (read == 0) 244 break; 245 } 246 247 return ret; 248 } 249 250 251 // #pragma mark - 252 253 254 status_t 255 CopyEngine::_CollectCopyInfo(const char* _source, sem_id cancelSemaphore, 256 off_t& bytesToCopy, uint64& itemsToCopy) 257 { 258 BEntry source(_source); 259 status_t ret = source.InitCheck(); 260 if (ret < B_OK) 261 return ret; 262 263 struct stat statInfo; 264 ret = source.GetStat(&statInfo); 265 if (ret < B_OK) 266 return ret; 267 268 SemaphoreLocker lock(cancelSemaphore); 269 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 270 // We are supposed to quit 271 return B_CANCELED; 272 } 273 274 if (fEntryFilter != NULL 275 && !fEntryFilter->ShouldCopyEntry(source, 276 _RelativeEntryPath(_source), statInfo)) { 277 // Skip this entry 278 return B_OK; 279 } 280 281 if (cancelSemaphore >= 0) 282 lock.Unlock(); 283 284 if (S_ISDIR(statInfo.st_mode)) { 285 BDirectory srcFolder(&source); 286 ret = srcFolder.InitCheck(); 287 if (ret < B_OK) 288 return ret; 289 290 BEntry entry; 291 while (srcFolder.GetNextEntry(&entry) == B_OK) { 292 BPath entryPath; 293 ret = entry.GetPath(&entryPath); 294 if (ret < B_OK) 295 return ret; 296 297 ret = _CollectCopyInfo(entryPath.Path(), cancelSemaphore, 298 bytesToCopy, itemsToCopy); 299 if (ret < B_OK) 300 return ret; 301 } 302 } else if (S_ISLNK(statInfo.st_mode)) { 303 // link, ignore size 304 } else { 305 bytesToCopy += statInfo.st_size; 306 } 307 308 itemsToCopy++; 309 return B_OK; 310 } 311 312 313 status_t 314 CopyEngine::_Copy(BEntry &source, BEntry &destination, 315 sem_id cancelSemaphore, bool copyAttributes) 316 { 317 struct stat sourceInfo; 318 status_t ret = source.GetStat(&sourceInfo); 319 if (ret != B_OK) 320 return ret; 321 322 SemaphoreLocker lock(cancelSemaphore); 323 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 324 // We are supposed to quit 325 return B_CANCELED; 326 } 327 328 BPath sourcePath(&source); 329 ret = sourcePath.InitCheck(); 330 if (ret != B_OK) 331 return ret; 332 333 BPath destPath(&destination); 334 ret = destPath.InitCheck(); 335 if (ret != B_OK) 336 return ret; 337 338 const char *relativeSourcePath = _RelativeEntryPath(sourcePath.Path()); 339 if (fEntryFilter != NULL 340 && !fEntryFilter->ShouldCopyEntry(source, relativeSourcePath, 341 sourceInfo)) { 342 // Silently skip the filtered entry. 343 return B_OK; 344 } 345 346 if (cancelSemaphore >= 0) 347 lock.Unlock(); 348 349 bool copyAttributesToTarget = copyAttributes; 350 // attributes of the current source to the destination will be copied 351 // when copyAttributes is set to true, but there may be exceptions, so 352 // allow the recursively used copyAttribute parameter to be overridden 353 // for the current target. 354 if (S_ISDIR(sourceInfo.st_mode)) { 355 BDirectory sourceDirectory(&source); 356 ret = sourceDirectory.InitCheck(); 357 if (ret != B_OK) 358 return ret; 359 360 if (destination.Exists()) { 361 if (destination.IsDirectory()) { 362 // Do not overwrite attributes on folders that exist. 363 // This should work better when the install target 364 // already contains a Haiku installation. 365 copyAttributesToTarget = false; 366 } else { 367 ret = destination.Remove(); 368 } 369 370 if (ret != B_OK) { 371 fprintf(stderr, "Failed to make room for folder '%s': " 372 "%s\n", sourcePath.Path(), strerror(ret)); 373 return ret; 374 } 375 } 376 377 ret = create_directory(destPath.Path(), 0777); 378 // Make sure the target path exists, it may have been deleted if 379 // the existing destination was a file instead of a directory. 380 if (ret != B_OK && ret != B_FILE_EXISTS) { 381 fprintf(stderr, "Could not create '%s': %s\n", destPath.Path(), 382 strerror(ret)); 383 return ret; 384 } 385 386 BDirectory destDirectory(&destination); 387 ret = destDirectory.InitCheck(); 388 if (ret != B_OK) 389 return ret; 390 391 BEntry entry; 392 while (sourceDirectory.GetNextEntry(&entry) == B_OK) { 393 BEntry dest(&destDirectory, entry.Name()); 394 ret = dest.InitCheck(); 395 if (ret != B_OK) 396 return ret; 397 ret = _Copy(entry, dest, cancelSemaphore, copyAttributes); 398 if (ret != B_OK) 399 return ret; 400 } 401 } else { 402 if (destination.Exists()) { 403 if (destination.IsDirectory()) 404 ret = CopyEngine::RemoveFolder(destination); 405 else 406 ret = destination.Remove(); 407 if (ret != B_OK) { 408 fprintf(stderr, "Failed to make room for entry '%s': " 409 "%s\n", sourcePath.Path(), strerror(ret)); 410 return ret; 411 } 412 } 413 414 fItemsCopied++; 415 BPath destDirectory; 416 ret = destPath.GetParent(&destDirectory); 417 if (ret != B_OK) 418 return ret; 419 fCurrentTargetFolder = destDirectory.Path(); 420 fCurrentItem = sourcePath.Leaf(); 421 _UpdateProgress(); 422 423 if (S_ISLNK(sourceInfo.st_mode)) { 424 // copy symbolic links 425 BSymLink srcLink(&source); 426 ret = srcLink.InitCheck(); 427 if (ret != B_OK) 428 return ret; 429 430 char linkPath[B_PATH_NAME_LENGTH]; 431 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 432 if (read < 0) 433 return (status_t)read; 434 435 BDirectory dstFolder; 436 ret = destination.GetParent(&dstFolder); 437 if (ret != B_OK) 438 return ret; 439 ret = dstFolder.CreateSymLink(sourcePath.Leaf(), linkPath, NULL); 440 if (ret != B_OK) 441 return ret; 442 } else { 443 // copy file data 444 // NOTE: Do not pass the locker, we simply keep holding the lock! 445 ret = _CopyData(source, destination); 446 if (ret != B_OK) 447 return ret; 448 } 449 } 450 451 if (copyAttributesToTarget) { 452 // copy attributes to the current target 453 BNode sourceNode(&source); 454 BNode targetNode(&destination); 455 char attrName[B_ATTR_NAME_LENGTH]; 456 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 457 attr_info info; 458 if (sourceNode.GetAttrInfo(attrName, &info) != B_OK) 459 continue; 460 size_t size = 4096; 461 uint8 buffer[size]; 462 off_t offset = 0; 463 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 464 offset, buffer, std::min((off_t)size, info.size)); 465 // NOTE: It's important to still write the attribute even if 466 // we have read 0 bytes! 467 while (read >= 0) { 468 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 469 offset += read; 470 read = sourceNode.ReadAttr(attrName, info.type, 471 offset, buffer, std::min((off_t)size, info.size - offset)); 472 if (read == 0) 473 break; 474 } 475 } 476 477 // copy basic attributes 478 destination.SetPermissions(sourceInfo.st_mode); 479 destination.SetOwner(sourceInfo.st_uid); 480 destination.SetGroup(sourceInfo.st_gid); 481 destination.SetModificationTime(sourceInfo.st_mtime); 482 destination.SetCreationTime(sourceInfo.st_crtime); 483 } 484 485 return B_OK; 486 } 487 488 489 const char* 490 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 491 { 492 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 493 fAbsoluteSourcePath.Length()) != 0) { 494 return absoluteSourcePath; 495 } 496 497 const char* relativePath 498 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 499 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 500 } 501 502 503 void 504 CopyEngine::_UpdateProgress() 505 { 506 if (fProgressReporter == NULL) 507 return; 508 509 uint64 items = 0; 510 if (fLastItemsCopied < fItemsCopied) { 511 items = fItemsCopied - fLastItemsCopied; 512 fLastItemsCopied = fItemsCopied; 513 } 514 515 off_t bytes = 0; 516 if (fLastBytesRead < fBytesRead) { 517 bytes = fBytesRead - fLastBytesRead; 518 fLastBytesRead = fBytesRead; 519 } 520 521 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 522 fCurrentTargetFolder); 523 } 524 525 526 int32 527 CopyEngine::_WriteThreadEntry(void* cookie) 528 { 529 CopyEngine* engine = (CopyEngine*)cookie; 530 engine->_WriteThread(); 531 return B_OK; 532 } 533 534 535 void 536 CopyEngine::_WriteThread() 537 { 538 bigtime_t bufferWaitTimeout = 100000; 539 540 while (!fQuitting) { 541 542 bigtime_t now = system_time(); 543 544 Buffer* buffer = NULL; 545 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 546 if (ret == B_TIMED_OUT) { 547 // no buffer, timeout 548 continue; 549 } else if (ret == B_NO_INIT) { 550 // real error 551 return; 552 } else if (ret != B_OK) { 553 // no buffer, queue error 554 snooze(10000); 555 continue; 556 } 557 558 if (!buffer->deleteFile) { 559 ssize_t written = buffer->file->Write(buffer->buffer, 560 buffer->validBytes); 561 if (written != (ssize_t)buffer->validBytes) { 562 // TODO: this should somehow be propagated back 563 // to the main thread! 564 fprintf(stderr, "Failed to write data: %s\n", 565 strerror((status_t)written)); 566 } 567 fBytesWritten += written; 568 } 569 570 delete buffer; 571 572 // measure performance 573 fTimeWritten += system_time() - now; 574 } 575 576 double megaBytes = (double)fBytesWritten / (1024 * 1024); 577 double seconds = (double)fTimeWritten / 1000000; 578 if (seconds > 0) { 579 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 580 megaBytes / seconds); 581 } 582 } 583