1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * Copyright 2014, Axel Dörfler, axeld@pinc-software.de 4 * All rights reserved. Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "CopyEngine.h" 9 10 #include <new> 11 12 #include <math.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <sys/resource.h> 16 17 #include <Directory.h> 18 #include <fs_attr.h> 19 #include <NodeInfo.h> 20 #include <Path.h> 21 #include <SymLink.h> 22 23 #include "SemaphoreLocker.h" 24 #include "ProgressReporter.h" 25 26 27 using std::nothrow; 28 29 30 // #pragma mark - EntryFilter 31 32 33 CopyEngine::EntryFilter::~EntryFilter() 34 { 35 } 36 37 38 // #pragma mark - CopyEngine 39 40 41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 42 : 43 fBufferQueue(), 44 fWriterThread(-1), 45 fQuitting(false), 46 47 fAbsoluteSourcePath(), 48 49 fBytesRead(0), 50 fLastBytesRead(0), 51 fItemsCopied(0), 52 fLastItemsCopied(0), 53 fTimeRead(0), 54 55 fBytesWritten(0), 56 fTimeWritten(0), 57 58 fCurrentTargetFolder(NULL), 59 fCurrentItem(NULL), 60 61 fProgressReporter(reporter), 62 fEntryFilter(entryFilter) 63 { 64 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 65 B_NORMAL_PRIORITY, this); 66 67 if (fWriterThread >= B_OK) 68 resume_thread(fWriterThread); 69 70 // ask for a bunch more file descriptors so that nested copying works well 71 struct rlimit rl; 72 rl.rlim_cur = 512; 73 rl.rlim_max = RLIM_SAVED_MAX; 74 setrlimit(RLIMIT_NOFILE, &rl); 75 } 76 77 78 CopyEngine::~CopyEngine() 79 { 80 while (fBufferQueue.Size() > 0) 81 snooze(10000); 82 83 fQuitting = true; 84 if (fWriterThread >= B_OK) { 85 int32 exitValue; 86 wait_for_thread(fWriterThread, &exitValue); 87 } 88 } 89 90 91 void 92 CopyEngine::ResetTargets(const char* source) 93 { 94 // TODO: One could subtract the bytes/items which were added to the 95 // ProgressReporter before resetting them... 96 97 fAbsoluteSourcePath = source; 98 99 fBytesRead = 0; 100 fLastBytesRead = 0; 101 fItemsCopied = 0; 102 fLastItemsCopied = 0; 103 fTimeRead = 0; 104 105 fBytesWritten = 0; 106 fTimeWritten = 0; 107 108 fCurrentTargetFolder = NULL; 109 fCurrentItem = NULL; 110 } 111 112 113 status_t 114 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 115 { 116 off_t bytesToCopy = 0; 117 uint64 itemsToCopy = 0; 118 status_t ret = _CollectCopyInfo(source, cancelSemaphore, bytesToCopy, 119 itemsToCopy); 120 if (ret == B_OK && fProgressReporter != NULL) 121 fProgressReporter->AddItems(itemsToCopy, bytesToCopy); 122 return ret; 123 } 124 125 126 status_t 127 CopyEngine::Copy(const char* _source, const char* _destination, 128 sem_id cancelSemaphore, bool copyAttributes) 129 { 130 status_t ret; 131 132 BEntry source(_source); 133 ret = source.InitCheck(); 134 if (ret != B_OK) 135 return ret; 136 137 BEntry destination(_destination); 138 ret = destination.InitCheck(); 139 if (ret != B_OK) 140 return ret; 141 142 return _Copy(source, destination, cancelSemaphore, copyAttributes); 143 } 144 145 146 status_t 147 CopyEngine::_CopyData(const BEntry& _source, const BEntry& _destination, 148 sem_id cancelSemaphore) 149 { 150 SemaphoreLocker lock(cancelSemaphore); 151 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 152 // We are supposed to quit 153 return B_CANCELED; 154 } 155 156 BFile source(&_source, B_READ_ONLY); 157 status_t ret = source.InitCheck(); 158 if (ret < B_OK) 159 return ret; 160 161 BFile* destination = new (nothrow) BFile(&_destination, 162 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 163 ret = destination->InitCheck(); 164 if (ret < B_OK) { 165 delete destination; 166 return ret; 167 } 168 169 int32 loopIteration = 0; 170 171 while (true) { 172 if (fBufferQueue.Size() >= BUFFER_COUNT) { 173 // the queue is "full", just wait a bit, the 174 // write thread will empty it 175 snooze(1000); 176 continue; 177 } 178 179 // allocate buffer 180 Buffer* buffer = new (nothrow) Buffer(destination); 181 if (!buffer || !buffer->buffer) { 182 delete destination; 183 delete buffer; 184 fprintf(stderr, "reading loop: out of memory\n"); 185 return B_NO_MEMORY; 186 } 187 188 // fill buffer 189 ssize_t read = source.Read(buffer->buffer, buffer->size); 190 if (read < 0) { 191 ret = (status_t)read; 192 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 193 delete buffer; 194 delete destination; 195 break; 196 } 197 198 fBytesRead += read; 199 loopIteration += 1; 200 if (loopIteration % 2 == 0) 201 _UpdateProgress(); 202 203 buffer->deleteFile = read == 0; 204 if (read > 0) 205 buffer->validBytes = (size_t)read; 206 else 207 buffer->validBytes = 0; 208 209 // enqueue the buffer 210 ret = fBufferQueue.Push(buffer); 211 if (ret < B_OK) { 212 buffer->deleteFile = false; 213 delete buffer; 214 delete destination; 215 return ret; 216 } 217 218 // quit if done 219 if (read == 0) 220 break; 221 } 222 223 return ret; 224 } 225 226 227 // #pragma mark - 228 229 230 status_t 231 CopyEngine::_CollectCopyInfo(const char* _source, sem_id cancelSemaphore, 232 off_t& bytesToCopy, uint64& itemsToCopy) 233 { 234 BEntry source(_source); 235 status_t ret = source.InitCheck(); 236 if (ret < B_OK) 237 return ret; 238 239 struct stat statInfo; 240 ret = source.GetStat(&statInfo); 241 if (ret < B_OK) 242 return ret; 243 244 SemaphoreLocker lock(cancelSemaphore); 245 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 246 // We are supposed to quit 247 return B_CANCELED; 248 } 249 250 if (fEntryFilter != NULL 251 && !fEntryFilter->ShouldCopyEntry(source, 252 _RelativeEntryPath(_source), statInfo)) { 253 // Skip this entry 254 return B_OK; 255 } 256 257 if (cancelSemaphore >= 0) 258 lock.Unlock(); 259 260 if (S_ISDIR(statInfo.st_mode)) { 261 BDirectory srcFolder(&source); 262 ret = srcFolder.InitCheck(); 263 if (ret < B_OK) 264 return ret; 265 266 BEntry entry; 267 while (srcFolder.GetNextEntry(&entry) == B_OK) { 268 BPath entryPath; 269 ret = entry.GetPath(&entryPath); 270 if (ret < B_OK) 271 return ret; 272 273 ret = _CollectCopyInfo(entryPath.Path(), cancelSemaphore, 274 bytesToCopy, itemsToCopy); 275 if (ret < B_OK) 276 return ret; 277 } 278 } else if (S_ISLNK(statInfo.st_mode)) { 279 // link, ignore size 280 } else { 281 bytesToCopy += statInfo.st_size; 282 } 283 284 itemsToCopy++; 285 return B_OK; 286 } 287 288 289 status_t 290 CopyEngine::_Copy(BEntry &source, BEntry &destination, 291 sem_id cancelSemaphore, bool copyAttributes) 292 { 293 struct stat sourceInfo; 294 status_t ret = source.GetStat(&sourceInfo); 295 if (ret != B_OK) 296 return ret; 297 298 SemaphoreLocker lock(cancelSemaphore); 299 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 300 // We are supposed to quit 301 return B_CANCELED; 302 } 303 304 BPath sourcePath(&source); 305 ret = sourcePath.InitCheck(); 306 if (ret != B_OK) 307 return ret; 308 309 BPath destPath(&destination); 310 ret = destPath.InitCheck(); 311 if (ret != B_OK) 312 return ret; 313 314 const char *relativeSourcePath = _RelativeEntryPath(sourcePath.Path()); 315 if (fEntryFilter != NULL 316 && !fEntryFilter->ShouldCopyEntry(source, relativeSourcePath, 317 sourceInfo)) { 318 // Silently skip the filtered entry. 319 return B_OK; 320 } 321 322 if (cancelSemaphore >= 0) 323 lock.Unlock(); 324 325 bool copyAttributesToTarget = copyAttributes; 326 // attributes of the current source to the destination will be copied 327 // when copyAttributes is set to true, but there may be exceptions, so 328 // allow the recursively used copyAttribute parameter to be overridden 329 // for the current target. 330 if (S_ISDIR(sourceInfo.st_mode)) { 331 BDirectory sourceDirectory(&source); 332 ret = sourceDirectory.InitCheck(); 333 if (ret != B_OK) 334 return ret; 335 336 if (destination.Exists()) { 337 if (destination.IsDirectory()) { 338 if (fEntryFilter 339 && fEntryFilter->ShouldClobberFolder(source, 340 relativeSourcePath, sourceInfo)) { 341 ret = _RemoveFolder(destination); 342 } else { 343 // Do not overwrite attributes on folders that exist. 344 // This should work better when the install target 345 // already contains a Haiku installation. 346 copyAttributesToTarget = false; 347 } 348 } else { 349 ret = destination.Remove(); 350 } 351 352 if (ret != B_OK) { 353 fprintf(stderr, "Failed to make room for folder '%s': " 354 "%s\n", sourcePath.Path(), strerror(ret)); 355 return ret; 356 } 357 } else { 358 ret = create_directory(destPath.Path(), 0777); 359 if (ret != B_OK && ret != B_FILE_EXISTS) { 360 fprintf(stderr, "Could not create '%s': %s\n", destPath.Path(), 361 strerror(ret)); 362 return ret; 363 } 364 } 365 366 BDirectory destDirectory(&destination); 367 ret = destDirectory.InitCheck(); 368 if (ret != B_OK) 369 return ret; 370 371 BEntry entry; 372 while (sourceDirectory.GetNextEntry(&entry) == B_OK) { 373 BEntry dest(&destDirectory, entry.Name()); 374 ret = dest.InitCheck(); 375 if (ret != B_OK) 376 return ret; 377 ret = _Copy(entry, dest, cancelSemaphore, copyAttributes); 378 if (ret != B_OK) 379 return ret; 380 } 381 } else { 382 if (destination.Exists()) { 383 if (destination.IsDirectory()) 384 ret = _RemoveFolder(destination); 385 else 386 ret = destination.Remove(); 387 if (ret != B_OK) { 388 fprintf(stderr, "Failed to make room for entry '%s': " 389 "%s\n", sourcePath.Path(), strerror(ret)); 390 return ret; 391 } 392 } 393 394 fItemsCopied++; 395 BPath destDirectory; 396 ret = destPath.GetParent(&destDirectory); 397 if (ret != B_OK) 398 return ret; 399 fCurrentTargetFolder = destDirectory.Path(); 400 fCurrentItem = sourcePath.Leaf(); 401 _UpdateProgress(); 402 403 if (S_ISLNK(sourceInfo.st_mode)) { 404 // copy symbolic links 405 BSymLink srcLink(&source); 406 ret = srcLink.InitCheck(); 407 if (ret != B_OK) 408 return ret; 409 410 char linkPath[B_PATH_NAME_LENGTH]; 411 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 412 if (read < 0) 413 return (status_t)read; 414 415 BDirectory dstFolder; 416 ret = destination.GetParent(&dstFolder); 417 if (ret != B_OK) 418 return ret; 419 ret = dstFolder.CreateSymLink(sourcePath.Leaf(), linkPath, NULL); 420 if (ret != B_OK) 421 return ret; 422 } else { 423 // copy file data 424 // NOTE: Do not pass the locker, we simply keep holding the lock! 425 ret = _CopyData(source, destination); 426 if (ret != B_OK) 427 return ret; 428 } 429 } 430 431 if (copyAttributesToTarget) { 432 // copy attributes to the current target 433 BNode sourceNode(&source); 434 BNode targetNode(&destination); 435 char attrName[B_ATTR_NAME_LENGTH]; 436 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 437 attr_info info; 438 if (sourceNode.GetAttrInfo(attrName, &info) != B_OK) 439 continue; 440 size_t size = 4096; 441 uint8 buffer[size]; 442 off_t offset = 0; 443 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 444 offset, buffer, std::min((off_t)size, info.size)); 445 // NOTE: It's important to still write the attribute even if 446 // we have read 0 bytes! 447 while (read >= 0) { 448 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 449 offset += read; 450 read = sourceNode.ReadAttr(attrName, info.type, 451 offset, buffer, std::min((off_t)size, info.size - offset)); 452 if (read == 0) 453 break; 454 } 455 } 456 457 // copy basic attributes 458 destination.SetPermissions(sourceInfo.st_mode); 459 destination.SetOwner(sourceInfo.st_uid); 460 destination.SetGroup(sourceInfo.st_gid); 461 destination.SetModificationTime(sourceInfo.st_mtime); 462 destination.SetCreationTime(sourceInfo.st_crtime); 463 } 464 465 return B_OK; 466 } 467 468 469 status_t 470 CopyEngine::_RemoveFolder(BEntry& entry) 471 { 472 BDirectory directory(&entry); 473 status_t ret = directory.InitCheck(); 474 if (ret != B_OK) 475 return ret; 476 477 BEntry subEntry; 478 while (directory.GetNextEntry(&subEntry) == B_OK) { 479 if (subEntry.IsDirectory()) { 480 ret = _RemoveFolder(subEntry); 481 if (ret != B_OK) 482 return ret; 483 } else { 484 ret = subEntry.Remove(); 485 if (ret != B_OK) 486 return ret; 487 } 488 } 489 return entry.Remove(); 490 } 491 492 493 const char* 494 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 495 { 496 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 497 fAbsoluteSourcePath.Length()) != 0) { 498 return absoluteSourcePath; 499 } 500 501 const char* relativePath 502 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 503 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 504 } 505 506 507 void 508 CopyEngine::_UpdateProgress() 509 { 510 if (fProgressReporter == NULL) 511 return; 512 513 uint64 items = 0; 514 if (fLastItemsCopied < fItemsCopied) { 515 items = fItemsCopied - fLastItemsCopied; 516 fLastItemsCopied = fItemsCopied; 517 } 518 519 off_t bytes = 0; 520 if (fLastBytesRead < fBytesRead) { 521 bytes = fBytesRead - fLastBytesRead; 522 fLastBytesRead = fBytesRead; 523 } 524 525 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 526 fCurrentTargetFolder); 527 } 528 529 530 int32 531 CopyEngine::_WriteThreadEntry(void* cookie) 532 { 533 CopyEngine* engine = (CopyEngine*)cookie; 534 engine->_WriteThread(); 535 return B_OK; 536 } 537 538 539 void 540 CopyEngine::_WriteThread() 541 { 542 bigtime_t bufferWaitTimeout = 100000; 543 544 while (!fQuitting) { 545 546 bigtime_t now = system_time(); 547 548 Buffer* buffer = NULL; 549 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 550 if (ret == B_TIMED_OUT) { 551 // no buffer, timeout 552 continue; 553 } else if (ret == B_NO_INIT) { 554 // real error 555 return; 556 } else if (ret != B_OK) { 557 // no buffer, queue error 558 snooze(10000); 559 continue; 560 } 561 562 if (!buffer->deleteFile) { 563 ssize_t written = buffer->file->Write(buffer->buffer, 564 buffer->validBytes); 565 if (written != (ssize_t)buffer->validBytes) { 566 // TODO: this should somehow be propagated back 567 // to the main thread! 568 fprintf(stderr, "Failed to write data: %s\n", 569 strerror((status_t)written)); 570 } 571 fBytesWritten += written; 572 } 573 574 delete buffer; 575 576 // measure performance 577 fTimeWritten += system_time() - now; 578 } 579 580 double megaBytes = (double)fBytesWritten / (1024 * 1024); 581 double seconds = (double)fTimeWritten / 1000000; 582 if (seconds > 0) { 583 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 584 megaBytes / seconds); 585 } 586 } 587