1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * Copyright 2014, Axel Dörfler, axeld@pinc-software.de 4 * All rights reserved. Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "CopyEngine.h" 9 10 #include <new> 11 12 #include <math.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <sys/resource.h> 16 17 #include <Directory.h> 18 #include <fs_attr.h> 19 #include <NodeInfo.h> 20 #include <Path.h> 21 #include <SymLink.h> 22 23 #include "SemaphoreLocker.h" 24 #include "ProgressReporter.h" 25 26 27 using std::nothrow; 28 29 30 // #pragma mark - EntryFilter 31 32 33 CopyEngine::EntryFilter::~EntryFilter() 34 { 35 } 36 37 38 // #pragma mark - CopyEngine 39 40 41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 42 : 43 fBufferQueue(), 44 fWriterThread(-1), 45 fQuitting(false), 46 47 fAbsoluteSourcePath(), 48 49 fBytesRead(0), 50 fLastBytesRead(0), 51 fItemsCopied(0), 52 fLastItemsCopied(0), 53 fTimeRead(0), 54 55 fBytesWritten(0), 56 fTimeWritten(0), 57 58 fBytesToCopy(0), 59 fItemsToCopy(0), 60 61 fCurrentTargetFolder(NULL), 62 fCurrentItem(NULL), 63 64 fProgressReporter(reporter), 65 fEntryFilter(entryFilter) 66 { 67 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 68 B_NORMAL_PRIORITY, this); 69 70 if (fWriterThread >= B_OK) 71 resume_thread(fWriterThread); 72 73 // ask for a bunch more file descriptors so that nested copying works well 74 struct rlimit rl; 75 rl.rlim_cur = 512; 76 rl.rlim_max = RLIM_SAVED_MAX; 77 setrlimit(RLIMIT_NOFILE, &rl); 78 } 79 80 81 CopyEngine::~CopyEngine() 82 { 83 while (fBufferQueue.Size() > 0) 84 snooze(10000); 85 86 fQuitting = true; 87 if (fWriterThread >= B_OK) { 88 int32 exitValue; 89 wait_for_thread(fWriterThread, &exitValue); 90 } 91 } 92 93 94 void 95 CopyEngine::ResetTargets(const char* source) 96 { 97 // TODO: One could subtract the bytes/items which were added to the 98 // ProgressReporter before resetting them... 99 100 fAbsoluteSourcePath = source; 101 102 fBytesRead = 0; 103 fLastBytesRead = 0; 104 fItemsCopied = 0; 105 fLastItemsCopied = 0; 106 fTimeRead = 0; 107 108 fBytesWritten = 0; 109 fTimeWritten = 0; 110 111 fBytesToCopy = 0; 112 fItemsToCopy = 0; 113 114 fCurrentTargetFolder = NULL; 115 fCurrentItem = NULL; 116 } 117 118 119 status_t 120 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 121 { 122 int32 level = 0; 123 status_t ret = _CollectCopyInfo(source, level, cancelSemaphore); 124 if (ret == B_OK && fProgressReporter != NULL) 125 fProgressReporter->AddItems(fItemsToCopy, fBytesToCopy); 126 return ret; 127 } 128 129 130 status_t 131 CopyEngine::CopyFolder(const char* source, const char* destination, 132 sem_id cancelSemaphore) 133 { 134 int32 level = 0; 135 return _CopyFolder(source, destination, level, cancelSemaphore); 136 } 137 138 139 status_t 140 CopyEngine::CopyFile(const BEntry& _source, const BEntry& _destination, 141 sem_id cancelSemaphore) 142 { 143 SemaphoreLocker lock(cancelSemaphore); 144 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 145 // We are supposed to quit 146 return B_CANCELED; 147 } 148 149 BFile source(&_source, B_READ_ONLY); 150 status_t ret = source.InitCheck(); 151 if (ret < B_OK) 152 return ret; 153 154 BFile* destination = new (nothrow) BFile(&_destination, 155 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 156 ret = destination->InitCheck(); 157 if (ret < B_OK) { 158 delete destination; 159 return ret; 160 } 161 162 int32 loopIteration = 0; 163 164 while (true) { 165 if (fBufferQueue.Size() >= BUFFER_COUNT) { 166 // the queue is "full", just wait a bit, the 167 // write thread will empty it 168 snooze(1000); 169 continue; 170 } 171 172 // allocate buffer 173 Buffer* buffer = new (nothrow) Buffer(destination); 174 if (!buffer || !buffer->buffer) { 175 delete destination; 176 delete buffer; 177 fprintf(stderr, "reading loop: out of memory\n"); 178 return B_NO_MEMORY; 179 } 180 181 // fill buffer 182 ssize_t read = source.Read(buffer->buffer, buffer->size); 183 if (read < 0) { 184 ret = (status_t)read; 185 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 186 delete buffer; 187 delete destination; 188 break; 189 } 190 191 fBytesRead += read; 192 loopIteration += 1; 193 if (loopIteration % 2 == 0) 194 _UpdateProgress(); 195 196 buffer->deleteFile = read == 0; 197 if (read > 0) 198 buffer->validBytes = (size_t)read; 199 else 200 buffer->validBytes = 0; 201 202 // enqueue the buffer 203 ret = fBufferQueue.Push(buffer); 204 if (ret < B_OK) { 205 buffer->deleteFile = false; 206 delete buffer; 207 delete destination; 208 return ret; 209 } 210 211 // quit if done 212 if (read == 0) 213 break; 214 } 215 216 return ret; 217 } 218 219 220 // #pragma mark - 221 222 223 status_t 224 CopyEngine::_CollectCopyInfo(const char* _source, int32& level, 225 sem_id cancelSemaphore) 226 { 227 level++; 228 229 BDirectory source(_source); 230 status_t ret = source.InitCheck(); 231 if (ret < B_OK) 232 return ret; 233 234 BEntry entry; 235 while (source.GetNextEntry(&entry) == B_OK) { 236 SemaphoreLocker lock(cancelSemaphore); 237 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 238 // We are supposed to quit 239 return B_CANCELED; 240 } 241 242 struct stat statInfo; 243 entry.GetStat(&statInfo); 244 245 BPath sourceEntryPath; 246 status_t ret = entry.GetPath(&sourceEntryPath); 247 if (ret < B_OK) 248 return ret; 249 250 if (fEntryFilter != NULL 251 && !fEntryFilter->ShouldCopyEntry(entry, 252 _RelativeEntryPath(sourceEntryPath.Path()), statInfo, level)) { 253 continue; 254 } 255 256 if (S_ISDIR(statInfo.st_mode)) { 257 // handle recursive directory copy 258 BPath srcFolder; 259 ret = entry.GetPath(&srcFolder); 260 if (ret < B_OK) 261 return ret; 262 263 if (cancelSemaphore >= 0) 264 lock.Unlock(); 265 266 ret = _CollectCopyInfo(srcFolder.Path(), level, cancelSemaphore); 267 if (ret < B_OK) 268 return ret; 269 } else if (S_ISLNK(statInfo.st_mode)) { 270 // link, ignore size 271 } else { 272 // file data 273 fBytesToCopy += statInfo.st_size; 274 } 275 276 fItemsToCopy++; 277 } 278 279 level--; 280 return B_OK; 281 } 282 283 284 status_t 285 CopyEngine::_CopyFolder(const char* _source, const char* _destination, 286 int32& level, sem_id cancelSemaphore) 287 { 288 level++; 289 fCurrentTargetFolder = _destination; 290 291 BDirectory source(_source); 292 status_t ret = source.InitCheck(); 293 if (ret < B_OK) 294 return ret; 295 296 ret = create_directory(_destination, 0777); 297 if (ret < B_OK && ret != B_FILE_EXISTS) { 298 fprintf(stderr, "Could not create '%s': %s\n", _destination, 299 strerror(ret)); 300 return ret; 301 } 302 303 BDirectory destination(_destination); 304 ret = destination.InitCheck(); 305 if (ret < B_OK) 306 return ret; 307 308 BEntry entry; 309 while (source.GetNextEntry(&entry) == B_OK) { 310 SemaphoreLocker lock(cancelSemaphore); 311 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 312 // We are supposed to quit 313 return B_CANCELED; 314 } 315 316 const char* name = entry.Name(); 317 BPath sourceEntryPath; 318 status_t ret = entry.GetPath(&sourceEntryPath); 319 if (ret != B_OK) 320 return ret; 321 const char* relativeSourceEntryPath 322 = _RelativeEntryPath(sourceEntryPath.Path()); 323 324 struct stat statInfo; 325 entry.GetStat(&statInfo); 326 327 if (fEntryFilter != NULL 328 && !fEntryFilter->ShouldCopyEntry(entry, relativeSourceEntryPath, 329 statInfo, level)) { 330 continue; 331 } 332 333 fItemsCopied++; 334 fCurrentItem = name; 335 _UpdateProgress(); 336 337 BEntry copy(&destination, name); 338 bool copyAttributes = true; 339 340 if (S_ISDIR(statInfo.st_mode)) { 341 // handle recursive directory copy 342 343 if (copy.Exists()) { 344 ret = B_OK; 345 if (copy.IsDirectory()) { 346 if (fEntryFilter 347 && fEntryFilter->ShouldClobberFolder(entry, 348 relativeSourceEntryPath, statInfo, level)) { 349 ret = _RemoveFolder(copy); 350 } else { 351 // Do not overwrite attributes on folders that exist. 352 // This should work better when the install target 353 // already contains a Haiku installation. 354 copyAttributes = false; 355 } 356 } else 357 ret = copy.Remove(); 358 359 if (ret != B_OK) { 360 fprintf(stderr, "Failed to make room for folder '%s': " 361 "%s\n", name, strerror(ret)); 362 return ret; 363 } 364 } 365 366 BPath dstFolder; 367 ret = copy.GetPath(&dstFolder); 368 if (ret < B_OK) 369 return ret; 370 371 if (cancelSemaphore >= 0) 372 lock.Unlock(); 373 374 ret = _CopyFolder(sourceEntryPath.Path(), dstFolder.Path(), level, 375 cancelSemaphore); 376 if (ret < B_OK) 377 return ret; 378 379 if (cancelSemaphore >= 0 && !lock.Lock()) { 380 // We are supposed to quit 381 return B_CANCELED; 382 } 383 } else { 384 if (copy.Exists()) { 385 if (copy.IsDirectory()) 386 ret = _RemoveFolder(copy); 387 else 388 ret = copy.Remove(); 389 if (ret != B_OK) { 390 fprintf(stderr, "Failed to make room for entry '%s': " 391 "%s\n", name, strerror(ret)); 392 return ret; 393 } 394 } 395 if (S_ISLNK(statInfo.st_mode)) { 396 // copy symbolic links 397 BSymLink srcLink(&entry); 398 if (ret < B_OK) 399 return ret; 400 401 char linkPath[B_PATH_NAME_LENGTH]; 402 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 403 if (read < 0) 404 return (status_t)read; 405 406 // just in case it already exists... 407 copy.Remove(); 408 409 BSymLink dstLink; 410 ret = destination.CreateSymLink(name, linkPath, &dstLink); 411 if (ret < B_OK) 412 return ret; 413 } else { 414 // copy file data 415 // NOTE: Do not pass the locker, we simply keep holding the lock! 416 ret = CopyFile(entry, copy); 417 if (ret < B_OK) 418 return ret; 419 } 420 } 421 422 if (!copyAttributes) 423 continue; 424 425 ret = copy.SetTo(&destination, name); 426 if (ret != B_OK) 427 return ret; 428 429 // copy attributes 430 BNode sourceNode(&entry); 431 BNode targetNode(©); 432 char attrName[B_ATTR_NAME_LENGTH]; 433 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 434 attr_info info; 435 if (sourceNode.GetAttrInfo(attrName, &info) < B_OK) 436 continue; 437 size_t size = 4096; 438 uint8 buffer[size]; 439 off_t offset = 0; 440 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 441 offset, buffer, std::min((off_t)size, info.size)); 442 // NOTE: It's important to still write the attribute even if 443 // we have read 0 bytes! 444 while (read >= 0) { 445 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 446 offset += read; 447 read = sourceNode.ReadAttr(attrName, info.type, 448 offset, buffer, std::min((off_t)size, info.size - offset)); 449 if (read == 0) 450 break; 451 } 452 } 453 454 // copy basic attributes 455 copy.SetPermissions(statInfo.st_mode); 456 copy.SetOwner(statInfo.st_uid); 457 copy.SetGroup(statInfo.st_gid); 458 copy.SetModificationTime(statInfo.st_mtime); 459 copy.SetCreationTime(statInfo.st_crtime); 460 } 461 462 level--; 463 return B_OK; 464 } 465 466 467 status_t 468 CopyEngine::_RemoveFolder(BEntry& entry) 469 { 470 BDirectory directory(&entry); 471 status_t ret = directory.InitCheck(); 472 if (ret != B_OK) 473 return ret; 474 475 BEntry subEntry; 476 while (directory.GetNextEntry(&subEntry) == B_OK) { 477 if (subEntry.IsDirectory()) { 478 ret = _RemoveFolder(subEntry); 479 if (ret != B_OK) 480 return ret; 481 } else { 482 ret = subEntry.Remove(); 483 if (ret != B_OK) 484 return ret; 485 } 486 } 487 return entry.Remove(); 488 } 489 490 491 const char* 492 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 493 { 494 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 495 fAbsoluteSourcePath.Length()) != 0) { 496 return absoluteSourcePath; 497 } 498 499 const char* relativePath 500 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 501 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 502 } 503 504 505 void 506 CopyEngine::_UpdateProgress() 507 { 508 if (fProgressReporter == NULL) 509 return; 510 511 uint64 items = 0; 512 if (fLastItemsCopied < fItemsCopied) { 513 items = fItemsCopied - fLastItemsCopied; 514 fLastItemsCopied = fItemsCopied; 515 } 516 517 off_t bytes = 0; 518 if (fLastBytesRead < fBytesRead) { 519 bytes = fBytesRead - fLastBytesRead; 520 fLastBytesRead = fBytesRead; 521 } 522 523 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 524 fCurrentTargetFolder); 525 } 526 527 528 int32 529 CopyEngine::_WriteThreadEntry(void* cookie) 530 { 531 CopyEngine* engine = (CopyEngine*)cookie; 532 engine->_WriteThread(); 533 return B_OK; 534 } 535 536 537 void 538 CopyEngine::_WriteThread() 539 { 540 bigtime_t bufferWaitTimeout = 100000; 541 542 while (!fQuitting) { 543 544 bigtime_t now = system_time(); 545 546 Buffer* buffer = NULL; 547 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 548 if (ret == B_TIMED_OUT) { 549 // no buffer, timeout 550 continue; 551 } else if (ret == B_NO_INIT) { 552 // real error 553 return; 554 } else if (ret != B_OK) { 555 // no buffer, queue error 556 snooze(10000); 557 continue; 558 } 559 560 if (!buffer->deleteFile) { 561 ssize_t written = buffer->file->Write(buffer->buffer, 562 buffer->validBytes); 563 if (written != (ssize_t)buffer->validBytes) { 564 // TODO: this should somehow be propagated back 565 // to the main thread! 566 fprintf(stderr, "Failed to write data: %s\n", 567 strerror((status_t)written)); 568 } 569 fBytesWritten += written; 570 } 571 572 delete buffer; 573 574 // measure performance 575 fTimeWritten += system_time() - now; 576 } 577 578 double megaBytes = (double)fBytesWritten / (1024 * 1024); 579 double seconds = (fTimeWritten / 1000000); 580 if (seconds > 0) { 581 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 582 megaBytes / seconds); 583 } 584 } 585