1 /* 2 * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de> 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 #include "CopyEngine.h" 7 8 #include <new> 9 10 #include <math.h> 11 #include <stdio.h> 12 #include <string.h> 13 #include <sys/resource.h> 14 15 #include <Directory.h> 16 #include <fs_attr.h> 17 #include <NodeInfo.h> 18 #include <Path.h> 19 #include <SymLink.h> 20 21 #include "SemaphoreLocker.h" 22 #include "ProgressReporter.h" 23 24 25 using std::nothrow; 26 27 28 // #pragma mark - EntryFilter 29 30 31 CopyEngine::EntryFilter::~EntryFilter() 32 { 33 } 34 35 36 // #pragma mark - CopyEngine 37 38 39 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter) 40 : 41 fBufferQueue(), 42 fWriterThread(-1), 43 fQuitting(false), 44 45 fAbsoluteSourcePath(), 46 47 fBytesRead(0), 48 fLastBytesRead(0), 49 fItemsCopied(0), 50 fLastItemsCopied(0), 51 fTimeRead(0), 52 53 fBytesWritten(0), 54 fTimeWritten(0), 55 56 fBytesToCopy(0), 57 fItemsToCopy(0), 58 59 fCurrentTargetFolder(NULL), 60 fCurrentItem(NULL), 61 62 fProgressReporter(reporter), 63 fEntryFilter(entryFilter) 64 { 65 fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer", 66 B_NORMAL_PRIORITY, this); 67 68 if (fWriterThread >= B_OK) 69 resume_thread(fWriterThread); 70 71 // ask for a bunch more file descriptors so that nested copying works well 72 struct rlimit rl; 73 rl.rlim_cur = 512; 74 rl.rlim_max = RLIM_SAVED_MAX; 75 setrlimit(RLIMIT_NOFILE, &rl); 76 } 77 78 79 CopyEngine::~CopyEngine() 80 { 81 while (fBufferQueue.Size() > 0) 82 snooze(10000); 83 84 fQuitting = true; 85 if (fWriterThread >= B_OK) { 86 int32 exitValue; 87 wait_for_thread(fWriterThread, &exitValue); 88 } 89 } 90 91 92 void 93 CopyEngine::ResetTargets(const char* source) 94 { 95 // TODO: One could subtract the bytes/items which were added to the 96 // ProgressReporter before resetting them... 97 98 fAbsoluteSourcePath = source; 99 100 fBytesRead = 0; 101 fLastBytesRead = 0; 102 fItemsCopied = 0; 103 fLastItemsCopied = 0; 104 fTimeRead = 0; 105 106 fBytesWritten = 0; 107 fTimeWritten = 0; 108 109 fBytesToCopy = 0; 110 fItemsToCopy = 0; 111 112 fCurrentTargetFolder = NULL; 113 fCurrentItem = NULL; 114 } 115 116 117 status_t 118 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore) 119 { 120 int32 level = 0; 121 status_t ret = _CollectCopyInfo(source, level, cancelSemaphore); 122 if (ret == B_OK && fProgressReporter != NULL) 123 fProgressReporter->AddItems(fItemsToCopy, fBytesToCopy); 124 return ret; 125 } 126 127 128 status_t 129 CopyEngine::CopyFolder(const char* source, const char* destination, 130 sem_id cancelSemaphore) 131 { 132 int32 level = 0; 133 return _CopyFolder(source, destination, level, cancelSemaphore); 134 } 135 136 137 status_t 138 CopyEngine::CopyFile(const BEntry& _source, const BEntry& _destination, 139 sem_id cancelSemaphore) 140 { 141 SemaphoreLocker lock(cancelSemaphore); 142 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 143 // We are supposed to quit 144 return B_CANCELED; 145 } 146 147 BFile source(&_source, B_READ_ONLY); 148 status_t ret = source.InitCheck(); 149 if (ret < B_OK) 150 return ret; 151 152 BFile* destination = new (nothrow) BFile(&_destination, 153 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE); 154 ret = destination->InitCheck(); 155 if (ret < B_OK) { 156 delete destination; 157 return ret; 158 } 159 160 int32 loopIteration = 0; 161 162 while (true) { 163 if (fBufferQueue.Size() >= BUFFER_COUNT) { 164 // the queue is "full", just wait a bit, the 165 // write thread will empty it 166 snooze(1000); 167 continue; 168 } 169 170 // allocate buffer 171 Buffer* buffer = new (nothrow) Buffer(destination); 172 if (!buffer || !buffer->buffer) { 173 delete destination; 174 delete buffer; 175 fprintf(stderr, "reading loop: out of memory\n"); 176 return B_NO_MEMORY; 177 } 178 179 // fill buffer 180 ssize_t read = source.Read(buffer->buffer, buffer->size); 181 if (read < 0) { 182 ret = (status_t)read; 183 fprintf(stderr, "Failed to read data: %s\n", strerror(ret)); 184 delete buffer; 185 delete destination; 186 break; 187 } 188 189 fBytesRead += read; 190 loopIteration += 1; 191 if (loopIteration % 2 == 0) 192 _UpdateProgress(); 193 194 buffer->deleteFile = read == 0; 195 if (read > 0) 196 buffer->validBytes = (size_t)read; 197 else 198 buffer->validBytes = 0; 199 200 // enqueue the buffer 201 ret = fBufferQueue.Push(buffer); 202 if (ret < B_OK) { 203 buffer->deleteFile = false; 204 delete buffer; 205 delete destination; 206 return ret; 207 } 208 209 // quit if done 210 if (read == 0) 211 break; 212 } 213 214 return ret; 215 } 216 217 218 // #pragma mark - 219 220 221 status_t 222 CopyEngine::_CollectCopyInfo(const char* _source, int32& level, 223 sem_id cancelSemaphore) 224 { 225 level++; 226 227 BDirectory source(_source); 228 status_t ret = source.InitCheck(); 229 if (ret < B_OK) 230 return ret; 231 232 BEntry entry; 233 while (source.GetNextEntry(&entry) == B_OK) { 234 SemaphoreLocker lock(cancelSemaphore); 235 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 236 // We are supposed to quit 237 return B_CANCELED; 238 } 239 240 struct stat statInfo; 241 entry.GetStat(&statInfo); 242 243 BPath sourceEntryPath; 244 status_t ret = entry.GetPath(&sourceEntryPath); 245 if (ret < B_OK) 246 return ret; 247 248 if (fEntryFilter != NULL 249 && !fEntryFilter->ShouldCopyEntry(entry, 250 _RelativeEntryPath(sourceEntryPath.Path()), statInfo, level)) { 251 continue; 252 } 253 254 if (S_ISDIR(statInfo.st_mode)) { 255 // handle recursive directory copy 256 BPath srcFolder; 257 ret = entry.GetPath(&srcFolder); 258 if (ret < B_OK) 259 return ret; 260 261 if (cancelSemaphore >= 0) 262 lock.Unlock(); 263 264 ret = _CollectCopyInfo(srcFolder.Path(), level, cancelSemaphore); 265 if (ret < B_OK) 266 return ret; 267 } else if (S_ISLNK(statInfo.st_mode)) { 268 // link, ignore size 269 } else { 270 // file data 271 fBytesToCopy += statInfo.st_size; 272 } 273 274 fItemsToCopy++; 275 } 276 277 level--; 278 return B_OK; 279 } 280 281 282 status_t 283 CopyEngine::_CopyFolder(const char* _source, const char* _destination, 284 int32& level, sem_id cancelSemaphore) 285 { 286 level++; 287 fCurrentTargetFolder = _destination; 288 289 BDirectory source(_source); 290 status_t ret = source.InitCheck(); 291 if (ret < B_OK) 292 return ret; 293 294 ret = create_directory(_destination, 0777); 295 if (ret < B_OK && ret != B_FILE_EXISTS) { 296 fprintf(stderr, "Could not create '%s': %s\n", _destination, 297 strerror(ret)); 298 return ret; 299 } 300 301 BDirectory destination(_destination); 302 ret = destination.InitCheck(); 303 if (ret < B_OK) 304 return ret; 305 306 BEntry entry; 307 while (source.GetNextEntry(&entry) == B_OK) { 308 SemaphoreLocker lock(cancelSemaphore); 309 if (cancelSemaphore >= 0 && !lock.IsLocked()) { 310 // We are supposed to quit 311 return B_CANCELED; 312 } 313 314 const char* name = entry.Name(); 315 BPath sourceEntryPath; 316 status_t ret = entry.GetPath(&sourceEntryPath); 317 if (ret != B_OK) 318 return ret; 319 const char* relativeSourceEntryPath 320 = _RelativeEntryPath(sourceEntryPath.Path()); 321 322 struct stat statInfo; 323 entry.GetStat(&statInfo); 324 325 if (fEntryFilter != NULL 326 && !fEntryFilter->ShouldCopyEntry(entry, relativeSourceEntryPath, 327 statInfo, level)) { 328 continue; 329 } 330 331 fItemsCopied++; 332 fCurrentItem = name; 333 _UpdateProgress(); 334 335 BEntry copy(&destination, name); 336 bool copyAttributes = true; 337 338 if (S_ISDIR(statInfo.st_mode)) { 339 // handle recursive directory copy 340 341 if (copy.Exists()) { 342 ret = B_OK; 343 if (copy.IsDirectory()) { 344 if (fEntryFilter 345 && fEntryFilter->ShouldClobberFolder(entry, 346 relativeSourceEntryPath, statInfo, level)) { 347 ret = _RemoveFolder(copy); 348 } else { 349 // Do not overwrite attributes on folders that exist. 350 // This should work better when the install target 351 // already contains a Haiku installation. 352 copyAttributes = false; 353 } 354 } else 355 ret = copy.Remove(); 356 357 if (ret != B_OK) { 358 fprintf(stderr, "Failed to make room for folder '%s': " 359 "%s\n", name, strerror(ret)); 360 return ret; 361 } 362 } 363 364 BPath dstFolder; 365 ret = copy.GetPath(&dstFolder); 366 if (ret < B_OK) 367 return ret; 368 369 if (cancelSemaphore >= 0) 370 lock.Unlock(); 371 372 ret = _CopyFolder(sourceEntryPath.Path(), dstFolder.Path(), level, 373 cancelSemaphore); 374 if (ret < B_OK) 375 return ret; 376 377 if (cancelSemaphore >= 0 && !lock.Lock()) { 378 // We are supposed to quit 379 return B_CANCELED; 380 } 381 } else { 382 if (copy.Exists()) { 383 if (copy.IsDirectory()) 384 ret = _RemoveFolder(copy); 385 else 386 ret = copy.Remove(); 387 if (ret != B_OK) { 388 fprintf(stderr, "Failed to make room for entry '%s': " 389 "%s\n", name, strerror(ret)); 390 return ret; 391 } 392 } 393 if (S_ISLNK(statInfo.st_mode)) { 394 // copy symbolic links 395 BSymLink srcLink(&entry); 396 if (ret < B_OK) 397 return ret; 398 399 char linkPath[B_PATH_NAME_LENGTH]; 400 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1); 401 if (read < 0) 402 return (status_t)read; 403 404 // just in case it already exists... 405 copy.Remove(); 406 407 BSymLink dstLink; 408 ret = destination.CreateSymLink(name, linkPath, &dstLink); 409 if (ret < B_OK) 410 return ret; 411 } else { 412 // copy file data 413 // NOTE: Do not pass the locker, we simply keep holding the lock! 414 ret = CopyFile(entry, copy); 415 if (ret < B_OK) 416 return ret; 417 } 418 } 419 420 if (!copyAttributes) 421 continue; 422 423 ret = copy.SetTo(&destination, name); 424 if (ret != B_OK) 425 return ret; 426 427 // copy attributes 428 BNode sourceNode(&entry); 429 BNode targetNode(©); 430 char attrName[B_ATTR_NAME_LENGTH]; 431 while (sourceNode.GetNextAttrName(attrName) == B_OK) { 432 attr_info info; 433 if (sourceNode.GetAttrInfo(attrName, &info) < B_OK) 434 continue; 435 size_t size = 4096; 436 uint8 buffer[size]; 437 off_t offset = 0; 438 ssize_t read = sourceNode.ReadAttr(attrName, info.type, 439 offset, buffer, std::min((off_t)size, info.size)); 440 // NOTE: It's important to still write the attribute even if 441 // we have read 0 bytes! 442 while (read >= 0) { 443 targetNode.WriteAttr(attrName, info.type, offset, buffer, read); 444 offset += read; 445 read = sourceNode.ReadAttr(attrName, info.type, 446 offset, buffer, std::min((off_t)size, info.size - offset)); 447 if (read == 0) 448 break; 449 } 450 } 451 452 // copy basic attributes 453 copy.SetPermissions(statInfo.st_mode); 454 copy.SetOwner(statInfo.st_uid); 455 copy.SetGroup(statInfo.st_gid); 456 copy.SetModificationTime(statInfo.st_mtime); 457 copy.SetCreationTime(statInfo.st_crtime); 458 } 459 460 level--; 461 return B_OK; 462 } 463 464 465 status_t 466 CopyEngine::_RemoveFolder(BEntry& entry) 467 { 468 BDirectory directory(&entry); 469 status_t ret = directory.InitCheck(); 470 if (ret != B_OK) 471 return ret; 472 473 BEntry subEntry; 474 while (directory.GetNextEntry(&subEntry) == B_OK) { 475 if (subEntry.IsDirectory()) { 476 ret = _RemoveFolder(subEntry); 477 if (ret != B_OK) 478 return ret; 479 } else { 480 ret = subEntry.Remove(); 481 if (ret != B_OK) 482 return ret; 483 } 484 } 485 return entry.Remove(); 486 } 487 488 489 const char* 490 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const 491 { 492 if (strncmp(absoluteSourcePath, fAbsoluteSourcePath, 493 fAbsoluteSourcePath.Length()) != 0) { 494 return absoluteSourcePath; 495 } 496 497 const char* relativePath 498 = absoluteSourcePath + fAbsoluteSourcePath.Length(); 499 return relativePath[0] == '/' ? relativePath + 1 : relativePath; 500 } 501 502 503 void 504 CopyEngine::_UpdateProgress() 505 { 506 if (fProgressReporter == NULL) 507 return; 508 509 uint64 items = 0; 510 if (fLastItemsCopied < fItemsCopied) { 511 items = fItemsCopied - fLastItemsCopied; 512 fLastItemsCopied = fItemsCopied; 513 } 514 515 off_t bytes = 0; 516 if (fLastBytesRead < fBytesRead) { 517 bytes = fBytesRead - fLastBytesRead; 518 fLastBytesRead = fBytesRead; 519 } 520 521 fProgressReporter->ItemsWritten(items, bytes, fCurrentItem, 522 fCurrentTargetFolder); 523 } 524 525 526 int32 527 CopyEngine::_WriteThreadEntry(void* cookie) 528 { 529 CopyEngine* engine = (CopyEngine*)cookie; 530 engine->_WriteThread(); 531 return B_OK; 532 } 533 534 535 void 536 CopyEngine::_WriteThread() 537 { 538 bigtime_t bufferWaitTimeout = 100000; 539 540 while (!fQuitting) { 541 542 bigtime_t now = system_time(); 543 544 Buffer* buffer = NULL; 545 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout); 546 if (ret == B_TIMED_OUT) { 547 // no buffer, timeout 548 continue; 549 } else if (ret == B_NO_INIT) { 550 // real error 551 return; 552 } else if (ret != B_OK) { 553 // no buffer, queue error 554 snooze(10000); 555 continue; 556 } 557 558 if (!buffer->deleteFile) { 559 ssize_t written = buffer->file->Write(buffer->buffer, 560 buffer->validBytes); 561 if (written != (ssize_t)buffer->validBytes) { 562 // TODO: this should somehow be propagated back 563 // to the main thread! 564 fprintf(stderr, "Failed to write data: %s\n", 565 strerror((status_t)written)); 566 } 567 fBytesWritten += written; 568 } 569 570 delete buffer; 571 572 // measure performance 573 fTimeWritten += system_time() - now; 574 } 575 576 double megaBytes = (double)fBytesWritten / (1024 * 1024); 577 double seconds = (fTimeWritten / 1000000); 578 if (seconds > 0) { 579 printf("%.2f MB written (%.2f MB/s)\n", megaBytes, 580 megaBytes / seconds); 581 } 582 } 583