1 /* 2 * Copyright 2017-2018, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 #include "AbstractServerProcess.h" 6 7 #include <unistd.h> 8 #include <errno.h> 9 #include <string.h> 10 11 #include <AutoDeleter.h> 12 #include <Autolock.h> 13 #include <FileIO.h> 14 #include <HttpTime.h> 15 #include <Locker.h> 16 #include <UrlProtocolRoster.h> 17 18 #include <support/ZlibCompressionAlgorithm.h> 19 20 #include "HaikuDepotConstants.h" 21 #include "Logger.h" 22 #include "ServerHelper.h" 23 #include "ServerSettings.h" 24 #include "StandardMetaDataJsonEventListener.h" 25 #include "StorageUtils.h" 26 #include "ToFileUrlProtocolListener.h" 27 28 29 #define MAX_REDIRECTS 3 30 #define MAX_FAILURES 2 31 32 33 // 30 seconds 34 #define TIMEOUT_MICROSECONDS 3e+7 35 36 37 AbstractServerProcess::AbstractServerProcess( 38 AbstractServerProcessListener* listener, uint32 options) 39 : 40 fLock(), 41 fListener(listener), 42 fWasStopped(false), 43 fProcessState(SERVER_PROCESS_INITIAL), 44 fErrorStatus(B_OK), 45 fOptions(options), 46 fRequest(NULL) 47 { 48 } 49 50 51 AbstractServerProcess::~AbstractServerProcess() 52 { 53 } 54 55 56 bool 57 AbstractServerProcess::HasOption(uint32 flag) 58 { 59 return (fOptions & flag) == flag; 60 } 61 62 63 bool 64 AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready) 65 { 66 return 67 !HasOption(SERVER_PROCESS_NO_NETWORKING) 68 && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready); 69 } 70 71 72 status_t 73 AbstractServerProcess::Run() 74 { 75 { 76 BAutolock locker(&fLock); 77 78 if (ProcessState() != SERVER_PROCESS_INITIAL) { 79 printf("cannot start server process as it is not idle"); 80 return B_NOT_ALLOWED; 81 } 82 83 fProcessState = SERVER_PROCESS_RUNNING; 84 } 85 86 SetErrorStatus(RunInternal()); 87 88 SetProcessState(SERVER_PROCESS_COMPLETE); 89 90 // this process may be part of a larger bulk-load process and 91 // if so, the process orchestration needs to know when this 92 // process has completed. 93 94 if (fListener != NULL) 95 fListener->ServerProcessExited(); 96 97 return ErrorStatus(); 98 } 99 100 101 bool 102 AbstractServerProcess::WasStopped() 103 { 104 BAutolock locker(&fLock); 105 return fWasStopped; 106 } 107 108 109 status_t 110 AbstractServerProcess::ErrorStatus() 111 { 112 BAutolock locker(&fLock); 113 return fErrorStatus; 114 } 115 116 117 status_t 118 AbstractServerProcess::Stop() 119 { 120 BAutolock locker(&fLock); 121 fWasStopped = true; 122 return StopInternal(); 123 } 124 125 126 status_t 127 AbstractServerProcess::StopInternal() 128 { 129 if (fRequest != NULL) { 130 return fRequest->Stop(); 131 } 132 133 return B_NOT_ALLOWED; 134 } 135 136 137 bool 138 AbstractServerProcess::IsRunning() 139 { 140 return ProcessState() == SERVER_PROCESS_RUNNING; 141 } 142 143 144 void 145 AbstractServerProcess::SetErrorStatus(status_t value) 146 { 147 BAutolock locker(&fLock); 148 149 if (fErrorStatus == B_OK) { 150 fErrorStatus = value; 151 } 152 } 153 154 155 void 156 AbstractServerProcess::SetProcessState(process_state value) 157 { 158 BAutolock locker(&fLock); 159 fProcessState = value; 160 } 161 162 163 process_state 164 AbstractServerProcess::ProcessState() 165 { 166 BAutolock locker(&fLock); 167 return fProcessState; 168 } 169 170 171 status_t 172 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const 173 { 174 BPath metaDataPath; 175 BString jsonPath; 176 177 GetStandardMetaDataPath(metaDataPath); 178 GetStandardMetaDataJsonPath(jsonPath); 179 180 return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath); 181 } 182 183 184 status_t 185 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue, 186 const BPath& metaDataPath, const BString& jsonPath) const 187 { 188 headerValue.SetTo(""); 189 struct stat s; 190 191 if (-1 == stat(metaDataPath.Path(), &s)) { 192 if (ENOENT != errno) 193 return B_ERROR; 194 195 return B_FILE_NOT_FOUND; 196 } 197 198 if (s.st_size == 0) 199 return B_BAD_VALUE; 200 201 StandardMetaData metaData; 202 status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath); 203 204 if (result == B_OK) { 205 206 // An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000' 207 208 BDateTime modifiedDateTime = metaData 209 .GetDataModifiedTimestampAsDateTime(); 210 BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime); 211 headerValue.SetTo(modifiedHttpTime 212 .ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE)); 213 } else { 214 fprintf(stderr, "unable to parse the meta-data date and time from [%s]" 215 " - cannot set the 'If-Modified-Since' header\n", 216 metaDataPath.Path()); 217 } 218 219 return result; 220 } 221 222 223 status_t 224 AbstractServerProcess::PopulateMetaData( 225 StandardMetaData& metaData, const BPath& path, 226 const BString& jsonPath) const 227 { 228 StandardMetaDataJsonEventListener listener(jsonPath, metaData); 229 status_t result = ParseJsonFromFileWithListener(&listener, path); 230 231 if (result != B_OK) 232 return result; 233 234 result = listener.ErrorStatus(); 235 236 if (result != B_OK) 237 return result; 238 239 if (!metaData.IsPopulated()) { 240 fprintf(stderr, "the meta data was read from [%s], but no values " 241 "were extracted\n", path.Path()); 242 return B_BAD_DATA; 243 } 244 245 return B_OK; 246 } 247 248 249 bool 250 AbstractServerProcess::LooksLikeGzip(const char *pathStr) const 251 { 252 int l = strlen(pathStr); 253 return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3); 254 } 255 256 257 /*! Note that a B_OK return code from this method may not indicate that the 258 listening process went well. One has to see if there was an error in 259 the listener. 260 */ 261 262 status_t 263 AbstractServerProcess::ParseJsonFromFileWithListener( 264 BJsonEventListener *listener, 265 const BPath& path) const 266 { 267 const char* pathStr = path.Path(); 268 FILE* file = fopen(pathStr, "rb"); 269 270 if (file == NULL) { 271 fprintf(stderr, "unable to find the meta data file at [%s]\n", 272 path.Path()); 273 return B_FILE_NOT_FOUND; 274 } 275 276 BFileIO rawInput(file, true); // takes ownership 277 278 // if the file extension ends with '.gz' then the data will be 279 // compressed and the algorithm needs to decompress the data as 280 // it is parsed. 281 282 if (LooksLikeGzip(pathStr)) { 283 BDataIO* gzDecompressedInput = NULL; 284 BZlibDecompressionParameters* zlibDecompressionParameters 285 = new BZlibDecompressionParameters(); 286 287 status_t result = BZlibCompressionAlgorithm() 288 .CreateDecompressingInputStream(&rawInput, 289 zlibDecompressionParameters, gzDecompressedInput); 290 291 if (B_OK != result) 292 return result; 293 294 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput); 295 BPrivate::BJson::Parse(gzDecompressedInput, listener); 296 } else { 297 BPrivate::BJson::Parse(&rawInput, listener); 298 } 299 300 return B_OK; 301 } 302 303 304 /*! In order to reduce the chance of failure half way through downloading a 305 large file, this method will download the file to a temporary file and 306 then it can rename the file to the final target file. 307 */ 308 309 status_t 310 AbstractServerProcess::DownloadToLocalFileAtomically( 311 const BPath& targetFilePath, 312 const BUrl& url) 313 { 314 BPath temporaryFilePath(tmpnam(NULL), NULL, true); 315 status_t result = DownloadToLocalFile( 316 temporaryFilePath, url, 0, 0); 317 318 // not copying if the data has not changed because the data will be 319 // zero length. This is if the result is APP_ERR_NOT_MODIFIED. 320 if (result == B_OK) { 321 322 // if the file is zero length then assume that something has 323 // gone wrong. 324 off_t size; 325 bool hasFile; 326 327 result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL, 328 &size); 329 330 if (result == B_OK && hasFile && size > 0) { 331 if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) { 332 printf("[%s] did rename [%s] --> [%s]\n", 333 Name(), temporaryFilePath.Path(), targetFilePath.Path()); 334 result = B_IO_ERROR; 335 } 336 } 337 } 338 339 return result; 340 } 341 342 343 status_t 344 AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, 345 const BUrl& url, uint32 redirects, uint32 failures) 346 { 347 if (WasStopped()) 348 return B_CANCELED; 349 350 if (redirects > MAX_REDIRECTS) { 351 fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS); 352 return B_IO_ERROR; 353 } 354 355 if (failures > MAX_FAILURES) { 356 fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES); 357 return B_IO_ERROR; 358 } 359 360 fprintf(stdout, "[%s] will stream '%s' to [%s]\n", 361 Name(), url.UrlString().String(), targetFilePath.Path()); 362 363 ToFileUrlProtocolListener listener(targetFilePath, Name(), 364 Logger::IsTraceEnabled()); 365 366 BHttpHeaders headers; 367 ServerSettings::AugmentHeaders(headers); 368 369 BString ifModifiedSinceHeader; 370 status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue( 371 ifModifiedSinceHeader); 372 373 if (ifModifiedSinceHeaderStatus == B_OK && 374 ifModifiedSinceHeader.Length() > 0) { 375 headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); 376 } 377 378 thread_id thread; 379 380 { 381 fRequest = dynamic_cast<BHttpRequest *>( 382 BUrlProtocolRoster::MakeRequest(url, &listener)); 383 fRequest->SetHeaders(headers); 384 fRequest->SetMaxRedirections(0); 385 fRequest->SetTimeout(TIMEOUT_MICROSECONDS); 386 thread = fRequest->Run(); 387 } 388 389 wait_for_thread(thread, NULL); 390 391 const BHttpResult& result = dynamic_cast<const BHttpResult&>( 392 fRequest->Result()); 393 int32 statusCode = result.StatusCode(); 394 const BHttpHeaders responseHeaders = result.Headers(); 395 const char *locationC = responseHeaders["Location"]; 396 BString location; 397 398 if (locationC != NULL) 399 location.SetTo(locationC); 400 401 delete fRequest; 402 fRequest = NULL; 403 404 if (BHttpRequest::IsSuccessStatusCode(statusCode)) { 405 fprintf(stdout, "[%s] did complete streaming data [%" 406 B_PRIdSSIZE " bytes]\n", Name(), listener.ContentLength()); 407 return B_OK; 408 } else if (statusCode == B_HTTP_STATUS_NOT_MODIFIED) { 409 fprintf(stdout, "[%s] remote data has not changed since [%s]\n", 410 Name(), ifModifiedSinceHeader.String()); 411 return HD_ERR_NOT_MODIFIED; 412 } else if (statusCode == B_HTTP_STATUS_PRECONDITION_FAILED) { 413 ServerHelper::NotifyClientTooOld(responseHeaders); 414 return HD_CLIENT_TOO_OLD; 415 } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) { 416 if (location.Length() != 0) { 417 BUrl redirectUrl(result.Url(), location); 418 fprintf(stdout, "[%s] will redirect to; %s\n", 419 Name(), redirectUrl.UrlString().String()); 420 return DownloadToLocalFile(targetFilePath, redirectUrl, 421 redirects + 1, 0); 422 } 423 424 fprintf(stdout, "[%s] unable to find 'Location' header for redirect\n", 425 Name()); 426 return B_IO_ERROR; 427 } else { 428 if (statusCode == 0 || (statusCode / 100) == 5) { 429 fprintf(stdout, "error response from server [%" B_PRId32 "] --> " 430 "retry...\n", statusCode); 431 return DownloadToLocalFile(targetFilePath, url, redirects, 432 failures + 1); 433 } 434 435 fprintf(stdout, "[%s] unexpected response from server [%" B_PRId32 "]\n", 436 Name(), statusCode); 437 return B_IO_ERROR; 438 } 439 } 440 441 442 status_t 443 AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath) 444 { 445 if (0 == remove(currentFilePath.Path())) 446 return B_OK; 447 448 return B_IO_ERROR; 449 } 450 451 452 /*! When a file is damaged or corrupted in some way, the file should be 'moved 453 aside' so that it is not involved in the next update. This method will 454 create such an alternative 'damaged' file location and move this file to 455 that location. 456 */ 457 458 status_t 459 AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath) 460 { 461 BPath damagedFilePath; 462 BString damagedLeaf; 463 464 damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf()); 465 currentFilePath.GetParent(&damagedFilePath); 466 damagedFilePath.Append(damagedLeaf.String()); 467 468 if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) { 469 printf("[%s] unable to move damaged file [%s] aside to [%s]\n", 470 Name(), currentFilePath.Path(), damagedFilePath.Path()); 471 return B_IO_ERROR; 472 } 473 474 printf("[%s] did move damaged file [%s] aside to [%s]\n", 475 Name(), currentFilePath.Path(), damagedFilePath.Path()); 476 477 return B_OK; 478 } 479 480 481 bool 482 AbstractServerProcess::IsSuccess(status_t e) { 483 return e == B_OK || e == HD_ERR_NOT_MODIFIED; 484 } 485