1 /* 2 * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "AbstractServerProcess.h" 8 9 #include <unistd.h> 10 #include <errno.h> 11 #include <string.h> 12 13 #include <AutoDeleter.h> 14 #include <File.h> 15 #include <FileIO.h> 16 #include <HttpTime.h> 17 #include <UrlProtocolRoster.h> 18 19 #include <support/ZlibCompressionAlgorithm.h> 20 21 #include "DataIOUtils.h" 22 #include "HaikuDepotConstants.h" 23 #include "Logger.h" 24 #include "ServerHelper.h" 25 #include "ServerSettings.h" 26 #include "StandardMetaDataJsonEventListener.h" 27 #include "StorageUtils.h" 28 #include "LoggingUrlProtocolListener.h" 29 30 31 using namespace BPrivate::Network; 32 33 34 #define MAX_REDIRECTS 3 35 #define MAX_FAILURES 2 36 37 38 // 30 seconds 39 #define TIMEOUT_MICROSECONDS 3e+7 40 41 42 AbstractServerProcess::AbstractServerProcess(uint32 options) 43 : 44 AbstractProcess(), 45 fOptions(options), 46 fRequest(NULL) 47 { 48 } 49 50 51 AbstractServerProcess::~AbstractServerProcess() 52 { 53 } 54 55 56 bool 57 AbstractServerProcess::HasOption(uint32 flag) 58 { 59 return (fOptions & flag) == flag; 60 } 61 62 63 bool 64 AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready) 65 { 66 return 67 !HasOption(SERVER_PROCESS_NO_NETWORKING) 68 && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready); 69 } 70 71 72 status_t 73 AbstractServerProcess::StopInternal() 74 { 75 if (fRequest != NULL) { 76 return fRequest->Stop(); 77 } 78 79 return AbstractProcess::StopInternal(); 80 } 81 82 83 status_t 84 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const 85 { 86 BPath metaDataPath; 87 BString jsonPath; 88 89 status_t result = GetStandardMetaDataPath(metaDataPath); 90 91 if (result != B_OK) 92 return result; 93 94 GetStandardMetaDataJsonPath(jsonPath); 95 96 return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath); 97 } 98 99 100 status_t 101 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue, 102 const BPath& metaDataPath, const BString& jsonPath) const 103 { 104 headerValue.SetTo(""); 105 struct stat s; 106 107 if (-1 == stat(metaDataPath.Path(), &s)) { 108 if (ENOENT != errno) 109 return B_ERROR; 110 111 return B_ENTRY_NOT_FOUND; 112 } 113 114 if (s.st_size == 0) 115 return B_BAD_VALUE; 116 117 StandardMetaData metaData; 118 status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath); 119 120 if (result == B_OK) 121 SetIfModifiedSinceHeaderValueFromMetaData(headerValue, metaData); 122 else { 123 HDERROR("unable to parse the meta-data date and time from [%s]" 124 " - cannot set the 'If-Modified-Since' header", 125 metaDataPath.Path()); 126 } 127 128 return result; 129 } 130 131 132 /*static*/ void 133 AbstractServerProcess::SetIfModifiedSinceHeaderValueFromMetaData( 134 BString& headerValue, 135 const StandardMetaData& metaData) 136 { 137 // An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000' 138 BDateTime modifiedDateTime = metaData 139 .GetDataModifiedTimestampAsDateTime(); 140 BHttpTime modifiedHttpTime(modifiedDateTime); 141 headerValue.SetTo(modifiedHttpTime 142 .ToString(B_HTTP_TIME_FORMAT_COOKIE)); 143 } 144 145 146 status_t 147 AbstractServerProcess::PopulateMetaData( 148 StandardMetaData& metaData, const BPath& path, 149 const BString& jsonPath) const 150 { 151 StandardMetaDataJsonEventListener listener(jsonPath, metaData); 152 status_t result = ParseJsonFromFileWithListener(&listener, path); 153 154 if (result != B_OK) 155 return result; 156 157 result = listener.ErrorStatus(); 158 159 if (result != B_OK) 160 return result; 161 162 if (!metaData.IsPopulated()) { 163 HDERROR("the meta data was read from [%s], but no values " 164 "were extracted", path.Path()); 165 return B_BAD_DATA; 166 } 167 168 return B_OK; 169 } 170 171 172 /* static */ bool 173 AbstractServerProcess::_LooksLikeGzip(const char *pathStr) 174 { 175 int l = strlen(pathStr); 176 return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3); 177 } 178 179 180 /*! Note that a B_OK return code from this method may not indicate that the 181 listening process went well. One has to see if there was an error in 182 the listener. 183 */ 184 185 status_t 186 AbstractServerProcess::ParseJsonFromFileWithListener( 187 BJsonEventListener *listener, 188 const BPath& path) const 189 { 190 const char* pathStr = path.Path(); 191 FILE* file = fopen(pathStr, "rb"); 192 193 if (file == NULL) { 194 HDERROR("[%s] unable to find the meta data file at [%s]", Name(), 195 path.Path()); 196 return B_ENTRY_NOT_FOUND; 197 } 198 199 BFileIO rawInput(file, true); // takes ownership 200 201 // if the file extension ends with '.gz' then the data will be 202 // compressed and the algorithm needs to decompress the data as 203 // it is parsed. 204 205 if (_LooksLikeGzip(pathStr)) { 206 BDataIO* gzDecompressedInput = NULL; 207 BZlibDecompressionParameters* zlibDecompressionParameters 208 = new BZlibDecompressionParameters(); 209 210 status_t result = BZlibCompressionAlgorithm() 211 .CreateDecompressingInputStream(&rawInput, 212 zlibDecompressionParameters, gzDecompressedInput); 213 214 if (B_OK != result) 215 return result; 216 217 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput); 218 BPrivate::BJson::Parse(gzDecompressedInput, listener); 219 } else { 220 BPrivate::BJson::Parse(&rawInput, listener); 221 } 222 223 return B_OK; 224 } 225 226 227 /*! In order to reduce the chance of failure half way through downloading a 228 large file, this method will download the file to a temporary file and 229 then it can rename the file to the final target file. 230 */ 231 232 status_t 233 AbstractServerProcess::DownloadToLocalFileAtomically( 234 const BPath& targetFilePath, 235 const BUrl& url) 236 { 237 BPath temporaryFilePath(tmpnam(NULL), NULL, true); 238 status_t result = DownloadToLocalFile( 239 temporaryFilePath, url, 0, 0); 240 241 // if the data is coming in as .gz, but is not stored as .gz then 242 // the data should be decompressed in the temporary file location 243 // before being shifted into place. 244 245 if (result == B_OK 246 && _LooksLikeGzip(url.Path()) 247 && !_LooksLikeGzip(targetFilePath.Path())) 248 result = _DeGzipInSitu(temporaryFilePath); 249 250 // not copying if the data has not changed because the data will be 251 // zero length. This is if the result is APP_ERR_NOT_MODIFIED. 252 if (result == B_OK) { 253 // if the file is zero length then assume that something has 254 // gone wrong. 255 off_t size; 256 bool hasFile; 257 258 result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL, 259 &size); 260 261 if (result == B_OK && hasFile && size > 0) { 262 if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) { 263 HDINFO("[%s] did rename [%s] --> [%s]", 264 Name(), temporaryFilePath.Path(), targetFilePath.Path()); 265 result = B_IO_ERROR; 266 } 267 } 268 } 269 270 return result; 271 } 272 273 274 /*static*/ status_t 275 AbstractServerProcess::_DeGzipInSitu(const BPath& path) 276 { 277 const char* tmpPath = tmpnam(NULL); 278 status_t result = B_OK; 279 280 { 281 BFile file(path.Path(), O_RDONLY); 282 BFile tmpFile(tmpPath, O_WRONLY | O_CREAT); 283 284 BDataIO* gzDecompressedInput = NULL; 285 BZlibDecompressionParameters* zlibDecompressionParameters 286 = new BZlibDecompressionParameters(); 287 288 result = BZlibCompressionAlgorithm() 289 .CreateDecompressingInputStream(&file, 290 zlibDecompressionParameters, gzDecompressedInput); 291 292 if (result == B_OK) { 293 ObjectDeleter<BDataIO> gzDecompressedInputDeleter( 294 gzDecompressedInput); 295 result = DataIOUtils::CopyAll(&tmpFile, gzDecompressedInput); 296 } 297 } 298 299 if (result == B_OK) { 300 if (rename(tmpPath, path.Path()) != 0) { 301 HDERROR("unable to move the uncompressed data into place"); 302 result = B_ERROR; 303 } 304 } 305 else { 306 HDERROR("it was not possible to decompress the data"); 307 } 308 309 return result; 310 } 311 312 313 status_t 314 AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, 315 const BUrl& url, uint32 redirects, uint32 failures) 316 { 317 if (WasStopped()) 318 return B_CANCELED; 319 320 if (redirects > MAX_REDIRECTS) { 321 HDINFO("[%s] exceeded %d redirects --> failure", Name(), 322 MAX_REDIRECTS); 323 return B_IO_ERROR; 324 } 325 326 if (failures > MAX_FAILURES) { 327 HDINFO("[%s] exceeded %d failures", Name(), MAX_FAILURES); 328 return B_IO_ERROR; 329 } 330 331 HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(), 332 targetFilePath.Path()); 333 334 LoggingUrlProtocolListener listener(Name(), Logger::IsTraceEnabled()); 335 BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT); 336 status_t err = targetFile.InitCheck(); 337 if (err != B_OK) 338 return err; 339 340 BHttpHeaders headers; 341 ServerSettings::AugmentHeaders(headers); 342 343 BString ifModifiedSinceHeader; 344 status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue( 345 ifModifiedSinceHeader); 346 347 if (ifModifiedSinceHeaderStatus == B_OK && 348 ifModifiedSinceHeader.Length() > 0) { 349 headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); 350 } 351 352 thread_id thread; 353 354 BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &targetFile, 355 &listener); 356 if (request == NULL) 357 return B_NO_MEMORY; 358 359 fRequest = dynamic_cast<BHttpRequest *>(request); 360 if (fRequest == NULL) { 361 delete request; 362 return B_ERROR; 363 } 364 fRequest->SetHeaders(headers); 365 fRequest->SetMaxRedirections(0); 366 fRequest->SetTimeout(TIMEOUT_MICROSECONDS); 367 fRequest->SetStopOnError(true); 368 thread = fRequest->Run(); 369 370 wait_for_thread(thread, NULL); 371 372 const BHttpResult& result = dynamic_cast<const BHttpResult&>( 373 fRequest->Result()); 374 int32 statusCode = result.StatusCode(); 375 const BHttpHeaders responseHeaders = result.Headers(); 376 const char *locationC = responseHeaders["Location"]; 377 BString location; 378 379 if (locationC != NULL) 380 location.SetTo(locationC); 381 382 delete fRequest; 383 fRequest = NULL; 384 385 if (BHttpRequest::IsSuccessStatusCode(statusCode)) { 386 HDINFO("[%s] did complete streaming data [%" 387 B_PRIdSSIZE " bytes]", Name(), listener.ContentLength()); 388 return B_OK; 389 } else if (statusCode == B_HTTP_STATUS_NOT_MODIFIED) { 390 HDINFO("[%s] remote data has not changed since [%s]", Name(), 391 ifModifiedSinceHeader.String()); 392 return HD_ERR_NOT_MODIFIED; 393 } else if (statusCode == B_HTTP_STATUS_PRECONDITION_FAILED) { 394 ServerHelper::NotifyClientTooOld(responseHeaders); 395 return HD_CLIENT_TOO_OLD; 396 } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) { 397 if (location.Length() != 0) { 398 BUrl redirectUrl(result.Url(), location); 399 HDINFO("[%s] will redirect to; %s", 400 Name(), redirectUrl.UrlString().String()); 401 return DownloadToLocalFile(targetFilePath, redirectUrl, 402 redirects + 1, 0); 403 } 404 405 HDERROR("[%s] unable to find 'Location' header for redirect", Name()); 406 return B_IO_ERROR; 407 } else { 408 if (statusCode == 0 || (statusCode / 100) == 5) { 409 HDERROR("error response from server [%" B_PRId32 "] --> retry...", 410 statusCode); 411 return DownloadToLocalFile(targetFilePath, url, redirects, 412 failures + 1); 413 } 414 415 HDERROR("[%s] unexpected response from server [%" B_PRId32 "]", 416 Name(), statusCode); 417 return B_IO_ERROR; 418 } 419 } 420 421 422 status_t 423 AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath) 424 { 425 if (0 == remove(currentFilePath.Path())) 426 return B_OK; 427 428 return B_IO_ERROR; 429 } 430 431 432 /*! When a file is damaged or corrupted in some way, the file should be 'moved 433 aside' so that it is not involved in the next update. This method will 434 create such an alternative 'damaged' file location and move this file to 435 that location. 436 */ 437 438 status_t 439 AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath) 440 { 441 BPath damagedFilePath; 442 BString damagedLeaf; 443 444 damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf()); 445 currentFilePath.GetParent(&damagedFilePath); 446 damagedFilePath.Append(damagedLeaf.String()); 447 448 if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) { 449 HDERROR("[%s] unable to move damaged file [%s] aside to [%s]", 450 Name(), currentFilePath.Path(), damagedFilePath.Path()); 451 return B_IO_ERROR; 452 } 453 454 HDINFO("[%s] did move damaged file [%s] aside to [%s]", 455 Name(), currentFilePath.Path(), damagedFilePath.Path()); 456 457 return B_OK; 458 } 459 460 461 bool 462 AbstractServerProcess::IsSuccess(status_t e) { 463 return e == B_OK || e == HD_ERR_NOT_MODIFIED; 464 } 465