1 /* 2 * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "AbstractServerProcess.h" 8 9 #include <unistd.h> 10 #include <errno.h> 11 #include <string.h> 12 13 #include <AutoDeleter.h> 14 #include <FileIO.h> 15 #include <HttpTime.h> 16 #include <UrlProtocolRoster.h> 17 18 #include <support/ZlibCompressionAlgorithm.h> 19 20 #include "DataIOUtils.h" 21 #include "HaikuDepotConstants.h" 22 #include "Logger.h" 23 #include "ServerHelper.h" 24 #include "ServerSettings.h" 25 #include "StandardMetaDataJsonEventListener.h" 26 #include "StorageUtils.h" 27 #include "ToFileUrlProtocolListener.h" 28 29 30 #define MAX_REDIRECTS 3 31 #define MAX_FAILURES 2 32 33 34 // 30 seconds 35 #define TIMEOUT_MICROSECONDS 3e+7 36 37 38 AbstractServerProcess::AbstractServerProcess(uint32 options) 39 : 40 AbstractProcess(), 41 fOptions(options), 42 fRequest(NULL) 43 { 44 } 45 46 47 AbstractServerProcess::~AbstractServerProcess() 48 { 49 } 50 51 52 bool 53 AbstractServerProcess::HasOption(uint32 flag) 54 { 55 return (fOptions & flag) == flag; 56 } 57 58 59 bool 60 AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready) 61 { 62 return 63 !HasOption(SERVER_PROCESS_NO_NETWORKING) 64 && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready); 65 } 66 67 68 status_t 69 AbstractServerProcess::StopInternal() 70 { 71 if (fRequest != NULL) { 72 return fRequest->Stop(); 73 } 74 75 return AbstractProcess::StopInternal(); 76 } 77 78 79 status_t 80 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const 81 { 82 BPath metaDataPath; 83 BString jsonPath; 84 85 status_t result = GetStandardMetaDataPath(metaDataPath); 86 87 if (result != B_OK) 88 return result; 89 90 GetStandardMetaDataJsonPath(jsonPath); 91 92 return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath); 93 } 94 95 96 status_t 97 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue, 98 const BPath& metaDataPath, const BString& jsonPath) const 99 { 100 headerValue.SetTo(""); 101 struct stat s; 102 103 if (-1 == stat(metaDataPath.Path(), &s)) { 104 if (ENOENT != errno) 105 return B_ERROR; 106 107 return B_FILE_NOT_FOUND; 108 } 109 110 if (s.st_size == 0) 111 return B_BAD_VALUE; 112 113 StandardMetaData metaData; 114 status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath); 115 116 if (result == B_OK) 117 SetIfModifiedSinceHeaderValueFromMetaData(headerValue, metaData); 118 else { 119 HDERROR("unable to parse the meta-data date and time from [%s]" 120 " - cannot set the 'If-Modified-Since' header", 121 metaDataPath.Path()); 122 } 123 124 return result; 125 } 126 127 128 /*static*/ void 129 AbstractServerProcess::SetIfModifiedSinceHeaderValueFromMetaData( 130 BString& headerValue, 131 const StandardMetaData& metaData) 132 { 133 // An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000' 134 BDateTime modifiedDateTime = metaData 135 .GetDataModifiedTimestampAsDateTime(); 136 BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime); 137 headerValue.SetTo(modifiedHttpTime 138 .ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE)); 139 } 140 141 142 status_t 143 AbstractServerProcess::PopulateMetaData( 144 StandardMetaData& metaData, const BPath& path, 145 const BString& jsonPath) const 146 { 147 StandardMetaDataJsonEventListener listener(jsonPath, metaData); 148 status_t result = ParseJsonFromFileWithListener(&listener, path); 149 150 if (result != B_OK) 151 return result; 152 153 result = listener.ErrorStatus(); 154 155 if (result != B_OK) 156 return result; 157 158 if (!metaData.IsPopulated()) { 159 HDERROR("the meta data was read from [%s], but no values " 160 "were extracted", path.Path()); 161 return B_BAD_DATA; 162 } 163 164 return B_OK; 165 } 166 167 168 /* static */ bool 169 AbstractServerProcess::_LooksLikeGzip(const char *pathStr) 170 { 171 int l = strlen(pathStr); 172 return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3); 173 } 174 175 176 /*! Note that a B_OK return code from this method may not indicate that the 177 listening process went well. One has to see if there was an error in 178 the listener. 179 */ 180 181 status_t 182 AbstractServerProcess::ParseJsonFromFileWithListener( 183 BJsonEventListener *listener, 184 const BPath& path) const 185 { 186 const char* pathStr = path.Path(); 187 FILE* file = fopen(pathStr, "rb"); 188 189 if (file == NULL) { 190 HDERROR("[%s] unable to find the meta data file at [%s]", Name(), 191 path.Path()); 192 return B_FILE_NOT_FOUND; 193 } 194 195 BFileIO rawInput(file, true); // takes ownership 196 197 // if the file extension ends with '.gz' then the data will be 198 // compressed and the algorithm needs to decompress the data as 199 // it is parsed. 200 201 if (_LooksLikeGzip(pathStr)) { 202 BDataIO* gzDecompressedInput = NULL; 203 BZlibDecompressionParameters* zlibDecompressionParameters 204 = new BZlibDecompressionParameters(); 205 206 status_t result = BZlibCompressionAlgorithm() 207 .CreateDecompressingInputStream(&rawInput, 208 zlibDecompressionParameters, gzDecompressedInput); 209 210 if (B_OK != result) 211 return result; 212 213 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput); 214 BPrivate::BJson::Parse(gzDecompressedInput, listener); 215 } else { 216 BPrivate::BJson::Parse(&rawInput, listener); 217 } 218 219 return B_OK; 220 } 221 222 223 /*! In order to reduce the chance of failure half way through downloading a 224 large file, this method will download the file to a temporary file and 225 then it can rename the file to the final target file. 226 */ 227 228 status_t 229 AbstractServerProcess::DownloadToLocalFileAtomically( 230 const BPath& targetFilePath, 231 const BUrl& url) 232 { 233 BPath temporaryFilePath(tmpnam(NULL), NULL, true); 234 status_t result = DownloadToLocalFile( 235 temporaryFilePath, url, 0, 0); 236 237 // if the data is coming in as .gz, but is not stored as .gz then 238 // the data should be decompressed in the temporary file location 239 // before being shifted into place. 240 241 if (result == B_OK 242 && _LooksLikeGzip(url.Path()) 243 && !_LooksLikeGzip(targetFilePath.Path())) 244 result = _DeGzipInSitu(temporaryFilePath); 245 246 // not copying if the data has not changed because the data will be 247 // zero length. This is if the result is APP_ERR_NOT_MODIFIED. 248 if (result == B_OK) { 249 // if the file is zero length then assume that something has 250 // gone wrong. 251 off_t size; 252 bool hasFile; 253 254 result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL, 255 &size); 256 257 if (result == B_OK && hasFile && size > 0) { 258 if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) { 259 HDINFO("[%s] did rename [%s] --> [%s]", 260 Name(), temporaryFilePath.Path(), targetFilePath.Path()); 261 result = B_IO_ERROR; 262 } 263 } 264 } 265 266 return result; 267 } 268 269 270 /*static*/ status_t 271 AbstractServerProcess::_DeGzipInSitu(const BPath& path) 272 { 273 const char* tmpPath = tmpnam(NULL); 274 status_t result = B_OK; 275 276 { 277 BFile file(path.Path(), O_RDONLY); 278 BFile tmpFile(tmpPath, O_WRONLY | O_CREAT); 279 280 BDataIO* gzDecompressedInput = NULL; 281 BZlibDecompressionParameters* zlibDecompressionParameters 282 = new BZlibDecompressionParameters(); 283 284 result = BZlibCompressionAlgorithm() 285 .CreateDecompressingInputStream(&file, 286 zlibDecompressionParameters, gzDecompressedInput); 287 288 if (result == B_OK) { 289 ObjectDeleter<BDataIO> gzDecompressedInputDeleter( 290 gzDecompressedInput); 291 result = DataIOUtils::CopyAll(&tmpFile, gzDecompressedInput); 292 } 293 } 294 295 if (result == B_OK) { 296 if (rename(tmpPath, path.Path()) != 0) { 297 HDERROR("unable to move the uncompressed data into place"); 298 result = B_ERROR; 299 } 300 } 301 else { 302 HDERROR("it was not possible to decompress the data"); 303 } 304 305 return result; 306 } 307 308 309 status_t 310 AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, 311 const BUrl& url, uint32 redirects, uint32 failures) 312 { 313 if (WasStopped()) 314 return B_CANCELED; 315 316 if (redirects > MAX_REDIRECTS) { 317 HDINFO("[%s] exceeded %d redirects --> failure", Name(), 318 MAX_REDIRECTS); 319 return B_IO_ERROR; 320 } 321 322 if (failures > MAX_FAILURES) { 323 HDINFO("[%s] exceeded %d failures", Name(), MAX_FAILURES); 324 return B_IO_ERROR; 325 } 326 327 HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(), 328 targetFilePath.Path()); 329 330 ToFileUrlProtocolListener listener(targetFilePath, Name(), 331 Logger::IsTraceEnabled()); 332 333 BHttpHeaders headers; 334 ServerSettings::AugmentHeaders(headers); 335 336 BString ifModifiedSinceHeader; 337 status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue( 338 ifModifiedSinceHeader); 339 340 if (ifModifiedSinceHeaderStatus == B_OK && 341 ifModifiedSinceHeader.Length() > 0) { 342 headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); 343 } 344 345 thread_id thread; 346 347 BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &listener); 348 if (request == NULL) 349 return B_NO_MEMORY; 350 351 fRequest = dynamic_cast<BHttpRequest *>(request); 352 if (fRequest == NULL) { 353 delete request; 354 return B_ERROR; 355 } 356 fRequest->SetHeaders(headers); 357 fRequest->SetMaxRedirections(0); 358 fRequest->SetTimeout(TIMEOUT_MICROSECONDS); 359 thread = fRequest->Run(); 360 361 wait_for_thread(thread, NULL); 362 363 const BHttpResult& result = dynamic_cast<const BHttpResult&>( 364 fRequest->Result()); 365 int32 statusCode = result.StatusCode(); 366 const BHttpHeaders responseHeaders = result.Headers(); 367 const char *locationC = responseHeaders["Location"]; 368 BString location; 369 370 if (locationC != NULL) 371 location.SetTo(locationC); 372 373 delete fRequest; 374 fRequest = NULL; 375 376 if (BHttpRequest::IsSuccessStatusCode(statusCode)) { 377 HDINFO("[%s] did complete streaming data [%" 378 B_PRIdSSIZE " bytes]", Name(), listener.ContentLength()); 379 return B_OK; 380 } else if (statusCode == B_HTTP_STATUS_NOT_MODIFIED) { 381 HDINFO("[%s] remote data has not changed since [%s]", Name(), 382 ifModifiedSinceHeader.String()); 383 return HD_ERR_NOT_MODIFIED; 384 } else if (statusCode == B_HTTP_STATUS_PRECONDITION_FAILED) { 385 ServerHelper::NotifyClientTooOld(responseHeaders); 386 return HD_CLIENT_TOO_OLD; 387 } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) { 388 if (location.Length() != 0) { 389 BUrl redirectUrl(result.Url(), location); 390 HDINFO("[%s] will redirect to; %s", 391 Name(), redirectUrl.UrlString().String()); 392 return DownloadToLocalFile(targetFilePath, redirectUrl, 393 redirects + 1, 0); 394 } 395 396 HDERROR("[%s] unable to find 'Location' header for redirect", Name()); 397 return B_IO_ERROR; 398 } else { 399 if (statusCode == 0 || (statusCode / 100) == 5) { 400 HDERROR("error response from server [%" B_PRId32 "] --> retry...", 401 statusCode); 402 return DownloadToLocalFile(targetFilePath, url, redirects, 403 failures + 1); 404 } 405 406 HDERROR("[%s] unexpected response from server [%" B_PRId32 "]", 407 Name(), statusCode); 408 return B_IO_ERROR; 409 } 410 } 411 412 413 status_t 414 AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath) 415 { 416 if (0 == remove(currentFilePath.Path())) 417 return B_OK; 418 419 return B_IO_ERROR; 420 } 421 422 423 /*! When a file is damaged or corrupted in some way, the file should be 'moved 424 aside' so that it is not involved in the next update. This method will 425 create such an alternative 'damaged' file location and move this file to 426 that location. 427 */ 428 429 status_t 430 AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath) 431 { 432 BPath damagedFilePath; 433 BString damagedLeaf; 434 435 damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf()); 436 currentFilePath.GetParent(&damagedFilePath); 437 damagedFilePath.Append(damagedLeaf.String()); 438 439 if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) { 440 HDERROR("[%s] unable to move damaged file [%s] aside to [%s]", 441 Name(), currentFilePath.Path(), damagedFilePath.Path()); 442 return B_IO_ERROR; 443 } 444 445 HDINFO("[%s] did move damaged file [%s] aside to [%s]", 446 Name(), currentFilePath.Path(), damagedFilePath.Path()); 447 448 return B_OK; 449 } 450 451 452 bool 453 AbstractServerProcess::IsSuccess(status_t e) { 454 return e == B_OK || e == HD_ERR_NOT_MODIFIED; 455 } 456