1 /* 2 * Copyright 2017-2023, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "AbstractServerProcess.h" 8 9 #include <unistd.h> 10 #include <errno.h> 11 #include <string.h> 12 13 #include <BufferedDataIO.h> 14 #include <AutoDeleter.h> 15 #include <File.h> 16 #include <FileIO.h> 17 #include <HttpTime.h> 18 #include <UrlProtocolRoster.h> 19 20 #include <support/ZlibCompressionAlgorithm.h> 21 22 #include "DataIOUtils.h" 23 #include "HaikuDepotConstants.h" 24 #include "Logger.h" 25 #include "ServerHelper.h" 26 #include "ServerSettings.h" 27 #include "StandardMetaDataJsonEventListener.h" 28 #include "StorageUtils.h" 29 #include "LoggingUrlProtocolListener.h" 30 31 32 using namespace BPrivate::Network; 33 34 35 #define MAX_REDIRECTS 3 36 #define MAX_FAILURES 2 37 38 39 // 30 seconds 40 #define TIMEOUT_MICROSECONDS 3e+7 41 42 43 const size_t kFileBufferSize = 10 * 1024; 44 45 46 AbstractServerProcess::AbstractServerProcess(uint32 options) 47 : 48 AbstractProcess(), 49 fOptions(options), 50 fRequest(NULL) 51 { 52 } 53 54 55 AbstractServerProcess::~AbstractServerProcess() 56 { 57 } 58 59 60 bool 61 AbstractServerProcess::HasOption(uint32 flag) 62 { 63 return (fOptions & flag) == flag; 64 } 65 66 67 bool 68 AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready) 69 { 70 return 71 !HasOption(SERVER_PROCESS_NO_NETWORKING) 72 && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready); 73 } 74 75 76 status_t 77 AbstractServerProcess::StopInternal() 78 { 79 if (fRequest != NULL) { 80 return fRequest->Stop(); 81 } 82 83 return AbstractProcess::StopInternal(); 84 } 85 86 87 status_t 88 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const 89 { 90 BPath metaDataPath; 91 BString jsonPath; 92 93 status_t result = GetStandardMetaDataPath(metaDataPath); 94 95 if (result != B_OK) 96 return result; 97 98 GetStandardMetaDataJsonPath(jsonPath); 99 100 return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath); 101 } 102 103 104 status_t 105 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue, 106 const BPath& metaDataPath, const BString& jsonPath) const 107 { 108 headerValue.SetTo(""); 109 struct stat s; 110 111 if (-1 == stat(metaDataPath.Path(), &s)) { 112 if (ENOENT != errno) 113 return B_ERROR; 114 115 return B_ENTRY_NOT_FOUND; 116 } 117 118 if (s.st_size == 0) 119 return B_BAD_VALUE; 120 121 StandardMetaData metaData; 122 status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath); 123 124 if (result == B_OK) 125 SetIfModifiedSinceHeaderValueFromMetaData(headerValue, metaData); 126 else { 127 HDERROR("unable to parse the meta-data date and time from [%s]" 128 " - cannot set the 'If-Modified-Since' header", 129 metaDataPath.Path()); 130 } 131 132 return result; 133 } 134 135 136 /*static*/ void 137 AbstractServerProcess::SetIfModifiedSinceHeaderValueFromMetaData( 138 BString& headerValue, 139 const StandardMetaData& metaData) 140 { 141 // An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000' 142 BDateTime modifiedDateTime = metaData 143 .GetDataModifiedTimestampAsDateTime(); 144 BHttpTime modifiedHttpTime(modifiedDateTime); 145 headerValue.SetTo(modifiedHttpTime 146 .ToString(B_HTTP_TIME_FORMAT_COOKIE)); 147 } 148 149 150 status_t 151 AbstractServerProcess::PopulateMetaData( 152 StandardMetaData& metaData, const BPath& path, 153 const BString& jsonPath) const 154 { 155 StandardMetaDataJsonEventListener listener(jsonPath, metaData); 156 status_t result = ParseJsonFromFileWithListener(&listener, path); 157 158 if (result != B_OK) 159 return result; 160 161 result = listener.ErrorStatus(); 162 163 if (result != B_OK) 164 return result; 165 166 if (!metaData.IsPopulated()) { 167 HDERROR("the meta data was read from [%s], but no values " 168 "were extracted", path.Path()); 169 return B_BAD_DATA; 170 } 171 172 return B_OK; 173 } 174 175 176 /* static */ bool 177 AbstractServerProcess::_LooksLikeGzip(const char *pathStr) 178 { 179 int l = strlen(pathStr); 180 return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3); 181 } 182 183 184 /*! Note that a B_OK return code from this method may not indicate that the 185 listening process went well. One has to see if there was an error in 186 the listener. 187 */ 188 189 status_t 190 AbstractServerProcess::ParseJsonFromFileWithListener( 191 BJsonEventListener *listener, 192 const BPath& path) const 193 { 194 const char* pathStr = path.Path(); 195 FILE* file = fopen(pathStr, "rb"); 196 197 if (file == NULL) { 198 HDERROR("[%s] unable to find the meta data file at [%s]", Name(), 199 path.Path()); 200 return B_ENTRY_NOT_FOUND; 201 } 202 203 BFileIO rawInput(file, true); // takes ownership 204 205 BDataIO* bufferedRawInput = new BBufferedDataIO(rawInput, kFileBufferSize, false, true); 206 ObjectDeleter<BDataIO> bufferedRawInputDeleter(bufferedRawInput); 207 208 // if the file extension ends with '.gz' then the data will be 209 // compressed and the algorithm needs to decompress the data as 210 // it is parsed. 211 212 if (_LooksLikeGzip(pathStr)) { 213 BDataIO* gzDecompressedInput = NULL; 214 BZlibDecompressionParameters* zlibDecompressionParameters 215 = new BZlibDecompressionParameters(); 216 217 status_t result = BZlibCompressionAlgorithm() 218 .CreateDecompressingInputStream(bufferedRawInput, 219 zlibDecompressionParameters, gzDecompressedInput); 220 221 if (B_OK != result) 222 return result; 223 224 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput); 225 BPrivate::BJson::Parse(gzDecompressedInput, listener); 226 } else { 227 BPrivate::BJson::Parse(bufferedRawInput, listener); 228 } 229 230 return B_OK; 231 } 232 233 234 /*! In order to reduce the chance of failure half way through downloading a 235 large file, this method will download the file to a temporary file and 236 then it can rename the file to the final target file. 237 */ 238 239 status_t 240 AbstractServerProcess::DownloadToLocalFileAtomically( 241 const BPath& targetFilePath, 242 const BUrl& url) 243 { 244 BPath temporaryFilePath(tmpnam(NULL), NULL, true); 245 status_t result = DownloadToLocalFile( 246 temporaryFilePath, url, 0, 0); 247 248 // if the data is coming in as .gz, but is not stored as .gz then 249 // the data should be decompressed in the temporary file location 250 // before being shifted into place. 251 252 if (result == B_OK 253 && _LooksLikeGzip(url.Path()) 254 && !_LooksLikeGzip(targetFilePath.Path())) 255 result = _DeGzipInSitu(temporaryFilePath); 256 257 // not copying if the data has not changed because the data will be 258 // zero length. This is if the result is APP_ERR_NOT_MODIFIED. 259 if (result == B_OK) { 260 // if the file is zero length then assume that something has 261 // gone wrong. 262 off_t size; 263 bool hasFile; 264 265 result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL, 266 &size); 267 268 if (result == B_OK && hasFile && size > 0) { 269 if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) { 270 HDINFO("[%s] did rename [%s] --> [%s]", 271 Name(), temporaryFilePath.Path(), targetFilePath.Path()); 272 result = B_IO_ERROR; 273 } 274 } 275 } 276 277 return result; 278 } 279 280 281 /*static*/ status_t 282 AbstractServerProcess::_DeGzipInSitu(const BPath& path) 283 { 284 const char* tmpPath = tmpnam(NULL); 285 status_t result = B_OK; 286 287 { 288 BFile file(path.Path(), O_RDONLY); 289 BFile tmpFile(tmpPath, O_WRONLY | O_CREAT); 290 291 BDataIO* gzDecompressedInput = NULL; 292 BZlibDecompressionParameters* zlibDecompressionParameters 293 = new BZlibDecompressionParameters(); 294 295 result = BZlibCompressionAlgorithm() 296 .CreateDecompressingInputStream(&file, 297 zlibDecompressionParameters, gzDecompressedInput); 298 299 if (result == B_OK) { 300 ObjectDeleter<BDataIO> gzDecompressedInputDeleter( 301 gzDecompressedInput); 302 result = DataIOUtils::CopyAll(&tmpFile, gzDecompressedInput); 303 } 304 } 305 306 if (result == B_OK) { 307 if (rename(tmpPath, path.Path()) != 0) { 308 HDERROR("unable to move the uncompressed data into place"); 309 result = B_ERROR; 310 } 311 } 312 else { 313 HDERROR("it was not possible to decompress the data"); 314 } 315 316 return result; 317 } 318 319 320 status_t 321 AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, 322 const BUrl& url, uint32 redirects, uint32 failures) 323 { 324 if (WasStopped()) 325 return B_CANCELED; 326 327 if (redirects > MAX_REDIRECTS) { 328 HDINFO("[%s] exceeded %d redirects --> failure", Name(), 329 MAX_REDIRECTS); 330 return B_IO_ERROR; 331 } 332 333 if (failures > MAX_FAILURES) { 334 HDINFO("[%s] exceeded %d failures", Name(), MAX_FAILURES); 335 return B_IO_ERROR; 336 } 337 338 HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(), 339 targetFilePath.Path()); 340 341 LoggingUrlProtocolListener listener(Name(), Logger::IsTraceEnabled()); 342 BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT); 343 status_t err = targetFile.InitCheck(); 344 if (err != B_OK) 345 return err; 346 347 BHttpHeaders headers; 348 ServerSettings::AugmentHeaders(headers); 349 350 BString ifModifiedSinceHeader; 351 status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue( 352 ifModifiedSinceHeader); 353 354 if (ifModifiedSinceHeaderStatus == B_OK && 355 ifModifiedSinceHeader.Length() > 0) { 356 headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); 357 } 358 359 thread_id thread; 360 361 BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &targetFile, 362 &listener); 363 if (request == NULL) 364 return B_NO_MEMORY; 365 366 fRequest = dynamic_cast<BHttpRequest *>(request); 367 if (fRequest == NULL) { 368 delete request; 369 return B_ERROR; 370 } 371 fRequest->SetHeaders(headers); 372 fRequest->SetMaxRedirections(0); 373 fRequest->SetTimeout(TIMEOUT_MICROSECONDS); 374 fRequest->SetStopOnError(true); 375 thread = fRequest->Run(); 376 377 wait_for_thread(thread, NULL); 378 379 const BHttpResult& result = dynamic_cast<const BHttpResult&>( 380 fRequest->Result()); 381 int32 statusCode = result.StatusCode(); 382 const BHttpHeaders responseHeaders = result.Headers(); 383 const char *locationC = responseHeaders["Location"]; 384 BString location; 385 386 if (locationC != NULL) 387 location.SetTo(locationC); 388 389 delete fRequest; 390 fRequest = NULL; 391 392 if (BHttpRequest::IsSuccessStatusCode(statusCode)) { 393 HDINFO("[%s] did complete streaming data [%" B_PRIdSSIZE " bytes]", Name(), 394 listener.ContentLength()); 395 return B_OK; 396 } else if (statusCode == B_HTTP_STATUS_NOT_MODIFIED) { 397 HDINFO("[%s] remote data has not changed since [%s] so was not downloaded", Name(), 398 ifModifiedSinceHeader.String()); 399 return HD_ERR_NOT_MODIFIED; 400 } else if (statusCode == B_HTTP_STATUS_PRECONDITION_FAILED) { 401 ServerHelper::NotifyClientTooOld(responseHeaders); 402 return HD_CLIENT_TOO_OLD; 403 } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) { 404 if (location.Length() != 0) { 405 BUrl redirectUrl(result.Url(), location); 406 HDINFO("[%s] will redirect to; %s", Name(), redirectUrl.UrlString().String()); 407 return DownloadToLocalFile(targetFilePath, redirectUrl, redirects + 1, 0); 408 } 409 410 HDERROR("[%s] unable to find 'Location' header for redirect", Name()); 411 return B_IO_ERROR; 412 } else { 413 if (statusCode == 0 || (statusCode / 100) == 5) { 414 HDERROR("error response from server [%" B_PRId32 "] --> retry...", statusCode); 415 return DownloadToLocalFile(targetFilePath, url, redirects, failures + 1); 416 } 417 418 HDERROR("[%s] unexpected response from server [%" B_PRId32 "]", 419 Name(), statusCode); 420 return B_IO_ERROR; 421 } 422 } 423 424 425 status_t 426 AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath) 427 { 428 if (0 == remove(currentFilePath.Path())) 429 return B_OK; 430 431 return B_IO_ERROR; 432 } 433 434 435 /*! When a file is damaged or corrupted in some way, the file should be 'moved 436 aside' so that it is not involved in the next update. This method will 437 create such an alternative 'damaged' file location and move this file to 438 that location. 439 */ 440 441 status_t 442 AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath) 443 { 444 BPath damagedFilePath; 445 BString damagedLeaf; 446 447 damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf()); 448 currentFilePath.GetParent(&damagedFilePath); 449 damagedFilePath.Append(damagedLeaf.String()); 450 451 if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) { 452 HDERROR("[%s] unable to move damaged file [%s] aside to [%s]", 453 Name(), currentFilePath.Path(), damagedFilePath.Path()); 454 return B_IO_ERROR; 455 } 456 457 HDINFO("[%s] did move damaged file [%s] aside to [%s]", 458 Name(), currentFilePath.Path(), damagedFilePath.Path()); 459 460 return B_OK; 461 } 462 463 464 bool 465 AbstractServerProcess::IsSuccess(status_t e) { 466 return e == B_OK || e == HD_ERR_NOT_MODIFIED; 467 } 468