1 /* 2 * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "AbstractServerProcess.h" 8 9 #include <unistd.h> 10 #include <errno.h> 11 #include <string.h> 12 13 #include <AutoDeleter.h> 14 #include <FileIO.h> 15 #include <HttpTime.h> 16 #include <UrlProtocolRoster.h> 17 18 #include <support/ZlibCompressionAlgorithm.h> 19 20 #include "DataIOUtils.h" 21 #include "HaikuDepotConstants.h" 22 #include "Logger.h" 23 #include "ServerHelper.h" 24 #include "ServerSettings.h" 25 #include "StandardMetaDataJsonEventListener.h" 26 #include "StorageUtils.h" 27 #include "ToFileUrlProtocolListener.h" 28 29 30 #define MAX_REDIRECTS 3 31 #define MAX_FAILURES 2 32 33 34 // 30 seconds 35 #define TIMEOUT_MICROSECONDS 3e+7 36 37 38 AbstractServerProcess::AbstractServerProcess(uint32 options) 39 : 40 AbstractProcess(), 41 fOptions(options), 42 fRequest(NULL) 43 { 44 } 45 46 47 AbstractServerProcess::~AbstractServerProcess() 48 { 49 } 50 51 52 bool 53 AbstractServerProcess::HasOption(uint32 flag) 54 { 55 return (fOptions & flag) == flag; 56 } 57 58 59 bool 60 AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready) 61 { 62 return 63 !HasOption(SERVER_PROCESS_NO_NETWORKING) 64 && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready); 65 } 66 67 68 status_t 69 AbstractServerProcess::StopInternal() 70 { 71 if (fRequest != NULL) { 72 return fRequest->Stop(); 73 } 74 75 return AbstractProcess::StopInternal(); 76 } 77 78 79 status_t 80 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const 81 { 82 BPath metaDataPath; 83 BString jsonPath; 84 85 status_t result = GetStandardMetaDataPath(metaDataPath); 86 87 if (result != B_OK) 88 return result; 89 90 GetStandardMetaDataJsonPath(jsonPath); 91 92 return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath); 93 } 94 95 96 status_t 97 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue, 98 const BPath& metaDataPath, const BString& jsonPath) const 99 { 100 headerValue.SetTo(""); 101 struct stat s; 102 103 if (-1 == stat(metaDataPath.Path(), &s)) { 104 if (ENOENT != errno) 105 return B_ERROR; 106 107 return B_FILE_NOT_FOUND; 108 } 109 110 if (s.st_size == 0) 111 return B_BAD_VALUE; 112 113 StandardMetaData metaData; 114 status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath); 115 116 if (result == B_OK) 117 SetIfModifiedSinceHeaderValueFromMetaData(headerValue, metaData); 118 else { 119 HDERROR("unable to parse the meta-data date and time from [%s]" 120 " - cannot set the 'If-Modified-Since' header", 121 metaDataPath.Path()); 122 } 123 124 return result; 125 } 126 127 128 /*static*/ void 129 AbstractServerProcess::SetIfModifiedSinceHeaderValueFromMetaData( 130 BString& headerValue, 131 const StandardMetaData& metaData) 132 { 133 // An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000' 134 BDateTime modifiedDateTime = metaData 135 .GetDataModifiedTimestampAsDateTime(); 136 BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime); 137 headerValue.SetTo(modifiedHttpTime 138 .ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE)); 139 } 140 141 142 status_t 143 AbstractServerProcess::PopulateMetaData( 144 StandardMetaData& metaData, const BPath& path, 145 const BString& jsonPath) const 146 { 147 StandardMetaDataJsonEventListener listener(jsonPath, metaData); 148 status_t result = ParseJsonFromFileWithListener(&listener, path); 149 150 if (result != B_OK) 151 return result; 152 153 result = listener.ErrorStatus(); 154 155 if (result != B_OK) 156 return result; 157 158 if (!metaData.IsPopulated()) { 159 HDERROR("the meta data was read from [%s], but no values " 160 "were extracted", path.Path()); 161 return B_BAD_DATA; 162 } 163 164 return B_OK; 165 } 166 167 168 /* static */ bool 169 AbstractServerProcess::_LooksLikeGzip(const char *pathStr) 170 { 171 int l = strlen(pathStr); 172 return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3); 173 } 174 175 176 /*! Note that a B_OK return code from this method may not indicate that the 177 listening process went well. One has to see if there was an error in 178 the listener. 179 */ 180 181 status_t 182 AbstractServerProcess::ParseJsonFromFileWithListener( 183 BJsonEventListener *listener, 184 const BPath& path) const 185 { 186 const char* pathStr = path.Path(); 187 FILE* file = fopen(pathStr, "rb"); 188 189 if (file == NULL) { 190 HDERROR("[%s] unable to find the meta data file at [%s]", Name(), 191 path.Path()); 192 return B_FILE_NOT_FOUND; 193 } 194 195 BFileIO rawInput(file, true); // takes ownership 196 197 // if the file extension ends with '.gz' then the data will be 198 // compressed and the algorithm needs to decompress the data as 199 // it is parsed. 200 201 if (_LooksLikeGzip(pathStr)) { 202 BDataIO* gzDecompressedInput = NULL; 203 BZlibDecompressionParameters* zlibDecompressionParameters 204 = new BZlibDecompressionParameters(); 205 206 status_t result = BZlibCompressionAlgorithm() 207 .CreateDecompressingInputStream(&rawInput, 208 zlibDecompressionParameters, gzDecompressedInput); 209 210 if (B_OK != result) 211 return result; 212 213 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput); 214 BPrivate::BJson::Parse(gzDecompressedInput, listener); 215 } else { 216 BPrivate::BJson::Parse(&rawInput, listener); 217 } 218 219 return B_OK; 220 } 221 222 223 /*! In order to reduce the chance of failure half way through downloading a 224 large file, this method will download the file to a temporary file and 225 then it can rename the file to the final target file. 226 */ 227 228 status_t 229 AbstractServerProcess::DownloadToLocalFileAtomically( 230 const BPath& targetFilePath, 231 const BUrl& url) 232 { 233 BPath temporaryFilePath(tmpnam(NULL), NULL, true); 234 status_t result = DownloadToLocalFile( 235 temporaryFilePath, url, 0, 0); 236 237 // if the data is coming in as .gz, but is not stored as .gz then 238 // the data should be decompressed in the temporary file location 239 // before being shifted into place. 240 241 if (result == B_OK 242 && _LooksLikeGzip(url.Path()) 243 && !_LooksLikeGzip(targetFilePath.Path())) 244 result = _DeGzipInSitu(temporaryFilePath); 245 246 // not copying if the data has not changed because the data will be 247 // zero length. This is if the result is APP_ERR_NOT_MODIFIED. 248 if (result == B_OK) { 249 // if the file is zero length then assume that something has 250 // gone wrong. 251 off_t size; 252 bool hasFile; 253 254 result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL, 255 &size); 256 257 if (result == B_OK && hasFile && size > 0) { 258 if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) { 259 HDINFO("[%s] did rename [%s] --> [%s]", 260 Name(), temporaryFilePath.Path(), targetFilePath.Path()); 261 result = B_IO_ERROR; 262 } 263 } 264 } 265 266 return result; 267 } 268 269 270 /*static*/ status_t 271 AbstractServerProcess::_DeGzipInSitu(const BPath& path) 272 { 273 const char* tmpPath = tmpnam(NULL); 274 status_t result = B_OK; 275 276 { 277 BFile file(path.Path(), O_RDONLY); 278 BFile tmpFile(tmpPath, O_WRONLY | O_CREAT); 279 280 BDataIO* gzDecompressedInput = NULL; 281 BZlibDecompressionParameters* zlibDecompressionParameters 282 = new BZlibDecompressionParameters(); 283 284 result = BZlibCompressionAlgorithm() 285 .CreateDecompressingInputStream(&file, 286 zlibDecompressionParameters, gzDecompressedInput); 287 288 if (result == B_OK) { 289 ObjectDeleter<BDataIO> gzDecompressedInputDeleter( 290 gzDecompressedInput); 291 result = DataIOUtils::CopyAll(&tmpFile, gzDecompressedInput); 292 } 293 } 294 295 if (result == B_OK) { 296 if (rename(tmpPath, path.Path()) != 0) { 297 HDERROR("unable to move the uncompressed data into place"); 298 result = B_ERROR; 299 } 300 } 301 else { 302 HDERROR("it was not possible to decompress the data"); 303 } 304 305 return result; 306 } 307 308 309 status_t 310 AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, 311 const BUrl& url, uint32 redirects, uint32 failures) 312 { 313 if (WasStopped()) 314 return B_CANCELED; 315 316 if (redirects > MAX_REDIRECTS) { 317 HDINFO("[%s] exceeded %d redirects --> failure", Name(), 318 MAX_REDIRECTS); 319 return B_IO_ERROR; 320 } 321 322 if (failures > MAX_FAILURES) { 323 HDINFO("[%s] exceeded %d failures", Name(), MAX_FAILURES); 324 return B_IO_ERROR; 325 } 326 327 HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(), 328 targetFilePath.Path()); 329 330 ToFileUrlProtocolListener listener(targetFilePath, Name(), 331 Logger::IsTraceEnabled()); 332 333 BHttpHeaders headers; 334 ServerSettings::AugmentHeaders(headers); 335 336 BString ifModifiedSinceHeader; 337 status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue( 338 ifModifiedSinceHeader); 339 340 if (ifModifiedSinceHeaderStatus == B_OK && 341 ifModifiedSinceHeader.Length() > 0) { 342 headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); 343 } 344 345 thread_id thread; 346 347 { 348 fRequest = dynamic_cast<BHttpRequest *>( 349 BUrlProtocolRoster::MakeRequest(url, &listener)); 350 fRequest->SetHeaders(headers); 351 fRequest->SetMaxRedirections(0); 352 fRequest->SetTimeout(TIMEOUT_MICROSECONDS); 353 thread = fRequest->Run(); 354 } 355 356 wait_for_thread(thread, NULL); 357 358 const BHttpResult& result = dynamic_cast<const BHttpResult&>( 359 fRequest->Result()); 360 int32 statusCode = result.StatusCode(); 361 const BHttpHeaders responseHeaders = result.Headers(); 362 const char *locationC = responseHeaders["Location"]; 363 BString location; 364 365 if (locationC != NULL) 366 location.SetTo(locationC); 367 368 delete fRequest; 369 fRequest = NULL; 370 371 if (BHttpRequest::IsSuccessStatusCode(statusCode)) { 372 HDINFO("[%s] did complete streaming data [%" 373 B_PRIdSSIZE " bytes]", Name(), listener.ContentLength()); 374 return B_OK; 375 } else if (statusCode == B_HTTP_STATUS_NOT_MODIFIED) { 376 HDINFO("[%s] remote data has not changed since [%s]", Name(), 377 ifModifiedSinceHeader.String()); 378 return HD_ERR_NOT_MODIFIED; 379 } else if (statusCode == B_HTTP_STATUS_PRECONDITION_FAILED) { 380 ServerHelper::NotifyClientTooOld(responseHeaders); 381 return HD_CLIENT_TOO_OLD; 382 } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) { 383 if (location.Length() != 0) { 384 BUrl redirectUrl(result.Url(), location); 385 HDINFO("[%s] will redirect to; %s", 386 Name(), redirectUrl.UrlString().String()); 387 return DownloadToLocalFile(targetFilePath, redirectUrl, 388 redirects + 1, 0); 389 } 390 391 HDERROR("[%s] unable to find 'Location' header for redirect", Name()); 392 return B_IO_ERROR; 393 } else { 394 if (statusCode == 0 || (statusCode / 100) == 5) { 395 HDERROR("error response from server [%" B_PRId32 "] --> retry...", 396 statusCode); 397 return DownloadToLocalFile(targetFilePath, url, redirects, 398 failures + 1); 399 } 400 401 HDERROR("[%s] unexpected response from server [%" B_PRId32 "]", 402 Name(), statusCode); 403 return B_IO_ERROR; 404 } 405 } 406 407 408 status_t 409 AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath) 410 { 411 if (0 == remove(currentFilePath.Path())) 412 return B_OK; 413 414 return B_IO_ERROR; 415 } 416 417 418 /*! When a file is damaged or corrupted in some way, the file should be 'moved 419 aside' so that it is not involved in the next update. This method will 420 create such an alternative 'damaged' file location and move this file to 421 that location. 422 */ 423 424 status_t 425 AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath) 426 { 427 BPath damagedFilePath; 428 BString damagedLeaf; 429 430 damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf()); 431 currentFilePath.GetParent(&damagedFilePath); 432 damagedFilePath.Append(damagedLeaf.String()); 433 434 if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) { 435 HDERROR("[%s] unable to move damaged file [%s] aside to [%s]", 436 Name(), currentFilePath.Path(), damagedFilePath.Path()); 437 return B_IO_ERROR; 438 } 439 440 HDINFO("[%s] did move damaged file [%s] aside to [%s]", 441 Name(), currentFilePath.Path(), damagedFilePath.Path()); 442 443 return B_OK; 444 } 445 446 447 bool 448 AbstractServerProcess::IsSuccess(status_t e) { 449 return e == B_OK || e == HD_ERR_NOT_MODIFIED; 450 } 451