1 /*
2 * Copyright 2017-2023, Andrew Lindesay <apl@lindesay.co.nz>.
3 * All rights reserved. Distributed under the terms of the MIT License.
4 */
5
6
7 #include "AbstractServerProcess.h"
8
9 #include <unistd.h>
10 #include <errno.h>
11 #include <string.h>
12
13 #include <BufferedDataIO.h>
14 #include <AutoDeleter.h>
15 #include <File.h>
16 #include <FileIO.h>
17 #include <HttpTime.h>
18 #include <UrlProtocolRoster.h>
19
20 #include <support/ZlibCompressionAlgorithm.h>
21
22 #include "DataIOUtils.h"
23 #include "HaikuDepotConstants.h"
24 #include "Logger.h"
25 #include "ServerHelper.h"
26 #include "ServerSettings.h"
27 #include "StandardMetaDataJsonEventListener.h"
28 #include "StorageUtils.h"
29 #include "LoggingUrlProtocolListener.h"
30
31
32 using namespace BPrivate::Network;
33
34
35 #define MAX_REDIRECTS 3
36 #define MAX_FAILURES 2
37
38
39 // 30 seconds
40 #define TIMEOUT_MICROSECONDS 3e+7
41
42
43 const size_t kFileBufferSize = 10 * 1024;
44
45
AbstractServerProcess(uint32 options)46 AbstractServerProcess::AbstractServerProcess(uint32 options)
47 :
48 AbstractProcess(),
49 fOptions(options),
50 fRequest(NULL)
51 {
52 }
53
54
~AbstractServerProcess()55 AbstractServerProcess::~AbstractServerProcess()
56 {
57 }
58
59
60 bool
HasOption(uint32 flag)61 AbstractServerProcess::HasOption(uint32 flag)
62 {
63 return (fOptions & flag) == flag;
64 }
65
66
67 bool
ShouldAttemptNetworkDownload(bool hasDataAlready)68 AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready)
69 {
70 return
71 !HasOption(SERVER_PROCESS_NO_NETWORKING)
72 && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready);
73 }
74
75
76 status_t
StopInternal()77 AbstractServerProcess::StopInternal()
78 {
79 if (fRequest != NULL) {
80 return fRequest->Stop();
81 }
82
83 return AbstractProcess::StopInternal();
84 }
85
86
87 status_t
IfModifiedSinceHeaderValue(BString & headerValue) const88 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const
89 {
90 BPath metaDataPath;
91 BString jsonPath;
92
93 status_t result = GetStandardMetaDataPath(metaDataPath);
94
95 if (result != B_OK)
96 return result;
97
98 GetStandardMetaDataJsonPath(jsonPath);
99
100 return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath);
101 }
102
103
104 status_t
IfModifiedSinceHeaderValue(BString & headerValue,const BPath & metaDataPath,const BString & jsonPath) const105 AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue,
106 const BPath& metaDataPath, const BString& jsonPath) const
107 {
108 headerValue.SetTo("");
109 struct stat s;
110
111 if (-1 == stat(metaDataPath.Path(), &s)) {
112 if (ENOENT != errno)
113 return B_ERROR;
114
115 return B_ENTRY_NOT_FOUND;
116 }
117
118 if (s.st_size == 0)
119 return B_BAD_VALUE;
120
121 StandardMetaData metaData;
122 status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath);
123
124 if (result == B_OK)
125 SetIfModifiedSinceHeaderValueFromMetaData(headerValue, metaData);
126 else {
127 HDERROR("unable to parse the meta-data date and time from [%s]"
128 " - cannot set the 'If-Modified-Since' header",
129 metaDataPath.Path());
130 }
131
132 return result;
133 }
134
135
136 /*static*/ void
SetIfModifiedSinceHeaderValueFromMetaData(BString & headerValue,const StandardMetaData & metaData)137 AbstractServerProcess::SetIfModifiedSinceHeaderValueFromMetaData(
138 BString& headerValue,
139 const StandardMetaData& metaData)
140 {
141 // An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000'
142 BDateTime modifiedDateTime = metaData
143 .GetDataModifiedTimestampAsDateTime();
144 BHttpTime modifiedHttpTime(modifiedDateTime);
145 headerValue.SetTo(modifiedHttpTime
146 .ToString(B_HTTP_TIME_FORMAT_COOKIE));
147 }
148
149
150 status_t
PopulateMetaData(StandardMetaData & metaData,const BPath & path,const BString & jsonPath) const151 AbstractServerProcess::PopulateMetaData(
152 StandardMetaData& metaData, const BPath& path,
153 const BString& jsonPath) const
154 {
155 StandardMetaDataJsonEventListener listener(jsonPath, metaData);
156 status_t result = ParseJsonFromFileWithListener(&listener, path);
157
158 if (result != B_OK)
159 return result;
160
161 result = listener.ErrorStatus();
162
163 if (result != B_OK)
164 return result;
165
166 if (!metaData.IsPopulated()) {
167 HDERROR("the meta data was read from [%s], but no values "
168 "were extracted", path.Path());
169 return B_BAD_DATA;
170 }
171
172 return B_OK;
173 }
174
175
176 /* static */ bool
_LooksLikeGzip(const char * pathStr)177 AbstractServerProcess::_LooksLikeGzip(const char *pathStr)
178 {
179 int l = strlen(pathStr);
180 return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3);
181 }
182
183
184 /*! Note that a B_OK return code from this method may not indicate that the
185 listening process went well. One has to see if there was an error in
186 the listener.
187 */
188
189 status_t
ParseJsonFromFileWithListener(BJsonEventListener * listener,const BPath & path) const190 AbstractServerProcess::ParseJsonFromFileWithListener(
191 BJsonEventListener *listener,
192 const BPath& path) const
193 {
194 const char* pathStr = path.Path();
195 FILE* file = fopen(pathStr, "rb");
196
197 if (file == NULL) {
198 HDERROR("[%s] unable to find the meta data file at [%s]", Name(),
199 path.Path());
200 return B_ENTRY_NOT_FOUND;
201 }
202
203 BFileIO rawInput(file, true); // takes ownership
204
205 BDataIO* bufferedRawInput = new BBufferedDataIO(rawInput, kFileBufferSize, false, true);
206 ObjectDeleter<BDataIO> bufferedRawInputDeleter(bufferedRawInput);
207
208 // if the file extension ends with '.gz' then the data will be
209 // compressed and the algorithm needs to decompress the data as
210 // it is parsed.
211
212 if (_LooksLikeGzip(pathStr)) {
213 BDataIO* gzDecompressedInput = NULL;
214 BZlibDecompressionParameters* zlibDecompressionParameters
215 = new BZlibDecompressionParameters();
216
217 status_t result = BZlibCompressionAlgorithm()
218 .CreateDecompressingInputStream(bufferedRawInput,
219 zlibDecompressionParameters, gzDecompressedInput);
220
221 if (B_OK != result)
222 return result;
223
224 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput);
225 BPrivate::BJson::Parse(gzDecompressedInput, listener);
226 } else {
227 BPrivate::BJson::Parse(bufferedRawInput, listener);
228 }
229
230 return B_OK;
231 }
232
233
234 /*! In order to reduce the chance of failure half way through downloading a
235 large file, this method will download the file to a temporary file and
236 then it can rename the file to the final target file.
237 */
238
239 status_t
DownloadToLocalFileAtomically(const BPath & targetFilePath,const BUrl & url)240 AbstractServerProcess::DownloadToLocalFileAtomically(
241 const BPath& targetFilePath,
242 const BUrl& url)
243 {
244 BPath temporaryFilePath(tmpnam(NULL), NULL, true);
245 status_t result = DownloadToLocalFile(
246 temporaryFilePath, url, 0, 0);
247
248 // if the data is coming in as .gz, but is not stored as .gz then
249 // the data should be decompressed in the temporary file location
250 // before being shifted into place.
251
252 if (result == B_OK
253 && _LooksLikeGzip(url.Path())
254 && !_LooksLikeGzip(targetFilePath.Path()))
255 result = _DeGzipInSitu(temporaryFilePath);
256
257 // not copying if the data has not changed because the data will be
258 // zero length. This is if the result is APP_ERR_NOT_MODIFIED.
259 if (result == B_OK) {
260 // if the file is zero length then assume that something has
261 // gone wrong.
262 off_t size;
263 bool hasFile;
264
265 result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL,
266 &size);
267
268 if (result == B_OK && hasFile && size > 0) {
269 if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) {
270 HDINFO("[%s] did rename [%s] --> [%s]",
271 Name(), temporaryFilePath.Path(), targetFilePath.Path());
272 result = B_IO_ERROR;
273 }
274 }
275 }
276
277 return result;
278 }
279
280
281 /*static*/ status_t
_DeGzipInSitu(const BPath & path)282 AbstractServerProcess::_DeGzipInSitu(const BPath& path)
283 {
284 const char* tmpPath = tmpnam(NULL);
285 status_t result = B_OK;
286
287 {
288 BFile file(path.Path(), O_RDONLY);
289 BFile tmpFile(tmpPath, O_WRONLY | O_CREAT);
290
291 BDataIO* gzDecompressedInput = NULL;
292 BZlibDecompressionParameters* zlibDecompressionParameters
293 = new BZlibDecompressionParameters();
294
295 result = BZlibCompressionAlgorithm()
296 .CreateDecompressingInputStream(&file,
297 zlibDecompressionParameters, gzDecompressedInput);
298
299 if (result == B_OK) {
300 ObjectDeleter<BDataIO> gzDecompressedInputDeleter(
301 gzDecompressedInput);
302 result = DataIOUtils::CopyAll(&tmpFile, gzDecompressedInput);
303 }
304 }
305
306 if (result == B_OK) {
307 if (rename(tmpPath, path.Path()) != 0) {
308 HDERROR("unable to move the uncompressed data into place");
309 result = B_ERROR;
310 }
311 }
312 else {
313 HDERROR("it was not possible to decompress the data");
314 }
315
316 return result;
317 }
318
319
320 status_t
DownloadToLocalFile(const BPath & targetFilePath,const BUrl & url,uint32 redirects,uint32 failures)321 AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
322 const BUrl& url, uint32 redirects, uint32 failures)
323 {
324 if (WasStopped())
325 return B_CANCELED;
326
327 if (redirects > MAX_REDIRECTS) {
328 HDINFO("[%s] exceeded %d redirects --> failure", Name(),
329 MAX_REDIRECTS);
330 return B_IO_ERROR;
331 }
332
333 if (failures > MAX_FAILURES) {
334 HDINFO("[%s] exceeded %d failures", Name(), MAX_FAILURES);
335 return B_IO_ERROR;
336 }
337
338 HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(),
339 targetFilePath.Path());
340
341 LoggingUrlProtocolListener listener(Name(), Logger::IsTraceEnabled());
342 BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT);
343 status_t err = targetFile.InitCheck();
344 if (err != B_OK)
345 return err;
346
347 BHttpHeaders headers;
348 ServerSettings::AugmentHeaders(headers);
349
350 BString ifModifiedSinceHeader;
351 status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue(
352 ifModifiedSinceHeader);
353
354 if (ifModifiedSinceHeaderStatus == B_OK &&
355 ifModifiedSinceHeader.Length() > 0) {
356 headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
357 }
358
359 thread_id thread;
360
361 BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &targetFile,
362 &listener);
363 if (request == NULL)
364 return B_NO_MEMORY;
365
366 fRequest = dynamic_cast<BHttpRequest *>(request);
367 if (fRequest == NULL) {
368 delete request;
369 return B_ERROR;
370 }
371 fRequest->SetHeaders(headers);
372 fRequest->SetMaxRedirections(0);
373 fRequest->SetTimeout(TIMEOUT_MICROSECONDS);
374 fRequest->SetStopOnError(true);
375 thread = fRequest->Run();
376
377 wait_for_thread(thread, NULL);
378
379 const BHttpResult& result = dynamic_cast<const BHttpResult&>(
380 fRequest->Result());
381 int32 statusCode = result.StatusCode();
382 const BHttpHeaders responseHeaders = result.Headers();
383 const char *locationC = responseHeaders["Location"];
384 BString location;
385
386 if (locationC != NULL)
387 location.SetTo(locationC);
388
389 delete fRequest;
390 fRequest = NULL;
391
392 if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
393 HDINFO("[%s] did complete streaming data [%" B_PRIdSSIZE " bytes]", Name(),
394 listener.ContentLength());
395 return B_OK;
396 } else if (statusCode == B_HTTP_STATUS_NOT_MODIFIED) {
397 HDINFO("[%s] remote data has not changed since [%s] so was not downloaded", Name(),
398 ifModifiedSinceHeader.String());
399 return HD_ERR_NOT_MODIFIED;
400 } else if (statusCode == B_HTTP_STATUS_PRECONDITION_FAILED) {
401 ServerHelper::NotifyClientTooOld(responseHeaders);
402 return HD_CLIENT_TOO_OLD;
403 } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
404 if (location.Length() != 0) {
405 BUrl redirectUrl(result.Url(), location);
406 HDINFO("[%s] will redirect to; %s", Name(), redirectUrl.UrlString().String());
407 return DownloadToLocalFile(targetFilePath, redirectUrl, redirects + 1, 0);
408 }
409
410 HDERROR("[%s] unable to find 'Location' header for redirect", Name());
411 return B_IO_ERROR;
412 } else {
413 if (statusCode == 0 || (statusCode / 100) == 5) {
414 HDERROR("error response from server [%" B_PRId32 "] --> retry...", statusCode);
415 return DownloadToLocalFile(targetFilePath, url, redirects, failures + 1);
416 }
417
418 HDERROR("[%s] unexpected response from server [%" B_PRId32 "]",
419 Name(), statusCode);
420 return B_IO_ERROR;
421 }
422 }
423
424
425 status_t
DeleteLocalFile(const BPath & currentFilePath)426 AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath)
427 {
428 if (0 == remove(currentFilePath.Path()))
429 return B_OK;
430
431 return B_IO_ERROR;
432 }
433
434
435 /*! When a file is damaged or corrupted in some way, the file should be 'moved
436 aside' so that it is not involved in the next update. This method will
437 create such an alternative 'damaged' file location and move this file to
438 that location.
439 */
440
441 status_t
MoveDamagedFileAside(const BPath & currentFilePath)442 AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath)
443 {
444 BPath damagedFilePath;
445 BString damagedLeaf;
446
447 damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf());
448 currentFilePath.GetParent(&damagedFilePath);
449 damagedFilePath.Append(damagedLeaf.String());
450
451 if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) {
452 HDERROR("[%s] unable to move damaged file [%s] aside to [%s]",
453 Name(), currentFilePath.Path(), damagedFilePath.Path());
454 return B_IO_ERROR;
455 }
456
457 HDINFO("[%s] did move damaged file [%s] aside to [%s]",
458 Name(), currentFilePath.Path(), damagedFilePath.Path());
459
460 return B_OK;
461 }
462
463
464 bool
IsSuccess(status_t e)465 AbstractServerProcess::IsSuccess(status_t e) {
466 return e == B_OK || e == HD_ERR_NOT_MODIFIED;
467 }
468