xref: /haiku/src/kits/network/libnetservices/HttpRequest.cpp (revision ef240bfffa9bd342a276eb73691a6996aa254156) 
1 /*
2  * Copyright 2010-2021 Haiku Inc. All rights reserved.
3  * Distributed under the terms of the MIT License.
4  *
5  * Authors:
6  *		Christophe Huriaux, c.huriaux@gmail.com
7  *		Niels Sascha Reedijk, niels.reedijk@gmail.com
8  *		Adrien Destugues, pulkomandy@pulkomandy.tk
9  *		Stephan Aßmus, superstippi@gmx.de
10  */
11 
12 
13 #include <HttpRequest.h>
14 
15 #include <arpa/inet.h>
16 #include <stdio.h>
17 
18 #include <cstdlib>
19 #include <deque>
20 #include <new>
21 
22 #include <AutoDeleter.h>
23 #include <Certificate.h>
24 #include <Debug.h>
25 #include <DynamicBuffer.h>
26 #include <File.h>
27 #include <ProxySecureSocket.h>
28 #include <Socket.h>
29 #include <SecureSocket.h>
30 #include <StackOrHeapArray.h>
31 #include <ZlibCompressionAlgorithm.h>
32 
33 using namespace BPrivate::Network;
34 
35 
36 static const int32 kHttpBufferSize = 4096;
37 
38 
39 namespace BPrivate {
40 
41 	class CheckedSecureSocket: public BSecureSocket
42 	{
43 		public:
44 			CheckedSecureSocket(BHttpRequest* request);
45 
46 			bool			CertificateVerificationFailed(BCertificate& certificate,
47 					const char* message);
48 
49 		private:
50 			BHttpRequest*	fRequest;
51 	};
52 
53 
54 	CheckedSecureSocket::CheckedSecureSocket(BHttpRequest* request)
55 		:
56 		BSecureSocket(),
57 		fRequest(request)
58 	{
59 	}
60 
61 
62 	bool
63 	CheckedSecureSocket::CertificateVerificationFailed(BCertificate& certificate,
64 		const char* message)
65 	{
66 		return fRequest->_CertificateVerificationFailed(certificate, message);
67 	}
68 
69 
70 	class CheckedProxySecureSocket: public BProxySecureSocket
71 	{
72 		public:
73 			CheckedProxySecureSocket(const BNetworkAddress& proxy, BHttpRequest* request);
74 
75 			bool			CertificateVerificationFailed(BCertificate& certificate,
76 					const char* message);
77 
78 		private:
79 			BHttpRequest*	fRequest;
80 	};
81 
82 
83 	CheckedProxySecureSocket::CheckedProxySecureSocket(const BNetworkAddress& proxy,
84 		BHttpRequest* request)
85 		:
86 		BProxySecureSocket(proxy),
87 		fRequest(request)
88 	{
89 	}
90 
91 
92 	bool
93 	CheckedProxySecureSocket::CertificateVerificationFailed(BCertificate& certificate,
94 		const char* message)
95 	{
96 		return fRequest->_CertificateVerificationFailed(certificate, message);
97 	}
98 };
99 
100 
101 BHttpRequest::BHttpRequest(const BUrl& url, BDataIO* output, bool ssl,
102 	const char* protocolName, BUrlProtocolListener* listener,
103 	BUrlContext* context)
104 	:
105 	BNetworkRequest(url, output, listener, context, "BUrlProtocol.HTTP",
106 		protocolName),
107 	fSSL(ssl),
108 	fRequestMethod(B_HTTP_GET),
109 	fHttpVersion(B_HTTP_11),
110 	fResult(url),
111 	fRequestStatus(kRequestInitialState),
112 	fOptHeaders(NULL),
113 	fOptPostFields(NULL),
114 	fOptInputData(NULL),
115 	fOptInputDataSize(-1),
116 	fOptRangeStart(-1),
117 	fOptRangeEnd(-1),
118 	fOptFollowLocation(true)
119 {
120 	_ResetOptions();
121 	fSocket = NULL;
122 }
123 
124 
125 BHttpRequest::BHttpRequest(const BHttpRequest& other)
126 	:
127 	BNetworkRequest(other.Url(), other.Output(), other.fListener,
128 		other.fContext, "BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"),
129 	fSSL(other.fSSL),
130 	fRequestMethod(other.fRequestMethod),
131 	fHttpVersion(other.fHttpVersion),
132 	fResult(other.fUrl),
133 	fRequestStatus(kRequestInitialState),
134 	fOptHeaders(NULL),
135 	fOptPostFields(NULL),
136 	fOptInputData(NULL),
137 	fOptInputDataSize(-1),
138 	fOptRangeStart(other.fOptRangeStart),
139 	fOptRangeEnd(other.fOptRangeEnd),
140 	fOptFollowLocation(other.fOptFollowLocation)
141 {
142 	_ResetOptions();
143 		// FIXME some options may be copied from other instead.
144 	fSocket = NULL;
145 }
146 
147 
148 BHttpRequest::~BHttpRequest()
149 {
150 	Stop();
151 
152 	delete fSocket;
153 
154 	delete fOptInputData;
155 	delete fOptHeaders;
156 	delete fOptPostFields;
157 }
158 
159 
160 void
161 BHttpRequest::SetMethod(const char* const method)
162 {
163 	fRequestMethod = method;
164 }
165 
166 
167 void
168 BHttpRequest::SetFollowLocation(bool follow)
169 {
170 	fOptFollowLocation = follow;
171 }
172 
173 
174 void
175 BHttpRequest::SetMaxRedirections(int8 redirections)
176 {
177 	fOptMaxRedirs = redirections;
178 }
179 
180 
181 void
182 BHttpRequest::SetReferrer(const BString& referrer)
183 {
184 	fOptReferer = referrer;
185 }
186 
187 
188 void
189 BHttpRequest::SetUserAgent(const BString& agent)
190 {
191 	fOptUserAgent = agent;
192 }
193 
194 
195 void
196 BHttpRequest::SetDiscardData(bool discard)
197 {
198 	fOptDiscardData = discard;
199 }
200 
201 
202 void
203 BHttpRequest::SetDisableListener(bool disable)
204 {
205 	fOptDisableListener = disable;
206 }
207 
208 
209 void
210 BHttpRequest::SetAutoReferrer(bool enable)
211 {
212 	fOptAutoReferer = enable;
213 }
214 
215 
216 void
217 BHttpRequest::SetStopOnError(bool stop)
218 {
219 	fOptStopOnError = stop;
220 }
221 
222 
223 void
224 BHttpRequest::SetUserName(const BString& name)
225 {
226 	fOptUsername = name;
227 }
228 
229 
230 void
231 BHttpRequest::SetPassword(const BString& password)
232 {
233 	fOptPassword = password;
234 }
235 
236 
237 void
238 BHttpRequest::SetRangeStart(off_t position)
239 {
240 	// This field is used within the transfer loop, so only
241 	// allow setting it before sending the request.
242 	if (fRequestStatus == kRequestInitialState)
243 		fOptRangeStart = position;
244 }
245 
246 
247 void
248 BHttpRequest::SetRangeEnd(off_t position)
249 {
250 	// This field could be used in the transfer loop, so only
251 	// allow setting it before sending the request.
252 	if (fRequestStatus == kRequestInitialState)
253 		fOptRangeEnd = position;
254 }
255 
256 
257 void
258 BHttpRequest::SetPostFields(const BHttpForm& fields)
259 {
260 	AdoptPostFields(new(std::nothrow) BHttpForm(fields));
261 }
262 
263 
264 void
265 BHttpRequest::SetHeaders(const BHttpHeaders& headers)
266 {
267 	AdoptHeaders(new(std::nothrow) BHttpHeaders(headers));
268 }
269 
270 
271 void
272 BHttpRequest::AdoptPostFields(BHttpForm* const fields)
273 {
274 	delete fOptPostFields;
275 	fOptPostFields = fields;
276 
277 	if (fOptPostFields != NULL)
278 		fRequestMethod = B_HTTP_POST;
279 }
280 
281 
282 void
283 BHttpRequest::AdoptInputData(BDataIO* const data, const ssize_t size)
284 {
285 	delete fOptInputData;
286 	fOptInputData = data;
287 	fOptInputDataSize = size;
288 }
289 
290 
291 void
292 BHttpRequest::AdoptHeaders(BHttpHeaders* const headers)
293 {
294 	delete fOptHeaders;
295 	fOptHeaders = headers;
296 }
297 
298 
299 /*static*/ bool
300 BHttpRequest::IsInformationalStatusCode(int16 code)
301 {
302 	return (code >= B_HTTP_STATUS__INFORMATIONAL_BASE)
303 		&& (code <  B_HTTP_STATUS__INFORMATIONAL_END);
304 }
305 
306 
307 /*static*/ bool
308 BHttpRequest::IsSuccessStatusCode(int16 code)
309 {
310 	return (code >= B_HTTP_STATUS__SUCCESS_BASE)
311 		&& (code <  B_HTTP_STATUS__SUCCESS_END);
312 }
313 
314 
315 /*static*/ bool
316 BHttpRequest::IsRedirectionStatusCode(int16 code)
317 {
318 	return (code >= B_HTTP_STATUS__REDIRECTION_BASE)
319 		&& (code <  B_HTTP_STATUS__REDIRECTION_END);
320 }
321 
322 
323 /*static*/ bool
324 BHttpRequest::IsClientErrorStatusCode(int16 code)
325 {
326 	return (code >= B_HTTP_STATUS__CLIENT_ERROR_BASE)
327 		&& (code <  B_HTTP_STATUS__CLIENT_ERROR_END);
328 }
329 
330 
331 /*static*/ bool
332 BHttpRequest::IsServerErrorStatusCode(int16 code)
333 {
334 	return (code >= B_HTTP_STATUS__SERVER_ERROR_BASE)
335 		&& (code <  B_HTTP_STATUS__SERVER_ERROR_END);
336 }
337 
338 
339 /*static*/ int16
340 BHttpRequest::StatusCodeClass(int16 code)
341 {
342 	if (BHttpRequest::IsInformationalStatusCode(code))
343 		return B_HTTP_STATUS_CLASS_INFORMATIONAL;
344 	else if (BHttpRequest::IsSuccessStatusCode(code))
345 		return B_HTTP_STATUS_CLASS_SUCCESS;
346 	else if (BHttpRequest::IsRedirectionStatusCode(code))
347 		return B_HTTP_STATUS_CLASS_REDIRECTION;
348 	else if (BHttpRequest::IsClientErrorStatusCode(code))
349 		return B_HTTP_STATUS_CLASS_CLIENT_ERROR;
350 	else if (BHttpRequest::IsServerErrorStatusCode(code))
351 		return B_HTTP_STATUS_CLASS_SERVER_ERROR;
352 
353 	return B_HTTP_STATUS_CLASS_INVALID;
354 }
355 
356 
357 const BUrlResult&
358 BHttpRequest::Result() const
359 {
360 	return fResult;
361 }
362 
363 
364 status_t
365 BHttpRequest::Stop()
366 {
367 
368 	if (fSocket != NULL) {
369 		fSocket->Disconnect();
370 			// Unlock any pending connect, read or write operation.
371 	}
372 	return BNetworkRequest::Stop();
373 }
374 
375 
376 void
377 BHttpRequest::_ResetOptions()
378 {
379 	delete fOptPostFields;
380 	delete fOptHeaders;
381 
382 	fOptFollowLocation = true;
383 	fOptMaxRedirs = 8;
384 	fOptReferer = "";
385 	fOptUserAgent = "Services Kit (Haiku)";
386 	fOptUsername = "";
387 	fOptPassword = "";
388 	fOptAuthMethods = B_HTTP_AUTHENTICATION_BASIC | B_HTTP_AUTHENTICATION_DIGEST
389 		| B_HTTP_AUTHENTICATION_IE_DIGEST;
390 	fOptHeaders = NULL;
391 	fOptPostFields = NULL;
392 	fOptSetCookies = true;
393 	fOptDiscardData = false;
394 	fOptDisableListener = false;
395 	fOptAutoReferer = true;
396 }
397 
398 
399 status_t
400 BHttpRequest::_ProtocolLoop()
401 {
402 	// Initialize the request redirection loop
403 	int8 maxRedirs = fOptMaxRedirs;
404 	bool newRequest;
405 
406 	do {
407 		newRequest = false;
408 
409 		// Result reset
410 		fHeaders.Clear();
411 		_ResultHeaders().Clear();
412 
413 		BString host = fUrl.Host();
414 		int port = fSSL ? 443 : 80;
415 
416 		if (fUrl.HasPort())
417 			port = fUrl.Port();
418 
419 		if (fContext->UseProxy()) {
420 			host = fContext->GetProxyHost();
421 			port = fContext->GetProxyPort();
422 		}
423 
424 		status_t result = fInputBuffer.InitCheck();
425 		if (result != B_OK)
426 			return result;
427 
428 		if (!_ResolveHostName(host, port)) {
429 			_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR,
430 				"Unable to resolve hostname (%s), aborting.",
431 					fUrl.Host().String());
432 			return B_SERVER_NOT_FOUND;
433 		}
434 
435 		status_t requestStatus = _MakeRequest();
436 		if (requestStatus != B_OK)
437 			return requestStatus;
438 
439 		// Prepare the referer for the next request if needed
440 		if (fOptAutoReferer)
441 			fOptReferer = fUrl.UrlString();
442 
443 		switch (StatusCodeClass(fResult.StatusCode())) {
444 			case B_HTTP_STATUS_CLASS_INFORMATIONAL:
445 				// Header 100:continue should have been
446 				// handled in the _MakeRequest read loop
447 				break;
448 
449 			case B_HTTP_STATUS_CLASS_SUCCESS:
450 				break;
451 
452 			case B_HTTP_STATUS_CLASS_REDIRECTION:
453 			{
454 				// Redirection has been explicitly disabled
455 				if (!fOptFollowLocation)
456 					break;
457 
458 				int code = fResult.StatusCode();
459 				if (code == B_HTTP_STATUS_MOVED_PERMANENTLY
460 					|| code == B_HTTP_STATUS_FOUND
461 					|| code == B_HTTP_STATUS_SEE_OTHER
462 					|| code == B_HTTP_STATUS_TEMPORARY_REDIRECT) {
463 					BString locationUrl = fHeaders["Location"];
464 
465 					fUrl = BUrl(fUrl, locationUrl);
466 
467 					// 302 and 303 redirections also convert POST requests to GET
468 					// (and remove the posted form data)
469 					if ((code == B_HTTP_STATUS_FOUND
470 						|| code == B_HTTP_STATUS_SEE_OTHER)
471 						&& fRequestMethod == B_HTTP_POST) {
472 						SetMethod(B_HTTP_GET);
473 						delete fOptPostFields;
474 						fOptPostFields = NULL;
475 						delete fOptInputData;
476 						fOptInputData = NULL;
477 						fOptInputDataSize = 0;
478 					}
479 
480 					if (--maxRedirs > 0) {
481 						newRequest = true;
482 
483 						// Redirections may need a switch from http to https.
484 						if (fUrl.Protocol() == "https")
485 							fSSL = true;
486 						else if (fUrl.Protocol() == "http")
487 							fSSL = false;
488 
489 						_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
490 							"Following: %s\n",
491 							fUrl.UrlString().String());
492 					}
493 				}
494 				break;
495 			}
496 
497 			case B_HTTP_STATUS_CLASS_CLIENT_ERROR:
498 				if (fResult.StatusCode() == B_HTTP_STATUS_UNAUTHORIZED) {
499 					BHttpAuthentication* authentication
500 						= &fContext->GetAuthentication(fUrl);
501 					status_t status = B_OK;
502 
503 					if (authentication->Method() == B_HTTP_AUTHENTICATION_NONE) {
504 						// There is no authentication context for this
505 						// url yet, so let's create one.
506 						BHttpAuthentication newAuth;
507 						newAuth.Initialize(fHeaders["WWW-Authenticate"]);
508 						fContext->AddAuthentication(fUrl, newAuth);
509 
510 						// Get the copy of the authentication we just added.
511 						// That copy is owned by the BUrlContext and won't be
512 						// deleted (unlike the temporary object above)
513 						authentication = &fContext->GetAuthentication(fUrl);
514 					}
515 
516 					newRequest = false;
517 					if (fOptUsername.Length() > 0 && status == B_OK) {
518 						// If we received an username and password, add them
519 						// to the request. This will either change the
520 						// credentials for an existing request, or set them
521 						// for a new one we created just above.
522 						//
523 						// If this request handles HTTP redirections, it will
524 						// also automatically retry connecting and send the
525 						// login information.
526 						authentication->SetUserName(fOptUsername);
527 						authentication->SetPassword(fOptPassword);
528 						newRequest = true;
529 					}
530 				}
531 				break;
532 
533 			case B_HTTP_STATUS_CLASS_SERVER_ERROR:
534 				break;
535 
536 			default:
537 			case B_HTTP_STATUS_CLASS_INVALID:
538 				break;
539 		}
540 	} while (newRequest);
541 
542 	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
543 		"%" B_PRId32 " headers and %" B_PRIuSIZE " bytes of data remaining",
544 		fHeaders.CountHeaders(), fInputBuffer.Size());
545 
546 	if (fResult.StatusCode() == 404)
547 		return B_RESOURCE_NOT_FOUND;
548 
549 	return B_OK;
550 }
551 
552 
553 status_t
554 BHttpRequest::_MakeRequest()
555 {
556 	delete fSocket;
557 
558 	if (fSSL) {
559 		if (fContext->UseProxy()) {
560 			BNetworkAddress proxy(fContext->GetProxyHost(), fContext->GetProxyPort());
561 			fSocket = new(std::nothrow) BPrivate::CheckedProxySecureSocket(proxy, this);
562 		} else
563 			fSocket = new(std::nothrow) BPrivate::CheckedSecureSocket(this);
564 	} else
565 		fSocket = new(std::nothrow) BSocket();
566 
567 	if (fSocket == NULL)
568 		return B_NO_MEMORY;
569 
570 	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
571 		fUrl.Authority().String(), fRemoteAddr.Port());
572 	status_t connectError = fSocket->Connect(fRemoteAddr);
573 
574 	if (connectError != B_OK) {
575 		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
576 			strerror(connectError));
577 		return connectError;
578 	}
579 
580 	//! ProtocolHook:ConnectionOpened
581 	if (fListener != NULL)
582 		fListener->ConnectionOpened(this);
583 
584 	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
585 		"Connection opened, sending request.");
586 
587 	BString requestHeaders;
588 	requestHeaders.Append(_SerializeRequest());
589 	requestHeaders.Append(_SerializeHeaders());
590 	requestHeaders.Append("\r\n");
591 	fSocket->Write(requestHeaders.String(), requestHeaders.Length());
592 	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");
593 
594 	_SendPostData();
595 	fRequestStatus = kRequestInitialState;
596 
597 
598 
599 	// Receive loop
600 	bool disableListener = false;
601 	bool receiveEnd = false;
602 	bool parseEnd = false;
603 	bool readByChunks = false;
604 	bool decompress = false;
605 	status_t readError = B_OK;
606 	ssize_t bytesRead = 0;
607 	off_t bytesReceived = 0;
608 	off_t bytesTotal = 0;
609 	size_t previousBufferSize = 0;
610 	off_t bytesUnpacked = 0;
611 	char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize];
612 	ArrayDeleter<char> inputTempBufferDeleter(inputTempBuffer);
613 	ssize_t inputTempSize = kHttpBufferSize;
614 	ssize_t chunkSize = -1;
615 	DynamicBuffer decompressorStorage;
616 	BDataIO* decompressingStream;
617 	ObjectDeleter<BDataIO> decompressingStreamDeleter;
618 
619 	while (!fQuit && !(receiveEnd && parseEnd)) {
620 		if ((!receiveEnd) && (fInputBuffer.Size() == previousBufferSize)) {
621 			BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize);
622 			bytesRead = fSocket->Read(chunk, kHttpBufferSize);
623 
624 			if (bytesRead < 0) {
625 				readError = bytesRead;
626 				break;
627 			} else if (bytesRead == 0) {
628 				// Check if we got the expected number of bytes.
629 				// Exceptions:
630 				// - If the content-length is not known (bytesTotal is 0), for
631 				//   example in the case of a chunked transfer, we can't know
632 				// - If the request method is "HEAD" which explicitly asks the
633 				//   server to not send any data (only the headers)
634 				if (bytesTotal > 0 && bytesReceived != bytesTotal) {
635 					readError = B_IO_ERROR;
636 					break;
637 				}
638 				receiveEnd = true;
639 			}
640 
641 			fInputBuffer.AppendData(chunk, bytesRead);
642 		} else
643 			bytesRead = 0;
644 
645 		previousBufferSize = fInputBuffer.Size();
646 
647 		if (fRequestStatus < kRequestStatusReceived) {
648 			_ParseStatus();
649 
650 			if (fOptFollowLocation
651 					&& IsRedirectionStatusCode(fResult.StatusCode()))
652 				disableListener = true;
653 
654 			if (fOptStopOnError
655 					&& fResult.StatusCode() >= B_HTTP_STATUS_CLASS_CLIENT_ERROR)
656 			{
657 				fQuit = true;
658 				break;
659 			}
660 
661 			//! ProtocolHook:ResponseStarted
662 			if (fRequestStatus >= kRequestStatusReceived && fListener != NULL
663 					&& !disableListener)
664 				fListener->ResponseStarted(this);
665 		}
666 
667 		if (fRequestStatus < kRequestHeadersReceived) {
668 			_ParseHeaders();
669 
670 			if (fRequestStatus >= kRequestHeadersReceived) {
671 				_ResultHeaders() = fHeaders;
672 
673 				// Parse received cookies
674 				if (fContext != NULL) {
675 					for (int32 i = 0;  i < fHeaders.CountHeaders(); i++) {
676 						if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) {
677 							fContext->GetCookieJar().AddCookie(
678 								fHeaders.HeaderAt(i).Value(), fUrl);
679 						}
680 					}
681 				}
682 
683 				//! ProtocolHook:HeadersReceived
684 				if (fListener != NULL && !disableListener)
685 					fListener->HeadersReceived(this);
686 
687 
688 				if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
689 					readByChunks = true;
690 
691 				BString contentEncoding(fHeaders["Content-Encoding"]);
692 				// We don't advertise "deflate" support (see above), but we
693 				// still try to decompress it, if a server ever sends a deflate
694 				// stream despite it not being in our Accept-Encoding list.
695 				if (contentEncoding == "gzip"
696 						|| contentEncoding == "deflate") {
697 					decompress = true;
698 					readError = BZlibCompressionAlgorithm()
699 						.CreateDecompressingOutputStream(&decompressorStorage,
700 							NULL, decompressingStream);
701 					if (readError != B_OK)
702 						break;
703 
704 					decompressingStreamDeleter.SetTo(decompressingStream);
705 				}
706 
707 				int32 index = fHeaders.HasHeader("Content-Length");
708 				if (index != B_ERROR)
709 					bytesTotal = atoll(fHeaders.HeaderAt(index).Value());
710 				else
711 					bytesTotal = -1;
712 
713 				if (fRequestMethod == B_HTTP_HEAD
714 					|| fResult.StatusCode() == 204) {
715 					// In the case of a HEAD request or if the server replies
716 					// 204 ("no content"), we don't expect to receive anything
717 					// more, and the socket will be closed.
718 					receiveEnd = true;
719 				}
720 			}
721 		}
722 
723 		if (fRequestStatus >= kRequestHeadersReceived) {
724 			// If Transfer-Encoding is chunked, we should read a complete
725 			// chunk in buffer before handling it
726 			if (readByChunks) {
727 				if (chunkSize >= 0) {
728 					if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) {
729 							// 2 more bytes to handle the closing CR+LF
730 						bytesRead = chunkSize;
731 						if (inputTempSize < chunkSize + 2) {
732 							inputTempSize = chunkSize + 2;
733 							inputTempBuffer
734 								= new(std::nothrow) char[inputTempSize];
735 							inputTempBufferDeleter.SetTo(inputTempBuffer);
736 						}
737 
738 						if (inputTempBuffer == NULL) {
739 							readError = B_NO_MEMORY;
740 							break;
741 						}
742 
743 						fInputBuffer.RemoveData(inputTempBuffer,
744 							chunkSize + 2);
745 						chunkSize = -1;
746 					} else {
747 						// Not enough data, try again later
748 						bytesRead = -1;
749 					}
750 				} else {
751 					BString chunkHeader;
752 					if (_GetLine(chunkHeader) == B_ERROR) {
753 						chunkSize = -1;
754 						bytesRead = -1;
755 					} else {
756 						// Format of a chunk header:
757 						// <chunk size in hex>[; optional data]
758 						int32 semiColonIndex = chunkHeader.FindFirst(';', 0);
759 
760 						// Cut-off optional data if present
761 						if (semiColonIndex != -1) {
762 							chunkHeader.Remove(semiColonIndex,
763 								chunkHeader.Length() - semiColonIndex);
764 						}
765 
766 						chunkSize = strtol(chunkHeader.String(), NULL, 16);
767 						if (chunkSize == 0)
768 							fRequestStatus = kRequestContentReceived;
769 
770 						bytesRead = -1;
771 					}
772 				}
773 
774 				// A chunk of 0 bytes indicates the end of the chunked transfer
775 				if (bytesRead == 0)
776 					receiveEnd = true;
777 			} else {
778 				bytesRead = fInputBuffer.Size();
779 
780 				if (bytesRead > 0) {
781 					if (inputTempSize < bytesRead) {
782 						inputTempSize = bytesRead;
783 						inputTempBuffer = new(std::nothrow) char[bytesRead];
784 						inputTempBufferDeleter.SetTo(inputTempBuffer);
785 					}
786 
787 					if (inputTempBuffer == NULL) {
788 						readError = B_NO_MEMORY;
789 						break;
790 					}
791 					fInputBuffer.RemoveData(inputTempBuffer, bytesRead);
792 				}
793 			}
794 
795 			if (bytesRead >= 0) {
796 				bytesReceived += bytesRead;
797 
798 				if (fOutput != NULL && !disableListener) {
799 					if (decompress) {
800 						readError = decompressingStream->WriteExactly(
801 							inputTempBuffer, bytesRead);
802 						if (readError != B_OK)
803 							break;
804 
805 						ssize_t size = decompressorStorage.Size();
806 						BStackOrHeapArray<char, 4096> buffer(size);
807 						size = decompressorStorage.Read(buffer, size);
808 						if (size > 0) {
809 							size_t written = 0;
810 							readError = fOutput->WriteExactly(buffer,
811 								size, &written);
812 							if (fListener != NULL && written > 0)
813 								fListener->BytesWritten(this, written);
814 							if (readError != B_OK)
815 								break;
816 							bytesUnpacked += size;
817 						}
818 					} else if (bytesRead > 0) {
819 						size_t written = 0;
820 						readError = fOutput->WriteExactly(inputTempBuffer,
821 							bytesRead, &written);
822 						if (fListener != NULL && written > 0)
823 							fListener->BytesWritten(this, written);
824 						if (readError != B_OK)
825 							break;
826 					}
827 				}
828 
829 				if (fListener != NULL && !disableListener)
830 					fListener->DownloadProgress(this, bytesReceived,
831 						std::max((off_t)0, bytesTotal));
832 
833 				if (bytesTotal >= 0 && bytesReceived >= bytesTotal)
834 					receiveEnd = true;
835 
836 				if (decompress && receiveEnd && !disableListener) {
837 					readError = decompressingStream->Flush();
838 
839 					if (readError == B_BUFFER_OVERFLOW)
840 						readError = B_OK;
841 
842 					if (readError != B_OK)
843 						break;
844 
845 					ssize_t size = decompressorStorage.Size();
846 					BStackOrHeapArray<char, 4096> buffer(size);
847 					size = decompressorStorage.Read(buffer, size);
848 					if (fOutput != NULL && size > 0 && !disableListener) {
849 						size_t written = 0;
850 						readError = fOutput->WriteExactly(buffer, size,
851 							&written);
852 						if (fListener != NULL && written > 0)
853 							fListener->BytesWritten(this, written);
854 						if (readError != B_OK)
855 							break;
856 						bytesUnpacked += size;
857 					}
858 				}
859 			}
860 		}
861 
862 		parseEnd = (fInputBuffer.Size() == 0);
863 	}
864 
865 	fSocket->Disconnect();
866 
867 	if (readError != B_OK)
868 		return readError;
869 
870 	return fQuit ? B_INTERRUPTED : B_OK;
871 }
872 
873 
874 void
875 BHttpRequest::_ParseStatus()
876 {
877 	// Status line should be formatted like: HTTP/M.m SSS ...
878 	// With:   M = Major version of the protocol
879 	//         m = Minor version of the protocol
880 	//       SSS = three-digit status code of the response
881 	//       ... = additional text info
882 	BString statusLine;
883 	if (_GetLine(statusLine) == B_ERROR)
884 		return;
885 
886 	if (statusLine.CountChars() < 12)
887 		return;
888 
889 	fRequestStatus = kRequestStatusReceived;
890 
891 	BString statusCodeStr;
892 	BString statusText;
893 	statusLine.CopyInto(statusCodeStr, 9, 3);
894 	_SetResultStatusCode(atoi(statusCodeStr.String()));
895 
896 	statusLine.CopyInto(_ResultStatusText(), 13, statusLine.Length() - 13);
897 
898 	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Status line received: Code %d (%s)",
899 		atoi(statusCodeStr.String()), _ResultStatusText().String());
900 }
901 
902 
903 void
904 BHttpRequest::_ParseHeaders()
905 {
906 	BString currentHeader;
907 	while (_GetLine(currentHeader) != B_ERROR) {
908 		// An empty line means the end of the header section
909 		if (currentHeader.Length() == 0) {
910 			fRequestStatus = kRequestHeadersReceived;
911 			return;
912 		}
913 
914 		_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_IN, "%s",
915 			currentHeader.String());
916 		fHeaders.AddHeader(currentHeader.String());
917 	}
918 }
919 
920 
921 BString
922 BHttpRequest::_SerializeRequest()
923 {
924 	BString request(fRequestMethod);
925 	request << ' ';
926 
927 	if (fContext->UseProxy()) {
928 		// When there is a proxy, the request must include the host and port so
929 		// the proxy knows where to send the request.
930 		request << Url().Protocol() << "://" << Url().Host();
931 		if (Url().HasPort())
932 			request << ':' << Url().Port();
933 	}
934 
935 	if (Url().HasPath() && Url().Path().Length() > 0)
936 		request << Url().Path();
937 	else
938 		request << '/';
939 
940 	if (Url().HasRequest())
941 		request << '?' << Url().Request();
942 
943 	switch (fHttpVersion) {
944 		case B_HTTP_11:
945 			request << " HTTP/1.1\r\n";
946 			break;
947 
948 		default:
949 		case B_HTTP_10:
950 			request << " HTTP/1.0\r\n";
951 			break;
952 	}
953 
954 	_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", request.String());
955 
956 	return request;
957 }
958 
959 
960 BString
961 BHttpRequest::_SerializeHeaders()
962 {
963 	BHttpHeaders outputHeaders;
964 
965 	// HTTP 1.1 additional headers
966 	if (fHttpVersion == B_HTTP_11) {
967 		BString host = Url().Host();
968 		if (Url().HasPort() && !_IsDefaultPort())
969 			host << ':' << Url().Port();
970 
971 		outputHeaders.AddHeader("Host", host);
972 
973 		outputHeaders.AddHeader("Accept", "*/*");
974 		outputHeaders.AddHeader("Accept-Encoding", "gzip");
975 			// Allows the server to compress data using the "gzip" format.
976 			// "deflate" is not supported, because there are two interpretations
977 			// of what it means (the RFC and Microsoft products), and we don't
978 			// want to handle this. Very few websites support only deflate,
979 			// and most of them will send gzip, or at worst, uncompressed data.
980 
981 		outputHeaders.AddHeader("Connection", "close");
982 			// Let the remote server close the connection after response since
983 			// we don't handle multiple request on a single connection
984 	}
985 
986 	// Classic HTTP headers
987 	if (fOptUserAgent.CountChars() > 0)
988 		outputHeaders.AddHeader("User-Agent", fOptUserAgent.String());
989 
990 	if (fOptReferer.CountChars() > 0)
991 		outputHeaders.AddHeader("Referer", fOptReferer.String());
992 
993 	// Optional range requests headers
994 	if (fOptRangeStart != -1 || fOptRangeEnd != -1) {
995 		if (fOptRangeStart == -1)
996 			fOptRangeStart = 0;
997 		BString range;
998 		if (fOptRangeEnd != -1) {
999 			range.SetToFormat("bytes=%" B_PRIdOFF "-%" B_PRIdOFF,
1000 				fOptRangeStart, fOptRangeEnd);
1001 		} else {
1002 			range.SetToFormat("bytes=%" B_PRIdOFF "-", fOptRangeStart);
1003 		}
1004 		outputHeaders.AddHeader("Range", range.String());
1005 	}
1006 
1007 	// Authentication
1008 	if (fContext != NULL) {
1009 		BHttpAuthentication& authentication = fContext->GetAuthentication(fUrl);
1010 		if (authentication.Method() != B_HTTP_AUTHENTICATION_NONE) {
1011 			if (fOptUsername.Length() > 0) {
1012 				authentication.SetUserName(fOptUsername);
1013 				authentication.SetPassword(fOptPassword);
1014 			}
1015 
1016 			BString request(fRequestMethod);
1017 			outputHeaders.AddHeader("Authorization",
1018 				authentication.Authorization(fUrl, request));
1019 		}
1020 	}
1021 
1022 	// Required headers for POST data
1023 	if (fOptPostFields != NULL && fRequestMethod == B_HTTP_POST) {
1024 		BString contentType;
1025 
1026 		switch (fOptPostFields->GetFormType()) {
1027 			case B_HTTP_FORM_MULTIPART:
1028 				contentType << "multipart/form-data; boundary="
1029 					<< fOptPostFields->GetMultipartBoundary() << "";
1030 				break;
1031 
1032 			case B_HTTP_FORM_URL_ENCODED:
1033 				contentType << "application/x-www-form-urlencoded";
1034 				break;
1035 		}
1036 
1037 		outputHeaders.AddHeader("Content-Type", contentType);
1038 		outputHeaders.AddHeader("Content-Length",
1039 			fOptPostFields->ContentLength());
1040 	} else if (fOptInputData != NULL
1041 			&& (fRequestMethod == B_HTTP_POST
1042 			|| fRequestMethod == B_HTTP_PUT)) {
1043 		if (fOptInputDataSize >= 0)
1044 			outputHeaders.AddHeader("Content-Length", fOptInputDataSize);
1045 		else
1046 			outputHeaders.AddHeader("Transfer-Encoding", "chunked");
1047 	}
1048 
1049 	// Optional headers specified by the user
1050 	if (fOptHeaders != NULL) {
1051 		for (int32 headerIndex = 0; headerIndex < fOptHeaders->CountHeaders();
1052 				headerIndex++) {
1053 			BHttpHeader& optHeader = (*fOptHeaders)[headerIndex];
1054 			int32 replaceIndex = outputHeaders.HasHeader(optHeader.Name());
1055 
1056 			// Add or replace the current option header to the
1057 			// output header list
1058 			if (replaceIndex == -1)
1059 				outputHeaders.AddHeader(optHeader.Name(), optHeader.Value());
1060 			else
1061 				outputHeaders[replaceIndex].SetValue(optHeader.Value());
1062 		}
1063 	}
1064 
1065 	// Context cookies
1066 	if (fOptSetCookies && fContext != NULL) {
1067 		BString cookieString;
1068 
1069 		BNetworkCookieJar::UrlIterator iterator
1070 			= fContext->GetCookieJar().GetUrlIterator(fUrl);
1071 		const BNetworkCookie* cookie = iterator.Next();
1072 		if (cookie != NULL) {
1073 			while (true) {
1074 				cookieString << cookie->RawCookie(false);
1075 				cookie = iterator.Next();
1076 				if (cookie == NULL)
1077 					break;
1078 				cookieString << "; ";
1079 			}
1080 
1081 			outputHeaders.AddHeader("Cookie", cookieString);
1082 		}
1083 	}
1084 
1085 	// Write output headers to output stream
1086 	BString headerData;
1087 
1088 	for (int32 headerIndex = 0; headerIndex < outputHeaders.CountHeaders();
1089 			headerIndex++) {
1090 		const char* header = outputHeaders.HeaderAt(headerIndex).Header();
1091 
1092 		headerData << header;
1093 		headerData << "\r\n";
1094 
1095 		_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", header);
1096 	}
1097 
1098 	return headerData;
1099 }
1100 
1101 
1102 void
1103 BHttpRequest::_SendPostData()
1104 {
1105 	if (fRequestMethod == B_HTTP_POST && fOptPostFields != NULL) {
1106 		if (fOptPostFields->GetFormType() != B_HTTP_FORM_MULTIPART) {
1107 			BString outputBuffer = fOptPostFields->RawData();
1108 			_EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
1109 				"%s", outputBuffer.String());
1110 			fSocket->Write(outputBuffer.String(), outputBuffer.Length());
1111 		} else {
1112 			for (BHttpForm::Iterator it = fOptPostFields->GetIterator();
1113 				const BHttpFormData* currentField = it.Next();
1114 				) {
1115 				_EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
1116 					it.MultipartHeader().String());
1117 				fSocket->Write(it.MultipartHeader().String(),
1118 					it.MultipartHeader().Length());
1119 
1120 				switch (currentField->Type()) {
1121 					default:
1122 					case B_HTTPFORM_UNKNOWN:
1123 						ASSERT(0);
1124 						break;
1125 
1126 					case B_HTTPFORM_STRING:
1127 						fSocket->Write(currentField->String().String(),
1128 							currentField->String().Length());
1129 						break;
1130 
1131 					case B_HTTPFORM_FILE:
1132 						{
1133 							BFile upFile(currentField->File().Path(),
1134 								B_READ_ONLY);
1135 							char readBuffer[kHttpBufferSize];
1136 							ssize_t readSize;
1137 							off_t totalSize;
1138 
1139 							if (upFile.GetSize(&totalSize) != B_OK)
1140 								ASSERT(0);
1141 
1142 							readSize = upFile.Read(readBuffer,
1143 								sizeof(readBuffer));
1144 							while (readSize > 0) {
1145 								fSocket->Write(readBuffer, readSize);
1146 								readSize = upFile.Read(readBuffer,
1147 									sizeof(readBuffer));
1148 								fListener->UploadProgress(this, readSize,
1149 									std::max((off_t)0, totalSize));
1150 							}
1151 
1152 							break;
1153 						}
1154 					case B_HTTPFORM_BUFFER:
1155 						fSocket->Write(currentField->Buffer(),
1156 							currentField->BufferSize());
1157 						break;
1158 				}
1159 
1160 				fSocket->Write("\r\n", 2);
1161 			}
1162 
1163 			BString footer = fOptPostFields->GetMultipartFooter();
1164 			fSocket->Write(footer.String(), footer.Length());
1165 		}
1166 	} else if ((fRequestMethod == B_HTTP_POST || fRequestMethod == B_HTTP_PUT)
1167 		&& fOptInputData != NULL) {
1168 
1169 		// If the input data is seekable, we rewind it for each new request.
1170 		BPositionIO* seekableData
1171 			= dynamic_cast<BPositionIO*>(fOptInputData);
1172 		if (seekableData)
1173 			seekableData->Seek(0, SEEK_SET);
1174 
1175 		for (;;) {
1176 			char outputTempBuffer[kHttpBufferSize];
1177 			ssize_t read = fOptInputData->Read(outputTempBuffer,
1178 				sizeof(outputTempBuffer));
1179 
1180 			if (read <= 0)
1181 				break;
1182 
1183 			if (fOptInputDataSize < 0) {
1184 				// Input data size unknown, so we have to use chunked transfer
1185 				char hexSize[18];
1186 				// The string does not need to be NULL terminated.
1187 				size_t hexLength = snprintf(hexSize, sizeof(hexSize), "%lx\r\n",
1188 					read);
1189 
1190 				fSocket->Write(hexSize, hexLength);
1191 				fSocket->Write(outputTempBuffer, read);
1192 				fSocket->Write("\r\n", 2);
1193 			} else {
1194 				fSocket->Write(outputTempBuffer, read);
1195 			}
1196 		}
1197 
1198 		if (fOptInputDataSize < 0) {
1199 			// Chunked transfer terminating sequence
1200 			fSocket->Write("0\r\n\r\n", 5);
1201 		}
1202 	}
1203 
1204 }
1205 
1206 
1207 BHttpHeaders&
1208 BHttpRequest::_ResultHeaders()
1209 {
1210 	return fResult.fHeaders;
1211 }
1212 
1213 
1214 void
1215 BHttpRequest::_SetResultStatusCode(int32 statusCode)
1216 {
1217 	fResult.fStatusCode = statusCode;
1218 }
1219 
1220 
1221 BString&
1222 BHttpRequest::_ResultStatusText()
1223 {
1224 	return fResult.fStatusString;
1225 }
1226 
1227 
1228 bool
1229 BHttpRequest::_CertificateVerificationFailed(BCertificate& certificate,
1230 	const char* message)
1231 {
1232 	if (fContext->HasCertificateException(certificate))
1233 		return true;
1234 
1235 	if (fListener != NULL
1236 		&& fListener->CertificateVerificationFailed(this, certificate, message)) {
1237 		// User asked us to continue anyway, let's add a temporary exception for this certificate
1238 		fContext->AddCertificateException(certificate);
1239 		return true;
1240 	}
1241 
1242 	return false;
1243 }
1244 
1245 
1246 bool
1247 BHttpRequest::_IsDefaultPort()
1248 {
1249 	if (fSSL && Url().Port() == 443)
1250 		return true;
1251 	if (!fSSL && Url().Port() == 80)
1252 		return true;
1253 	return false;
1254 }
1255