network/libnetservices/HttpRequest.cpp

/*
 * Copyright 2010-2021 Haiku Inc. All rights reserved.
 * Distributed under the terms of the MIT License.
 *
 * Authors:
 *		Christophe Huriaux, c.huriaux@gmail.com
 *		Niels Sascha Reedijk, niels.reedijk@gmail.com
 *		Adrien Destugues, pulkomandy@pulkomandy.tk
 *		Stephan Aßmus, superstippi@gmx.de
 */


#include <HttpRequest.h>

#include <arpa/inet.h>
#include <stdio.h>

#include <cstdlib>
#include <deque>
#include <new>

#include <AutoDeleter.h>
#include <Certificate.h>
#include <Debug.h>
#include <DynamicBuffer.h>
#include <File.h>
#include <ProxySecureSocket.h>
#include <Socket.h>
#include <SecureSocket.h>
#include <StackOrHeapArray.h>
#include <ZlibCompressionAlgorithm.h>


static const int32 kHttpBufferSize = 4096;


#ifndef LIBNETAPI_DEPRECATED
using namespace BPrivate::Network;
#endif


namespace BPrivate {

	class CheckedSecureSocket: public BSecureSocket
	{
		public:
			CheckedSecureSocket(BHttpRequest* request);

			bool			CertificateVerificationFailed(BCertificate& certificate,
					const char* message);

		private:
			BHttpRequest*	fRequest;
	};


	CheckedSecureSocket::CheckedSecureSocket(BHttpRequest* request)
		:
		BSecureSocket(),
		fRequest(request)
	{
	}


	bool
	CheckedSecureSocket::CertificateVerificationFailed(BCertificate& certificate,
		const char* message)
	{
		return fRequest->_CertificateVerificationFailed(certificate, message);
	}


	class CheckedProxySecureSocket: public BProxySecureSocket
	{
		public:
			CheckedProxySecureSocket(const BNetworkAddress& proxy, BHttpRequest* request);

			bool			CertificateVerificationFailed(BCertificate& certificate,
					const char* message);

		private:
			BHttpRequest*	fRequest;
	};


	CheckedProxySecureSocket::CheckedProxySecureSocket(const BNetworkAddress& proxy,
		BHttpRequest* request)
		:
		BProxySecureSocket(proxy),
		fRequest(request)
	{
	}


	bool
	CheckedProxySecureSocket::CertificateVerificationFailed(BCertificate& certificate,
		const char* message)
	{
		return fRequest->_CertificateVerificationFailed(certificate, message);
	}
};


#ifdef LIBNETAPI_DEPRECATED
BHttpRequest::BHttpRequest(const BUrl& url, bool ssl, const char* protocolName,
	BUrlProtocolListener* listener, BUrlContext* context)
	:
	BNetworkRequest(url, listener, context, "BUrlProtocol.HTTP", protocolName),
	fSSL(ssl),
	fRequestMethod(B_HTTP_GET),
	fHttpVersion(B_HTTP_11),
	fResult(url),
	fRequestStatus(kRequestInitialState),
	fOptHeaders(NULL),
	fOptPostFields(NULL),
	fOptInputData(NULL),
	fOptInputDataSize(-1),
	fOptRangeStart(-1),
	fOptRangeEnd(-1),
	fOptFollowLocation(true)
{
	_ResetOptions();
	fSocket = NULL;
}


BHttpRequest::BHttpRequest(const BHttpRequest& other)
	:
	BNetworkRequest(other.Url(), other.fListener, other.fContext,
		"BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"),
	fSSL(other.fSSL),
	fRequestMethod(other.fRequestMethod),
	fHttpVersion(other.fHttpVersion),
	fResult(other.fUrl),
	fRequestStatus(kRequestInitialState),
	fOptHeaders(NULL),
	fOptPostFields(NULL),
	fOptInputData(NULL),
	fOptInputDataSize(-1),
	fOptRangeStart(other.fOptRangeStart),
	fOptRangeEnd(other.fOptRangeEnd),
	fOptFollowLocation(other.fOptFollowLocation)
{
	_ResetOptions();
		// FIXME some options may be copied from other instead.
	fSocket = NULL;
}

#else

BHttpRequest::BHttpRequest(const BUrl& url, BDataIO* output, bool ssl,
	const char* protocolName, BUrlProtocolListener* listener,
	BUrlContext* context)
	:
	BNetworkRequest(url, output, listener, context, "BUrlProtocol.HTTP",
		protocolName),
	fSSL(ssl),
	fRequestMethod(B_HTTP_GET),
	fHttpVersion(B_HTTP_11),
	fResult(url),
	fRequestStatus(kRequestInitialState),
	fOptHeaders(NULL),
	fOptPostFields(NULL),
	fOptInputData(NULL),
	fOptInputDataSize(-1),
	fOptRangeStart(-1),
	fOptRangeEnd(-1),
	fOptFollowLocation(true)
{
	_ResetOptions();
	fSocket = NULL;
}


BHttpRequest::BHttpRequest(const BHttpRequest& other)
	:
	BNetworkRequest(other.Url(), other.Output(), other.fListener,
		other.fContext, "BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"),
	fSSL(other.fSSL),
	fRequestMethod(other.fRequestMethod),
	fHttpVersion(other.fHttpVersion),
	fResult(other.fUrl),
	fRequestStatus(kRequestInitialState),
	fOptHeaders(NULL),
	fOptPostFields(NULL),
	fOptInputData(NULL),
	fOptInputDataSize(-1),
	fOptRangeStart(other.fOptRangeStart),
	fOptRangeEnd(other.fOptRangeEnd),
	fOptFollowLocation(other.fOptFollowLocation)
{
	_ResetOptions();
		// FIXME some options may be copied from other instead.
	fSocket = NULL;
}
#endif // LIBNETAPI_DEPRECATED


BHttpRequest::~BHttpRequest()
{
	Stop();

	delete fSocket;

	delete fOptInputData;
	delete fOptHeaders;
	delete fOptPostFields;
}


void
BHttpRequest::SetMethod(const char* const method)
{
	fRequestMethod = method;
}


void
BHttpRequest::SetFollowLocation(bool follow)
{
	fOptFollowLocation = follow;
}


void
BHttpRequest::SetMaxRedirections(int8 redirections)
{
	fOptMaxRedirs = redirections;
}


void
BHttpRequest::SetReferrer(const BString& referrer)
{
	fOptReferer = referrer;
}


void
BHttpRequest::SetUserAgent(const BString& agent)
{
	fOptUserAgent = agent;
}


void
BHttpRequest::SetDiscardData(bool discard)
{
	fOptDiscardData = discard;
}


void
BHttpRequest::SetDisableListener(bool disable)
{
	fOptDisableListener = disable;
}


void
BHttpRequest::SetAutoReferrer(bool enable)
{
	fOptAutoReferer = enable;
}


#ifndef LIBNETAPI_DEPRECATED
void
BHttpRequest::SetStopOnError(bool stop)
{
	fOptStopOnError = stop;
}
#endif


void
BHttpRequest::SetUserName(const BString& name)
{
	fOptUsername = name;
}


void
BHttpRequest::SetPassword(const BString& password)
{
	fOptPassword = password;
}


void
BHttpRequest::SetRangeStart(off_t position)
{
	// This field is used within the transfer loop, so only
	// allow setting it before sending the request.
	if (fRequestStatus == kRequestInitialState)
		fOptRangeStart = position;
}


void
BHttpRequest::SetRangeEnd(off_t position)
{
	// This field could be used in the transfer loop, so only
	// allow setting it before sending the request.
	if (fRequestStatus == kRequestInitialState)
		fOptRangeEnd = position;
}


void
BHttpRequest::SetPostFields(const BHttpForm& fields)
{
	AdoptPostFields(new(std::nothrow) BHttpForm(fields));
}


void
BHttpRequest::SetHeaders(const BHttpHeaders& headers)
{
	AdoptHeaders(new(std::nothrow) BHttpHeaders(headers));
}


void
BHttpRequest::AdoptPostFields(BHttpForm* const fields)
{
	delete fOptPostFields;
	fOptPostFields = fields;

	if (fOptPostFields != NULL)
		fRequestMethod = B_HTTP_POST;
}


void
BHttpRequest::AdoptInputData(BDataIO* const data, const ssize_t size)
{
	delete fOptInputData;
	fOptInputData = data;
	fOptInputDataSize = size;
}


void
BHttpRequest::AdoptHeaders(BHttpHeaders* const headers)
{
	delete fOptHeaders;
	fOptHeaders = headers;
}


/*static*/ bool
BHttpRequest::IsInformationalStatusCode(int16 code)
{
	return (code >= B_HTTP_STATUS__INFORMATIONAL_BASE)
		&& (code <  B_HTTP_STATUS__INFORMATIONAL_END);
}


/*static*/ bool
BHttpRequest::IsSuccessStatusCode(int16 code)
{
	return (code >= B_HTTP_STATUS__SUCCESS_BASE)
		&& (code <  B_HTTP_STATUS__SUCCESS_END);
}


/*static*/ bool
BHttpRequest::IsRedirectionStatusCode(int16 code)
{
	return (code >= B_HTTP_STATUS__REDIRECTION_BASE)
		&& (code <  B_HTTP_STATUS__REDIRECTION_END);
}


/*static*/ bool
BHttpRequest::IsClientErrorStatusCode(int16 code)
{
	return (code >= B_HTTP_STATUS__CLIENT_ERROR_BASE)
		&& (code <  B_HTTP_STATUS__CLIENT_ERROR_END);
}


/*static*/ bool
BHttpRequest::IsServerErrorStatusCode(int16 code)
{
	return (code >= B_HTTP_STATUS__SERVER_ERROR_BASE)
		&& (code <  B_HTTP_STATUS__SERVER_ERROR_END);
}


/*static*/ int16
BHttpRequest::StatusCodeClass(int16 code)
{
	if (BHttpRequest::IsInformationalStatusCode(code))
		return B_HTTP_STATUS_CLASS_INFORMATIONAL;
	else if (BHttpRequest::IsSuccessStatusCode(code))
		return B_HTTP_STATUS_CLASS_SUCCESS;
	else if (BHttpRequest::IsRedirectionStatusCode(code))
		return B_HTTP_STATUS_CLASS_REDIRECTION;
	else if (BHttpRequest::IsClientErrorStatusCode(code))
		return B_HTTP_STATUS_CLASS_CLIENT_ERROR;
	else if (BHttpRequest::IsServerErrorStatusCode(code))
		return B_HTTP_STATUS_CLASS_SERVER_ERROR;

	return B_HTTP_STATUS_CLASS_INVALID;
}


const BUrlResult&
BHttpRequest::Result() const
{
	return fResult;
}


status_t
BHttpRequest::Stop()
{
	if (fSocket != NULL) {
		fSocket->Disconnect();
			// Unlock any pending connect, read or write operation.
	}
	return BNetworkRequest::Stop();
}


void
BHttpRequest::_ResetOptions()
{
	delete fOptPostFields;
	delete fOptHeaders;

	fOptFollowLocation = true;
	fOptMaxRedirs = 8;
	fOptReferer = "";
	fOptUserAgent = "Services Kit (Haiku)";
	fOptUsername = "";
	fOptPassword = "";
	fOptAuthMethods = B_HTTP_AUTHENTICATION_BASIC | B_HTTP_AUTHENTICATION_DIGEST
		| B_HTTP_AUTHENTICATION_IE_DIGEST;
	fOptHeaders = NULL;
	fOptPostFields = NULL;
	fOptSetCookies = true;
	fOptDiscardData = false;
	fOptDisableListener = false;
	fOptAutoReferer = true;
}


status_t
BHttpRequest::_ProtocolLoop()
{
	// Initialize the request redirection loop
	int8 maxRedirs = fOptMaxRedirs;
	bool newRequest;

	do {
		newRequest = false;

		// Result reset
		fHeaders.Clear();
		_ResultHeaders().Clear();

		BString host = fUrl.Host();
		int port = fSSL ? 443 : 80;

		if (fUrl.HasPort())
			port = fUrl.Port();

		if (fContext->UseProxy()) {
			host = fContext->GetProxyHost();
			port = fContext->GetProxyPort();
		}

		status_t result = fInputBuffer.InitCheck();
		if (result != B_OK)
			return result;

		if (!_ResolveHostName(host, port)) {
			_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR,
				"Unable to resolve hostname (%s), aborting.",
					fUrl.Host().String());
			return B_SERVER_NOT_FOUND;
		}

		status_t requestStatus = _MakeRequest();
		if (requestStatus != B_OK)
			return requestStatus;

		// Prepare the referer for the next request if needed
		if (fOptAutoReferer)
			fOptReferer = fUrl.UrlString();

		switch (StatusCodeClass(fResult.StatusCode())) {
			case B_HTTP_STATUS_CLASS_INFORMATIONAL:
				// Header 100:continue should have been
				// handled in the _MakeRequest read loop
				break;

			case B_HTTP_STATUS_CLASS_SUCCESS:
				break;

			case B_HTTP_STATUS_CLASS_REDIRECTION:
			{
				// Redirection has been explicitly disabled
				if (!fOptFollowLocation)
					break;

				int code = fResult.StatusCode();
				if (code == B_HTTP_STATUS_MOVED_PERMANENTLY
					|| code == B_HTTP_STATUS_FOUND
					|| code == B_HTTP_STATUS_SEE_OTHER
					|| code == B_HTTP_STATUS_TEMPORARY_REDIRECT) {
					BString locationUrl = fHeaders["Location"];

					fUrl = BUrl(fUrl, locationUrl);

					// 302 and 303 redirections also convert POST requests to GET
					// (and remove the posted form data)
					if ((code == B_HTTP_STATUS_FOUND
						|| code == B_HTTP_STATUS_SEE_OTHER)
						&& fRequestMethod == B_HTTP_POST) {
						SetMethod(B_HTTP_GET);
						delete fOptPostFields;
						fOptPostFields = NULL;
						delete fOptInputData;
						fOptInputData = NULL;
						fOptInputDataSize = 0;
					}

					if (--maxRedirs > 0) {
						newRequest = true;

						// Redirections may need a switch from http to https.
						if (fUrl.Protocol() == "https")
							fSSL = true;
						else if (fUrl.Protocol() == "http")
							fSSL = false;

						_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
							"Following: %s\n",
							fUrl.UrlString().String());
					}
				}
				break;
			}

			case B_HTTP_STATUS_CLASS_CLIENT_ERROR:
				if (fResult.StatusCode() == B_HTTP_STATUS_UNAUTHORIZED) {
					BHttpAuthentication* authentication
						= &fContext->GetAuthentication(fUrl);
					status_t status = B_OK;

					if (authentication->Method() == B_HTTP_AUTHENTICATION_NONE) {
						// There is no authentication context for this
						// url yet, so let's create one.
						BHttpAuthentication newAuth;
						newAuth.Initialize(fHeaders["WWW-Authenticate"]);
						fContext->AddAuthentication(fUrl, newAuth);

						// Get the copy of the authentication we just added.
						// That copy is owned by the BUrlContext and won't be
						// deleted (unlike the temporary object above)
						authentication = &fContext->GetAuthentication(fUrl);
					}

					newRequest = false;
					if (fOptUsername.Length() > 0 && status == B_OK) {
						// If we received an username and password, add them
						// to the request. This will either change the
						// credentials for an existing request, or set them
						// for a new one we created just above.
						//
						// If this request handles HTTP redirections, it will
						// also automatically retry connecting and send the
						// login information.
						authentication->SetUserName(fOptUsername);
						authentication->SetPassword(fOptPassword);
						newRequest = true;
					}
				}
				break;

			case B_HTTP_STATUS_CLASS_SERVER_ERROR:
				break;

			default:
			case B_HTTP_STATUS_CLASS_INVALID:
				break;
		}
	} while (newRequest);

	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
		"%ld headers and %ld bytes of data remaining",
		fHeaders.CountHeaders(), fInputBuffer.Size());

	if (fResult.StatusCode() == 404)
		return B_RESOURCE_NOT_FOUND;

	return B_OK;
}


#ifdef LIBNETAPI_DEPRECATED
status_t
BHttpRequest::_MakeRequest()
{
	delete fSocket;

	if (fSSL) {
		if (fContext->UseProxy()) {
			BNetworkAddress proxy(fContext->GetProxyHost(), fContext->GetProxyPort());
			fSocket = new(std::nothrow) BPrivate::CheckedProxySecureSocket(proxy, this);
		} else
			fSocket = new(std::nothrow) BPrivate::CheckedSecureSocket(this);
	} else
		fSocket = new(std::nothrow) BSocket();

	if (fSocket == NULL)
		return B_NO_MEMORY;

	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
		fUrl.Authority().String(), fRemoteAddr.Port());
	status_t connectError = fSocket->Connect(fRemoteAddr);

	if (connectError != B_OK) {
		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
			strerror(connectError));
		return connectError;
	}

	//! ProtocolHook:ConnectionOpened
	if (fListener != NULL)
		fListener->ConnectionOpened(this);

	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
		"Connection opened, sending request.");

	BString requestHeaders;
	requestHeaders.Append(_SerializeRequest());
	requestHeaders.Append(_SerializeHeaders());
	requestHeaders.Append("\r\n");
	fSocket->Write(requestHeaders.String(), requestHeaders.Length());
	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");

	_SendPostData();
	fRequestStatus = kRequestInitialState;


	// Receive loop
	bool disableListener = false;
	bool receiveEnd = false;
	bool parseEnd = false;
	bool readByChunks = false;
	bool decompress = false;
	status_t readError = B_OK;
	ssize_t bytesRead = 0;
	off_t bytesReceived = 0;
	off_t bytesTotal = 0;
	size_t previousBufferSize = 0;
	off_t bytesUnpacked = 0;
	char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize];
	ArrayDeleter<char> inputTempBufferDeleter(inputTempBuffer);
	ssize_t inputTempSize = kHttpBufferSize;
	ssize_t chunkSize = -1;
	DynamicBuffer decompressorStorage;
	BDataIO* decompressingStream;
	ObjectDeleter<BDataIO> decompressingStreamDeleter;

	while (!fQuit && !(receiveEnd && parseEnd)) {
		if ((!receiveEnd) && (fInputBuffer.Size() == previousBufferSize)) {
			BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize);
			bytesRead = fSocket->Read(chunk, kHttpBufferSize);

			if (bytesRead < 0) {
				readError = bytesRead;
				break;
			} else if (bytesRead == 0) {
				// Check if we got the expected number of bytes.
				// Exceptions:
				// - If the content-length is not known (bytesTotal is 0), for
				//   example in the case of a chunked transfer, we can't know
				// - If the request method is "HEAD" which explicitly asks the
				//   server to not send any data (only the headers)
				if (bytesTotal > 0 && bytesReceived != bytesTotal) {
					readError = B_IO_ERROR;
					break;
				}
				receiveEnd = true;
			}

			fInputBuffer.AppendData(chunk, bytesRead);
		} else
			bytesRead = 0;

		previousBufferSize = fInputBuffer.Size();

		if (fRequestStatus < kRequestStatusReceived) {
			_ParseStatus();

			//! ProtocolHook:ResponseStarted
			if (fRequestStatus >= kRequestStatusReceived && fListener != NULL
					&& !disableListener)
				fListener->ResponseStarted(this);
		}

		if (fRequestStatus < kRequestHeadersReceived) {
			_ParseHeaders();

			if (fRequestStatus >= kRequestHeadersReceived) {
				_ResultHeaders() = fHeaders;

				// Parse received cookies
				if (fContext != NULL) {
					for (int32 i = 0;  i < fHeaders.CountHeaders(); i++) {
						if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) {
							fContext->GetCookieJar().AddCookie(
								fHeaders.HeaderAt(i).Value(), fUrl);
						}
					}
				}

				//! ProtocolHook:HeadersReceived
				if (fListener != NULL && !disableListener)
					fListener->HeadersReceived(this, fResult);


				if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
					readByChunks = true;

				BString contentEncoding(fHeaders["Content-Encoding"]);
				// We don't advertise "deflate" support (see above), but we
				// still try to decompress it, if a server ever sends a deflate
				// stream despite it not being in our Accept-Encoding list.
				if (contentEncoding == "gzip"
						|| contentEncoding == "deflate") {
					decompress = true;
					readError = BZlibCompressionAlgorithm()
						.CreateDecompressingOutputStream(&decompressorStorage,
							NULL, decompressingStream);
					if (readError != B_OK)
						break;

					decompressingStreamDeleter.SetTo(decompressingStream);
				}

				int32 index = fHeaders.HasHeader("Content-Length");
				if (index != B_ERROR)
					bytesTotal = atoll(fHeaders.HeaderAt(index).Value());
				else
					bytesTotal = -1;

				if (fRequestMethod == B_HTTP_HEAD
					|| fResult.StatusCode() == 204) {
					// In the case of a HEAD request or if the server replies
					// 204 ("no content"), we don't expect to receive anything
					// more, and the socket will be closed.
					receiveEnd = true;
				}
			}
		}

		if (fRequestStatus >= kRequestHeadersReceived) {
			// If Transfer-Encoding is chunked, we should read a complete
			// chunk in buffer before handling it
			if (readByChunks) {
				if (chunkSize >= 0) {
					if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) {
							// 2 more bytes to handle the closing CR+LF
						bytesRead = chunkSize;
						if (inputTempSize < chunkSize + 2) {
							inputTempSize = chunkSize + 2;
							inputTempBuffer
								= new(std::nothrow) char[inputTempSize];
							inputTempBufferDeleter.SetTo(inputTempBuffer);
						}

						if (inputTempBuffer == NULL) {
							readError = B_NO_MEMORY;
							break;
						}

						fInputBuffer.RemoveData(inputTempBuffer,
							chunkSize + 2);
						chunkSize = -1;
					} else {
						// Not enough data, try again later
						bytesRead = -1;
					}
				} else {
					BString chunkHeader;
					if (_GetLine(chunkHeader) == B_ERROR) {
						chunkSize = -1;
						bytesRead = -1;
					} else {
						// Format of a chunk header:
						// <chunk size in hex>[; optional data]
						int32 semiColonIndex = chunkHeader.FindFirst(';', 0);

						// Cut-off optional data if present
						if (semiColonIndex != -1) {
							chunkHeader.Remove(semiColonIndex,
								chunkHeader.Length() - semiColonIndex);
						}

						chunkSize = strtol(chunkHeader.String(), NULL, 16);
						if (chunkSize == 0)
							fRequestStatus = kRequestContentReceived;

						bytesRead = -1;
					}
				}

				// A chunk of 0 bytes indicates the end of the chunked transfer
				if (bytesRead == 0)
					receiveEnd = true;
			} else {
				bytesRead = fInputBuffer.Size();

				if (bytesRead > 0) {
					if (inputTempSize < bytesRead) {
						inputTempSize = bytesRead;
						inputTempBuffer = new(std::nothrow) char[bytesRead];
						inputTempBufferDeleter.SetTo(inputTempBuffer);
					}

					if (inputTempBuffer == NULL) {
						readError = B_NO_MEMORY;
						break;
					}
					fInputBuffer.RemoveData(inputTempBuffer, bytesRead);
				}
			}

			if (bytesRead >= 0) {
				bytesReceived += bytesRead;

				if (fListener != NULL && !disableListener) {
					if (decompress) {
						readError = decompressingStream->WriteExactly(
							inputTempBuffer, bytesRead);
						if (readError != B_OK)
							break;

						ssize_t size = decompressorStorage.Size();
						BStackOrHeapArray<char, 4096> buffer(size);
						size = decompressorStorage.Read(buffer, size);
						_NotifyDataReceived(buffer, bytesUnpacked, size,
							bytesReceived, bytesTotal);
						bytesUnpacked += size;
					} else if (bytesRead > 0) {
						_NotifyDataReceived(inputTempBuffer,
							bytesReceived - bytesRead, bytesRead,
							bytesReceived, bytesTotal);
					}
				}

				if (bytesTotal >= 0 && bytesReceived >= bytesTotal)
					receiveEnd = true;

				if (decompress && receiveEnd && !disableListener) {
					readError = decompressingStream->Flush();

					if (readError == B_BUFFER_OVERFLOW)
						readError = B_OK;

					if (readError != B_OK)
						break;

					ssize_t size = decompressorStorage.Size();
					BStackOrHeapArray<char, 4096> buffer(size);
					size = decompressorStorage.Read(buffer, size);
					if (fListener != NULL) {
						_NotifyDataReceived(buffer, bytesUnpacked, size,
							bytesReceived, bytesTotal);
						bytesUnpacked += size;
					}
				}
			}
		}

		parseEnd = (fInputBuffer.Size() == 0);
	}

	fSocket->Disconnect();

	if (readError != B_OK)
		return readError;

	return fQuit ? B_INTERRUPTED : B_OK;
}

#else

status_t
BHttpRequest::_MakeRequest()
{
	delete fSocket;

	if (fSSL) {
		if (fContext->UseProxy()) {
			BNetworkAddress proxy(fContext->GetProxyHost(), fContext->GetProxyPort());
			fSocket = new(std::nothrow) BPrivate::CheckedProxySecureSocket(proxy, this);
		} else
			fSocket = new(std::nothrow) BPrivate::CheckedSecureSocket(this);
	} else
		fSocket = new(std::nothrow) BSocket();

	if (fSocket == NULL)
		return B_NO_MEMORY;

	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
		fUrl.Authority().String(), fRemoteAddr.Port());
	status_t connectError = fSocket->Connect(fRemoteAddr);

	if (connectError != B_OK) {
		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
			strerror(connectError));
		return connectError;
	}

	//! ProtocolHook:ConnectionOpened
	if (fListener != NULL)
		fListener->ConnectionOpened(this);

	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
		"Connection opened, sending request.");

	BString requestHeaders;
	requestHeaders.Append(_SerializeRequest());
	requestHeaders.Append(_SerializeHeaders());
	requestHeaders.Append("\r\n");
	fSocket->Write(requestHeaders.String(), requestHeaders.Length());
	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");

	_SendPostData();
	fRequestStatus = kRequestInitialState;


	// Receive loop
	bool disableListener = false;
	bool receiveEnd = false;
	bool parseEnd = false;
	bool readByChunks = false;
	bool decompress = false;
	status_t readError = B_OK;
	ssize_t bytesRead = 0;
	off_t bytesReceived = 0;
	off_t bytesTotal = 0;
	size_t previousBufferSize = 0;
	off_t bytesUnpacked = 0;
	char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize];
	ArrayDeleter<char> inputTempBufferDeleter(inputTempBuffer);
	ssize_t inputTempSize = kHttpBufferSize;
	ssize_t chunkSize = -1;
	DynamicBuffer decompressorStorage;
	BDataIO* decompressingStream;
	ObjectDeleter<BDataIO> decompressingStreamDeleter;

	while (!fQuit && !(receiveEnd && parseEnd)) {
		if ((!receiveEnd) && (fInputBuffer.Size() == previousBufferSize)) {
			BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize);
			bytesRead = fSocket->Read(chunk, kHttpBufferSize);

			if (bytesRead < 0) {
				readError = bytesRead;
				break;
			} else if (bytesRead == 0) {
				// Check if we got the expected number of bytes.
				// Exceptions:
				// - If the content-length is not known (bytesTotal is 0), for
				//   example in the case of a chunked transfer, we can't know
				// - If the request method is "HEAD" which explicitly asks the
				//   server to not send any data (only the headers)
				if (bytesTotal > 0 && bytesReceived != bytesTotal) {
					readError = B_IO_ERROR;
					break;
				}
				receiveEnd = true;
			}

			fInputBuffer.AppendData(chunk, bytesRead);
		} else
			bytesRead = 0;

		previousBufferSize = fInputBuffer.Size();

		if (fRequestStatus < kRequestStatusReceived) {
			_ParseStatus();

			if (fOptFollowLocation
					&& IsRedirectionStatusCode(fResult.StatusCode()))
				disableListener = true;

			if (fOptStopOnError
					&& fResult.StatusCode() >= B_HTTP_STATUS_CLASS_CLIENT_ERROR)
			{
				fQuit = true;
				break;
			}

			//! ProtocolHook:ResponseStarted
			if (fRequestStatus >= kRequestStatusReceived && fListener != NULL
					&& !disableListener)
				fListener->ResponseStarted(this);
		}

		if (fRequestStatus < kRequestHeadersReceived) {
			_ParseHeaders();

			if (fRequestStatus >= kRequestHeadersReceived) {
				_ResultHeaders() = fHeaders;

				// Parse received cookies
				if (fContext != NULL) {
					for (int32 i = 0;  i < fHeaders.CountHeaders(); i++) {
						if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) {
							fContext->GetCookieJar().AddCookie(
								fHeaders.HeaderAt(i).Value(), fUrl);
						}
					}
				}

				//! ProtocolHook:HeadersReceived
				if (fListener != NULL && !disableListener)
					fListener->HeadersReceived(this);


				if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
					readByChunks = true;

				BString contentEncoding(fHeaders["Content-Encoding"]);
				// We don't advertise "deflate" support (see above), but we
				// still try to decompress it, if a server ever sends a deflate
				// stream despite it not being in our Accept-Encoding list.
				if (contentEncoding == "gzip"
						|| contentEncoding == "deflate") {
					decompress = true;
					readError = BZlibCompressionAlgorithm()
						.CreateDecompressingOutputStream(&decompressorStorage,
							NULL, decompressingStream);
					if (readError != B_OK)
						break;

					decompressingStreamDeleter.SetTo(decompressingStream);
				}

				int32 index = fHeaders.HasHeader("Content-Length");
				if (index != B_ERROR)
					bytesTotal = atoll(fHeaders.HeaderAt(index).Value());
				else
					bytesTotal = -1;

				if (fRequestMethod == B_HTTP_HEAD
					|| fResult.StatusCode() == 204) {
					// In the case of a HEAD request or if the server replies
					// 204 ("no content"), we don't expect to receive anything
					// more, and the socket will be closed.
					receiveEnd = true;
				}
			}
		}

		if (fRequestStatus >= kRequestHeadersReceived) {
			// If Transfer-Encoding is chunked, we should read a complete
			// chunk in buffer before handling it
			if (readByChunks) {
				if (chunkSize >= 0) {
					if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) {
							// 2 more bytes to handle the closing CR+LF
						bytesRead = chunkSize;
						if (inputTempSize < chunkSize + 2) {
							inputTempSize = chunkSize + 2;
							inputTempBuffer
								= new(std::nothrow) char[inputTempSize];
							inputTempBufferDeleter.SetTo(inputTempBuffer);
						}

						if (inputTempBuffer == NULL) {
							readError = B_NO_MEMORY;
							break;
						}

						fInputBuffer.RemoveData(inputTempBuffer,
							chunkSize + 2);
						chunkSize = -1;
					} else {
						// Not enough data, try again later
						bytesRead = -1;
					}
				} else {
					BString chunkHeader;
					if (_GetLine(chunkHeader) == B_ERROR) {
						chunkSize = -1;
						bytesRead = -1;
					} else {
						// Format of a chunk header:
						// <chunk size in hex>[; optional data]
						int32 semiColonIndex = chunkHeader.FindFirst(';', 0);

						// Cut-off optional data if present
						if (semiColonIndex != -1) {
							chunkHeader.Remove(semiColonIndex,
								chunkHeader.Length() - semiColonIndex);
						}

						chunkSize = strtol(chunkHeader.String(), NULL, 16);
						if (chunkSize == 0)
							fRequestStatus = kRequestContentReceived;

						bytesRead = -1;
					}
				}

				// A chunk of 0 bytes indicates the end of the chunked transfer
				if (bytesRead == 0)
					receiveEnd = true;
			} else {
				bytesRead = fInputBuffer.Size();

				if (bytesRead > 0) {
					if (inputTempSize < bytesRead) {
						inputTempSize = bytesRead;
						inputTempBuffer = new(std::nothrow) char[bytesRead];
						inputTempBufferDeleter.SetTo(inputTempBuffer);
					}

					if (inputTempBuffer == NULL) {
						readError = B_NO_MEMORY;
						break;
					}
					fInputBuffer.RemoveData(inputTempBuffer, bytesRead);
				}
			}

			if (bytesRead >= 0) {
				bytesReceived += bytesRead;

				if (fOutput != NULL && !disableListener) {
					if (decompress) {
						readError = decompressingStream->WriteExactly(
							inputTempBuffer, bytesRead);
						if (readError != B_OK)
							break;

						ssize_t size = decompressorStorage.Size();
						BStackOrHeapArray<char, 4096> buffer(size);
						size = decompressorStorage.Read(buffer, size);
						if (size > 0) {
							size_t written = 0;
							readError = fOutput->WriteExactly(buffer,
								size, &written);
							if (fListener != NULL && written > 0)
								fListener->BytesWritten(this, written);
							if (readError != B_OK)
								break;
							bytesUnpacked += size;
						}
					} else if (bytesRead > 0) {
						size_t written = 0;
						readError = fOutput->WriteExactly(inputTempBuffer,
							bytesRead, &written);
						if (fListener != NULL && written > 0)
							fListener->BytesWritten(this, written);
						if (readError != B_OK)
							break;
					}
				}

				if (fListener != NULL && !disableListener)
					fListener->DownloadProgress(this, bytesReceived,
						std::max((off_t)0, bytesTotal));

				if (bytesTotal >= 0 && bytesReceived >= bytesTotal)
					receiveEnd = true;

				if (decompress && receiveEnd && !disableListener) {
					readError = decompressingStream->Flush();

					if (readError == B_BUFFER_OVERFLOW)
						readError = B_OK;

					if (readError != B_OK)
						break;

					ssize_t size = decompressorStorage.Size();
					BStackOrHeapArray<char, 4096> buffer(size);
					size = decompressorStorage.Read(buffer, size);
					if (fOutput != NULL && size > 0 && !disableListener) {
						size_t written = 0;
						readError = fOutput->WriteExactly(buffer, size,
							&written);
						if (fListener != NULL && written > 0)
							fListener->BytesWritten(this, written);
						if (readError != B_OK)
							break;
						bytesUnpacked += size;
					}
				}
			}
		}

		parseEnd = (fInputBuffer.Size() == 0);
	}

	fSocket->Disconnect();

	if (readError != B_OK)
		return readError;

	return fQuit ? B_INTERRUPTED : B_OK;
}
#endif // LIBNETAPI_DEPRECATED


void
BHttpRequest::_ParseStatus()
{
	// Status line should be formatted like: HTTP/M.m SSS ...
	// With:   M = Major version of the protocol
	//         m = Minor version of the protocol
	//       SSS = three-digit status code of the response
	//       ... = additional text info
	BString statusLine;
	if (_GetLine(statusLine) == B_ERROR)
		return;

	if (statusLine.CountChars() < 12)
		return;

	fRequestStatus = kRequestStatusReceived;

	BString statusCodeStr;
	BString statusText;
	statusLine.CopyInto(statusCodeStr, 9, 3);
	_SetResultStatusCode(atoi(statusCodeStr.String()));

	statusLine.CopyInto(_ResultStatusText(), 13, statusLine.Length() - 13);

	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Status line received: Code %d (%s)",
		atoi(statusCodeStr.String()), _ResultStatusText().String());
}


void
BHttpRequest::_ParseHeaders()
{
	BString currentHeader;
	while (_GetLine(currentHeader) != B_ERROR) {
		// An empty line means the end of the header section
		if (currentHeader.Length() == 0) {
			fRequestStatus = kRequestHeadersReceived;
			return;
		}

		_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_IN, "%s",
			currentHeader.String());
		fHeaders.AddHeader(currentHeader.String());
	}
}


BString
BHttpRequest::_SerializeRequest()
{
	BString request(fRequestMethod);
	request << ' ';

	if (fContext->UseProxy()) {
		// When there is a proxy, the request must include the host and port so
		// the proxy knows where to send the request.
		request << Url().Protocol() << "://" << Url().Host();
		if (Url().HasPort())
			request << ':' << Url().Port();
	}

	if (Url().HasPath() && Url().Path().Length() > 0)
		request << Url().Path();
	else
		request << '/';

	if (Url().HasRequest())
		request << '?' << Url().Request();

	switch (fHttpVersion) {
		case B_HTTP_11:
			request << " HTTP/1.1\r\n";
			break;

		default:
		case B_HTTP_10:
			request << " HTTP/1.0\r\n";
			break;
	}

	_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", request.String());

	return request;
}


BString
BHttpRequest::_SerializeHeaders()
{
	BHttpHeaders outputHeaders;

	// HTTP 1.1 additional headers
	if (fHttpVersion == B_HTTP_11) {
		BString host = Url().Host();
		if (Url().HasPort() && !_IsDefaultPort())
			host << ':' << Url().Port();

		outputHeaders.AddHeader("Host", host);

		outputHeaders.AddHeader("Accept", "*/*");
		outputHeaders.AddHeader("Accept-Encoding", "gzip");
			// Allows the server to compress data using the "gzip" format.
			// "deflate" is not supported, because there are two interpretations
			// of what it means (the RFC and Microsoft products), and we don't
			// want to handle this. Very few websites support only deflate,
			// and most of them will send gzip, or at worst, uncompressed data.

		outputHeaders.AddHeader("Connection", "close");
			// Let the remote server close the connection after response since
			// we don't handle multiple request on a single connection
	}

	// Classic HTTP headers
	if (fOptUserAgent.CountChars() > 0)
		outputHeaders.AddHeader("User-Agent", fOptUserAgent.String());

	if (fOptReferer.CountChars() > 0)
		outputHeaders.AddHeader("Referer", fOptReferer.String());

	// Optional range requests headers
	if (fOptRangeStart != -1 || fOptRangeEnd != -1) {
		if (fOptRangeStart == -1)
			fOptRangeStart = 0;
		BString range;
		if (fOptRangeEnd != -1) {
			range.SetToFormat("bytes=%" B_PRIdOFF "-%" B_PRIdOFF,
				fOptRangeStart, fOptRangeEnd);
		} else {
			range.SetToFormat("bytes=%" B_PRIdOFF "-", fOptRangeStart);
		}
		outputHeaders.AddHeader("Range", range.String());
	}

	// Authentication
	if (fContext != NULL) {
		BHttpAuthentication& authentication = fContext->GetAuthentication(fUrl);
		if (authentication.Method() != B_HTTP_AUTHENTICATION_NONE) {
			if (fOptUsername.Length() > 0) {
				authentication.SetUserName(fOptUsername);
				authentication.SetPassword(fOptPassword);
			}

			BString request(fRequestMethod);
			outputHeaders.AddHeader("Authorization",
				authentication.Authorization(fUrl, request));
		}
	}

	// Required headers for POST data
	if (fOptPostFields != NULL && fRequestMethod == B_HTTP_POST) {
		BString contentType;

		switch (fOptPostFields->GetFormType()) {
			case B_HTTP_FORM_MULTIPART:
				contentType << "multipart/form-data; boundary="
					<< fOptPostFields->GetMultipartBoundary() << "";
				break;

			case B_HTTP_FORM_URL_ENCODED:
				contentType << "application/x-www-form-urlencoded";
				break;
		}

		outputHeaders.AddHeader("Content-Type", contentType);
		outputHeaders.AddHeader("Content-Length",
			fOptPostFields->ContentLength());
	} else if (fOptInputData != NULL
			&& (fRequestMethod == B_HTTP_POST
			|| fRequestMethod == B_HTTP_PUT)) {
		if (fOptInputDataSize >= 0)
			outputHeaders.AddHeader("Content-Length", fOptInputDataSize);
		else
			outputHeaders.AddHeader("Transfer-Encoding", "chunked");
	}

	// Optional headers specified by the user
	if (fOptHeaders != NULL) {
		for (int32 headerIndex = 0; headerIndex < fOptHeaders->CountHeaders();
				headerIndex++) {
			BHttpHeader& optHeader = (*fOptHeaders)[headerIndex];
			int32 replaceIndex = outputHeaders.HasHeader(optHeader.Name());

			// Add or replace the current option header to the
			// output header list
			if (replaceIndex == -1)
				outputHeaders.AddHeader(optHeader.Name(), optHeader.Value());
			else
				outputHeaders[replaceIndex].SetValue(optHeader.Value());
		}
	}

	// Context cookies
	if (fOptSetCookies && fContext != NULL) {
		BString cookieString;

		BNetworkCookieJar::UrlIterator iterator
			= fContext->GetCookieJar().GetUrlIterator(fUrl);
		const BNetworkCookie* cookie = iterator.Next();
		if (cookie != NULL) {
			while (true) {
				cookieString << cookie->RawCookie(false);
				cookie = iterator.Next();
				if (cookie == NULL)
					break;
				cookieString << "; ";
			}

			outputHeaders.AddHeader("Cookie", cookieString);
		}
	}

	// Write output headers to output stream
	BString headerData;

	for (int32 headerIndex = 0; headerIndex < outputHeaders.CountHeaders();
			headerIndex++) {
		const char* header = outputHeaders.HeaderAt(headerIndex).Header();

		headerData << header;
		headerData << "\r\n";

		_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", header);
	}

	return headerData;
}


void
BHttpRequest::_SendPostData()
{
	if (fRequestMethod == B_HTTP_POST && fOptPostFields != NULL) {
		if (fOptPostFields->GetFormType() != B_HTTP_FORM_MULTIPART) {
			BString outputBuffer = fOptPostFields->RawData();
			_EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
				"%s", outputBuffer.String());
			fSocket->Write(outputBuffer.String(), outputBuffer.Length());
		} else {
			for (BHttpForm::Iterator it = fOptPostFields->GetIterator();
				const BHttpFormData* currentField = it.Next();
				) {
				_EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
					it.MultipartHeader().String());
				fSocket->Write(it.MultipartHeader().String(),
					it.MultipartHeader().Length());

				switch (currentField->Type()) {
					default:
					case B_HTTPFORM_UNKNOWN:
						ASSERT(0);
						break;

					case B_HTTPFORM_STRING:
						fSocket->Write(currentField->String().String(),
							currentField->String().Length());
						break;

					case B_HTTPFORM_FILE:
						{
							BFile upFile(currentField->File().Path(),
								B_READ_ONLY);
							char readBuffer[kHttpBufferSize];
							ssize_t readSize;
							off_t totalSize;

							if (upFile.GetSize(&totalSize) != B_OK)
								ASSERT(0);

							readSize = upFile.Read(readBuffer,
								sizeof(readBuffer));
							while (readSize > 0) {
								fSocket->Write(readBuffer, readSize);
								readSize = upFile.Read(readBuffer,
									sizeof(readBuffer));
								fListener->UploadProgress(this, readSize,
									std::max((off_t)0, totalSize));
							}

							break;
						}
					case B_HTTPFORM_BUFFER:
						fSocket->Write(currentField->Buffer(),
							currentField->BufferSize());
						break;
				}

				fSocket->Write("\r\n", 2);
			}

			BString footer = fOptPostFields->GetMultipartFooter();
			fSocket->Write(footer.String(), footer.Length());
		}
	} else if ((fRequestMethod == B_HTTP_POST || fRequestMethod == B_HTTP_PUT)
		&& fOptInputData != NULL) {

		// If the input data is seekable, we rewind it for each new request.
		BPositionIO* seekableData
			= dynamic_cast<BPositionIO*>(fOptInputData);
		if (seekableData)
			seekableData->Seek(0, SEEK_SET);

		for (;;) {
			char outputTempBuffer[kHttpBufferSize];
			ssize_t read = fOptInputData->Read(outputTempBuffer,
				sizeof(outputTempBuffer));

			if (read <= 0)
				break;

			if (fOptInputDataSize < 0) {
				// Input data size unknown, so we have to use chunked transfer
				char hexSize[18];
				// The string does not need to be NULL terminated.
				size_t hexLength = snprintf(hexSize, sizeof(hexSize), "%lx\r\n",
					read);

				fSocket->Write(hexSize, hexLength);
				fSocket->Write(outputTempBuffer, read);
				fSocket->Write("\r\n", 2);
			} else {
				fSocket->Write(outputTempBuffer, read);
			}
		}

		if (fOptInputDataSize < 0) {
			// Chunked transfer terminating sequence
			fSocket->Write("0\r\n\r\n", 5);
		}
	}

}


BHttpHeaders&
BHttpRequest::_ResultHeaders()
{
	return fResult.fHeaders;
}


void
BHttpRequest::_SetResultStatusCode(int32 statusCode)
{
	fResult.fStatusCode = statusCode;
}


BString&
BHttpRequest::_ResultStatusText()
{
	return fResult.fStatusString;
}


bool
BHttpRequest::_CertificateVerificationFailed(BCertificate& certificate,
	const char* message)
{
	if (fContext->HasCertificateException(certificate))
		return true;

	if (fListener != NULL
		&& fListener->CertificateVerificationFailed(this, certificate, message)) {
		// User asked us to continue anyway, let's add a temporary exception for this certificate
		fContext->AddCertificateException(certificate);
		return true;
	}

	return false;
}


bool
BHttpRequest::_IsDefaultPort()
{
	if (fSSL && Url().Port() == 443)
		return true;
	if (!fSSL && Url().Port() == 80)
		return true;
	return false;
}


#ifdef LIBNETAPI_DEPRECATED
void
BHttpRequest::_NotifyDataReceived(const char* data, off_t pos, ssize_t size,
	off_t bytesReceived, ssize_t bytesTotal)
{
	if (fListener == NULL || size <= 0)
		return;
	if (fOptRangeStart > 0) {
		pos += fOptRangeStart;
		// bytesReceived and bytesTotal refer to the requested range,
		// so that should technically not be adjusted for the range start.
		// For displaying progress to the user, this is not ideal, though.
		// But only for the case where we request the remainder of a file.
		// Range requests can also be used to request any portion of a
		// resource, so not modifying them is technically more correct.
		// We can use a little trick, though: We know when the remainder
		// is requested, because then fOptRangeEnd is -1.
		if (fOptRangeEnd == -1) {
			bytesReceived += fOptRangeStart;
			if (bytesTotal > 0)
				bytesTotal += fOptRangeStart;
		}
	}
	fListener->DataReceived(this, data, pos, size);
	fListener->DownloadProgress(this, bytesReceived,
		std::max((ssize_t)0, bytesTotal));
}
#endif