1 /* 2 * Copyright 2010-2021 Haiku Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Christophe Huriaux, c.huriaux@gmail.com 7 * Niels Sascha Reedijk, niels.reedijk@gmail.com 8 * Adrien Destugues, pulkomandy@pulkomandy.tk 9 * Stephan Aßmus, superstippi@gmx.de 10 */ 11 12 13 #include <HttpRequest.h> 14 15 #include <arpa/inet.h> 16 #include <stdio.h> 17 18 #include <cstdlib> 19 #include <deque> 20 #include <new> 21 22 #include <AutoDeleter.h> 23 #include <Certificate.h> 24 #include <Debug.h> 25 #include <DynamicBuffer.h> 26 #include <File.h> 27 #include <ProxySecureSocket.h> 28 #include <Socket.h> 29 #include <SecureSocket.h> 30 #include <StackOrHeapArray.h> 31 #include <ZlibCompressionAlgorithm.h> 32 33 34 static const int32 kHttpBufferSize = 4096; 35 36 37 #ifndef LIBNETAPI_DEPRECATED 38 using namespace BPrivate::Network; 39 #endif 40 41 42 namespace BPrivate { 43 44 class CheckedSecureSocket: public BSecureSocket 45 { 46 public: 47 CheckedSecureSocket(BHttpRequest* request); 48 49 bool CertificateVerificationFailed(BCertificate& certificate, 50 const char* message); 51 52 private: 53 BHttpRequest* fRequest; 54 }; 55 56 57 CheckedSecureSocket::CheckedSecureSocket(BHttpRequest* request) 58 : 59 BSecureSocket(), 60 fRequest(request) 61 { 62 } 63 64 65 bool 66 CheckedSecureSocket::CertificateVerificationFailed(BCertificate& certificate, 67 const char* message) 68 { 69 return fRequest->_CertificateVerificationFailed(certificate, message); 70 } 71 72 73 class CheckedProxySecureSocket: public BProxySecureSocket 74 { 75 public: 76 CheckedProxySecureSocket(const BNetworkAddress& proxy, BHttpRequest* request); 77 78 bool CertificateVerificationFailed(BCertificate& certificate, 79 const char* message); 80 81 private: 82 BHttpRequest* fRequest; 83 }; 84 85 86 CheckedProxySecureSocket::CheckedProxySecureSocket(const BNetworkAddress& proxy, 87 BHttpRequest* request) 88 : 89 BProxySecureSocket(proxy), 90 fRequest(request) 91 { 92 } 93 94 95 bool 96 CheckedProxySecureSocket::CertificateVerificationFailed(BCertificate& certificate, 97 const char* message) 98 { 99 return fRequest->_CertificateVerificationFailed(certificate, message); 100 } 101 }; 102 103 104 BHttpRequest::BHttpRequest(const BUrl& url, bool ssl, const char* protocolName, 105 BUrlProtocolListener* listener, BUrlContext* context) 106 : 107 BNetworkRequest(url, listener, context, "BUrlProtocol.HTTP", protocolName), 108 fSSL(ssl), 109 fRequestMethod(B_HTTP_GET), 110 fHttpVersion(B_HTTP_11), 111 fResult(url), 112 fRequestStatus(kRequestInitialState), 113 fOptHeaders(NULL), 114 fOptPostFields(NULL), 115 fOptInputData(NULL), 116 fOptInputDataSize(-1), 117 fOptRangeStart(-1), 118 fOptRangeEnd(-1), 119 fOptFollowLocation(true) 120 { 121 _ResetOptions(); 122 fSocket = NULL; 123 } 124 125 126 BHttpRequest::BHttpRequest(const BHttpRequest& other) 127 : 128 BNetworkRequest(other.Url(), other.fListener, other.fContext, 129 "BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"), 130 fSSL(other.fSSL), 131 fRequestMethod(other.fRequestMethod), 132 fHttpVersion(other.fHttpVersion), 133 fResult(other.fUrl), 134 fRequestStatus(kRequestInitialState), 135 fOptHeaders(NULL), 136 fOptPostFields(NULL), 137 fOptInputData(NULL), 138 fOptInputDataSize(-1), 139 fOptRangeStart(other.fOptRangeStart), 140 fOptRangeEnd(other.fOptRangeEnd), 141 fOptFollowLocation(other.fOptFollowLocation) 142 { 143 _ResetOptions(); 144 // FIXME some options may be copied from other instead. 145 fSocket = NULL; 146 } 147 148 149 BHttpRequest::~BHttpRequest() 150 { 151 Stop(); 152 153 delete fSocket; 154 155 delete fOptInputData; 156 delete fOptHeaders; 157 delete fOptPostFields; 158 } 159 160 161 void 162 BHttpRequest::SetMethod(const char* const method) 163 { 164 fRequestMethod = method; 165 } 166 167 168 void 169 BHttpRequest::SetFollowLocation(bool follow) 170 { 171 fOptFollowLocation = follow; 172 } 173 174 175 void 176 BHttpRequest::SetMaxRedirections(int8 redirections) 177 { 178 fOptMaxRedirs = redirections; 179 } 180 181 182 void 183 BHttpRequest::SetReferrer(const BString& referrer) 184 { 185 fOptReferer = referrer; 186 } 187 188 189 void 190 BHttpRequest::SetUserAgent(const BString& agent) 191 { 192 fOptUserAgent = agent; 193 } 194 195 196 void 197 BHttpRequest::SetDiscardData(bool discard) 198 { 199 fOptDiscardData = discard; 200 } 201 202 203 void 204 BHttpRequest::SetDisableListener(bool disable) 205 { 206 fOptDisableListener = disable; 207 } 208 209 210 void 211 BHttpRequest::SetAutoReferrer(bool enable) 212 { 213 fOptAutoReferer = enable; 214 } 215 216 217 void 218 BHttpRequest::SetUserName(const BString& name) 219 { 220 fOptUsername = name; 221 } 222 223 224 void 225 BHttpRequest::SetPassword(const BString& password) 226 { 227 fOptPassword = password; 228 } 229 230 231 void 232 BHttpRequest::SetRangeStart(off_t position) 233 { 234 // This field is used within the transfer loop, so only 235 // allow setting it before sending the request. 236 if (fRequestStatus == kRequestInitialState) 237 fOptRangeStart = position; 238 } 239 240 241 void 242 BHttpRequest::SetRangeEnd(off_t position) 243 { 244 // This field could be used in the transfer loop, so only 245 // allow setting it before sending the request. 246 if (fRequestStatus == kRequestInitialState) 247 fOptRangeEnd = position; 248 } 249 250 251 void 252 BHttpRequest::SetPostFields(const BHttpForm& fields) 253 { 254 AdoptPostFields(new(std::nothrow) BHttpForm(fields)); 255 } 256 257 258 void 259 BHttpRequest::SetHeaders(const BHttpHeaders& headers) 260 { 261 AdoptHeaders(new(std::nothrow) BHttpHeaders(headers)); 262 } 263 264 265 void 266 BHttpRequest::AdoptPostFields(BHttpForm* const fields) 267 { 268 delete fOptPostFields; 269 fOptPostFields = fields; 270 271 if (fOptPostFields != NULL) 272 fRequestMethod = B_HTTP_POST; 273 } 274 275 276 void 277 BHttpRequest::AdoptInputData(BDataIO* const data, const ssize_t size) 278 { 279 delete fOptInputData; 280 fOptInputData = data; 281 fOptInputDataSize = size; 282 } 283 284 285 void 286 BHttpRequest::AdoptHeaders(BHttpHeaders* const headers) 287 { 288 delete fOptHeaders; 289 fOptHeaders = headers; 290 } 291 292 293 /*static*/ bool 294 BHttpRequest::IsInformationalStatusCode(int16 code) 295 { 296 return (code >= B_HTTP_STATUS__INFORMATIONAL_BASE) 297 && (code < B_HTTP_STATUS__INFORMATIONAL_END); 298 } 299 300 301 /*static*/ bool 302 BHttpRequest::IsSuccessStatusCode(int16 code) 303 { 304 return (code >= B_HTTP_STATUS__SUCCESS_BASE) 305 && (code < B_HTTP_STATUS__SUCCESS_END); 306 } 307 308 309 /*static*/ bool 310 BHttpRequest::IsRedirectionStatusCode(int16 code) 311 { 312 return (code >= B_HTTP_STATUS__REDIRECTION_BASE) 313 && (code < B_HTTP_STATUS__REDIRECTION_END); 314 } 315 316 317 /*static*/ bool 318 BHttpRequest::IsClientErrorStatusCode(int16 code) 319 { 320 return (code >= B_HTTP_STATUS__CLIENT_ERROR_BASE) 321 && (code < B_HTTP_STATUS__CLIENT_ERROR_END); 322 } 323 324 325 /*static*/ bool 326 BHttpRequest::IsServerErrorStatusCode(int16 code) 327 { 328 return (code >= B_HTTP_STATUS__SERVER_ERROR_BASE) 329 && (code < B_HTTP_STATUS__SERVER_ERROR_END); 330 } 331 332 333 /*static*/ int16 334 BHttpRequest::StatusCodeClass(int16 code) 335 { 336 if (BHttpRequest::IsInformationalStatusCode(code)) 337 return B_HTTP_STATUS_CLASS_INFORMATIONAL; 338 else if (BHttpRequest::IsSuccessStatusCode(code)) 339 return B_HTTP_STATUS_CLASS_SUCCESS; 340 else if (BHttpRequest::IsRedirectionStatusCode(code)) 341 return B_HTTP_STATUS_CLASS_REDIRECTION; 342 else if (BHttpRequest::IsClientErrorStatusCode(code)) 343 return B_HTTP_STATUS_CLASS_CLIENT_ERROR; 344 else if (BHttpRequest::IsServerErrorStatusCode(code)) 345 return B_HTTP_STATUS_CLASS_SERVER_ERROR; 346 347 return B_HTTP_STATUS_CLASS_INVALID; 348 } 349 350 351 const BUrlResult& 352 BHttpRequest::Result() const 353 { 354 return fResult; 355 } 356 357 358 status_t 359 BHttpRequest::Stop() 360 { 361 if (fSocket != NULL) { 362 fSocket->Disconnect(); 363 // Unlock any pending connect, read or write operation. 364 } 365 return BNetworkRequest::Stop(); 366 } 367 368 369 void 370 BHttpRequest::_ResetOptions() 371 { 372 delete fOptPostFields; 373 delete fOptHeaders; 374 375 fOptFollowLocation = true; 376 fOptMaxRedirs = 8; 377 fOptReferer = ""; 378 fOptUserAgent = "Services Kit (Haiku)"; 379 fOptUsername = ""; 380 fOptPassword = ""; 381 fOptAuthMethods = B_HTTP_AUTHENTICATION_BASIC | B_HTTP_AUTHENTICATION_DIGEST 382 | B_HTTP_AUTHENTICATION_IE_DIGEST; 383 fOptHeaders = NULL; 384 fOptPostFields = NULL; 385 fOptSetCookies = true; 386 fOptDiscardData = false; 387 fOptDisableListener = false; 388 fOptAutoReferer = true; 389 } 390 391 392 status_t 393 BHttpRequest::_ProtocolLoop() 394 { 395 // Initialize the request redirection loop 396 int8 maxRedirs = fOptMaxRedirs; 397 bool newRequest; 398 399 do { 400 newRequest = false; 401 402 // Result reset 403 fHeaders.Clear(); 404 _ResultHeaders().Clear(); 405 406 BString host = fUrl.Host(); 407 int port = fSSL ? 443 : 80; 408 409 if (fUrl.HasPort()) 410 port = fUrl.Port(); 411 412 if (fContext->UseProxy()) { 413 host = fContext->GetProxyHost(); 414 port = fContext->GetProxyPort(); 415 } 416 417 status_t result = fInputBuffer.InitCheck(); 418 if (result != B_OK) 419 return result; 420 421 if (!_ResolveHostName(host, port)) { 422 _EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, 423 "Unable to resolve hostname (%s), aborting.", 424 fUrl.Host().String()); 425 return B_SERVER_NOT_FOUND; 426 } 427 428 status_t requestStatus = _MakeRequest(); 429 if (requestStatus != B_OK) 430 return requestStatus; 431 432 // Prepare the referer for the next request if needed 433 if (fOptAutoReferer) 434 fOptReferer = fUrl.UrlString(); 435 436 switch (StatusCodeClass(fResult.StatusCode())) { 437 case B_HTTP_STATUS_CLASS_INFORMATIONAL: 438 // Header 100:continue should have been 439 // handled in the _MakeRequest read loop 440 break; 441 442 case B_HTTP_STATUS_CLASS_SUCCESS: 443 break; 444 445 case B_HTTP_STATUS_CLASS_REDIRECTION: 446 { 447 // Redirection has been explicitly disabled 448 if (!fOptFollowLocation) 449 break; 450 451 int code = fResult.StatusCode(); 452 if (code == B_HTTP_STATUS_MOVED_PERMANENTLY 453 || code == B_HTTP_STATUS_FOUND 454 || code == B_HTTP_STATUS_SEE_OTHER 455 || code == B_HTTP_STATUS_TEMPORARY_REDIRECT) { 456 BString locationUrl = fHeaders["Location"]; 457 458 fUrl = BUrl(fUrl, locationUrl); 459 460 // 302 and 303 redirections also convert POST requests to GET 461 // (and remove the posted form data) 462 if ((code == B_HTTP_STATUS_FOUND 463 || code == B_HTTP_STATUS_SEE_OTHER) 464 && fRequestMethod == B_HTTP_POST) { 465 SetMethod(B_HTTP_GET); 466 delete fOptPostFields; 467 fOptPostFields = NULL; 468 delete fOptInputData; 469 fOptInputData = NULL; 470 fOptInputDataSize = 0; 471 } 472 473 if (--maxRedirs > 0) { 474 newRequest = true; 475 476 // Redirections may need a switch from http to https. 477 if (fUrl.Protocol() == "https") 478 fSSL = true; 479 else if (fUrl.Protocol() == "http") 480 fSSL = false; 481 482 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, 483 "Following: %s\n", 484 fUrl.UrlString().String()); 485 } 486 } 487 break; 488 } 489 490 case B_HTTP_STATUS_CLASS_CLIENT_ERROR: 491 if (fResult.StatusCode() == B_HTTP_STATUS_UNAUTHORIZED) { 492 BHttpAuthentication* authentication 493 = &fContext->GetAuthentication(fUrl); 494 status_t status = B_OK; 495 496 if (authentication->Method() == B_HTTP_AUTHENTICATION_NONE) { 497 // There is no authentication context for this 498 // url yet, so let's create one. 499 BHttpAuthentication newAuth; 500 newAuth.Initialize(fHeaders["WWW-Authenticate"]); 501 fContext->AddAuthentication(fUrl, newAuth); 502 503 // Get the copy of the authentication we just added. 504 // That copy is owned by the BUrlContext and won't be 505 // deleted (unlike the temporary object above) 506 authentication = &fContext->GetAuthentication(fUrl); 507 } 508 509 newRequest = false; 510 if (fOptUsername.Length() > 0 && status == B_OK) { 511 // If we received an username and password, add them 512 // to the request. This will either change the 513 // credentials for an existing request, or set them 514 // for a new one we created just above. 515 // 516 // If this request handles HTTP redirections, it will 517 // also automatically retry connecting and send the 518 // login information. 519 authentication->SetUserName(fOptUsername); 520 authentication->SetPassword(fOptPassword); 521 newRequest = true; 522 } 523 } 524 break; 525 526 case B_HTTP_STATUS_CLASS_SERVER_ERROR: 527 break; 528 529 default: 530 case B_HTTP_STATUS_CLASS_INVALID: 531 break; 532 } 533 } while (newRequest); 534 535 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, 536 "%ld headers and %ld bytes of data remaining", 537 fHeaders.CountHeaders(), fInputBuffer.Size()); 538 539 if (fResult.StatusCode() == 404) 540 return B_RESOURCE_NOT_FOUND; 541 542 return B_OK; 543 } 544 545 546 status_t 547 BHttpRequest::_MakeRequest() 548 { 549 delete fSocket; 550 551 if (fSSL) { 552 if (fContext->UseProxy()) { 553 BNetworkAddress proxy(fContext->GetProxyHost(), fContext->GetProxyPort()); 554 fSocket = new(std::nothrow) BPrivate::CheckedProxySecureSocket(proxy, this); 555 } else 556 fSocket = new(std::nothrow) BPrivate::CheckedSecureSocket(this); 557 } else 558 fSocket = new(std::nothrow) BSocket(); 559 560 if (fSocket == NULL) 561 return B_NO_MEMORY; 562 563 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.", 564 fUrl.Authority().String(), fRemoteAddr.Port()); 565 status_t connectError = fSocket->Connect(fRemoteAddr); 566 567 if (connectError != B_OK) { 568 _EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s", 569 strerror(connectError)); 570 return connectError; 571 } 572 573 //! ProtocolHook:ConnectionOpened 574 if (fListener != NULL) 575 fListener->ConnectionOpened(this); 576 577 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, 578 "Connection opened, sending request."); 579 580 BString requestHeaders; 581 requestHeaders.Append(_SerializeRequest()); 582 requestHeaders.Append(_SerializeHeaders()); 583 requestHeaders.Append("\r\n"); 584 fSocket->Write(requestHeaders.String(), requestHeaders.Length()); 585 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent."); 586 587 _SendPostData(); 588 fRequestStatus = kRequestInitialState; 589 590 591 592 // Receive loop 593 bool receiveEnd = false; 594 bool parseEnd = false; 595 bool readByChunks = false; 596 bool decompress = false; 597 status_t readError = B_OK; 598 ssize_t bytesRead = 0; 599 off_t bytesReceived = 0; 600 off_t bytesTotal = 0; 601 size_t previousBufferSize = 0; 602 off_t bytesUnpacked = 0; 603 char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize]; 604 ArrayDeleter<char> inputTempBufferDeleter(inputTempBuffer); 605 ssize_t inputTempSize = kHttpBufferSize; 606 ssize_t chunkSize = -1; 607 DynamicBuffer decompressorStorage; 608 BDataIO* decompressingStream; 609 ObjectDeleter<BDataIO> decompressingStreamDeleter; 610 611 while (!fQuit && !(receiveEnd && parseEnd)) { 612 if ((!receiveEnd) && (fInputBuffer.Size() == previousBufferSize)) { 613 BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize); 614 bytesRead = fSocket->Read(chunk, kHttpBufferSize); 615 616 if (bytesRead < 0) { 617 readError = bytesRead; 618 break; 619 } else if (bytesRead == 0) { 620 // Check if we got the expected number of bytes. 621 // Exceptions: 622 // - If the content-length is not known (bytesTotal is 0), for 623 // example in the case of a chunked transfer, we can't know 624 // - If the request method is "HEAD" which explicitly asks the 625 // server to not send any data (only the headers) 626 if (bytesTotal > 0 && bytesReceived != bytesTotal 627 && fRequestMethod != B_HTTP_HEAD) { 628 readError = B_IO_ERROR; 629 break; 630 } 631 receiveEnd = true; 632 } 633 634 fInputBuffer.AppendData(chunk, bytesRead); 635 } else 636 bytesRead = 0; 637 638 previousBufferSize = fInputBuffer.Size(); 639 640 if (fRequestStatus < kRequestStatusReceived) { 641 _ParseStatus(); 642 643 //! ProtocolHook:ResponseStarted 644 if (fRequestStatus >= kRequestStatusReceived && fListener != NULL) 645 fListener->ResponseStarted(this); 646 } 647 648 if (fRequestStatus < kRequestHeadersReceived) { 649 _ParseHeaders(); 650 651 if (fRequestStatus >= kRequestHeadersReceived) { 652 _ResultHeaders() = fHeaders; 653 654 // Parse received cookies 655 if (fContext != NULL) { 656 for (int32 i = 0; i < fHeaders.CountHeaders(); i++) { 657 if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) { 658 fContext->GetCookieJar().AddCookie( 659 fHeaders.HeaderAt(i).Value(), fUrl); 660 } 661 } 662 } 663 664 //! ProtocolHook:HeadersReceived 665 if (fListener != NULL) 666 fListener->HeadersReceived(this, fResult); 667 668 669 if (BString(fHeaders["Transfer-Encoding"]) == "chunked") 670 readByChunks = true; 671 672 BString contentEncoding(fHeaders["Content-Encoding"]); 673 // We don't advertise "deflate" support (see above), but we 674 // still try to decompress it, if a server ever sends a deflate 675 // stream despite it not being in our Accept-Encoding list. 676 if (contentEncoding == "gzip" 677 || contentEncoding == "deflate") { 678 decompress = true; 679 readError = BZlibCompressionAlgorithm() 680 .CreateDecompressingOutputStream(&decompressorStorage, 681 NULL, decompressingStream); 682 if (readError != B_OK) 683 break; 684 685 decompressingStreamDeleter.SetTo(decompressingStream); 686 } 687 688 int32 index = fHeaders.HasHeader("Content-Length"); 689 if (index != B_ERROR) 690 bytesTotal = atoll(fHeaders.HeaderAt(index).Value()); 691 else 692 bytesTotal = -1; 693 } 694 } 695 696 if (fRequestStatus >= kRequestHeadersReceived) { 697 // If Transfer-Encoding is chunked, we should read a complete 698 // chunk in buffer before handling it 699 if (readByChunks) { 700 if (chunkSize >= 0) { 701 if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) { 702 // 2 more bytes to handle the closing CR+LF 703 bytesRead = chunkSize; 704 if (inputTempSize < chunkSize + 2) { 705 inputTempSize = chunkSize + 2; 706 inputTempBuffer 707 = new(std::nothrow) char[inputTempSize]; 708 inputTempBufferDeleter.SetTo(inputTempBuffer); 709 } 710 711 if (inputTempBuffer == NULL) { 712 readError = B_NO_MEMORY; 713 break; 714 } 715 716 fInputBuffer.RemoveData(inputTempBuffer, 717 chunkSize + 2); 718 chunkSize = -1; 719 } else { 720 // Not enough data, try again later 721 bytesRead = -1; 722 } 723 } else { 724 BString chunkHeader; 725 if (_GetLine(chunkHeader) == B_ERROR) { 726 chunkSize = -1; 727 bytesRead = -1; 728 } else { 729 // Format of a chunk header: 730 // <chunk size in hex>[; optional data] 731 int32 semiColonIndex = chunkHeader.FindFirst(';', 0); 732 733 // Cut-off optional data if present 734 if (semiColonIndex != -1) { 735 chunkHeader.Remove(semiColonIndex, 736 chunkHeader.Length() - semiColonIndex); 737 } 738 739 chunkSize = strtol(chunkHeader.String(), NULL, 16); 740 if (chunkSize == 0) 741 fRequestStatus = kRequestContentReceived; 742 743 bytesRead = -1; 744 } 745 } 746 747 // A chunk of 0 bytes indicates the end of the chunked transfer 748 if (bytesRead == 0) 749 receiveEnd = true; 750 } else { 751 bytesRead = fInputBuffer.Size(); 752 753 if (bytesRead > 0) { 754 if (inputTempSize < bytesRead) { 755 inputTempSize = bytesRead; 756 inputTempBuffer = new(std::nothrow) char[bytesRead]; 757 inputTempBufferDeleter.SetTo(inputTempBuffer); 758 } 759 760 if (inputTempBuffer == NULL) { 761 readError = B_NO_MEMORY; 762 break; 763 } 764 fInputBuffer.RemoveData(inputTempBuffer, bytesRead); 765 } 766 } 767 768 if (bytesRead >= 0) { 769 bytesReceived += bytesRead; 770 771 if (fListener != NULL) { 772 if (decompress) { 773 readError = decompressingStream->WriteExactly( 774 inputTempBuffer, bytesRead); 775 if (readError != B_OK) 776 break; 777 778 ssize_t size = decompressorStorage.Size(); 779 BStackOrHeapArray<char, 4096> buffer(size); 780 size = decompressorStorage.Read(buffer, size); 781 _NotifyDataReceived(buffer, bytesUnpacked, size, 782 bytesReceived, bytesTotal); 783 bytesUnpacked += size; 784 } else if (bytesRead > 0) { 785 _NotifyDataReceived(inputTempBuffer, 786 bytesReceived - bytesRead, bytesRead, 787 bytesReceived, bytesTotal); 788 } 789 } 790 791 if (bytesTotal >= 0 && bytesReceived >= bytesTotal) 792 receiveEnd = true; 793 794 if (decompress && receiveEnd) { 795 readError = decompressingStream->Flush(); 796 797 if (readError == B_BUFFER_OVERFLOW) 798 readError = B_OK; 799 800 if (readError != B_OK) 801 break; 802 803 ssize_t size = decompressorStorage.Size(); 804 BStackOrHeapArray<char, 4096> buffer(size); 805 size = decompressorStorage.Read(buffer, size); 806 _NotifyDataReceived(buffer, bytesUnpacked, size, 807 bytesReceived, bytesTotal); 808 bytesUnpacked += size; 809 } 810 } 811 } 812 813 parseEnd = (fInputBuffer.Size() == 0); 814 } 815 816 fSocket->Disconnect(); 817 818 if (readError != B_OK) 819 return readError; 820 821 return fQuit ? B_INTERRUPTED : B_OK; 822 } 823 824 825 void 826 BHttpRequest::_ParseStatus() 827 { 828 // Status line should be formatted like: HTTP/M.m SSS ... 829 // With: M = Major version of the protocol 830 // m = Minor version of the protocol 831 // SSS = three-digit status code of the response 832 // ... = additional text info 833 BString statusLine; 834 if (_GetLine(statusLine) == B_ERROR) 835 return; 836 837 if (statusLine.CountChars() < 12) 838 return; 839 840 fRequestStatus = kRequestStatusReceived; 841 842 BString statusCodeStr; 843 BString statusText; 844 statusLine.CopyInto(statusCodeStr, 9, 3); 845 _SetResultStatusCode(atoi(statusCodeStr.String())); 846 847 statusLine.CopyInto(_ResultStatusText(), 13, statusLine.Length() - 13); 848 849 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Status line received: Code %d (%s)", 850 atoi(statusCodeStr.String()), _ResultStatusText().String()); 851 } 852 853 854 void 855 BHttpRequest::_ParseHeaders() 856 { 857 BString currentHeader; 858 while (_GetLine(currentHeader) != B_ERROR) { 859 // An empty line means the end of the header section 860 if (currentHeader.Length() == 0) { 861 fRequestStatus = kRequestHeadersReceived; 862 return; 863 } 864 865 _EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_IN, "%s", 866 currentHeader.String()); 867 fHeaders.AddHeader(currentHeader.String()); 868 } 869 } 870 871 872 BString 873 BHttpRequest::_SerializeRequest() 874 { 875 BString request(fRequestMethod); 876 request << ' '; 877 878 if (fContext->UseProxy()) { 879 // When there is a proxy, the request must include the host and port so 880 // the proxy knows where to send the request. 881 request << Url().Protocol() << "://" << Url().Host(); 882 if (Url().HasPort()) 883 request << ':' << Url().Port(); 884 } 885 886 if (Url().HasPath() && Url().Path().Length() > 0) 887 request << Url().Path(); 888 else 889 request << '/'; 890 891 if (Url().HasRequest()) 892 request << '?' << Url().Request(); 893 894 switch (fHttpVersion) { 895 case B_HTTP_11: 896 request << " HTTP/1.1\r\n"; 897 break; 898 899 default: 900 case B_HTTP_10: 901 request << " HTTP/1.0\r\n"; 902 break; 903 } 904 905 _EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", request.String()); 906 907 return request; 908 } 909 910 911 BString 912 BHttpRequest::_SerializeHeaders() 913 { 914 BHttpHeaders outputHeaders; 915 916 // HTTP 1.1 additional headers 917 if (fHttpVersion == B_HTTP_11) { 918 BString host = Url().Host(); 919 if (Url().HasPort() && !_IsDefaultPort()) 920 host << ':' << Url().Port(); 921 922 outputHeaders.AddHeader("Host", host); 923 924 outputHeaders.AddHeader("Accept", "*/*"); 925 outputHeaders.AddHeader("Accept-Encoding", "gzip"); 926 // Allows the server to compress data using the "gzip" format. 927 // "deflate" is not supported, because there are two interpretations 928 // of what it means (the RFC and Microsoft products), and we don't 929 // want to handle this. Very few websites support only deflate, 930 // and most of them will send gzip, or at worst, uncompressed data. 931 932 outputHeaders.AddHeader("Connection", "close"); 933 // Let the remote server close the connection after response since 934 // we don't handle multiple request on a single connection 935 } 936 937 // Classic HTTP headers 938 if (fOptUserAgent.CountChars() > 0) 939 outputHeaders.AddHeader("User-Agent", fOptUserAgent.String()); 940 941 if (fOptReferer.CountChars() > 0) 942 outputHeaders.AddHeader("Referer", fOptReferer.String()); 943 944 // Optional range requests headers 945 if (fOptRangeStart != -1 || fOptRangeEnd != -1) { 946 if (fOptRangeStart == -1) 947 fOptRangeStart = 0; 948 BString range; 949 if (fOptRangeEnd != -1) { 950 range.SetToFormat("bytes=%" B_PRIdOFF "-%" B_PRIdOFF, 951 fOptRangeStart, fOptRangeEnd); 952 } else { 953 range.SetToFormat("bytes=%" B_PRIdOFF "-", fOptRangeStart); 954 } 955 outputHeaders.AddHeader("Range", range.String()); 956 } 957 958 // Authentication 959 if (fContext != NULL) { 960 BHttpAuthentication& authentication = fContext->GetAuthentication(fUrl); 961 if (authentication.Method() != B_HTTP_AUTHENTICATION_NONE) { 962 if (fOptUsername.Length() > 0) { 963 authentication.SetUserName(fOptUsername); 964 authentication.SetPassword(fOptPassword); 965 } 966 967 BString request(fRequestMethod); 968 outputHeaders.AddHeader("Authorization", 969 authentication.Authorization(fUrl, request)); 970 } 971 } 972 973 // Required headers for POST data 974 if (fOptPostFields != NULL && fRequestMethod == B_HTTP_POST) { 975 BString contentType; 976 977 switch (fOptPostFields->GetFormType()) { 978 case B_HTTP_FORM_MULTIPART: 979 contentType << "multipart/form-data; boundary=" 980 << fOptPostFields->GetMultipartBoundary() << ""; 981 break; 982 983 case B_HTTP_FORM_URL_ENCODED: 984 contentType << "application/x-www-form-urlencoded"; 985 break; 986 } 987 988 outputHeaders.AddHeader("Content-Type", contentType); 989 outputHeaders.AddHeader("Content-Length", 990 fOptPostFields->ContentLength()); 991 } else if (fOptInputData != NULL 992 && (fRequestMethod == B_HTTP_POST 993 || fRequestMethod == B_HTTP_PUT)) { 994 if (fOptInputDataSize >= 0) 995 outputHeaders.AddHeader("Content-Length", fOptInputDataSize); 996 else 997 outputHeaders.AddHeader("Transfer-Encoding", "chunked"); 998 } 999 1000 // Optional headers specified by the user 1001 if (fOptHeaders != NULL) { 1002 for (int32 headerIndex = 0; headerIndex < fOptHeaders->CountHeaders(); 1003 headerIndex++) { 1004 BHttpHeader& optHeader = (*fOptHeaders)[headerIndex]; 1005 int32 replaceIndex = outputHeaders.HasHeader(optHeader.Name()); 1006 1007 // Add or replace the current option header to the 1008 // output header list 1009 if (replaceIndex == -1) 1010 outputHeaders.AddHeader(optHeader.Name(), optHeader.Value()); 1011 else 1012 outputHeaders[replaceIndex].SetValue(optHeader.Value()); 1013 } 1014 } 1015 1016 // Context cookies 1017 if (fOptSetCookies && fContext != NULL) { 1018 BString cookieString; 1019 1020 BNetworkCookieJar::UrlIterator iterator 1021 = fContext->GetCookieJar().GetUrlIterator(fUrl); 1022 const BNetworkCookie* cookie = iterator.Next(); 1023 if (cookie != NULL) { 1024 while (true) { 1025 cookieString << cookie->RawCookie(false); 1026 cookie = iterator.Next(); 1027 if (cookie == NULL) 1028 break; 1029 cookieString << "; "; 1030 } 1031 1032 outputHeaders.AddHeader("Cookie", cookieString); 1033 } 1034 } 1035 1036 // Write output headers to output stream 1037 BString headerData; 1038 1039 for (int32 headerIndex = 0; headerIndex < outputHeaders.CountHeaders(); 1040 headerIndex++) { 1041 const char* header = outputHeaders.HeaderAt(headerIndex).Header(); 1042 1043 headerData << header; 1044 headerData << "\r\n"; 1045 1046 _EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", header); 1047 } 1048 1049 return headerData; 1050 } 1051 1052 1053 void 1054 BHttpRequest::_SendPostData() 1055 { 1056 if (fRequestMethod == B_HTTP_POST && fOptPostFields != NULL) { 1057 if (fOptPostFields->GetFormType() != B_HTTP_FORM_MULTIPART) { 1058 BString outputBuffer = fOptPostFields->RawData(); 1059 _EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT, 1060 "%s", outputBuffer.String()); 1061 fSocket->Write(outputBuffer.String(), outputBuffer.Length()); 1062 } else { 1063 for (BHttpForm::Iterator it = fOptPostFields->GetIterator(); 1064 const BHttpFormData* currentField = it.Next(); 1065 ) { 1066 _EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT, 1067 it.MultipartHeader().String()); 1068 fSocket->Write(it.MultipartHeader().String(), 1069 it.MultipartHeader().Length()); 1070 1071 switch (currentField->Type()) { 1072 default: 1073 case B_HTTPFORM_UNKNOWN: 1074 ASSERT(0); 1075 break; 1076 1077 case B_HTTPFORM_STRING: 1078 fSocket->Write(currentField->String().String(), 1079 currentField->String().Length()); 1080 break; 1081 1082 case B_HTTPFORM_FILE: 1083 { 1084 BFile upFile(currentField->File().Path(), 1085 B_READ_ONLY); 1086 char readBuffer[kHttpBufferSize]; 1087 ssize_t readSize; 1088 off_t totalSize; 1089 1090 if (upFile.GetSize(&totalSize) != B_OK) 1091 ASSERT(0); 1092 1093 readSize = upFile.Read(readBuffer, 1094 sizeof(readBuffer)); 1095 while (readSize > 0) { 1096 fSocket->Write(readBuffer, readSize); 1097 readSize = upFile.Read(readBuffer, 1098 sizeof(readBuffer)); 1099 fListener->UploadProgress(this, readSize, 1100 std::max((off_t)0, totalSize)); 1101 } 1102 1103 break; 1104 } 1105 case B_HTTPFORM_BUFFER: 1106 fSocket->Write(currentField->Buffer(), 1107 currentField->BufferSize()); 1108 break; 1109 } 1110 1111 fSocket->Write("\r\n", 2); 1112 } 1113 1114 BString footer = fOptPostFields->GetMultipartFooter(); 1115 fSocket->Write(footer.String(), footer.Length()); 1116 } 1117 } else if ((fRequestMethod == B_HTTP_POST || fRequestMethod == B_HTTP_PUT) 1118 && fOptInputData != NULL) { 1119 1120 // If the input data is seekable, we rewind it for each new request. 1121 BPositionIO* seekableData 1122 = dynamic_cast<BPositionIO*>(fOptInputData); 1123 if (seekableData) 1124 seekableData->Seek(0, SEEK_SET); 1125 1126 for (;;) { 1127 char outputTempBuffer[kHttpBufferSize]; 1128 ssize_t read = fOptInputData->Read(outputTempBuffer, 1129 sizeof(outputTempBuffer)); 1130 1131 if (read <= 0) 1132 break; 1133 1134 if (fOptInputDataSize < 0) { 1135 // Input data size unknown, so we have to use chunked transfer 1136 char hexSize[18]; 1137 // The string does not need to be NULL terminated. 1138 size_t hexLength = snprintf(hexSize, sizeof(hexSize), "%lx\r\n", 1139 read); 1140 1141 fSocket->Write(hexSize, hexLength); 1142 fSocket->Write(outputTempBuffer, read); 1143 fSocket->Write("\r\n", 2); 1144 } else { 1145 fSocket->Write(outputTempBuffer, read); 1146 } 1147 } 1148 1149 if (fOptInputDataSize < 0) { 1150 // Chunked transfer terminating sequence 1151 fSocket->Write("0\r\n\r\n", 5); 1152 } 1153 } 1154 1155 } 1156 1157 1158 BHttpHeaders& 1159 BHttpRequest::_ResultHeaders() 1160 { 1161 return fResult.fHeaders; 1162 } 1163 1164 1165 void 1166 BHttpRequest::_SetResultStatusCode(int32 statusCode) 1167 { 1168 fResult.fStatusCode = statusCode; 1169 } 1170 1171 1172 BString& 1173 BHttpRequest::_ResultStatusText() 1174 { 1175 return fResult.fStatusString; 1176 } 1177 1178 1179 bool 1180 BHttpRequest::_CertificateVerificationFailed(BCertificate& certificate, 1181 const char* message) 1182 { 1183 if (fContext->HasCertificateException(certificate)) 1184 return true; 1185 1186 if (fListener != NULL 1187 && fListener->CertificateVerificationFailed(this, certificate, message)) { 1188 // User asked us to continue anyway, let's add a temporary exception for this certificate 1189 fContext->AddCertificateException(certificate); 1190 return true; 1191 } 1192 1193 return false; 1194 } 1195 1196 1197 bool 1198 BHttpRequest::_IsDefaultPort() 1199 { 1200 if (fSSL && Url().Port() == 443) 1201 return true; 1202 if (!fSSL && Url().Port() == 80) 1203 return true; 1204 return false; 1205 } 1206 1207 1208 void 1209 BHttpRequest::_NotifyDataReceived(const char* data, off_t pos, ssize_t size, 1210 off_t bytesReceived, ssize_t bytesTotal) 1211 { 1212 if (fListener == NULL || size <= 0) 1213 return; 1214 if (fOptRangeStart > 0) { 1215 pos += fOptRangeStart; 1216 // bytesReceived and bytesTotal refer to the requested range, 1217 // so that should technically not be adjusted for the range start. 1218 // For displaying progress to the user, this is not ideal, though. 1219 // But only for the case where we request the remainder of a file. 1220 // Range requests can also be used to request any portion of a 1221 // resource, so not modifying them is technically more correct. 1222 // We can use a little trick, though: We know when the remainder 1223 // is requested, because then fOptRangeEnd is -1. 1224 if (fOptRangeEnd == -1) { 1225 bytesReceived += fOptRangeStart; 1226 if (bytesTotal > 0) 1227 bytesTotal += fOptRangeStart; 1228 } 1229 } 1230 fListener->DataReceived(this, data, pos, size); 1231 fListener->DownloadProgress(this, bytesReceived, 1232 std::max((ssize_t)0, bytesTotal)); 1233 } 1234