HTTPClient should support large replies (RIPD-1366):

A full ledger on the production Ripple network could exceed the default maximum reply size for the HTTPClient code. Remove the reply size maximum for responses that include a Content-Length header.
waffcloud · Apr 19, 2017 · e52614a · e52614a
1 parent 10a7f5b
commit e52614a
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 16 deletions.
diff --git a/src/ripple/net/HTTPClient.h b/src/ripple/net/HTTPClient.h
@@ -45,7 +45,7 @@ class HTTPClient
         std::deque <std::string> deqSites,
         const unsigned short port,
         std::string const& strPath,
-        std::size_t responseMax,
+        std::size_t responseMax,    // if no Content-Length header
         std::chrono::seconds timeout,
         std::function <bool (const boost::system::error_code& ecResult, int iStatus, std::string const& strData)> complete,
         beast::Journal& j);
@@ -56,7 +56,7 @@ class HTTPClient
         std::string strSite,
         const unsigned short port,
         std::string const& strPath,
-        std::size_t responseMax,
+        std::size_t responseMax,    // if no Content-Length header
         std::chrono::seconds timeout,
         std::function <bool (const boost::system::error_code& ecResult, int iStatus, std::string const& strData)> complete,
         beast::Journal& j);
@@ -67,7 +67,7 @@ class HTTPClient
         std::string strSite,
         const unsigned short port,
         std::function <void (boost::asio::streambuf& sb, std::string const& strHost)> build,
-        std::size_t responseMax,
+        std::size_t responseMax,    // if no Content-Length header
         std::chrono::seconds timeout,
         std::function <bool (const boost::system::error_code& ecResult, int iStatus, std::string const& strData)> complete,
         beast::Journal& j);

diff --git a/src/ripple/net/impl/HTTPClient.cpp b/src/ripple/net/impl/HTTPClient.cpp
@@ -104,13 +104,13 @@ class HTTPClientImp
 public:
     HTTPClientImp (boost::asio::io_service& io_service,
         const unsigned short port,
-        std::size_t responseMax,
+        std::size_t responseSize,
         beast::Journal& j)
         : mSocket (io_service, httpClientSSLContext->context ())
         , mResolver (io_service)
         , mHeader (maxClientHeaderBytes)
         , mPort (port)
-        , mResponseMax (responseMax)
+        , mResponseSize (responseSize)
         , mDeadline (io_service)
         , j_ (j)
     {
@@ -411,27 +411,22 @@ class HTTPClientImp
             mBody = smMatch[1];
 
         if (boost::regex_match (strHeader, smMatch, reSize))
-        {
-            int size = beast::lexicalCastThrow <int> (std::string(smMatch[1]));
-
-            if (size < mResponseMax)
-                mResponseMax = size;
-        }
+            mResponseSize = beast::lexicalCastThrow <int> (std::string(smMatch[1]));
 
-        if (mResponseMax == 0)
+        if (mResponseSize == 0)
         {
             // no body wanted or available
             invokeComplete (ecResult, mStatus);
         }
-        else if (mBody.size () >= mResponseMax)
+        else if (mBody.size () >= mResponseSize)
         {
             // we got the whole thing
             invokeComplete (ecResult, mStatus, mBody);
         }
         else
         {
             mSocket.async_read (
-                mResponse.prepare (mResponseMax - mBody.size ()),
+                mResponse.prepare (mResponseSize - mBody.size ()),
                 boost::asio::transfer_all (),
                 std::bind (&HTTPClientImp::handleData,
                              shared_from_this (),
@@ -507,13 +502,13 @@ class HTTPClientImp
     bool                                                        mSSL;
     AutoSocket                                                  mSocket;
     boost::asio::ip::tcp::resolver                              mResolver;
-    std::shared_ptr<boost::asio::ip::tcp::resolver::query>    mQuery;
+    std::shared_ptr<boost::asio::ip::tcp::resolver::query>      mQuery;
     boost::asio::streambuf                                      mRequest;
     boost::asio::streambuf                                      mHeader;
     boost::asio::streambuf                                      mResponse;
     std::string                                                 mBody;
     const unsigned short                                        mPort;
-    int                                                         mResponseMax;
+    int                                                         mResponseSize;
     int                                                         mStatus;
     std::function<void (boost::asio::streambuf& sb, std::string const& strHost)>         mBuild;
     std::function<bool (const boost::system::error_code& ecResult, int iStatus, std::string const& strData)> mComplete;

diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp
@@ -1415,7 +1415,10 @@ void fromNetwork (
 
     // Send request
 
+    // Number of bytes to try to receive if no
+    // Content-Length header received
     const int RPC_REPLY_MAX_BYTES (256*1024*1024);
+
     using namespace std::chrono_literals;
     auto constexpr RPC_NOTIFY = 10min;