Updated http_client on Windows desktop to obey the client config

chunksize for handling the response body when explicitly set to by users.
This allows users to configure for large data to avoid the overhead of
processing small chunks of data at a time. The default still remains the
same.

Author: stgates

Release/include/cpprest/http_client.h

@@ -92,7 +92,7 @@ class http_client_config
     http_client_config() : 
         m_guarantee_order(false),
         m_timeout(utility::seconds(30)),
-        m_chunksize(64 * 1024)
+        m_chunksize(0)
 #if !defined(__cplusplus_winrt)
         , m_validate_certificates(true)
 #endif
@@ -178,7 +178,7 @@ class http_client_config
     /// <returns>The internal buffer size used by the http client when sending and receiving data from the network.</returns>
     size_t chunksize() const
     {
-        return m_chunksize;
+        return m_chunksize == 0 ? 64 * 1024 : m_chunksize;
     }
 
     /// <summary>
@@ -191,6 +191,16 @@ class http_client_config
         m_chunksize = size;
     }
 
+    /// <summary>
+    /// Returns true if the default chunk size is in use.
+    /// <remarks>If true, implementations are allowed to choose whatever size is best.</remarks>
+    /// </summary>
+    /// <returns>True if default, false if set by user.</returns>
+    bool is_default_chunksize() const
+    {
+        return m_chunksize == 0;
+    }
+
 #if !defined(__cplusplus_winrt)
     /// <summary>
     /// Gets the server certificate validation property.

Release/src/http/client/http_win7.cpp

@@ -696,6 +696,44 @@ class winhttp_client : public _http_client_communicator
         return has_proxy_credentials || has_server_credentials;
     }
 
+    // Helper function to query/read next part of response data from winhttp.
+    static void read_next_response_chunk(winhttp_request_context *pContext, DWORD bytesRead, bool firstRead=false)
+    {
+        const bool defaultChunkSize = pContext->m_http_client->client_config().is_default_chunksize();
+
+        // If user specified a chunk size then read in chunks instead of using query data avaliable.
+        if (defaultChunkSize)
+        {
+            if (!WinHttpQueryDataAvailable(pContext->m_request_handle, nullptr))
+            {
+                pContext->report_error(GetLastError(), _XPLATSTR("Error querying for http body data"));
+            }
+        }
+        else
+        {
+            // If bytes read is less than the chunksize this request is done.
+            const size_t chunkSize = pContext->m_http_client->client_config().chunksize();
+            if (bytesRead < chunkSize && !firstRead)
+            {
+                pContext->complete_request((size_t) pContext->m_downloaded);
+            }
+            else
+            {
+                auto writebuf = pContext->_get_writebuffer();
+                pContext->allocate_reply_space(writebuf.alloc(chunkSize), chunkSize);
+
+                if (!WinHttpReadData(
+                    pContext->m_request_handle,
+                    (LPVOID) pContext->m_body_data.get(),
+                    (DWORD) chunkSize,
+                    nullptr))
+                {
+                    pContext->report_error(GetLastError(), _XPLATSTR("Error receiving http response body chunk"));
+                }
+            }
+        }
+    }
+
     static void _transfer_encoding_chunked_write_data(_In_ winhttp_request_context * p_request_context)
     {
         const size_t chunk_size = p_request_context->m_http_client->client_config().chunksize();
@@ -1186,11 +1224,7 @@ class winhttp_client : public _http_client_communicator
                     // If none of them is specified, the message length should be determined by the server closing the connection.
                     // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
 
-                    // WINHTTP_CALLBACK_STATUS_DATA_AVAILABLE callback determines whether this function was successful and the value of the parameters.
-                    if(!WinHttpQueryDataAvailable(hRequestHandle, nullptr))
-                    {
-                        p_request_context->report_error(GetLastError(), _XPLATSTR("Error querying for http body data"));
-                    }
+                    read_next_response_chunk(p_request_context, 0, true);
                     break;
                 }
             case WINHTTP_CALLBACK_STATUS_DATA_AVAILABLE :
@@ -1201,9 +1235,6 @@ class winhttp_client : public _http_client_communicator
                     if(num_bytes > 0)
                     {
                         auto writebuf = p_request_context->_get_writebuffer();
-                        if ( !_check_streambuf(p_request_context, writebuf, _XPLATSTR("Output stream is not open")) )
-                            break;
-
                         p_request_context->allocate_reply_space(writebuf.alloc(num_bytes), num_bytes);
 
                         // Read in body all at once.
@@ -1237,68 +1268,58 @@ class winhttp_client : public _http_client_communicator
             case WINHTTP_CALLBACK_STATUS_READ_COMPLETE :
                 {
                     // Status information length contains the number of bytes read.
-                    // WinHTTP will always fill the whole buffer or read nothing.
-                    // If number of bytes read is zero than we have reached the end.
+                    const DWORD bytesRead = statusInfoLength;
 
-                    if(statusInfoLength > 0)
+                    // Report progress about downloaded bytes.
+                    auto progress = p_request_context->m_request._get_impl()->_progress_handler();
+                    p_request_context->m_downloaded += (size64_t) statusInfoLength;
+                    if (progress)
                     {
-                        auto progress = p_request_context->m_request._get_impl()->_progress_handler();
-                        p_request_context->m_downloaded += (size64_t)statusInfoLength;
-                        if ( progress )
+                        try { (*progress)(message_direction::download, p_request_context->m_downloaded); }
+                        catch (...)
                         {
-                            try { (*progress)(message_direction::download, p_request_context->m_downloaded); } catch(...)
+                            p_request_context->report_exception(std::current_exception());
+                            return;
+                        }
+                    }
+
+                    // If no bytes have been read, then this is the end of the response.
+                    if (bytesRead == 0)
+                    {
+                        p_request_context->complete_request((size_t) p_request_context->m_downloaded);
+                        break;
+                    }
+
+                    // If the data was allocated directly from the buffer then commit, otherwise we still
+                    // need to write to the response stream buffer.
+                    auto writebuf = p_request_context->_get_writebuffer();
+                    if (p_request_context->is_externally_allocated())
+                    {
+                        writebuf.commit(bytesRead);
+                        read_next_response_chunk(p_request_context, bytesRead);
+                    }
+                    else
+                    {
+                        writebuf.putn(p_request_context->m_body_data.get(), bytesRead).then(
+                            [hRequestHandle, p_request_context, bytesRead] (pplx::task<size_t> op)
+                        {
+                            size_t written = 0;
+                            try { written = op.get(); }
+                            catch (...)
                             {
                                 p_request_context->report_exception(std::current_exception());
                                 return;
                             }
-                        }
 
-                        auto writebuf = p_request_context->_get_writebuffer();
-                        
-                        if ( p_request_context->is_externally_allocated() )
-                        {
-                            writebuf.commit(statusInfoLength);
-
-                            // Look for more data
-                            if (!WinHttpQueryDataAvailable(hRequestHandle, nullptr))
+                            // If we couldn't write everything, it's time to exit.
+                            if (written != bytesRead)
                             {
-                                p_request_context->report_error(GetLastError(), _XPLATSTR("Error querying for http body chunk"));
+                                p_request_context->report_exception(std::runtime_error("response stream unexpectedly failed to write the requested number of bytes"));
                                 return;
                             }
-                        }
-                        else 
-                        {
-                            writebuf.putn(p_request_context->m_body_data.get(), statusInfoLength).then(
-                                [hRequestHandle, p_request_context, statusInfoLength]
-                            (pplx::task<size_t> op)
-                            {
-                                size_t written = 0;
-                                try { written = op.get(); } catch(...)
-                                {
-                                    p_request_context->report_exception(std::current_exception());
-                                    return;
-                                }
-
-                                // If we couldn't write everything, it's time to exit.
-                                if ( written != statusInfoLength ) 
-                                {
-                                    p_request_context->report_exception(std::runtime_error("response stream unexpectedly failed to write the requested number of bytes"));
-                                    return;
-                                }
-
-                                // Look for more data
-                                if (!WinHttpQueryDataAvailable(hRequestHandle, nullptr))
-                                {
-                                    p_request_context->report_error(GetLastError(), _XPLATSTR("Error querying for http body chunk"));
-                                    return;
-                                }
-                            });
-                        }
-                    }
-                    else
-                    {
-                        // Done reading so set task completion event and close the request handle.
-                        p_request_context->complete_request((size_t)p_request_context->m_downloaded);
+
+                            read_next_response_chunk(p_request_context, bytesRead);
+                        });
                     }
                     break;
                 }