Merge branch 'fix/make_scratch_buffer_dynamic' into 'master'

fix(esp_http_server): Make the http server scratch buffer dynamic

Closes IDF-11879

See merge request espressif/esp-idf!36651

Author: mahavirj

components/esp_http_server/Kconfig

@@ -1,11 +1,16 @@
 menu "HTTP Server"
 
+
     config HTTPD_MAX_REQ_HDR_LEN
-        int "Max HTTP Request Header Length"
-        default 512
+        int "HTTP Request Header Length limit"
+        default 1024
+        range 128 65536
         help
-            This sets the maximum supported size of headers section in HTTP request packet to be processed by the
-            server
+            This sets the default limit for the HTTP request header length. The limit can be
+            configured at run time by setting max_req_hdr_len member of httpd_config_t structure.
+            The memory allocated will depend on the actual header length. Hence keeping a sufficiently
+            large max header length is recommended.
+
 
     config HTTPD_MAX_URI_LEN
         int "Max HTTP URI Length"

components/esp_http_server/include/esp_http_server.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -55,6 +55,8 @@ initializer that should be kept in sync
         .stack_size         = 4096,                     \
         .core_id            = tskNO_AFFINITY,           \
         .task_caps          = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),       \
+        .max_req_hdr_len    = CONFIG_HTTPD_MAX_REQ_HDR_LEN,    \
+        .max_uri_len        = CONFIG_HTTPD_MAX_URI_LEN,        \
         .server_port        = 80,                       \
         .ctrl_port          = ESP_HTTPD_DEF_CTRL_PORT,  \
         .max_open_sockets   = 7,                        \
@@ -173,6 +175,13 @@ typedef struct httpd_config {
     BaseType_t  core_id;            /*!< The core the HTTP server task will run on */
     uint32_t    task_caps;          /*!< The memory capabilities to use when allocating the HTTP server task's stack */
 
+    /**
+     * Size limits for the header and URI buffers respectively.
+     * These are just limits, allocation would depend upon actual size of URI/header.
+     */
+    size_t max_req_hdr_len;    /*!< Size limit for the header buffer (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
+    size_t max_uri_len;    /*!< Size limit for the URI buffer By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
+
     /**
      * TCP Port number for receiving and transmitting HTTP traffic
      */
@@ -358,19 +367,13 @@ esp_err_t httpd_stop(httpd_handle_t handle);
  * @{
  */
 
-/* Max supported HTTP request header length */
-#define HTTPD_MAX_REQ_HDR_LEN CONFIG_HTTPD_MAX_REQ_HDR_LEN
-
-/* Max supported HTTP request URI length */
-#define HTTPD_MAX_URI_LEN CONFIG_HTTPD_MAX_URI_LEN
-
 /**
  * @brief HTTP Request Data Structure
  */
 typedef struct httpd_req {
     httpd_handle_t  handle;                     /*!< Handle to server instance */
     int             method;                     /*!< The type of HTTP request, -1 if unsupported method, HTTP_ANY for wildcard method to support every method */
-    const char      uri[HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
+    const char      uri[CONFIG_HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
     size_t          content_len;                /*!< Length of the request body */
     void           *aux;                        /*!< Internally used members */
 
@@ -611,10 +614,10 @@ typedef enum {
     /* Incoming payload is too large */
     HTTPD_413_CONTENT_TOO_LARGE,
 
-    /* URI length greater than CONFIG_HTTPD_MAX_URI_LEN */
+    /* URI length greater than CONFIG_CONFIG_HTTPD_MAX_URI_LEN */
     HTTPD_414_URI_TOO_LONG,
 
-    /* Headers section larger than CONFIG_HTTPD_MAX_REQ_HDR_LEN */
+    /* Headers section larger than CONFIG_CONFIG_HTTPD_MAX_REQ_HDR_LEN */
     HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE,
 
     /* Used internally for retrieving the total count of errors */

components/esp_http_server/src/esp_httpd_priv.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -31,13 +31,9 @@ extern "C" {
 #endif
 
 /* Size of request data block/chunk (not to be confused with chunked encoded data)
- * that is received and parsed in one turn of the parsing process. This should not
- * exceed the scratch buffer size and should at least be 8 bytes */
+ * that is received and parsed in one turn of the parsing process. */
 #define PARSER_BLOCK_SIZE  128
 
-/* Calculate the maximum size needed for the scratch buffer */
-#define HTTPD_SCRATCH_BUF  MAX(HTTPD_MAX_REQ_HDR_LEN, HTTPD_MAX_URI_LEN)
-
 /* Formats a log string to prepend context function name */
 #define LOG_FMT(x)      "%s: " x, __func__
 
@@ -88,7 +84,11 @@ struct sock_db {
  */
 struct httpd_req_aux {
     struct sock_db *sd;                             /*!< Pointer to socket database */
-    char            scratch[HTTPD_SCRATCH_BUF + 1]; /*!< Temporary buffer for our operations (1 byte extra for null termination) */
+    char           *scratch;                        /*!< Temporary buffer for our operations (1 byte extra for null termination) */
+    size_t          scratch_size_limit;             /*!< Scratch buffer size limit (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
+    size_t          scratch_cur_size;               /*!< Scratch buffer cur size (By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
+    size_t          max_req_hdr_len;             /*!< Header buffer size limit */
+    size_t          max_uri_len;             /*!< URI buffer size limit */
     size_t          remaining_len;                  /*!< Amount of data remaining to be fetched */
     char           *status;                         /*!< HTTP response's status code */
     char           *content_type;                   /*!< HTTP response's content type */

components/esp_http_server/src/httpd_parse.c

@@ -112,6 +112,8 @@ static esp_err_t cb_url(http_parser *parser,
                         const char *at, size_t length)
 {
     parser_data_t *parser_data = (parser_data_t *) parser->data;
+    httpd_req_t          *req   = parser_data->req;
+    struct httpd_req_aux *raux  = req->aux;
 
     if (parser_data->status == PARSING_IDLE) {
         ESP_LOGD(TAG, LOG_FMT("message begin"));
@@ -130,9 +132,9 @@ static esp_err_t cb_url(http_parser *parser,
     ESP_LOGD(TAG, LOG_FMT("processing url = %.*s"), (int)length, at);
 
     /* Update length of URL string */
-    if ((parser_data->last.length += length) > HTTPD_MAX_URI_LEN) {
+    if ((parser_data->last.length += length) > raux->max_uri_len) {
         ESP_LOGW(TAG, LOG_FMT("URI length (%"NEWLIB_NANO_COMPAT_FORMAT") greater than supported (%d)"),
-                 NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), HTTPD_MAX_URI_LEN);
+                 NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->max_uri_len);
         parser_data->error = HTTPD_414_URI_TOO_LONG;
         parser_data->status = PARSING_FAILED;
         return ESP_FAIL;
@@ -215,6 +217,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
         parser_data->last.at     = ra->scratch;
         parser_data->last.length = 0;
         parser_data->status      = PARSING_HDR_FIELD;
+        ra->scratch_size_limit   = ra->max_req_hdr_len;
 
         /* Stop parsing for now and give control to process */
         if (pause_parsing(parser, at) != ESP_OK) {
@@ -232,6 +235,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
         parser_data->last.at     = at;
         parser_data->last.length = 0;
         parser_data->status      = PARSING_HDR_FIELD;
+        ra->scratch_size_limit   = ra->max_req_hdr_len;
 
         /* Increment header count */
         ra->req_hdrs_count++;
@@ -480,16 +484,32 @@ static esp_err_t cb_no_body(http_parser *parser)
     return ESP_OK;
 }
 
-static int read_block(httpd_req_t *req, size_t offset, size_t length)
+static int read_block(httpd_req_t *req, http_parser *parser, size_t offset, size_t length)
 {
     struct httpd_req_aux *raux  = req->aux;
+    parser_data_t *parser_data = (parser_data_t *) parser->data;
 
     /* Limits the read to scratch buffer size */
-    ssize_t buf_len = MIN(length, (sizeof(raux->scratch) - offset));
+    ssize_t buf_len = MIN(length, (raux->scratch_size_limit - offset));
     if (buf_len <= 0) {
         return 0;
     }
-
+    /* Calculate the offset of the current position from the start of the buffer,
+     * as after reallocating the buffer, the base address of the buffer may change.
+     */
+    size_t at_offset = parser_data->last.at - raux->scratch;
+    /* Allocate the buffer according to offset and buf_len. Offset is
+       from where the reading will start and buf_len is till what length
+       the buffer will be read.
+    */
+    raux->scratch = (char*) realloc(raux->scratch, offset + buf_len);
+    if (raux->scratch == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate the scratch buffer");
+        return 0;
+    }
+    parser_data->last.at = raux->scratch + at_offset;
+    raux->scratch_cur_size = offset + buf_len;
+    ESP_LOGD(TAG, "scratch buf qsize = %d", raux->scratch_cur_size);
     /* Receive data into buffer. If data is pending (from unrecv) then return
      * immediately after receiving pending data, as pending data may just complete
      * this request packet. */
@@ -533,13 +553,14 @@ static int parse_block(http_parser *parser, size_t offset, size_t length)
          * parse means no more space left on buffer,
          * therefore it can be inferred that the
          * request URI/header must be too long */
-        ESP_LOGW(TAG, LOG_FMT("request URI/header too long"));
         switch (data->status) {
             case PARSING_URL:
+                ESP_LOGW(TAG, LOG_FMT("request URI too long"));
                 data->error = HTTPD_414_URI_TOO_LONG;
                 break;
             case PARSING_HDR_FIELD:
             case PARSING_HDR_VALUE:
+                ESP_LOGW(TAG, LOG_FMT("request header too long"));
                 data->error = HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE;
                 break;
             default:
@@ -632,7 +653,7 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
     offset = 0;
     do {
         /* Read block into scratch buffer */
-        if ((blk_len = read_block(r, offset, PARSER_BLOCK_SIZE)) < 0) {
+        if ((blk_len = read_block(r, &parser, offset, PARSER_BLOCK_SIZE)) < 0) {
             if (blk_len == HTTPD_SOCK_ERR_TIMEOUT) {
                 /* Retry read in case of non-fatal timeout error.
                  * read_block() ensures that the timeout error is
@@ -679,13 +700,17 @@ static void init_req(httpd_req_t *r, httpd_config_t *config)
 static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config)
 {
     ra->sd = 0;
-    memset(ra->scratch, 0, sizeof(ra->scratch));
     ra->remaining_len = 0;
     ra->status = 0;
     ra->content_type = 0;
     ra->first_chunk_sent = 0;
     ra->req_hdrs_count = 0;
     ra->resp_hdrs_count = 0;
+    ra->scratch = NULL;
+    ra->scratch_cur_size = 0;
+    ra->max_req_hdr_len = (config->max_req_hdr_len > 0) ? config->max_req_hdr_len : CONFIG_HTTPD_MAX_REQ_HDR_LEN;
+    ra->max_uri_len = (config->max_uri_len > 0) ? config->max_uri_len : CONFIG_HTTPD_MAX_URI_LEN;
+    ra->scratch_size_limit = ra->max_uri_len;
 #if CONFIG_HTTPD_WS_SUPPORT
     ra->ws_handshake_detect = false;
 #endif
@@ -716,6 +741,10 @@ static void httpd_req_cleanup(httpd_req_t *r)
 
     /* Clear out the request and request_aux structures */
     ra->sd = NULL;
+    free(ra->scratch);
+    ra->scratch = NULL;
+    ra->scratch_size_limit = 0;
+    ra->scratch_cur_size = 0;
     r->handle = NULL;
     r->aux = NULL;
     r->user_ctx = NULL;

components/esp_http_server/src/httpd_txrx.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -247,14 +247,26 @@ esp_err_t httpd_resp_send(httpd_req_t *r, const char *buf, ssize_t buf_len)
     /* Request headers are no longer available */
     ra->req_hdrs_count = 0;
 
-    /* Size of essential headers is limited by scratch buffer size */
-    if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_hdr_str,
-                 ra->status, ra->content_type, buf_len) >= sizeof(ra->scratch)) {
+    /* Calculate the size of the headers. +1 for the null terminator */
+    size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1;
+    if (required_size > ra->max_req_hdr_len) {
         return ESP_ERR_HTTPD_RESP_HDR;
     }
+    char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
+    if (res_buf == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate httpd send buffer");
+        return ESP_ERR_HTTPD_ALLOC_MEM;
+    }
+    ESP_LOGD(TAG, "httpd send buffer size = %d", strlen(res_buf));
 
-    /* Sending essential headers */
-    if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
+    esp_err_t ret = snprintf(res_buf, required_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
+    if (ret < 0 || ret >= required_size) {
+        free(res_buf);
+        return ESP_ERR_HTTPD_RESP_HDR;
+    }
+    ret = httpd_send_all(r, res_buf, strlen(res_buf));
+    free(res_buf);
+    if (ret != ESP_OK) {
         return ESP_ERR_HTTPD_RESP_SEND;
     }
 
@@ -320,15 +332,27 @@ esp_err_t httpd_resp_send_chunk(httpd_req_t *r, const char *buf, ssize_t buf_len
     /* Request headers are no longer available */
     ra->req_hdrs_count = 0;
 
+    /* Calculate the size of the headers. +1 for the null terminator */
+    size_t required_size = snprintf(NULL, 0, httpd_chunked_hdr_str, ra->status, ra->content_type) + 1;
+    if (required_size > ra->max_req_hdr_len) {
+        return ESP_ERR_HTTPD_RESP_HDR;
+    }
+    char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
+    if (res_buf == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate httpd send chunk buffer");
+        return ESP_ERR_HTTPD_ALLOC_MEM;
+    }
+    ESP_LOGD(TAG, "httpd send chunk buffer size = %d", strlen(res_buf));
     if (!ra->first_chunk_sent) {
-        /* Size of essential headers is limited by scratch buffer size */
-        if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_chunked_hdr_str,
-                     ra->status, ra->content_type) >= sizeof(ra->scratch)) {
+        esp_err_t ret = snprintf(res_buf, required_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
+        if (ret < 0 || ret >= required_size) {
+            free(res_buf);
             return ESP_ERR_HTTPD_RESP_HDR;
         }
-
-        /* Sending essential headers */
-        if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
+        /* Size of essential headers is limited by scratch buffer size */
+        ret = httpd_send_all(r, res_buf, strlen(res_buf));
+        free(res_buf);
+        if (ret != ESP_OK) {
             return ESP_ERR_HTTPD_RESP_SEND;
         }
 
@@ -650,7 +674,10 @@ esp_err_t httpd_req_async_handler_complete(httpd_req_t *r)
 
     struct httpd_req_aux *ra = r->aux;
     ra->sd->for_async_req = false;
-
+    free(ra->scratch);
+    ra->scratch = NULL;
+    ra->scratch_cur_size = 0;
+    ra->scratch_size_limit = 0;
     free(ra->resp_hdrs);
     free(r->aux);
     free(r);

components/esp_https_server/include/esp_https_server.h

@@ -150,6 +150,8 @@ typedef struct httpd_ssl_config httpd_ssl_config_t;
         .stack_size         = 10240,              \
         .core_id            = tskNO_AFFINITY,     \
         .task_caps          = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),       \
+        .max_req_hdr_len    = CONFIG_HTTPD_MAX_REQ_HDR_LEN,    \
+        .max_uri_len        = CONFIG_HTTPD_MAX_URI_LEN,        \
         .server_port        = 0,                  \
         .ctrl_port   = ESP_HTTPD_DEF_CTRL_PORT+1, \
         .max_open_sockets   = 4,                  \

docs/en/migration-guides/release-5.x/5.5/index.rst

@@ -8,3 +8,4 @@ Migration from 5.4 to 5.5
 
     system
     peripherals
+    protocols

docs/en/migration-guides/release-5.x/5.5/protocols.rst

@@ -0,0 +1,12 @@
+Protocols
+=========
+
+:link_to_translation:`zh_CN:[中文]`
+
+ESP HTTP SERVER
+---------------
+
+:ref:`CONFIG_HTTPD_MAX_REQ_HDR_LEN`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`CONFIG_HTTPD_MAX_REQ_HDR_LEN` option now defines the maximum limit for the memory that can be allocated internally for the HTTP request header. The actual memory allocated for the header will depend on the size of the header received in the HTTP request, rather than being fixed to this value which was the case previously. This provides more flexible memory usage based on the actual header size.

docs/zh_CN/migration-guides/release-5.x/5.5/index.rst

@@ -8,3 +8,4 @@
 
     system
     peripherals
+    protocols

docs/zh_CN/migration-guides/release-5.x/5.5/protocols.rst

@@ -0,0 +1,4 @@
+协议
+====
+
+:link_to_translation:`en:[English]`