feat(esp_http_server): Dynamically allocate http server's scratch buffer

In this commit, esp_http_server's http_parser scratch is made dynamic.
User is asked to give limit size for header and URI, according to which
scratch buufer allocates memory upto limits

Author: hrushikesh430

components/esp_http_server/Kconfig

@@ -1,15 +1,16 @@
 menu "HTTP Server"
 
+
     config HTTPD_MAX_REQ_HDR_LEN
         int "Max HTTP Request Header Length"
-        default 512
+        default 2000
         help
             This sets the maximum supported size of headers section in HTTP request packet to be processed by the
             server
 
     config HTTPD_MAX_URI_LEN
         int "Max HTTP URI Length"
-        default 512
+        default 2000
         help
             This sets the maximum supported size of HTTP request URI to be processed by the server
 

components/esp_http_server/include/esp_http_server.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -53,6 +53,8 @@ initializer that should be kept in sync
 #define HTTPD_DEFAULT_CONFIG() {                        \
         .task_priority      = tskIDLE_PRIORITY+5,       \
         .stack_size         = 4096,                     \
+        .hdr_buf_size_limit = HTTPD_MAX_REQ_HDR_LEN,    \
+        .uri_buf_size_limit = HTTPD_MAX_URI_LEN,        \
         .core_id            = tskNO_AFFINITY,           \
         .task_caps          = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),       \
         .server_port        = 80,                       \
@@ -173,6 +175,12 @@ typedef struct httpd_config {
     BaseType_t  core_id;            /*!< The core the HTTP server task will run on */
     uint32_t    task_caps;          /*!< The memory capabilities to use when allocating the HTTP server task's stack */
 
+    /**
+     * Size limits for the header and URI buffers respectively. These are just limits, actually allocation would be depend upon actual size of URI/header.
+     * These are set to the values defined in the Kconfig file.
+     */
+    uint16_t hdr_buf_size_limit;    /*!< Size limit for the header buffer */
+    uint16_t uri_buf_size_limit;    /*!< Size limit for the URI buffer */
     /**
      * TCP Port number for receiving and transmitting HTTP traffic
      */

components/esp_http_server/src/esp_httpd_priv.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -35,9 +35,6 @@ extern "C" {
  * exceed the scratch buffer size and should at least be 8 bytes */
 #define PARSER_BLOCK_SIZE  128
 
-/* Calculate the maximum size needed for the scratch buffer */
-#define HTTPD_SCRATCH_BUF  MAX(HTTPD_MAX_REQ_HDR_LEN, HTTPD_MAX_URI_LEN)
-
 /* Formats a log string to prepend context function name */
 #define LOG_FMT(x)      "%s: " x, __func__
 
@@ -88,7 +85,11 @@ struct sock_db {
  */
 struct httpd_req_aux {
     struct sock_db *sd;                             /*!< Pointer to socket database */
-    char            scratch[HTTPD_SCRATCH_BUF + 1]; /*!< Temporary buffer for our operations (1 byte extra for null termination) */
+    char           *scratch;                        /*!< Temporary buffer for our operations (1 byte extra for null termination) */
+    uint16_t        scratch_size_limit;             /*!< Scratch buffer size limit */
+    uint16_t        scratch_cur_size;               /*!< Scratch buffer cur size */
+    uint16_t        hdr_buf_size_limit;             /*!< Header buffer size limit */
+    uint16_t        uri_buf_size_limit;             /*!< URI buffer size limit */
     size_t          remaining_len;                  /*!< Amount of data remaining to be fetched */
     char           *status;                         /*!< HTTP response's status code */
     char           *content_type;                   /*!< HTTP response's content type */

components/esp_http_server/src/httpd_parse.c

@@ -112,7 +112,8 @@ static esp_err_t cb_url(http_parser *parser,
                         const char *at, size_t length)
 {
     parser_data_t *parser_data = (parser_data_t *) parser->data;
-
+    httpd_req_t          *req   = parser_data->req;
+    struct httpd_req_aux *raux  = req->aux;
     if (parser_data->status == PARSING_IDLE) {
         ESP_LOGD(TAG, LOG_FMT("message begin"));
 
@@ -130,9 +131,9 @@ static esp_err_t cb_url(http_parser *parser,
     ESP_LOGD(TAG, LOG_FMT("processing url = %.*s"), (int)length, at);
 
     /* Update length of URL string */
-    if ((parser_data->last.length += length) > HTTPD_MAX_URI_LEN) {
+    if ((parser_data->last.length += length) > raux->uri_buf_size_limit) {
         ESP_LOGW(TAG, LOG_FMT("URI length (%"NEWLIB_NANO_COMPAT_FORMAT") greater than supported (%d)"),
-                 NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), HTTPD_MAX_URI_LEN);
+                 NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->uri_buf_size_limit);
         parser_data->error = HTTPD_414_URI_TOO_LONG;
         parser_data->status = PARSING_FAILED;
         return ESP_FAIL;
@@ -215,7 +216,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
         parser_data->last.at     = ra->scratch;
         parser_data->last.length = 0;
         parser_data->status      = PARSING_HDR_FIELD;
-
+        ra->scratch_size_limit = ra->hdr_buf_size_limit;
         /* Stop parsing for now and give control to process */
         if (pause_parsing(parser, at) != ESP_OK) {
             parser_data->error = HTTPD_500_INTERNAL_SERVER_ERROR;
@@ -232,6 +233,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
         parser_data->last.at     = at;
         parser_data->last.length = 0;
         parser_data->status      = PARSING_HDR_FIELD;
+        ra->scratch_size_limit = ra->hdr_buf_size_limit;
 
         /* Increment header count */
         ra->req_hdrs_count++;
@@ -413,7 +415,6 @@ static esp_err_t cb_headers_complete(http_parser *parser)
 static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length)
 {
     parser_data_t *parser_data = (parser_data_t *) parser->data;
-
     /* Check previous status */
     if (parser_data->status != PARSING_BODY) {
         ESP_LOGE(TAG, LOG_FMT("unexpected state transition"));
@@ -445,7 +446,6 @@ static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length)
 static esp_err_t cb_no_body(http_parser *parser)
 {
     parser_data_t *parser_data = (parser_data_t *) parser->data;
-
     /* Check previous status */
     if (parser_data->status == PARSING_URL) {
         ESP_LOGD(TAG, LOG_FMT("no headers"));
@@ -485,11 +485,21 @@ static int read_block(httpd_req_t *req, size_t offset, size_t length)
     struct httpd_req_aux *raux  = req->aux;
 
     /* Limits the read to scratch buffer size */
-    ssize_t buf_len = MIN(length, (sizeof(raux->scratch) - offset));
+    ssize_t buf_len = MIN(length, (raux->scratch_size_limit - offset));
     if (buf_len <= 0) {
         return 0;
     }
-
+    if (raux->scratch == NULL && buf_len < raux->scratch_size_limit) {
+        raux->scratch = (char*) malloc(buf_len);
+    }
+    else if (raux->scratch != NULL && buf_len < raux->scratch_size_limit) {
+        raux->scratch = (char*) realloc(raux->scratch, raux->scratch_cur_size + buf_len);
+    }
+    if (raux->scratch == NULL) {
+        return 0;
+    }
+    raux->scratch_cur_size += buf_len;
+    ESP_LOGD(TAG, "scratch size = %d", raux->scratch_cur_size);
     /* Receive data into buffer. If data is pending (from unrecv) then return
      * immediately after receiving pending data, as pending data may just complete
      * this request packet. */
@@ -527,19 +537,20 @@ static int parse_block(http_parser *parser, size_t offset, size_t length)
     httpd_req_t          *req   = data->req;
     struct httpd_req_aux *raux  = req->aux;
     size_t nparsed = 0;
-
+    data->last.at = raux->scratch;
     if (!length) {
         /* Parsing is still happening but nothing to
          * parse means no more space left on buffer,
          * therefore it can be inferred that the
          * request URI/header must be too long */
-        ESP_LOGW(TAG, LOG_FMT("request URI/header too long"));
         switch (data->status) {
             case PARSING_URL:
+                ESP_LOGW(TAG, LOG_FMT("request URI too long"));
                 data->error = HTTPD_414_URI_TOO_LONG;
                 break;
             case PARSING_HDR_FIELD:
             case PARSING_HDR_VALUE:
+                ESP_LOGW(TAG, LOG_FMT("request header too long"));
                 data->error = HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE;
                 break;
             default:
@@ -649,7 +660,6 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
         /* This is used by the callbacks to track
          * data usage of the buffer */
         parser_data.raw_datalen = blk_len + offset;
-
         /* Parse data block from buffer */
         if ((offset = parse_block(&parser, offset, blk_len)) < 0) {
             /* HTTP error occurred.
@@ -679,13 +689,17 @@ static void init_req(httpd_req_t *r, httpd_config_t *config)
 static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config)
 {
     ra->sd = 0;
-    memset(ra->scratch, 0, sizeof(ra->scratch));
     ra->remaining_len = 0;
     ra->status = 0;
     ra->content_type = 0;
     ra->first_chunk_sent = 0;
     ra->req_hdrs_count = 0;
     ra->resp_hdrs_count = 0;
+    ra->scratch = NULL;
+    ra->scratch_cur_size = 0;
+    ra->hdr_buf_size_limit = config->hdr_buf_size_limit;
+    ra->uri_buf_size_limit = config->uri_buf_size_limit;
+    ra->scratch_size_limit = ra->uri_buf_size_limit;
 #if CONFIG_HTTPD_WS_SUPPORT
     ra->ws_handshake_detect = false;
 #endif
@@ -716,6 +730,10 @@ static void httpd_req_cleanup(httpd_req_t *r)
 
     /* Clear out the request and request_aux structures */
     ra->sd = NULL;
+    free(ra->scratch);
+    ra->scratch = NULL;
+    ra->scratch_cur_size = 0;
+    ra->scratch_size_limit = 0;
     r->handle = NULL;
     r->aux = NULL;
     r->user_ctx = NULL;

components/esp_http_server/src/httpd_txrx.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -132,7 +132,6 @@ int httpd_recv_with_opt(httpd_req_t *r, char *buf, size_t buf_len, bool halt_aft
         }
         return ret;
     }
-
     ESP_LOGD(TAG, LOG_FMT("received length = %"NEWLIB_NANO_COMPAT_FORMAT), NEWLIB_NANO_COMPAT_CAST((ret + pending_len)));
     return ret + pending_len;
 }
@@ -243,16 +242,27 @@ esp_err_t httpd_resp_send(httpd_req_t *r, const char *buf, ssize_t buf_len)
     if (buf_len == HTTPD_RESP_USE_STRLEN) {
         buf_len = strlen(buf);
     }
-
+    size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1;
+    if (required_size >= HTTPD_MAX_REQ_HDR_LEN) {
+        return ESP_ERR_HTTPD_RESP_HDR;
+    }
+    if (ra->scratch == NULL)
+        ra->scratch = malloc(required_size);
+    else {
+        ra->scratch = realloc(ra->scratch, required_size);
+    }
+    if (ra->scratch == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate scratch buffer");
+        return ESP_ERR_HTTPD_ALLOC_MEM;
+    }
+    ra->scratch_cur_size = required_size;
+    ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size);
     /* Request headers are no longer available */
     ra->req_hdrs_count = 0;
 
     /* Size of essential headers is limited by scratch buffer size */
-    if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_hdr_str,
-                 ra->status, ra->content_type, buf_len) >= sizeof(ra->scratch)) {
-        return ESP_ERR_HTTPD_RESP_HDR;
-    }
-
+    snprintf(ra->scratch, ra->scratch_cur_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
+    ESP_LOG_BUFFER_HEXDUMP(TAG, ra->scratch, strlen(ra->scratch), ESP_LOG_INFO);
     /* Sending essential headers */
     if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
         return ESP_ERR_HTTPD_RESP_SEND;
@@ -319,13 +329,24 @@ esp_err_t httpd_resp_send_chunk(httpd_req_t *r, const char *buf, ssize_t buf_len
 
     /* Request headers are no longer available */
     ra->req_hdrs_count = 0;
-
+    size_t required_size = snprintf(NULL, 0, httpd_chunked_hdr_str, ra->status, ra->content_type) + 1;
+    if (required_size >= HTTPD_MAX_REQ_HDR_LEN) {
+        return ESP_ERR_HTTPD_RESP_HDR;
+    }
+    if (ra->scratch == NULL)
+        ra->scratch = malloc(required_size);
+    else {
+        ra->scratch = realloc(ra->scratch, required_size);
+    }
+    if (ra->scratch == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate scratch buffer");
+        return ESP_ERR_HTTPD_ALLOC_MEM;
+    }
+    ra->scratch_cur_size = required_size;
+    ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size);
     if (!ra->first_chunk_sent) {
         /* Size of essential headers is limited by scratch buffer size */
-        if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_chunked_hdr_str,
-                     ra->status, ra->content_type) >= sizeof(ra->scratch)) {
-            return ESP_ERR_HTTPD_RESP_HDR;
-        }
+        snprintf(ra->scratch, ra->scratch_cur_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
 
         /* Sending essential headers */
         if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {

components/esp_https_server/src/https_server.c

@@ -377,7 +377,7 @@ static esp_err_t create_secure_context(const struct httpd_ssl_config *config, ht
 }
 
 /** Start the server */
-esp_err_t httpd_ssl_start(httpd_handle_t *pHandle, struct httpd_ssl_config *config)
+esp_err_t  httpd_ssl_start(httpd_handle_t *pHandle, struct httpd_ssl_config *config)
 {
     assert(config != NULL);
     assert(pHandle != NULL);

examples/protocols/https_server/simple/main/main.c

@@ -44,7 +44,6 @@ static esp_err_t root_get_handler(httpd_req_t *req)
 {
     httpd_resp_set_type(req, "text/html");
     httpd_resp_send(req, "<h1>Hello Secure World!</h1>", HTTPD_RESP_USE_STRLEN);
-
     return ESP_OK;
 }