feat(esp_http_server): Modified the calculation of buf_len

Modified the calculation of buf_len, so that scratch buffer allocates
the memory according to requirement and not extra (except the last read chunk)

Author: hrushikesh430

components/esp_http_server/Kconfig

@@ -2,15 +2,19 @@ menu "HTTP Server"
 
 
     config HTTPD_MAX_REQ_HDR_LEN
-        int "Max HTTP Request Header Length"
-        default 2000
+        int "HTTP Request Header Length limit"
+        default 1024
+        range 128 65536
         help
-            This sets the maximum supported size of headers section in HTTP request packet to be processed by the
-            server
+            This sets the default limit for the HTTP request header length. The limit can be
+            configured at run time by setting max_req_hdr_len member of httpd_config_t structure.
+            The memory allocated will depend on the actual header length. Hence keeping a sufficiently
+            large max header length is recommended.
+
 
     config HTTPD_MAX_URI_LEN
         int "Max HTTP URI Length"
-        default 2000
+        default 512
         help
             This sets the maximum supported size of HTTP request URI to be processed by the server
 

components/esp_http_server/include/esp_http_server.h

@@ -53,10 +53,10 @@ initializer that should be kept in sync
 #define HTTPD_DEFAULT_CONFIG() {                        \
         .task_priority      = tskIDLE_PRIORITY+5,       \
         .stack_size         = 4096,                     \
-        .hdr_buf_size_limit = HTTPD_MAX_REQ_HDR_LEN,    \
-        .uri_buf_size_limit = HTTPD_MAX_URI_LEN,        \
         .core_id            = tskNO_AFFINITY,           \
         .task_caps          = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),       \
+        .max_req_hdr_len    = CONFIG_HTTPD_MAX_REQ_HDR_LEN,    \
+        .max_uri_len        = CONFIG_HTTPD_MAX_URI_LEN,        \
         .server_port        = 80,                       \
         .ctrl_port          = ESP_HTTPD_DEF_CTRL_PORT,  \
         .max_open_sockets   = 7,                        \
@@ -176,11 +176,12 @@ typedef struct httpd_config {
     uint32_t    task_caps;          /*!< The memory capabilities to use when allocating the HTTP server task's stack */
 
     /**
-     * Size limits for the header and URI buffers respectively. These are just limits, actually allocation would be depend upon actual size of URI/header.
-     * These are set to the values defined in the Kconfig file.
+     * Size limits for the header and URI buffers respectively.
+     * These are just limits, allocation would depend upon actual size of URI/header.
      */
-    uint16_t hdr_buf_size_limit;    /*!< Size limit for the header buffer */
-    uint16_t uri_buf_size_limit;    /*!< Size limit for the URI buffer */
+    size_t max_req_hdr_len;    /*!< Size limit for the header buffer (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
+    size_t max_uri_len;    /*!< Size limit for the URI buffer By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
+
     /**
      * TCP Port number for receiving and transmitting HTTP traffic
      */
@@ -366,19 +367,13 @@ esp_err_t httpd_stop(httpd_handle_t handle);
  * @{
  */
 
-/* Max supported HTTP request header length */
-#define HTTPD_MAX_REQ_HDR_LEN CONFIG_HTTPD_MAX_REQ_HDR_LEN
-
-/* Max supported HTTP request URI length */
-#define HTTPD_MAX_URI_LEN CONFIG_HTTPD_MAX_URI_LEN
-
 /**
  * @brief HTTP Request Data Structure
  */
 typedef struct httpd_req {
     httpd_handle_t  handle;                     /*!< Handle to server instance */
     int             method;                     /*!< The type of HTTP request, -1 if unsupported method, HTTP_ANY for wildcard method to support every method */
-    const char      uri[HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
+    const char      uri[CONFIG_HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
     size_t          content_len;                /*!< Length of the request body */
     void           *aux;                        /*!< Internally used members */
 
@@ -619,10 +614,10 @@ typedef enum {
     /* Incoming payload is too large */
     HTTPD_413_CONTENT_TOO_LARGE,
 
-    /* URI length greater than CONFIG_HTTPD_MAX_URI_LEN */
+    /* URI length greater than CONFIG_CONFIG_HTTPD_MAX_URI_LEN */
     HTTPD_414_URI_TOO_LONG,
 
-    /* Headers section larger than CONFIG_HTTPD_MAX_REQ_HDR_LEN */
+    /* Headers section larger than CONFIG_CONFIG_HTTPD_MAX_REQ_HDR_LEN */
     HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE,
 
     /* Used internally for retrieving the total count of errors */

components/esp_http_server/src/esp_httpd_priv.h

@@ -31,8 +31,7 @@ extern "C" {
 #endif
 
 /* Size of request data block/chunk (not to be confused with chunked encoded data)
- * that is received and parsed in one turn of the parsing process. This should not
- * exceed the scratch buffer size and should at least be 8 bytes */
+ * that is received and parsed in one turn of the parsing process. */
 #define PARSER_BLOCK_SIZE  128
 
 /* Formats a log string to prepend context function name */
@@ -86,10 +85,10 @@ struct sock_db {
 struct httpd_req_aux {
     struct sock_db *sd;                             /*!< Pointer to socket database */
     char           *scratch;                        /*!< Temporary buffer for our operations (1 byte extra for null termination) */
-    uint16_t        scratch_size_limit;             /*!< Scratch buffer size limit */
-    uint16_t        scratch_cur_size;               /*!< Scratch buffer cur size */
-    uint16_t        hdr_buf_size_limit;             /*!< Header buffer size limit */
-    uint16_t        uri_buf_size_limit;             /*!< URI buffer size limit */
+    size_t          scratch_size_limit;             /*!< Scratch buffer size limit (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
+    size_t          scratch_cur_size;               /*!< Scratch buffer cur size (By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
+    size_t          max_req_hdr_len;             /*!< Header buffer size limit */
+    size_t          max_uri_len;             /*!< URI buffer size limit */
     size_t          remaining_len;                  /*!< Amount of data remaining to be fetched */
     char           *status;                         /*!< HTTP response's status code */
     char           *content_type;                   /*!< HTTP response's content type */

components/esp_http_server/src/httpd_parse.c

@@ -114,6 +114,7 @@ static esp_err_t cb_url(http_parser *parser,
     parser_data_t *parser_data = (parser_data_t *) parser->data;
     httpd_req_t          *req   = parser_data->req;
     struct httpd_req_aux *raux  = req->aux;
+
     if (parser_data->status == PARSING_IDLE) {
         ESP_LOGD(TAG, LOG_FMT("message begin"));
 
@@ -131,9 +132,9 @@ static esp_err_t cb_url(http_parser *parser,
     ESP_LOGD(TAG, LOG_FMT("processing url = %.*s"), (int)length, at);
 
     /* Update length of URL string */
-    if ((parser_data->last.length += length) > raux->uri_buf_size_limit) {
+    if ((parser_data->last.length += length) > raux->max_uri_len) {
         ESP_LOGW(TAG, LOG_FMT("URI length (%"NEWLIB_NANO_COMPAT_FORMAT") greater than supported (%d)"),
-                 NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->uri_buf_size_limit);
+                 NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->max_uri_len);
         parser_data->error = HTTPD_414_URI_TOO_LONG;
         parser_data->status = PARSING_FAILED;
         return ESP_FAIL;
@@ -216,7 +217,8 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
         parser_data->last.at     = ra->scratch;
         parser_data->last.length = 0;
         parser_data->status      = PARSING_HDR_FIELD;
-        ra->scratch_size_limit = ra->hdr_buf_size_limit;
+        ra->scratch_size_limit   = ra->max_req_hdr_len;
+
         /* Stop parsing for now and give control to process */
         if (pause_parsing(parser, at) != ESP_OK) {
             parser_data->error = HTTPD_500_INTERNAL_SERVER_ERROR;
@@ -233,7 +235,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
         parser_data->last.at     = at;
         parser_data->last.length = 0;
         parser_data->status      = PARSING_HDR_FIELD;
-        ra->scratch_size_limit = ra->hdr_buf_size_limit;
+        ra->scratch_size_limit   = ra->max_req_hdr_len;
 
         /* Increment header count */
         ra->req_hdrs_count++;
@@ -415,6 +417,7 @@ static esp_err_t cb_headers_complete(http_parser *parser)
 static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length)
 {
     parser_data_t *parser_data = (parser_data_t *) parser->data;
+
     /* Check previous status */
     if (parser_data->status != PARSING_BODY) {
         ESP_LOGE(TAG, LOG_FMT("unexpected state transition"));
@@ -446,6 +449,7 @@ static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length)
 static esp_err_t cb_no_body(http_parser *parser)
 {
     parser_data_t *parser_data = (parser_data_t *) parser->data;
+
     /* Check previous status */
     if (parser_data->status == PARSING_URL) {
         ESP_LOGD(TAG, LOG_FMT("no headers"));
@@ -480,26 +484,32 @@ static esp_err_t cb_no_body(http_parser *parser)
     return ESP_OK;
 }
 
-static int read_block(httpd_req_t *req, size_t offset, size_t length)
+static int read_block(httpd_req_t *req, http_parser *parser, size_t offset, size_t length)
 {
     struct httpd_req_aux *raux  = req->aux;
+    parser_data_t *parser_data = (parser_data_t *) parser->data;
 
     /* Limits the read to scratch buffer size */
     ssize_t buf_len = MIN(length, (raux->scratch_size_limit - offset));
     if (buf_len <= 0) {
         return 0;
     }
-    if (raux->scratch == NULL && buf_len < raux->scratch_size_limit) {
-        raux->scratch = (char*) malloc(buf_len);
-    }
-    else if (raux->scratch != NULL && buf_len < raux->scratch_size_limit) {
-        raux->scratch = (char*) realloc(raux->scratch, raux->scratch_cur_size + buf_len);
-    }
+    /* Calculate the offset of the current position from the start of the buffer,
+     * as after reallocating the buffer, the base address of the buffer may change.
+     */
+    size_t at_offset = parser_data->last.at - raux->scratch;
+    /* Allocate the buffer according to offset and buf_len. Offset is
+       from where the reading will start and buf_len is till what length
+       the buffer will be read.
+    */
+    raux->scratch = (char*) realloc(raux->scratch, offset + buf_len);
     if (raux->scratch == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate the scratch buffer");
         return 0;
     }
-    raux->scratch_cur_size += buf_len;
-    ESP_LOGD(TAG, "scratch size = %d", raux->scratch_cur_size);
+    parser_data->last.at = raux->scratch + at_offset;
+    raux->scratch_cur_size = offset + buf_len;
+    ESP_LOGD(TAG, "scratch buf qsize = %d", raux->scratch_cur_size);
     /* Receive data into buffer. If data is pending (from unrecv) then return
      * immediately after receiving pending data, as pending data may just complete
      * this request packet. */
@@ -537,7 +547,7 @@ static int parse_block(http_parser *parser, size_t offset, size_t length)
     httpd_req_t          *req   = data->req;
     struct httpd_req_aux *raux  = req->aux;
     size_t nparsed = 0;
-    data->last.at = raux->scratch;
+
     if (!length) {
         /* Parsing is still happening but nothing to
          * parse means no more space left on buffer,
@@ -643,7 +653,7 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
     offset = 0;
     do {
         /* Read block into scratch buffer */
-        if ((blk_len = read_block(r, offset, PARSER_BLOCK_SIZE)) < 0) {
+        if ((blk_len = read_block(r, &parser, offset, PARSER_BLOCK_SIZE)) < 0) {
             if (blk_len == HTTPD_SOCK_ERR_TIMEOUT) {
                 /* Retry read in case of non-fatal timeout error.
                  * read_block() ensures that the timeout error is
@@ -660,6 +670,7 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
         /* This is used by the callbacks to track
          * data usage of the buffer */
         parser_data.raw_datalen = blk_len + offset;
+
         /* Parse data block from buffer */
         if ((offset = parse_block(&parser, offset, blk_len)) < 0) {
             /* HTTP error occurred.
@@ -697,9 +708,9 @@ static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config)
     ra->resp_hdrs_count = 0;
     ra->scratch = NULL;
     ra->scratch_cur_size = 0;
-    ra->hdr_buf_size_limit = config->hdr_buf_size_limit;
-    ra->uri_buf_size_limit = config->uri_buf_size_limit;
-    ra->scratch_size_limit = ra->uri_buf_size_limit;
+    ra->max_req_hdr_len = (config->max_req_hdr_len > 0) ? config->max_req_hdr_len : CONFIG_HTTPD_MAX_REQ_HDR_LEN;
+    ra->max_uri_len = (config->max_uri_len > 0) ? config->max_uri_len : CONFIG_HTTPD_MAX_URI_LEN;
+    ra->scratch_size_limit = ra->max_uri_len;
 #if CONFIG_HTTPD_WS_SUPPORT
     ra->ws_handshake_detect = false;
 #endif
@@ -732,8 +743,8 @@ static void httpd_req_cleanup(httpd_req_t *r)
     ra->sd = NULL;
     free(ra->scratch);
     ra->scratch = NULL;
-    ra->scratch_cur_size = 0;
     ra->scratch_size_limit = 0;
+    ra->scratch_cur_size = 0;
     r->handle = NULL;
     r->aux = NULL;
     r->user_ctx = NULL;

components/esp_http_server/src/httpd_txrx.c

@@ -132,6 +132,7 @@ int httpd_recv_with_opt(httpd_req_t *r, char *buf, size_t buf_len, bool halt_aft
         }
         return ret;
     }
+
     ESP_LOGD(TAG, LOG_FMT("received length = %"NEWLIB_NANO_COMPAT_FORMAT), NEWLIB_NANO_COMPAT_CAST((ret + pending_len)));
     return ret + pending_len;
 }
@@ -242,29 +243,30 @@ esp_err_t httpd_resp_send(httpd_req_t *r, const char *buf, ssize_t buf_len)
     if (buf_len == HTTPD_RESP_USE_STRLEN) {
         buf_len = strlen(buf);
     }
+
+    /* Request headers are no longer available */
+    ra->req_hdrs_count = 0;
+
+    /* Calculate the size of the headers. +1 for the null terminator */
     size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1;
-    if (required_size >= HTTPD_MAX_REQ_HDR_LEN) {
+    if (required_size > ra->max_req_hdr_len) {
         return ESP_ERR_HTTPD_RESP_HDR;
     }
-    if (ra->scratch == NULL)
-        ra->scratch = malloc(required_size);
-    else {
-        ra->scratch = realloc(ra->scratch, required_size);
-    }
-    if (ra->scratch == NULL) {
-        ESP_LOGE(TAG, "Unable to allocate scratch buffer");
+    char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
+    if (res_buf == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate httpd send buffer");
         return ESP_ERR_HTTPD_ALLOC_MEM;
     }
-    ra->scratch_cur_size = required_size;
-    ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size);
-    /* Request headers are no longer available */
-    ra->req_hdrs_count = 0;
+    ESP_LOGD(TAG, "httpd send buffer size = %d", strlen(res_buf));
 
-    /* Size of essential headers is limited by scratch buffer size */
-    snprintf(ra->scratch, ra->scratch_cur_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
-    ESP_LOG_BUFFER_HEXDUMP(TAG, ra->scratch, strlen(ra->scratch), ESP_LOG_INFO);
-    /* Sending essential headers */
-    if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
+    esp_err_t ret = snprintf(res_buf, required_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
+    if (ret < 0 || ret >= required_size) {
+        free(res_buf);
+        return ESP_ERR_HTTPD_RESP_HDR;
+    }
+    ret = httpd_send_all(r, res_buf, strlen(res_buf));
+    free(res_buf);
+    if (ret != ESP_OK) {
         return ESP_ERR_HTTPD_RESP_SEND;
     }
 
@@ -329,27 +331,28 @@ esp_err_t httpd_resp_send_chunk(httpd_req_t *r, const char *buf, ssize_t buf_len
 
     /* Request headers are no longer available */
     ra->req_hdrs_count = 0;
+
+    /* Calculate the size of the headers. +1 for the null terminator */
     size_t required_size = snprintf(NULL, 0, httpd_chunked_hdr_str, ra->status, ra->content_type) + 1;
-    if (required_size >= HTTPD_MAX_REQ_HDR_LEN) {
+    if (required_size > ra->max_req_hdr_len) {
         return ESP_ERR_HTTPD_RESP_HDR;
     }
-    if (ra->scratch == NULL)
-        ra->scratch = malloc(required_size);
-    else {
-        ra->scratch = realloc(ra->scratch, required_size);
-    }
-    if (ra->scratch == NULL) {
-        ESP_LOGE(TAG, "Unable to allocate scratch buffer");
+    char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
+    if (res_buf == NULL) {
+        ESP_LOGE(TAG, "Unable to allocate httpd send chunk buffer");
         return ESP_ERR_HTTPD_ALLOC_MEM;
     }
-    ra->scratch_cur_size = required_size;
-    ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size);
+    ESP_LOGD(TAG, "httpd send chunk buffer size = %d", strlen(res_buf));
     if (!ra->first_chunk_sent) {
+        esp_err_t ret = snprintf(res_buf, required_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
+        if (ret < 0 || ret >= required_size) {
+            free(res_buf);
+            return ESP_ERR_HTTPD_RESP_HDR;
+        }
         /* Size of essential headers is limited by scratch buffer size */
-        snprintf(ra->scratch, ra->scratch_cur_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
-
-        /* Sending essential headers */
-        if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
+        ret = httpd_send_all(r, res_buf, strlen(res_buf));
+        free(res_buf);
+        if (ret != ESP_OK) {
             return ESP_ERR_HTTPD_RESP_SEND;
         }
 
@@ -671,7 +674,10 @@ esp_err_t httpd_req_async_handler_complete(httpd_req_t *r)
 
     struct httpd_req_aux *ra = r->aux;
     ra->sd->for_async_req = false;
-
+    free(ra->scratch);
+    ra->scratch = NULL;
+    ra->scratch_cur_size = 0;
+    ra->scratch_size_limit = 0;
     free(ra->resp_hdrs);
     free(r->aux);
     free(r);

components/esp_https_server/include/esp_https_server.h

@@ -150,6 +150,8 @@ typedef struct httpd_ssl_config httpd_ssl_config_t;
         .stack_size         = 10240,              \
         .core_id            = tskNO_AFFINITY,     \
         .task_caps          = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),       \
+        .max_req_hdr_len    = CONFIG_HTTPD_MAX_REQ_HDR_LEN,    \
+        .max_uri_len        = CONFIG_HTTPD_MAX_URI_LEN,        \
         .server_port        = 0,                  \
         .ctrl_port   = ESP_HTTPD_DEF_CTRL_PORT+1, \
         .max_open_sockets   = 4,                  \

components/esp_https_server/src/https_server.c

@@ -377,7 +377,7 @@ static esp_err_t create_secure_context(const struct httpd_ssl_config *config, ht
 }
 
 /** Start the server */
-esp_err_t  httpd_ssl_start(httpd_handle_t *pHandle, struct httpd_ssl_config *config)
+esp_err_t httpd_ssl_start(httpd_handle_t *pHandle, struct httpd_ssl_config *config)
 {
     assert(config != NULL);
     assert(pHandle != NULL);

docs/en/migration-guides/release-5.x/5.5/index.rst

@@ -8,3 +8,4 @@ Migration from 5.4 to 5.5
 
     system
     peripherals
+    protocols