Merge branch 'master' into stricten_chunk_parser

 Conflicts:
	include/proxy/http/HttpTunnel.h
	src/proxy/http/HttpTunnel.cc

Author: maskit

CMakeLists.txt

@@ -24,7 +24,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
 endif()
 
 cmake_minimum_required(VERSION 3.20..3.27)
-project(ats VERSION 10.1.0)
+project(ats VERSION 10.2.0)
 
 set(TS_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
 set(TS_VERSION_MINOR ${PROJECT_VERSION_MINOR})

doc/admin-guide/files/index.en.rst

@@ -57,7 +57,7 @@ Configuration Files
    Defines custom log file formats, filters, and processing options.
 
 :doc:`parent.config.en`
-   Configures parent proxies in hierarchical caching layouts.
+   Configures parent proxies in hierarchical caching layouts. Deprecated in `TS` 10.1, and replaced with strategies.yaml in 11.
 
 :doc:`plugin.config.en`
    Control runtime loadable plugins available to |TS|, as well as their
@@ -83,7 +83,7 @@ Configuration Files
    Configures all storage devices and paths to be used for the |TS| cache.
 
 :doc:`strategies.yaml.en`
-   Configures NextHop strategies used with `remap.config`
+   Configures NextHop strategies used with `remap.config` and replaces parent.config.
 
 :doc:`volume.config.en`
     Defines cache space usage by individual protocols.

doc/admin-guide/plugins/compress.en.rst

@@ -106,8 +106,21 @@ by the origin. Enabled by default.
 range-request
 -------------
 
-When set to ``true``, causes |TS| to compress responses to Range Requests.
-Disabled by default. Setting this to true while setting cache to false leads to delivering corrupted content.
+This config controls behavior of this plugin when a client send ``Range`` header and ``Accept-Encoding`` header in the same time.
+
+============== =================================================================
+Value          Description
+============== =================================================================
+ignore-range   Remove ``Range`` header if the request has both headers (Default)
+false          Same as ``ignore-range`` for compatiblity
+no-compression Remove ``Accept-Encoding`` header if the request has both headers
+none           Do nothing
+true           Same as ``none`` for compatibility
+============== =================================================================
+
+.. important::
+
+   Do NOT set this to ``none`` (or ``true``) if the cache config is set to ``false``. This combination will deliver corrupted content.
 
 compressible-content-type
 -------------------------

doc/admin-guide/plugins/header_rewrite.en.rst

@@ -140,6 +140,17 @@ with the word ``else``. The following example illustrates this::
 The ``else`` clause is not a condition, and does not take any flags, it is
 of course optional, but when specified must be followed by at least one operator.
 
+State variables
+---------------
+
+A set of state variables are also available for both conditions and operators.
+There are currently 16 flag states, 4 8-bit integers and one 16-bit integer states.
+These states are all transactional, meaning they are usable and persistent across
+all hooks.
+
+The flag states are numbers 0-15, the 8-bit integer states are numbered 0-3, and the
+one 16-bit integer state is number 0.
+
 Conditions
 ----------
 
@@ -457,9 +468,10 @@ As a special matcher, the inbound IP addresses can be matched against a list of
 
    cond %{INBOUND:REMOTE-ADDR} {192.168.201.0/24,10.0.0.0/8}
 
-Note that this will not work against the non-IP based conditions, such as the protocol families,
-and the configuration parser will error out. The format here is very specific, in particular no
-white spaces are allowed between the ranges.
+.. note::
+    This will not work against the non-IP based conditions, such as the protocol families,
+    and the configuration parser will error out. The format here is very specific, in particular no
+    white spaces are allowed between the ranges.
 
 IP
 ~~
@@ -569,6 +581,36 @@ RANDOM
 
 Generates a random integer from ``0`` up to (but not including) ``<n>``. Mathematically, ``[0,n)`` or ``0 <= r < n``.
 
+STATE-FLAG
+~~~~~~~~~~
+::
+
+      cond %{STATE-FLAG:<n>}
+
+This condition allows you to check the state of a flag. The ``<n>`` is the
+number of the flag, from 0 to 15. This condition returns a ``true`` or
+``false`` value, depending on the state of the flag.
+
+STATE-INT8
+~~~~~~~~~~
+::
+
+      cond %{STATE-INT8:<n>}
+
+This condition allows you to check the state of an 8-bit unsigned integer.
+The ``<n>`` is the number of the integer, from 0 to 3. The current value of
+the state integer is returned, and all 4 integers are initialized to 0.
+
+STATE-INT16
+~~~~~~~~~~~
+::
+
+      cond %{STATE-INT16<:0>}
+
+This condition allows you to check the state of an 16-bit unsigned integer.
+There's only one such integer, and its value is returned from this condition.
+As such, the index, ``0``, is optional here.
+
 STATUS
 ~~~~~~
 ::
@@ -920,6 +962,38 @@ location automatically. This operator supports `String concatenations`_ for
 ``READ_RESPONSE_HDR_HOOK`` (the default when the plugin is global), the
 ``SEND_RESPONSE_HDR_HOOK``, or the ``REMAP_PSEUDO_HOOK``.
 
+set-state-flag
+~~~~~~~~~~~~~~
+::
+
+  set-state-flag <n> <value>
+
+This operator allows you to set the state of a flag. The ``<n>`` is the
+number of the flag, from 0 to 15. The ``<value>`` is either ``true`` or ``false``,
+turning the flag on or off.
+
+set-state-int8
+~~~~~~~~~~~~~~
+::
+
+   set-state-int8 <n> <value>
+
+This operator allows you to set the state of an 8-bit unsigned integer.
+The ``<n>`` is the number of the integer, from 0 to 3. The ``<value>`` is an
+unsigned 8-bit integer, 0-255. It can also be a condition, in which case the
+value of the condition is used.
+
+set-state-int16
+~~~~~~~~~~~~~~~
+::
+
+   set-state-int16 0 <value>
+
+This operator allows you to set the state of a 16-bit unsigned integer.
+The ``<value>`` is an unsigned 16-bit integer as well, 0-65535. It can also
+be a condition, in which case thevalue of the condition is used. The index,
+0, is always required eventhough there is only one 16-bit integer state variable.
+
 set-status
 ~~~~~~~~~~
 ::
@@ -1438,6 +1512,11 @@ could each be tagged with a consistent name to make finding logs easier.::
 
 (Then in :file:`logging.yaml`, log ``%<{@PropertyName}cqh>``)
 
+.. note::
+    With the new ``state-flag``, ``state-int8`` and ``state-int16`` operators, you can
+    sometimes avoid setting internal ``@`` headers for passing information between hooks.
+    These internal state variables are much more efficient than setting and reading headers.
+
 Remove Client Query Parameters
 ------------------------------------
 

include/proxy/hdrs/HTTP.h

@@ -485,6 +485,7 @@ class HTTPHdr : public MIMEHdr
   mutable int        m_port                  = 0;     ///< Target port.
   mutable bool       m_target_cached         = false; ///< Whether host name and port are cached.
   mutable bool       m_target_in_url         = false; ///< Whether host name and port are in the URL.
+  mutable bool       m_100_continue_sent     = false; ///< Whether ATS sent a 100 Continue optimized response.
   mutable bool       m_100_continue_required = false; ///< Whether 100_continue is in the Expect header.
   /// Set if the port was effectively specified in the header.
   /// @c true if the target (in the URL or the HOST field) also specified

include/proxy/hdrs/MIME.h

@@ -143,7 +143,6 @@ struct MIMEField {
 
   /// @return The name of @a this field.
   std::string_view name_get() const;
-  const char      *name_get(int *length) const;
 
   /** Find the index of the value in the multi-value field.
 
@@ -161,7 +160,6 @@ struct MIMEField {
 
   /// @return The value of @a this field.
   std::string_view value_get() const;
-  const char      *value_get(int *length) const;
 
   int32_t  value_get_int() const;
   uint32_t value_get_uint() const;
@@ -940,17 +938,6 @@ bool     mime_parse_integer(const char *&buf, const char *end, int *integer);
  *                                                                     *
  ***********************************************************************/
 
-/*-------------------------------------------------------------------------
-  -------------------------------------------------------------------------*/
-
-inline const char *
-MIMEField::name_get(int *length) const
-{
-  auto name{this->name_get()};
-  *length = int(name.size());
-  return name.data();
-}
-
 /*-------------------------------------------------------------------------
   -------------------------------------------------------------------------*/
 
@@ -974,11 +961,9 @@ MIMEField::name_set(HdrHeap *heap, MIMEHdrImpl *mh, const char *name, int length
 inline bool
 MIMEField::name_is_valid(uint32_t invalid_char_bits) const
 {
-  const char *name;
-  int         length;
-
-  for (name = name_get(&length); length > 0; length--) {
-    if (ParseRules::is_type(name[length - 1], invalid_char_bits)) {
+  auto name{name_get()};
+  for (auto c : name) {
+    if (ParseRules::is_type(c, invalid_char_bits)) {
       return false;
     }
   }
@@ -988,14 +973,6 @@ MIMEField::name_is_valid(uint32_t invalid_char_bits) const
 /*-------------------------------------------------------------------------
   -------------------------------------------------------------------------*/
 
-inline const char *
-MIMEField::value_get(int *length) const
-{
-  auto value{this->value_get()};
-  *length = int(value.size());
-  return value.data();
-}
-
 inline int32_t
 MIMEField::value_get_int() const
 {
@@ -1083,11 +1060,9 @@ MIMEField::value_append(HdrHeap *heap, MIMEHdrImpl *mh, const char *value, int l
 inline bool
 MIMEField::value_is_valid(uint32_t invalid_char_bits) const
 {
-  const char *value;
-  int         length;
-
-  for (value = value_get(&length); length > 0; length--) {
-    if (ParseRules::is_type(value[length - 1], invalid_char_bits)) {
+  auto value{value_get()};
+  for (auto c : value) {
+    if (ParseRules::is_type(c, invalid_char_bits)) {
       return false;
     }
   }
@@ -1449,7 +1424,9 @@ MIMEHdr::value_get(const char *name, int name_length, int *value_length_return)
   const MIMEField *field = field_find(name, name_length);
 
   if (field) {
-    return field->value_get(value_length_return);
+    auto value{field->value_get()};
+    *value_length_return = static_cast<int>(value.length());
+    return value.data();
   }
   return nullptr;
 }
@@ -1581,12 +1558,10 @@ MIMEHdr::field_combine_dups(MIMEField *field, bool prepend_comma, const char sep
   MIMEField *current = field->m_next_dup;
 
   while (current) {
-    int         value_len = 0;
-    const char *value_str = current->value_get(&value_len);
-
-    if (value_len > 0) {
-      HdrHeap::HeapGuard guard(m_heap, value_str); // reference count the source string so it doesn't get moved
-      field->value_append(m_heap, m_mime, value_str, value_len, prepend_comma, separator);
+    auto value{current->value_get()};
+    if (value.length() > 0) {
+      HdrHeap::HeapGuard guard(m_heap, value.data()); // reference count the source string so it doesn't get moved
+      field->value_append(m_heap, m_mime, value.data(), value.length(), prepend_comma, separator);
     }
     field_delete(current, false); // don't delete duplicates
     current = field->m_next_dup;

include/proxy/http/HttpSM.h

@@ -153,7 +153,7 @@ class PostDataBuffers
 
   void            clear();
   void            init(IOBufferReader *ua_reader);
-  void            copy_partial_post_data();
+  int64_t         copy_partial_post_data(int64_t consumed_bytes);
   IOBufferReader *get_post_data_buffer_clone_reader();
   void
   set_post_data_buffer_done(bool done)
@@ -287,7 +287,7 @@ class HttpSM : public Continuation, public PluginUserArgs<TS_USER_ARGS_TXN>
   char   *redirect_url = nullptr; // url for force redirect (provide users a functionality to redirect to another url when needed)
   int     redirect_url_len  = 0;
   int     redirection_tries = 0; // To monitor number of redirections
-  int64_t transfered_bytes  = 0; // Added to calculate POST data
+  int64_t transferred_bytes = 0; // For handling buffering of request body data.
 
   BackgroundFill_t background_fill = BACKGROUND_FILL_NONE;
 
@@ -317,7 +317,7 @@ class HttpSM : public Continuation, public PluginUserArgs<TS_USER_ARGS_TXN>
   int64_t         postbuf_buffer_avail();
   void            postbuf_clear();
   void            disable_redirect();
-  void            postbuf_copy_partial_data();
+  int64_t         postbuf_copy_partial_data(int64_t consumed_bytes);
   void            postbuf_init(IOBufferReader *ua_reader);
   void            set_postbuf_done(bool done);
   IOBufferReader *get_postbuf_clone_reader();
@@ -764,10 +764,10 @@ HttpSM::disable_redirect()
   this->_postbuf.clear();
 }
 
-inline void
-HttpSM::postbuf_copy_partial_data()
+inline int64_t
+HttpSM::postbuf_copy_partial_data(int64_t consumed_bytes)
 {
-  this->_postbuf.copy_partial_post_data();
+  return this->_postbuf.copy_partial_post_data(consumed_bytes);
 }
 
 inline void