Add Zstandard compression support and update tests (#12201)

* Add comprehensive zstd compression support to ATS

This patch adds full support for the zstd (Zstandard) compression
algorithm throughout Apache Traffic Server, including build system
integration, compression plugin support, Accept-Encoding header
normalization, and comprehensive test coverage.

Build system and dependencies:
- Add CMake support for finding zstd library with new Findzstd.cmake
- Update Docker build files to include libzstd-dev package
- Add TS_HAS_ZSTD feature flag for conditional compilation

Core compression support:
- Extend compress plugin to support zstd compression alongside gzip
  and brotli
- Add zstd stream handling structures and functions
- Update compression configuration to include zstd in supported
  algorithms list
- Add zstd compression type constant and related infrastructure

Accept-Encoding header normalization:
- Extend proxy.config.http.normalize_ae configuration to support
  values 4 and 5 for zstd normalization
- Add zstd support to header normalization logic with proper
  priority handling (zstd > br > gzip)
- Update HTTP transaction cache matching to handle zstd encoding
- Add zstd token to header parsing infrastructure

API and infrastructure:
- Add TS_HTTP_VALUE_ZSTD and TS_HTTP_LEN_ZSTD constants
- Update MIME field handling to recognize zstd encoding
- Add zstd support to traffic_layout feature detection

Test coverage:
- Expand compress plugin tests to cover zstd compression scenarios
- Add zstd test cases to Accept-Encoding normalization tests
- Update golden files to include zstd compression test results
- Add new compress3.config for zstd-specific plugin configuration
- Test all combinations of zstd, br, and gzip in various scenarios

The implementation follows RFC 8878 standards for zstd compression
and maintains backward compatibility with existing gzip and brotli
compression functionality. All tests pass and the feature is
properly integrated with the existing caching and content negotiation
mechanisms.

Co-authored-by: JosiahWI <[email protected]>

Author: JakeChampion

CMakeLists.txt

@@ -359,6 +359,33 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR})
 set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS})
 find_package(ZLIB REQUIRED)
 
+find_package(zstd CONFIG QUIET)
+if(zstd_FOUND)
+
+  # Provide a compatibility target name if the upstream package does not export it
+  # Our code links against `zstd::zstd`; upstream zstd usually exports
+  # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed.
+  if(NOT TARGET zstd::zstd)
+    if(TARGET zstd::libzstd_shared)
+      set(_zstd_target zstd::libzstd_shared)
+    elseif(TARGET zstd::libzstd_static)
+      set(_zstd_target zstd::libzstd_static)
+    elseif(TARGET zstd::libzstd)
+      set(_zstd_target zstd::libzstd)
+    endif()
+    if(DEFINED _zstd_target)
+      add_library(zstd_zstd INTERFACE)
+      target_link_libraries(zstd_zstd INTERFACE ${_zstd_target})
+      add_library(zstd::zstd ALIAS zstd_zstd)
+      set(HAVE_ZSTD_H TRUE)
+    else()
+      set(HAVE_ZSTD_H FALSE)
+    endif()
+  endif()
+else()
+  set(HAVE_ZSTD_H FALSE)
+endif()
+
 # ncurses is used in traffic_top
 find_package(Curses)
 set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H})

ci/docker/deb/Dockerfile

@@ -55,7 +55,8 @@ RUN apt-get update; apt-get -y dist-upgrade; \
     apt-get -y install libssl-dev libexpat1-dev libpcre3-dev libcap-dev \
     libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \
     tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \
-    libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev; \
+    libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \
+    libzstd-dev; \
     # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed)
     apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \
     groupadd  -g 665 jenkins && \

ci/docker/yum/Dockerfile

@@ -52,7 +52,7 @@ RUN yum -y update; \
     # Devel packages that ATS needs
     yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \
     xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \
-    ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel \
+    ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \
     perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \
     # This is for autest stuff
     yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \

contrib/docker/ubuntu/noble/Dockerfile

@@ -48,6 +48,8 @@ RUN apt update \
     libpcre3-dev \
     hwloc \
     libbrotli-dev \
+    libzstd-dev \
+    luajit \
     libluajit-5.1-dev \
     libcap-dev \
     libmagick++-dev \

doc/admin-guide/files/records.yaml.en.rst

@@ -2170,10 +2170,14 @@ Proxy User Variables
          normalize as for value ``1``
    ``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE**
          normalize as for value ``2``
+   ``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE**
+         normalize as for value ``2``
+   ``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE**
+         normalize as for value ``4``
    ===== ======================================================================
 
    This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``).
-   Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli).
+   Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard).
 
 Security
 ========

doc/admin-guide/plugins/compress.en.rst

@@ -202,12 +202,59 @@ supported-algorithms
 
 Provides the compression algorithms that are supported, a comma separate list
 of values. This will allow |TS| to selectively support ``gzip``, ``deflate``,
-and brotli (``br``) compression. The default is ``gzip``. Multiple algorithms can
-be selected using ',' delimiter, for instance, ``supported-algorithms
-deflate,gzip,br``. Note that this list must **not** contain any white-spaces!
+brotli (``br``), and zstd (``zstd``) compression. The default is ``gzip``.
+Multiple algorithms can be selected using ',' delimiter, for instance,
+``supported-algorithms deflate,gzip,br,zstd``. Note that this list must **not**
+contain any white-spaces!
+
+============== =================================================================
+Algorithm      Description
+============== =================================================================
+gzip           Standard gzip compression (default, widely supported)
+deflate        Deflate compression (RFC 1951)
+br             Brotli compression (modern, efficient)
+zstd           Zstandard compression (fast, high compression ratio)
+============== =================================================================
 
 Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will
-be considered, and if it is ``2``, only br or gzip will be considered.
+be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``,
+only zstd, br, or gzip will be considered, and if it is ``5``, all combinations
+of zstd, br, and gzip will be considered.
+
+gzip-compression-level
+-----------------------
+
+Sets the compression level for gzip compression. Valid values are 1-9, where
+1 is fastest compression (lowest compression ratio) and 9 is slowest compression
+(highest compression ratio). The default is 6, which provides a good balance
+between compression speed and ratio.
+
+brotli-compression-level
+-------------------------
+
+Sets the compression level for Brotli compression. Valid values are 0-11, where
+0 is fastest compression (lowest compression ratio) and 11 is slowest compression
+(highest compression ratio). The default is 6, which provides a good balance
+between compression speed and ratio.
+
+brotli-lgwin
+------------
+
+Sets the window size for Brotli compression. Valid values are 10-24, where
+larger values provide better compression but use more memory. The default is 16.
+This parameter controls the sliding window size used during compression:
+
+- 10: 1KB window (fastest, least memory)
+- 16: 64KB window (default, good balance)
+- 24: 16MB window (slowest, most memory, best compression)
+
+zstd-compression-level
+----------------------
+
+Sets the compression level for Zstandard compression. Valid values are 1-22, where
+1 is fastest compression (lowest compression ratio) and 22 is slowest compression
+(highest compression ratio). The default is 12, which provides an excellent
+balance between compression speed and ratio for web content.
 
 Examples
 ========
@@ -224,6 +271,10 @@ might create a configuration with the following options::
    compressible-status-code 200, 206
    minimum-content-length 860
    flush false
+   gzip-compression-level 6
+   brotli-compression-level 6
+   brotli-lgwin 16
+   zstd-compression-level 12
 
    # Now set a configuration for www.example.com
    [www.example.com]
@@ -242,14 +293,38 @@ might create a configuration with the following options::
    flush true
    supported-algorithms gzip,deflate
 
-   # Supports brotli compression
+   # Supports brotli compression with custom settings
    [brotli.compress.com]
    enabled true
    compressible-content-type text/*
    compressible-content-type application/json
    content_type_ignore_parameters true
    flush true
    supported-algorithms br,gzip
+   brotli-compression-level 8
+   brotli-lgwin 20
+
+   # Supports zstd compression for high efficiency
+   [zstd.compress.com]
+   enabled true
+   compressible-content-type text/*
+   compressible-content-type application/json
+   compressible-content-type application/javascript
+   flush true
+   supported-algorithms zstd,gzip
+   zstd-compression-level 15
+
+   # Supports all compression algorithms with optimized settings
+   [all.compress.com]
+   enabled true
+   compressible-content-type text/*
+   compressible-content-type application/json
+   flush true
+   supported-algorithms zstd,br,gzip,deflate
+   gzip-compression-level 7
+   brotli-compression-level 9
+   brotli-lgwin 18
+   zstd-compression-level 10
 
    # This origin does it all
    [bar.example.com]

doc/developer-guide/plugins/http-headers/header-functions.en.rst

@@ -92,6 +92,9 @@ headers.
 ``TS_HTTP_VALUE_GZIP``
    "gzip"
 
+``TS_HTTP_VALUE_ZSTD``
+   "zstd"
+
 ``TS_HTTP_VALUE_IDENTITY``
    "identity"
 

doc/release-notes/whats-new.en.rst

@@ -185,6 +185,7 @@ Plugins
 * xdebug - ``--enable`` option to selectively enable features has been added
 * system_stats - Stats about memory have been added
 * slice plugin - This plugin was promoted to stable.
+* compress plugin - Added support for Zstandard (zstd) compression algorithm.
 
 JSON-RPC
 ^^^^^^^^

include/proxy/hdrs/HTTP.h

@@ -368,6 +368,7 @@ extern c_str_view HTTP_VALUE_COMPRESS;
 extern c_str_view HTTP_VALUE_DEFLATE;
 extern c_str_view HTTP_VALUE_GZIP;
 extern c_str_view HTTP_VALUE_BROTLI;
+extern c_str_view HTTP_VALUE_ZSTD;
 extern c_str_view HTTP_VALUE_IDENTITY;
 extern c_str_view HTTP_VALUE_KEEP_ALIVE;
 extern c_str_view HTTP_VALUE_MAX_AGE;

include/proxy/hdrs/MIME.h

@@ -602,6 +602,8 @@ extern c_str_view MIME_VALUE_COMPRESS;
 extern c_str_view MIME_VALUE_DEFLATE;
 extern c_str_view MIME_VALUE_GZIP;
 extern c_str_view MIME_VALUE_BROTLI;
+extern c_str_view MIME_VALUE_ZSTD;
+
 extern c_str_view MIME_VALUE_IDENTITY;
 extern c_str_view MIME_VALUE_KEEP_ALIVE;
 extern c_str_view MIME_VALUE_MAX_AGE;