diff --git a/CMakeLists.txt b/CMakeLists.txt index 654022f85dd..4fd1faefd45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,11 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) +find_package(zstd) +if(zstd_FOUND) + set(HAVE_ZSTD_H TRUE) +endif() + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 337356ca8c3..4e1398d15b7 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -55,7 +55,8 @@ RUN apt-get update; apt-get -y dist-upgrade; \ apt-get -y install libssl-dev libexpat1-dev libpcre3-dev libcap-dev \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ - libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev; \ + libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ + libzstd-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 85e9a7add64..5a160fb24ff 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \ diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake new file mode 100644 index 00000000000..17587dca39a --- /dev/null +++ b/cmake/Findzstd.cmake @@ -0,0 +1,53 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +# Findzstd.cmake +# +# This will define the following variables +# +# zstd_FOUND +# zstd_LIBRARY +# zstd_INCLUDE_DIRS +# +# and the following imported target +# +# zstd::zstd +# + +find_path(zstd_INCLUDE_DIR NAMES zstd.h) + +find_library(zstd_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(zstd_LIBRARY_RELEASE NAMES zstd zstd_static) + +mark_as_advanced(zstd_LIBRARY zstd_INCLUDE_DIR) + +include(SelectLibraryConfigurations) +select_library_configurations(zstd) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARY zstd_INCLUDE_DIR) + +if(zstd_FOUND) + set(zstd_INCLUDE_DIRS "${zstd_INCLUDE_DIR}") +endif() + +if(zstd_FOUND AND NOT TARGET zstd::zstd) + add_library(zstd::zstd INTERFACE IMPORTED) + target_include_directories(zstd::zstd INTERFACE ${zstd_INCLUDE_DIRS}) + target_link_libraries(zstd::zstd INTERFACE "${zstd_LIBRARY}") +endif() diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index 494cd222dc3..34e83f185ff 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2076,10 +2076,14 @@ Proxy User Variables normalize as for value ``1`` ``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE** normalize as for value ``2`` + ``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE** + normalize as for value ``2`` + ``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE** + normalize as for value ``4`` ===== ====================================================================== This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``). - Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli). + Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard). Security ======== diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index 81dec81dc8f..338e6101c11 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -192,12 +192,59 @@ supported-algorithms Provides the compression algorithms that are supported, a comma separate list of values. This will allow |TS| to selectively support ``gzip``, ``deflate``, -and brotli (``br``) compression. The default is ``gzip``. Multiple algorithms can -be selected using ',' delimiter, for instance, ``supported-algorithms -deflate,gzip,br``. Note that this list must **not** contain any white-spaces! +brotli (``br``), and zstd (``zstd``) compression. The default is ``gzip``. +Multiple algorithms can be selected using ',' delimiter, for instance, +``supported-algorithms deflate,gzip,br,zstd``. Note that this list must **not** +contain any white-spaces! + +============== ================================================================= +Algorithm Description +============== ================================================================= +gzip Standard gzip compression (default, widely supported) +deflate Deflate compression (RFC 1951) +br Brotli compression (modern, efficient) +zstd Zstandard compression (fast, high compression ratio) +============== ================================================================= Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will -be considered, and if it is ``2``, only br or gzip will be considered. +be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``, +only zstd, br, or gzip will be considered, and if it is ``5``, all combinations +of zstd, br, and gzip will be considered. + +gzip-compression-level +----------------------- + +Sets the compression level for gzip compression. Valid values are 1-9, where +1 is fastest compression (lowest compression ratio) and 9 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-compression-level +------------------------- + +Sets the compression level for Brotli compression. Valid values are 0-11, where +0 is fastest compression (lowest compression ratio) and 11 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-lgwin +------------ + +Sets the window size for Brotli compression. Valid values are 10-24, where +larger values provide better compression but use more memory. The default is 16. +This parameter controls the sliding window size used during compression: + +- 10: 1KB window (fastest, least memory) +- 16: 64KB window (default, good balance) +- 24: 16MB window (slowest, most memory, best compression) + +zstd-compression-level +---------------------- + +Sets the compression level for Zstandard compression. Valid values are 1-22, where +1 is fastest compression (lowest compression ratio) and 22 is slowest compression +(highest compression ratio). The default is 12, which provides an excellent +balance between compression speed and ratio for web content. Examples ======== @@ -214,6 +261,10 @@ might create a configuration with the following options:: compressible-status-code 200, 206 minimum-content-length 860 flush false + gzip-compression-level 6 + brotli-compression-level 6 + brotli-lgwin 16 + zstd-compression-level 12 # Now set a configuration for www.example.com [www.example.com] @@ -231,13 +282,37 @@ might create a configuration with the following options:: flush true supported-algorithms gzip,deflate - # Supports brotli compression + # Supports brotli compression with custom settings [brotli.compress.com] enabled true compressible-content-type text/* compressible-content-type application/json flush true supported-algorithms br,gzip + brotli-compression-level 8 + brotli-lgwin 20 + + # Supports zstd compression for high efficiency + [zstd.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + compressible-content-type application/javascript + flush true + supported-algorithms zstd,gzip + zstd-compression-level 15 + + # Supports all compression algorithms with optimized settings + [all.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + flush true + supported-algorithms zstd,br,gzip,deflate + gzip-compression-level 7 + brotli-compression-level 9 + brotli-lgwin 18 + zstd-compression-level 10 # This origin does it all [bar.example.com] diff --git a/doc/developer-guide/plugins/http-headers/header-functions.en.rst b/doc/developer-guide/plugins/http-headers/header-functions.en.rst index 9a27115434f..47d07459a2c 100644 --- a/doc/developer-guide/plugins/http-headers/header-functions.en.rst +++ b/doc/developer-guide/plugins/http-headers/header-functions.en.rst @@ -92,6 +92,9 @@ headers. ``TS_HTTP_VALUE_GZIP`` "gzip" +``TS_HTTP_VALUE_ZSTD`` + "zstd" + ``TS_HTTP_VALUE_IDENTITY`` "identity" diff --git a/doc/release-notes/whats-new.en.rst b/doc/release-notes/whats-new.en.rst index f00111e5513..0c64321ac8f 100644 --- a/doc/release-notes/whats-new.en.rst +++ b/doc/release-notes/whats-new.en.rst @@ -185,6 +185,7 @@ Plugins * xdebug - ``--enable`` option to selectively enable features has been added * system_stats - Stats about memory have been added * slice plugin - This plugin was promoted to stable. +* compress plugin - Added support for Zstandard (zstd) compression algorithm. JSON-RPC ^^^^^^^^ diff --git a/include/proxy/hdrs/HTTP.h b/include/proxy/hdrs/HTTP.h index 22d6c45a7b3..f3a3d879b92 100644 --- a/include/proxy/hdrs/HTTP.h +++ b/include/proxy/hdrs/HTTP.h @@ -374,6 +374,7 @@ extern const char *HTTP_VALUE_COMPRESS; extern const char *HTTP_VALUE_DEFLATE; extern const char *HTTP_VALUE_GZIP; extern const char *HTTP_VALUE_BROTLI; +extern const char *HTTP_VALUE_ZSTD; extern const char *HTTP_VALUE_IDENTITY; extern const char *HTTP_VALUE_KEEP_ALIVE; extern const char *HTTP_VALUE_MAX_AGE; @@ -399,6 +400,7 @@ extern int HTTP_LEN_COMPRESS; extern int HTTP_LEN_DEFLATE; extern int HTTP_LEN_GZIP; extern int HTTP_LEN_BROTLI; +extern int HTTP_LEN_ZSTD; extern int HTTP_LEN_IDENTITY; extern int HTTP_LEN_KEEP_ALIVE; extern int HTTP_LEN_MAX_AGE; diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index 23282845567..1b4cae23fc4 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -600,7 +600,6 @@ extern const char *MIME_VALUE_CLOSE; extern const char *MIME_VALUE_COMPRESS; extern const char *MIME_VALUE_DEFLATE; extern const char *MIME_VALUE_GZIP; -extern const char *MIME_VALUE_BROTLI; extern const char *MIME_VALUE_IDENTITY; extern const char *MIME_VALUE_KEEP_ALIVE; extern const char *MIME_VALUE_MAX_AGE; @@ -701,7 +700,6 @@ extern int MIME_LEN_CLOSE; extern int MIME_LEN_COMPRESS; extern int MIME_LEN_DEFLATE; extern int MIME_LEN_GZIP; -extern int MIME_LEN_BLOTLI; extern int MIME_LEN_IDENTITY; extern int MIME_LEN_KEEP_ALIVE; extern int MIME_LEN_MAX_AGE; diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index bd37127970d..05049a6d95b 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -1351,6 +1351,7 @@ extern const char *TS_HTTP_VALUE_COMPRESS; extern const char *TS_HTTP_VALUE_DEFLATE; extern const char *TS_HTTP_VALUE_GZIP; extern const char *TS_HTTP_VALUE_BROTLI; +extern const char *TS_HTTP_VALUE_ZSTD; extern const char *TS_HTTP_VALUE_IDENTITY; extern const char *TS_HTTP_VALUE_KEEP_ALIVE; extern const char *TS_HTTP_VALUE_MAX_AGE; @@ -1375,6 +1376,7 @@ extern int TS_HTTP_LEN_COMPRESS; extern int TS_HTTP_LEN_DEFLATE; extern int TS_HTTP_LEN_GZIP; extern int TS_HTTP_LEN_BROTLI; +extern int TS_HTTP_LEN_ZSTD; extern int TS_HTTP_LEN_IDENTITY; extern int TS_HTTP_LEN_KEEP_ALIVE; extern int TS_HTTP_LEN_MAX_AGE; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index fcbd98a5822..2d515338c8a 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -187,3 +187,5 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine YAMLCPP_LIB_VERSION "@YAMLCPP_LIB_VERSION@" #cmakedefine01 TS_HAS_CRIPTS + +#cmakedefine HAVE_ZSTD_H 1 diff --git a/plugins/compress/CMakeLists.txt b/plugins/compress/CMakeLists.txt index f630bf0d1c9..65b24ce9558 100644 --- a/plugins/compress/CMakeLists.txt +++ b/plugins/compress/CMakeLists.txt @@ -20,5 +20,10 @@ target_link_libraries(compress PRIVATE libswoc::libswoc) if(HAVE_BROTLI_ENCODE_H) target_link_libraries(compress PRIVATE brotli::brotlienc) endif() + +if(HAVE_ZSTD_H) + target_link_libraries(compress PRIVATE zstd::zstd) +endif() + verify_global_plugin(compress) verify_remap_plugin(compress) diff --git a/plugins/compress/README b/plugins/compress/README index 759add28e03..f963a8e05e9 100644 --- a/plugins/compress/README +++ b/plugins/compress/README @@ -1,7 +1,7 @@ What this plugin does: ===================== -This plugin compresses responses, via gzip or brotli, whichever is applicable +This plugin compresses responses, via gzip, deflate, brotli, or zstd (Zstandard), whichever is applicable it can compress origin responses as well as cached responses installation: @@ -24,4 +24,4 @@ compress.so /sample.compress.config After modifying plugin.config, restart traffic server (sudo traffic_ctl server restart) the configuration is re-read when a management update is given (sudo traffic_ctl config reload) -See sample.config.compress for an example configuration and the options that are available +See sample.compress.config for an example configuration and the options that are available diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 44e8235a0d0..7a9dd2688de 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -23,6 +23,9 @@ #include #include +#if HAVE_ZSTD_H +#include +#endif #include "ts/apidefs.h" #include "tscore/ink_config.h" @@ -62,14 +65,7 @@ namespace compress_ns DbgCtl dbg_ctl{TAG}; } -const int ZLIB_COMPRESSION_LEVEL = 6; -const char *dictionary = nullptr; - -// brotli compression quality 1-11. Testing proved level '6' -#if HAVE_BROTLI_ENCODE_H -const int BROTLI_COMPRESSION_LEVEL = 6; -const int BROTLI_LGW = 16; -#endif +const char *dictionary = nullptr; static const char *global_hidden_header_name = nullptr; @@ -136,8 +132,14 @@ handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) } } // namespace +// Forward declarations for ZSTD compression functions +#if HAVE_ZSTD_H +static void zstd_compress_init(Data *data, unsigned long long content_size = ZSTD_CONTENTSIZE_UNKNOWN); +static void zstd_compress_finish(Data *data); +static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length); +#endif static Data * -data_alloc(int compression_type, int compression_algorithms) +data_alloc(int compression_type, int compression_algorithms, HostConfiguration *hc) { Data *data; int err; @@ -150,6 +152,7 @@ data_alloc(int compression_type, int compression_algorithms) data->state = transform_state_initialized; data->compression_type = compression_type; data->compression_algorithms = compression_algorithms; + data->hc = hc; data->zstrm.next_in = Z_NULL; data->zstrm.avail_in = 0; data->zstrm.total_in = 0; @@ -166,7 +169,7 @@ data_alloc(int compression_type, int compression_algorithms) window_bits = WINDOW_BITS_DEFLATE; } - err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); + err = deflateInit2(&data->zstrm, data->hc->zlib_compression_level(), Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); if (err != Z_OK) { fatal("gzip-transform: ERROR: deflateInit (%d)!", err); @@ -186,8 +189,8 @@ data_alloc(int compression_type, int compression_algorithms) if (!data->bstrm.br) { fatal("Brotli Encoder Instance Failed"); } - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, BROTLI_COMPRESSION_LEVEL); - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, BROTLI_LGW); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, data->hc->brotli_compression_level()); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, data->hc->brotli_lgw_size()); data->bstrm.next_in = nullptr; data->bstrm.avail_in = 0; data->bstrm.total_in = 0; @@ -195,6 +198,22 @@ data_alloc(int compression_type, int compression_algorithms) data->bstrm.avail_out = 0; data->bstrm.total_out = 0; } +#endif +#if HAVE_ZSTD_H + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.next_in = nullptr; + data->zstrm_zstd.avail_in = 0; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.next_out = nullptr; + data->zstrm_zstd.avail_out = 0; + data->zstrm_zstd.total_out = 0; + if (compression_type & COMPRESSION_TYPE_ZSTD) { + debug("zstd compression. Create Zstd Compression Context."); + data->zstrm_zstd.cctx = ZSTD_createCCtx(); + if (!data->zstrm_zstd.cctx) { + fatal("Zstd Compression Context Creation Failed"); + } + } #endif return data; } @@ -216,6 +235,11 @@ data_destroy(Data *data) #if HAVE_BROTLI_ENCODE_H BrotliEncoderDestroyInstance(data->bstrm.br); #endif +#if HAVE_ZSTD_H + if (data->zstrm_zstd.cctx) { + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + } +#endif TSfree(data); } @@ -228,7 +252,10 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty const char *value = nullptr; int value_len = 0; // Delete Content-Encoding if present??? - if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { + if (compression_type & COMPRESSION_TYPE_ZSTD && (algorithm & ALGORITHM_ZSTD)) { + value = TS_HTTP_VALUE_ZSTD; + value_len = TS_HTTP_LEN_ZSTD; + } else if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { value = TS_HTTP_VALUE_BROTLI; value_len = TS_HTTP_LEN_BROTLI; } else if (compression_type & COMPRESSION_TYPE_GZIP && (algorithm & ALGORITHM_GZIP)) { @@ -240,7 +267,6 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty } if (value_len == 0) { - error("no need to add Content-Encoding header"); return TS_SUCCESS; } @@ -335,7 +361,6 @@ etag_header(TSMBuffer bufp, TSMLoc hdr_loc) return ret; } -// FIXME: some things are potentially compressible. those responses static void compress_transform_init(TSCont contp, Data *data) { @@ -361,6 +386,28 @@ compress_transform_init(TSCont contp, Data *data) data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX); } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD) { + // Try to get content length from response headers + unsigned long long content_size = ZSTD_CONTENTSIZE_UNKNOWN; + TSMLoc content_length_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH); + if (content_length_loc != TS_NULL_MLOC) { + unsigned int hdr_value = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, content_length_loc, -1); + if (hdr_value > 0) { + content_size = static_cast(hdr_value); + debug("Found content-length header: %llu", content_size); + } + TSHandleMLocRelease(bufp, hdr_loc, content_length_loc); + } + + zstd_compress_init(data, content_size); + if (!data->zstrm_zstd.cctx) { + TSError("Failed to create Zstandard compression context"); + return; + } + } +#endif + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); } @@ -466,6 +513,137 @@ brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_l } #endif +#if HAVE_ZSTD_H +static void +zstd_compress_init(Data *data, unsigned long long content_size) +{ + if (!data->zstrm_zstd.cctx) { + error("Failed to initialize Zstd compression context"); + return; + } + + // Set compression level + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); + if (ZSTD_isError(result)) { + error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); + return; + } + + // set zstd_btopt + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_strategy, ZSTD_btopt); + if (ZSTD_isError(result)) { + error("Failed to set Zstd btlazy2 option: %s", ZSTD_getErrorName(result)); + return; + } + + // Enable checksum for data integrity + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(result)) { + error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); + return; + } + + // Enable content size flag - this will include the content size in the frame header + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_contentSizeFlag, 1); + if (ZSTD_isError(result)) { + error("Failed to enable Zstd content size flag: %s", ZSTD_getErrorName(result)); + return; + } + + // Set the pledged source size if known + if (content_size != ZSTD_CONTENTSIZE_UNKNOWN) { + result = ZSTD_CCtx_setPledgedSrcSize(data->zstrm_zstd.cctx, content_size); + if (ZSTD_isError(result)) { + error("Failed to set Zstd pledged source size: %s", ZSTD_getErrorName(result)); + return; + } + debug("zstd compression context initialized with level %d and content size %llu", data->hc->zstd_compression_level(), + content_size); + } else { + debug("zstd compression context initialized with level %d (unknown content size)", data->hc->zstd_compression_level()); + } +} + +static void +zstd_compress_finish(Data *data) +{ + if (data->state == transform_state_output) { + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->state = transform_state_finished; + + // Finalize the zstd stream + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); + + if (ZSTD_isError(remaining)) { + error("zstd compression finish failed: %s", ZSTD_getErrorName(remaining)); + break; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (remaining == 0) { /* compression finished */ + break; + } + } + + debug("zstd-transform: Finished zstd compression"); + log_compression_ratio(data->zstrm_zstd.total_in, data->downstream_length); + } +} + +static void +zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + // Set up input buffer for zstd streaming + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; + data->zstrm_zstd.total_in += upstream_length; + + while (input.pos < input.size) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + // Set up output buffer for zstd streaming + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + // Compress the data using streaming API + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, ZSTD_e_continue); + + if (ZSTD_isError(result)) { + error("Zstd compression failed: %s", ZSTD_getErrorName(result)); + return; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + // If we have output space but no more input was consumed, break to avoid infinite loop + if (output.pos == 0 && input.pos < input.size) { + error("zstd-transform: no progress made in compression"); + break; + } + } +} +#endif + static void compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) { @@ -488,8 +666,13 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) upstream_length = amount; } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { + zstd_compress_one(data, upstream_buffer, upstream_length); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { brotli_transform_one(data, upstream_buffer, upstream_length); } else #endif @@ -575,8 +758,14 @@ brotli_transform_finish(Data *data) static void compress_transform_finish(Data *data) { +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { + zstd_compress_finish(data); + debug("compress_transform_finish: zstd compression finish"); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { brotli_transform_finish(data); debug("compress_transform_finish: brotli compression finish"); } else @@ -787,7 +976,14 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration continue; } - if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { + info("Accept-Encoding value [%.*s]", len, value); + + if (strncasecmp(value, "zstd", sizeof("zstd") - 1) == 0) { + if (*algorithms & ALGORITHM_ZSTD) { + compression_acceptable = 1; + } + *compress_type |= COMPRESSION_TYPE_ZSTD; + } else if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { if (*algorithms & ALGORITHM_BROTLI) { compression_acceptable = 1; } @@ -887,9 +1083,8 @@ compress_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type, } connp = TSTransformCreate(compress_transform, txnp); - data = data_alloc(compress_type, algorithms); + data = data_alloc(compress_type, algorithms, hc); data->txn = txnp; - data->hc = hc; TSContDataSet(connp, data); TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index dd95952f97e..d4311ec286c 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -108,7 +108,11 @@ enum ParserState { kParseRangeRequest, kParseFlush, kParseAllow, - kParseMinimumContentLength + kParseMinimumContentLength, + kParseGzipCompressionLevel, + kPaseBrotliCompressionLevel, + kParseBrotliLGWSize, + kParseZstdCompressionLevel, }; void @@ -229,6 +233,12 @@ HostConfiguration::add_compression_algorithms(string &line) string token = extractFirstToken(line, isCommaOrSpace); if (token.empty()) { break; + } else if (token == "zstd") { +#ifdef HAVE_ZSTD_H + compression_algorithms_ |= ALGORITHM_ZSTD; +#else + error("supported-algorithms: zstd support not compiled in."); +#endif } else if (token == "br") { #ifdef HAVE_BROTLI_ENCODE_H compression_algorithms_ |= ALGORITHM_BROTLI; @@ -240,7 +250,7 @@ HostConfiguration::add_compression_algorithms(string &line) } else if (token == "deflate") { compression_algorithms_ |= ALGORITHM_DEFLATE; } else { - error("Unknown compression type. Supported compression-algorithms ."); + error("Unknown compression type. Supported compression-algorithms ."); } } } @@ -383,6 +393,14 @@ Configuration::Parse(const char *path) state = kParseStart; } else if (token == "minimum-content-length") { state = kParseMinimumContentLength; + } else if (token == "gzip-compression-level") { + state = kParseGzipCompressionLevel; + } else if (token == "brotli-compression-level") { + state = kPaseBrotliCompressionLevel; + } else if (token == "brotli-lgwin") { + state = kParseBrotliLGWSize; + } else if (token == "zstd-compression-level") { + state = kParseZstdCompressionLevel; } else { warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno); } @@ -419,6 +437,46 @@ Configuration::Parse(const char *path) current_host_configuration->set_minimum_content_length(strtoul(token.c_str(), nullptr, 10)); state = kParseStart; break; + case kParseGzipCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 1 || level > 9) { + error("gzip-compression-level must be between 1 and 9, got %d", level); + } else { + current_host_configuration->set_gzip_compression_level(level); + } + state = kParseStart; + break; + } + case kPaseBrotliCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 0 || level > 11) { + error("brotli-compression-level must be between 0 and 11, got %d", level); + } else { + current_host_configuration->set_brotli_compression_level(level); + } + state = kParseStart; + break; + } + case kParseBrotliLGWSize: { + int lgw = strtol(token.c_str(), nullptr, 10); + if (lgw < 10 || lgw > 24) { + error("brotli-lgwin must be between 10 and 24, got %d", lgw); + } else { + current_host_configuration->set_brotli_lgw_size(lgw); + } + state = kParseStart; + break; + } + case kParseZstdCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 1 || level > 22) { + error("zstd-compression-level must be between 1 and 22, got %d", level); + } else { + current_host_configuration->set_zstd_compression_level(level); + } + state = kParseStart; + break; + } } } } diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index 22b2173c590..0db81784199 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -38,7 +38,8 @@ enum CompressionAlgorithm { ALGORITHM_DEFAULT = 0, ALGORITHM_DEFLATE = 1, ALGORITHM_GZIP = 2, - ALGORITHM_BROTLI = 4 // For bit manipulations + ALGORITHM_BROTLI = 4, + ALGORITHM_ZSTD = 8 }; enum class RangeRequestCtrl : int { @@ -58,7 +59,11 @@ class HostConfiguration : private atscppapi::noncopyable remove_accept_encoding_(false), flush_(false), compression_algorithms_(ALGORITHM_GZIP), - minimum_content_length_(1024) + minimum_content_length_(1024), + zlib_compression_level_(6), + brotli_compression_level_(6), + brotli_lgw_size_(16), + zstd_compression_level_(12) { } @@ -129,6 +134,51 @@ class HostConfiguration : private atscppapi::noncopyable minimum_content_length_ = x; } + unsigned int + zlib_compression_level() const + { + return zlib_compression_level_; + } + + void + set_gzip_compression_level(int level) + { + zlib_compression_level_ = level; + } + + unsigned int + brotli_compression_level() const + { + return brotli_compression_level_; + } + void + set_brotli_compression_level(int level) + { + brotli_compression_level_ = level; + } + + unsigned int + brotli_lgw_size() const + { + return brotli_lgw_size_; + } + void + set_brotli_lgw_size(unsigned int lgw) + { + brotli_lgw_size_ = lgw; + } + + int + zstd_compression_level() const + { + return zstd_compression_level_; + } + void + set_zstd_compression_level(int level) + { + zstd_compression_level_ = level; + } + void update_defaults(); void add_allow(const std::string &allow); void add_compressible_content_type(const std::string &content_type); @@ -148,6 +198,10 @@ class HostConfiguration : private atscppapi::noncopyable bool flush_; int compression_algorithms_; unsigned int minimum_content_length_; + unsigned int zlib_compression_level_; + unsigned int brotli_compression_level_; + unsigned int brotli_lgw_size_; + int zstd_compression_level_; RangeRequestCtrl range_request_ctl_ = RangeRequestCtrl::NO_COMPRESSION; StringContainer compressible_content_types_; diff --git a/plugins/compress/misc.cc b/plugins/compress/misc.cc index c0086b8c7be..29e0bf14ffc 100644 --- a/plugins/compress/misc.cc +++ b/plugins/compress/misc.cc @@ -84,8 +84,9 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo bool deflate = false; bool gzip = false; bool br = false; + bool zstd = false; // remove the accept encoding field(s), - // while finding out if gzip or deflate is supported. + // while finding out if gzip, brotli, deflate, or zstandard are supported. while (field) { int val_len; const char *values_ = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, -1, &val_len); @@ -100,6 +101,8 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo br = true; } else if (strcasecmp("deflate", next) == 0) { deflate = true; + } else if (strcasecmp("zstd", next) == 0) { + zstd = true; } } } @@ -111,9 +114,13 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo } // append a new accept-encoding field in the header - if (deflate || gzip || br) { + if (deflate || gzip || br || zstd) { TSMimeHdrFieldCreate(reqp, hdr_loc, &field); TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + if (zstd) { + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "zstd", strlen("zstd")); + info("normalized accept encoding to zstd"); + } if (br) { TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "br", strlen("br")); info("normalized accept encoding to br"); diff --git a/plugins/compress/misc.h b/plugins/compress/misc.h index 9491271df36..4a3da5d99a5 100644 --- a/plugins/compress/misc.h +++ b/plugins/compress/misc.h @@ -32,6 +32,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + #include "configuration.h" // zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html @@ -44,7 +48,8 @@ enum CompressionType { COMPRESSION_TYPE_DEFAULT = 0, COMPRESSION_TYPE_DEFLATE = 1, COMPRESSION_TYPE_GZIP = 2, - COMPRESSION_TYPE_BROTLI = 4 + COMPRESSION_TYPE_BROTLI = 4, + COMPRESSION_TYPE_ZSTD = 8, }; // this one is used to rename the accept encoding header @@ -70,6 +75,18 @@ using b_stream = struct { }; #endif +#if HAVE_ZSTD_H +using zstd_stream = struct { + ZSTD_CCtx *cctx; + const void *next_in; + size_t avail_in; + void *next_out; + size_t avail_out; + size_t total_in; + size_t total_out; +}; +#endif + using Data = struct { TSHttpTxn txn; Gzip::HostConfiguration *hc; @@ -84,6 +101,9 @@ using Data = struct { #if HAVE_BROTLI_ENCODE_H b_stream bstrm; #endif +#if HAVE_ZSTD_H + zstd_stream zstrm_zstd; +#endif }; voidpf gzip_alloc(voidpf opaque, uInt items, uInt size); diff --git a/plugins/compress/sample.compress.config b/plugins/compress/sample.compress.config index b1431a97794..451f8958349 100644 --- a/plugins/compress/sample.compress.config +++ b/plugins/compress/sample.compress.config @@ -37,6 +37,22 @@ # minimum-content-length: minimum content length for compression to be enabled (in bytes) # - this setting only applies if the origin response has a Content-Length header # +# gzip-compression-level: compression level for gzip (1-9, default 6) +# - 1 is fastest compression (lowest compression ratio) +# - 9 is slowest compression (highest compression ratio) +# +# brotli-compression-level: compression level for brotli (0-11, default 6) +# - 0 is fastest compression (lowest compression ratio) +# - 11 is slowest compression (highest compression ratio) +# +# brotli-lgwin: window size for brotli compression (10-24, default 16) +# - larger values provide better compression but use more memory +# - 10: 1KB window, 16: 64KB window, 24: 16MB window +# +# zstd-compression-level: compression level for zstandard (1-22, default 12) +# - 1 is fastest compression (lowest compression ratio) +# - 22 is slowest compression (highest compression ratio) +# ###################################################################### #first, we configure the default/global plugin behaviour @@ -56,7 +72,13 @@ allow !*/bla* minimum-content-length 1024 #supported algorithms -supported-algorithms br,gzip +supported-algorithms br,gzip,zstd + +# Compression level settings (optional) +gzip-compression-level 6 +brotli-compression-level 6 +brotli-lgwin 16 +zstd-compression-level 12 #override the global configuration for a host. #www.foo.nl does NOT inherit anything @@ -68,6 +90,12 @@ compressible-content-type text/* compressible-status-code 200,206,409 minimum-content-length 1024 +# Custom compression settings for this host +gzip-compression-level 8 +brotli-compression-level 9 +brotli-lgwin 20 +zstd-compression-level 15 + allow /this/*.js allow !/notthis/*.js allow !/notthat* diff --git a/src/api/InkAPIInternal.cc b/src/api/InkAPIInternal.cc index f3c6fc70520..c909d49eb4a 100644 --- a/src/api/InkAPIInternal.cc +++ b/src/api/InkAPIInternal.cc @@ -232,6 +232,7 @@ const char *TS_HTTP_VALUE_COMPRESS; const char *TS_HTTP_VALUE_DEFLATE; const char *TS_HTTP_VALUE_GZIP; const char *TS_HTTP_VALUE_BROTLI; +const char *TS_HTTP_VALUE_ZSTD; const char *TS_HTTP_VALUE_IDENTITY; const char *TS_HTTP_VALUE_KEEP_ALIVE; const char *TS_HTTP_VALUE_MAX_AGE; @@ -256,6 +257,7 @@ int TS_HTTP_LEN_COMPRESS; int TS_HTTP_LEN_DEFLATE; int TS_HTTP_LEN_GZIP; int TS_HTTP_LEN_BROTLI; +int TS_HTTP_LEN_ZSTD; int TS_HTTP_LEN_IDENTITY; int TS_HTTP_LEN_KEEP_ALIVE; int TS_HTTP_LEN_MAX_AGE; @@ -748,6 +750,7 @@ api_init() TS_HTTP_VALUE_DEFLATE = HTTP_VALUE_DEFLATE; TS_HTTP_VALUE_GZIP = HTTP_VALUE_GZIP; TS_HTTP_VALUE_BROTLI = HTTP_VALUE_BROTLI; + TS_HTTP_VALUE_ZSTD = HTTP_VALUE_ZSTD; TS_HTTP_VALUE_IDENTITY = HTTP_VALUE_IDENTITY; TS_HTTP_VALUE_KEEP_ALIVE = HTTP_VALUE_KEEP_ALIVE; TS_HTTP_VALUE_MAX_AGE = HTTP_VALUE_MAX_AGE; @@ -771,6 +774,7 @@ api_init() TS_HTTP_LEN_DEFLATE = HTTP_LEN_DEFLATE; TS_HTTP_LEN_GZIP = HTTP_LEN_GZIP; TS_HTTP_LEN_BROTLI = HTTP_LEN_BROTLI; + TS_HTTP_LEN_ZSTD = HTTP_LEN_ZSTD; TS_HTTP_LEN_IDENTITY = HTTP_LEN_IDENTITY; TS_HTTP_LEN_KEEP_ALIVE = HTTP_LEN_KEEP_ALIVE; TS_HTTP_LEN_MAX_AGE = HTTP_LEN_MAX_AGE; diff --git a/src/iocore/cache/HttpTransactCache.cc b/src/iocore/cache/HttpTransactCache.cc index 46514e594bd..2d3f3238404 100644 --- a/src/iocore/cache/HttpTransactCache.cc +++ b/src/iocore/cache/HttpTransactCache.cc @@ -979,60 +979,38 @@ HttpTransactCache::calculate_quality_of_accept_encoding_match(MIMEField *accept_ if (!content_field) { if (!match_accept_content_encoding("identity", accept_field, &wildcard_present, &wildcard_q, &q)) { // CE was not returned, and AE does not have identity - if (match_content_encoding(accept_field, "gzip") and match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } goto encoding_wildcard; } - // use q from identity match - } else { - // "Accept-encoding must correctly handle multiple content encoding" - // The combined quality factor is the product of all quality factors. - // (Note that there may be other possible choice, eg, min(), - // but I think multiplication is the best.) - // For example, if "content-encoding: a, b", and quality factors - // of a and b (in accept-encoding header) are q_a and q_b, resp, - // then the combined quality factor is (q_a * q_b). - // If any one of the content-encoding is not matched, - // then the q value will not be changed. - float combined_q = 1.0; + // Handle multiple content encodings - use minimum quality + float min_q = 1.0; // Start with maximum quality + bool found_match = false; + for (c_value = c_values_list.head; c_value; c_value = c_value->next) { float this_q = -1.0; if (!match_accept_content_encoding(c_value->str, accept_field, &wildcard_present, &wildcard_q, &this_q)) { goto encoding_wildcard; } - combined_q *= this_q; + if (this_q >= 0.0) { + found_match = false; + if (this_q < min_q) { + min_q = this_q; + } + } + } + if (found_match) { + q = min_q; + } else { + q = -1.0; } - q = combined_q; } encoding_wildcard: - // match the wildcard now // if ((q == -1.0) && (wildcard_present == true)) { - q = wildcard_q; - } - ///////////////////////////////////////////////////////////////////////// - // there was an Accept-Encoding, but it didn't match anything, at // - // any quality level --- if this is an identity-coded document, that's // - // still okay, but otherwise, this is just not a match at all. // - ///////////////////////////////////////////////////////////////////////// - if ((q == -1.0) && is_identity_encoding) { - if (match_content_encoding(accept_field, "gzip")) { - if (match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } else { - // always try to fetch GZIP content if we have not tried sending AE before - return -1.0f; - } - } else if (cached_accept_field && !match_content_encoding(cached_accept_field, "gzip")) { - return 0.001f; - } else { - return -1.0f; - } + return wildcard_q; } - // q = (float)-1.0; - return (q); + + return q; } /** diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index 389b98097ba..71f141abfd0 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc @@ -87,6 +87,7 @@ const char *HTTP_VALUE_COMPRESS; const char *HTTP_VALUE_DEFLATE; const char *HTTP_VALUE_GZIP; const char *HTTP_VALUE_BROTLI; +const char *HTTP_VALUE_ZSTD; const char *HTTP_VALUE_IDENTITY; const char *HTTP_VALUE_KEEP_ALIVE; const char *HTTP_VALUE_MAX_AGE; @@ -123,6 +124,7 @@ int HTTP_LEN_COMPRESS; int HTTP_LEN_DEFLATE; int HTTP_LEN_GZIP; int HTTP_LEN_BROTLI; +int HTTP_LEN_ZSTD; int HTTP_LEN_IDENTITY; int HTTP_LEN_KEEP_ALIVE; int HTTP_LEN_MAX_AGE; @@ -228,6 +230,7 @@ http_init() HTTP_VALUE_DEFLATE = hdrtoken_string_to_wks("deflate"); HTTP_VALUE_GZIP = hdrtoken_string_to_wks("gzip"); HTTP_VALUE_BROTLI = hdrtoken_string_to_wks("br"); + HTTP_VALUE_ZSTD = hdrtoken_string_to_wks("zstd"); HTTP_VALUE_IDENTITY = hdrtoken_string_to_wks("identity"); HTTP_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks("keep-alive"); HTTP_VALUE_MAX_AGE = hdrtoken_string_to_wks("max-age"); @@ -253,6 +256,7 @@ http_init() HTTP_LEN_DEFLATE = hdrtoken_wks_to_length(HTTP_VALUE_DEFLATE); HTTP_LEN_GZIP = hdrtoken_wks_to_length(HTTP_VALUE_GZIP); HTTP_LEN_BROTLI = hdrtoken_wks_to_length(HTTP_VALUE_BROTLI); + HTTP_LEN_ZSTD = hdrtoken_wks_to_length(HTTP_VALUE_ZSTD); HTTP_LEN_IDENTITY = hdrtoken_wks_to_length(HTTP_VALUE_IDENTITY); HTTP_LEN_KEEP_ALIVE = hdrtoken_wks_to_length(HTTP_VALUE_KEEP_ALIVE); HTTP_LEN_MAX_AGE = hdrtoken_wks_to_length(HTTP_VALUE_MAX_AGE); diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index 18b05f23ea6..9e46546ec1d 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc @@ -122,7 +122,10 @@ const char *const _hdrtoken_strs[] = { "Early-Data", // RFC-7932 - "br"}; + "br", + + // RFC-8878 + "zstd"}; HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {"file", HDRTOKEN_TYPE_SCHEME }, diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index 634e1bf3f73..2ce56858f47 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -164,6 +164,7 @@ const char *MIME_VALUE_COMPRESS; const char *MIME_VALUE_DEFLATE; const char *MIME_VALUE_GZIP; const char *MIME_VALUE_BROTLI; +const char *MIME_VALUE_ZSTD; const char *MIME_VALUE_IDENTITY; const char *MIME_VALUE_KEEP_ALIVE; const char *MIME_VALUE_MAX_AGE; @@ -926,6 +927,7 @@ mime_init() MIME_VALUE_DEFLATE = hdrtoken_string_to_wks("deflate"); MIME_VALUE_GZIP = hdrtoken_string_to_wks("gzip"); MIME_VALUE_BROTLI = hdrtoken_string_to_wks("br"); + MIME_VALUE_ZSTD = hdrtoken_string_to_wks("zstd"); MIME_VALUE_IDENTITY = hdrtoken_string_to_wks("identity"); MIME_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks("keep-alive"); MIME_VALUE_MAX_AGE = hdrtoken_string_to_wks("max-age"); diff --git a/src/proxy/http/HttpTransactHeaders.cc b/src/proxy/http/HttpTransactHeaders.cc index 7f730d7db15..809635e030c 100644 --- a/src/proxy/http/HttpTransactHeaders.cc +++ b/src/proxy/http/HttpTransactHeaders.cc @@ -1233,6 +1233,53 @@ HttpTransactHeaders::normalize_accept_encoding(const OverridableHttpConfigParams header->field_delete(ae_field); Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-br non-gzip Accept-Encoding"); } + } else if (normalize_ae == 4) { + // Force Accept-Encoding header to zstd or fallback to br/gzip or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd", 4); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br", 2); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip", 4); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } + } else if (normalize_ae == 5) { + // Force Accept-Encoding header to zstd,br,gzip combinations or individual algorithms or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, br, gzip", 14); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "zstd, br", 8); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, gzip", 10); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd", 4); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "br, gzip", 8); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br", 2); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip", 4); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } } else { static bool logged = false; diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index faf1e5aad0a..f6456d26e70 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -535,7 +535,7 @@ static const RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.http.allow_multi_range", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , // This defaults to a special invalid value so the HTTP transaction handling code can tell that it was not explicitly set. - {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-3]", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-5]", RECA_NULL} , // #################################################### diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index 56766955c62..b86d4f2573a 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -31,6 +31,10 @@ if(HAVE_BROTLI_ENCODE_H) target_link_libraries(traffic_layout PRIVATE brotli::brotlienc) endif() +if(HAVE_ZSTD_H) + target_link_libraries(traffic_layout PRIVATE zstd::zstd) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 33de584eff0..1672116c140 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -54,6 +54,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + // Produce output about compile time features, useful for checking how things were built static void print_feature(std::string_view name, int value, bool json, bool last = false) @@ -96,6 +100,11 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif +#if HAVE_ZSTD_H + print_feature("TS_HAS_ZSTD", 1, json); +#else + print_feature("TS_HAS_ZSTD", 0, json); +#endif #ifdef F_GETPIPE_SZ print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json); #else @@ -208,6 +217,11 @@ produce_versions(bool json) #else print_var("brotli", undef, json); #endif +#if HAVE_ZSTD_H + print_var("zstd", LBW().print("{}", ZSTD_versionString()).view(), json); +#else + print_var("zstd", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); diff --git a/tests/gold_tests/headers/normalize_ae.gold b/tests/gold_tests/headers/normalize_ae.gold index 5e44f966741..f9b2c675ae9 100644 --- a/tests/gold_tests/headers/normalize_ae.gold +++ b/tests/gold_tests/headers/normalize_ae.gold @@ -11,6 +11,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -24,6 +44,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -37,6 +77,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -50,6 +110,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -63,6 +143,92 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- X-Au-Test: www.no-oride.com ACCEPT-ENCODING MISSING - @@ -76,6 +242,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -89,6 +275,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -102,6 +308,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -115,6 +341,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -128,3 +374,89 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- diff --git a/tests/gold_tests/headers/normalize_ae.test.py b/tests/gold_tests/headers/normalize_ae.test.py index 313023150d6..9843b38dd93 100644 --- a/tests/gold_tests/headers/normalize_ae.test.py +++ b/tests/gold_tests/headers/normalize_ae.test.py @@ -40,6 +40,10 @@ server.addResponse("sessionlog.json", request_header, response_header) request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-2.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-4.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-5.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) # Define first ATS. Disable the cache to make sure each request is sent to the # origin server. @@ -65,6 +69,12 @@ def baselineTsSetup(ts): ts.Disk.remap_config.AddLine( 'map http://www.ae-3.com http://127.0.0.1:{0}'.format(server.Variables.Port) + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-4.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-5.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') baselineTsSetup(ts) @@ -124,6 +134,47 @@ def curlTail(hdrValue): tr.MakeCurlCommand(baseCurl + curlTail('gzip;q=0.3, whatever;q=0.666, br;q=0.7'), ts=ts) tr.Processes.Default.ReturnCode = 0 + # ZSTD-related tests for normalize_ae modes 4 and 5 + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('zstd') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('zstd, gzip') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('zstd, br') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('zstd, br, gzip') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('gzip, zstd, br') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('br, zstd') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('zstd;q=0.8, br;q=0.7, gzip;q=0.6') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('deflate, zstd') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('identity, zstd, compress') + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.Processes.Default.Command = baseCurl + curlTail('br, compress') + tr.Processes.Default.ReturnCode = 0 + def perTsTest(shouldWaitForUServer, ts): allAEHdrs(shouldWaitForUServer, True, ts, 'www.no-oride.com') @@ -131,6 +182,8 @@ def perTsTest(shouldWaitForUServer, ts): allAEHdrs(False, False, ts, 'www.ae-1.com') allAEHdrs(False, False, ts, 'www.ae-2.com') allAEHdrs(False, False, ts, 'www.ae-3.com') + allAEHdrs(False, False, ts, 'www.ae-4.com') + allAEHdrs(False, False, ts, 'www.ae-5.com') perTsTest(True, ts) diff --git a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py index 991d2fda32b..f83ae463fb9 100644 --- a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py +++ b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py @@ -45,6 +45,12 @@ ts.Disk.remap_config.AddLine( f"map http://www.ae-3.com http://127.0.0.1:{server.Variables.http_port}" + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-4.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-5.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache') ts.Disk.records_config.update( { @@ -53,7 +59,7 @@ 'proxy.config.http.response_via_str': 3, # the following variables could affect the results of alternate cache matching, # define them with their default values explicitly - 'proxy.config.cache.limits.http.max_alts': 5, + 'proxy.config.cache.limits.http.max_alts': 6, 'proxy.config.http.cache.ignore_accept_mismatch': 2, 'proxy.config.http.cache.ignore_accept_language_mismatch': 2, 'proxy.config.http.cache.ignore_accept_encoding_mismatch': 2, diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 57842341a59..4910af81027 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -96,7 +96,7 @@ sessions: - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header deflate would match the alternate of empty Accept-Encoding header + # load an alternate of deflate Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -110,16 +110,25 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, deflate ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Deflate-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Deflate-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress would match the alternate of empty Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -133,14 +142,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # load an alternate of gzip Accept-Encoding header - client-request: @@ -162,6 +180,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -176,7 +195,7 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress, gzip would match the alternate of gzip Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -190,14 +209,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Case 2 proxy.config.http.normalize_ae:1 @@ -322,6 +350,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -458,6 +487,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -492,6 +522,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -651,6 +682,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -686,6 +718,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, gzip ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] content: @@ -699,10 +732,6 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -722,6 +751,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, "br, gzip" ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] @@ -733,10 +763,8 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header - client-request: @@ -784,11 +812,6 @@ sessions: - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] - # NOTICE: This case should make Accept-Encoding header br, gzip;q=0.8 match - # the alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -808,6 +831,826 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 5 proxy.config.http.normalize_ae:4 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 41 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 42 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 43 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 44 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, zstd ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 45 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, br ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 46 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Accept-Encoding header zstd, br, compress, gzip would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 47 ] + - [ Accept-Encoding, "zstd, br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br, compress, gzip would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 48 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 49 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 410 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 6 proxy.config.http.normalize_ae:5 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 51 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 52 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 53 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 54 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, zstd ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 55 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Content-Encoding, br ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 56 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ Content-Encoding, gzip ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 57 ] + - [ Accept-Encoding, "zstd, compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Content-Encoding, zstd ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Br-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 58 ] + - [ Accept-Encoding, "zstd, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ Content-Encoding, zstd ] + - [ X-Response-Identifier, Zstd-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 59 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "br, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ Content-Encoding, br ] + - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Br, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 510 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, br would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 511 ] + - [ Accept-Encoding, "compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 512 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header zstd;q=1.1 would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 513 ] + - [ Accept-Encoding, "zstd;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br;q=1.1 would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 514 ] + - [ Accept-Encoding, "br;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 515 ] + - [ Accept-Encoding, "zstd, br;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 516 ] + - [ Accept-Encoding, "zstd, gzip;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 6745b65f0ae..c82efb26047 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -1,6 +1,6 @@ > GET ``/obj0 HTTP/1.1 -> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -32,14 +32,21 @@ < Content-Type: text/javascript < Content-Length: 1049 === +> GET ``/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== > GET ``/obj1 HTTP/1.1 -> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -48,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -64,9 +71,16 @@ < Content-Type: text/javascript < Content-Length: 1049 === +> GET ``/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== > GET ``/obj2 HTTP/1.1 -> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br @@ -80,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -98,6 +112,140 @@ < Content-Type: text/javascript < Content-Length: 1049 === +> GET ``/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET ``/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET ``/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 72 +=== +> GET ``/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET ``/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET ``/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET ``/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET ``/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 72 +=== +> GET ``/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 47 +=== +> GET ``/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET ``/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET ``/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET ``/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 72 +=== +> GET ``/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 47 +=== +> GET ``/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET ``/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666 > Accept-Encoding: gzip;q=0.666 @@ -105,7 +253,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -114,7 +262,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -123,7 +271,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -132,23 +280,23 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === -> GET ``obj0 HTTP/1.1 +> GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 > Accept-Encoding: gzip;q=0.0 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``obj0 HTTP/1.1 +> GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=-0.1 > Accept-Encoding: gzip;q=-0.1 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -157,7 +305,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > GET ``/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -175,7 +323,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === > POST ``/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -184,5 +332,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 72 === diff --git a/tests/gold_tests/pluginTest/compress/compress.test.py b/tests/gold_tests/pluginTest/compress/compress.test.py index fd34c6f7524..47f13c6b2a2 100644 --- a/tests/gold_tests/pluginTest/compress/compress.test.py +++ b/tests/gold_tests/pluginTest/compress/compress.test.py @@ -25,7 +25,8 @@ # Skip if plugins not present. # Test.SkipUnless( - Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI')) + Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI'), + Condition.HasATSFeature('TS_HAS_ZSTD')) server = Test.MakeOriginServer("server", options={'--load': f'{Test.TestDirectory}/compress_observer.py'}) @@ -43,7 +44,7 @@ "timestamp": "1469733493.993", "body": body } -for i in range(3): +for i in range(6): # add request/response to the server dictionary request_header = {"headers": f"GET /obj{i} HTTP/1.1\r\nHost: just.any.thing\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header, response_header) @@ -89,6 +90,7 @@ def curl_post(ts, idx, encodingList, out_path): ts.Setup.Copy("compress.config") ts.Setup.Copy("compress2.config") +ts.Setup.Copy("compress3.config") ts.Disk.remap_config.AddLine( f'map http://ae-0/ http://127.0.0.1:{server.Variables.Port}/' + @@ -103,7 +105,16 @@ def curl_post(ts, idx, encodingList, out_path): f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') ts.Disk.remap_config.AddLine( f'map http://ae-3/ http://127.0.0.1:{server.Variables.Port}/' + - f' @plugin=compress.so @pparam={Test.RunDirectory}/compress.config') + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-4/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-5/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') out_path_counter = 0 @@ -119,12 +130,12 @@ def get_out_path(): def get_verify_command(out_path, decrompressor): - return f"{decrompressor} -c {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" + return f"{decrompressor} {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" -for i in range(3): +for i in range(6): - tr = Test.AddTestRun(f'gzip, deflate, sdch, br: {i}') + tr = Test.AddTestRun(f'gzip, deflate, sdch, br, zstd: {i}') if (waitForTs): tr.Processes.Default.StartBefore(ts) waitForTs = False @@ -133,15 +144,21 @@ def get_verify_command(out_path, decrompressor): waitForServer = False tr.Processes.Default.ReturnCode = 0 out_path = get_out_path() - tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br', out_path), ts=ts) - tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br: {i}') + tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br, zstd', out_path), ts=ts) + tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br, zstd: {i}') tr.ReturnCode = 0 if i == 0: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") elif i == 1: - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") elif i == 2: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 3: + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 4: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + elif i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") tr = Test.AddTestRun(f'gzip: {i}') tr.Processes.Default.ReturnCode = 0 @@ -149,7 +166,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, i, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip: {i}') tr.ReturnCode = 0 - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun(f'br: {i}') tr.Processes.Default.ReturnCode = 0 @@ -160,7 +177,7 @@ def get_verify_command(out_path, decrompressor): if i == 1: tr.Processes.Default.Command = f"diff {out_path} {orig_path}" else: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun(f'deflate: {i}') tr.Processes.Default.ReturnCode = 0 @@ -170,6 +187,17 @@ def get_verify_command(out_path, decrompressor): tr.ReturnCode = 0 tr.Processes.Default.Command = f"diff {out_path} {orig_path}" + tr = Test.AddTestRun(f'zstd: {i}') + tr.Processes.Default.ReturnCode = 0 + out_path = get_out_path() + tr.Processes.Default.Command = curl(ts, i, "zstd", out_path) + tr = Test.AddTestRun(f'verify zstd: {i}') + tr.ReturnCode = 0 + if i == 4 or i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + else: + tr.Processes.Default.Command = get_verify_command(out_path, "cat") + # Test Accept-Encoding normalization. tr = Test.AddTestRun() @@ -178,7 +206,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -186,7 +214,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666x", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666x') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -194,7 +222,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=#0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=#0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -202,7 +230,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip; Q = 0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip; Q = 0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -218,7 +246,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=-0.1", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=-0.1') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -226,7 +254,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -234,7 +262,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, " br ; q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify br ; q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") +tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -242,7 +270,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666 , ", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666 , ') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # post tr = Test.AddTestRun() @@ -251,7 +279,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl_post(ts, 3, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip post') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # compress_long.log contains all the output from the curl commands. The tr removes the carriage returns for easier # readability. Curl seems to have a bug, where it will neglect to output an end of line before outputting an HTTP diff --git a/tests/gold_tests/pluginTest/compress/compress2.config b/tests/gold_tests/pluginTest/compress/compress2.config index c808d87fcfd..8fa674c687b 100644 --- a/tests/gold_tests/pluginTest/compress/compress2.config +++ b/tests/gold_tests/pluginTest/compress/compress2.config @@ -5,3 +5,6 @@ compressible-content-type application/x-javascript* compressible-content-type application/javascript* compressible-content-type application/json* supported-algorithms gzip, br +gzip-compression-level 7 +brotli-compression-level 8 +brotli-lgwin 18 diff --git a/tests/gold_tests/pluginTest/compress/compress3.config b/tests/gold_tests/pluginTest/compress/compress3.config new file mode 100644 index 00000000000..54ff0eed2d8 --- /dev/null +++ b/tests/gold_tests/pluginTest/compress/compress3.config @@ -0,0 +1,11 @@ +cache true +remove-accept-encoding true +compressible-content-type text/* +compressible-content-type application/x-javascript* +compressible-content-type application/javascript* +compressible-content-type application/json* +supported-algorithms zstd, br, gzip +gzip-compression-level 5 +brotli-compression-level 7 +brotli-lgwin 17 +zstd-compression-level 10 diff --git a/tests/gold_tests/pluginTest/compress/compress_userver.gold b/tests/gold_tests/pluginTest/compress/compress_userver.gold index 1ddc6a4f3b5..693bd03905e 100644 --- a/tests/gold_tests/pluginTest/compress/compress_userver.gold +++ b/tests/gold_tests/pluginTest/compress/compress_userver.gold @@ -1,15 +1,33 @@ -0/gzip, deflate, sdch, br +0/gzip, deflate, sdch, br, zstd 0/gzip 0/br 0/deflate -1/gzip, deflate, sdch, br +0/zstd +1/gzip, deflate, sdch, br, zstd 1/gzip 1/br 1/deflate -2/gzip, deflate, sdch, br +1/zstd +2/gzip, deflate, sdch, br, zstd 2/gzip 2/br 2/deflate +2/zstd +3/gzip, deflate, sdch, br, zstd +3/gzip +3/br +3/deflate +3/zstd +4/gzip, deflate, sdch, br, zstd +4/gzip +4/br +4/deflate +4/zstd +5/gzip, deflate, sdch, br, zstd +5/gzip +5/br +5/deflate +5/zstd 0/gzip;q=0.666 0/gzip;q=0.666x 0/gzip;q=#0.666