From b22cbee7c65e89d4e5dacd62d1821658cada1414 Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 01:30:58 +0100 Subject: [PATCH 01/29] Add comprehensive zstd compression support to ATS This patch adds full support for the zstd (Zstandard) compression algorithm throughout Apache Traffic Server, including build system integration, compression plugin support, Accept-Encoding header normalization, and comprehensive test coverage. Build system and dependencies: - Add CMake support for finding zstd library with new Findzstd.cmake - Update Docker build files to include libzstd-dev package - Add TS_HAS_ZSTD feature flag for conditional compilation Core compression support: - Extend compress plugin to support zstd compression alongside gzip and brotli - Add zstd stream handling structures and functions - Update compression configuration to include zstd in supported algorithms list - Add zstd compression type constant and related infrastructure Accept-Encoding header normalization: - Extend proxy.config.http.normalize_ae configuration to support values 4 and 5 for zstd normalization - Add zstd support to header normalization logic with proper priority handling (zstd > br > gzip) - Update HTTP transaction cache matching to handle zstd encoding - Add zstd token to header parsing infrastructure API and infrastructure: - Add TS_HTTP_VALUE_ZSTD and TS_HTTP_LEN_ZSTD constants - Update MIME field handling to recognize zstd encoding - Add zstd support to traffic_layout feature detection Test coverage: - Expand compress plugin tests to cover zstd compression scenarios - Add zstd test cases to Accept-Encoding normalization tests - Update golden files to include zstd compression test results - Add new compress3.config for zstd-specific plugin configuration - Test all combinations of zstd, br, and gzip in various scenarios The implementation follows RFC 8878 standards for zstd compression and maintains backward compatibility with existing gzip and brotli compression functionality. All tests pass and the feature is properly integrated with the existing caching and content negotiation mechanisms. --- CMakeLists.txt | 5 + ci/docker/deb/Dockerfile | 3 +- ci/docker/yum/Dockerfile | 2 +- cmake/Findzstd.cmake | 53 ++ contrib/docker/ubuntu/noble/Dockerfile | 1 + doc/admin-guide/files/records.yaml.en.rst | 6 +- doc/admin-guide/plugins/compress.en.rst | 37 +- .../http-headers/header-functions.en.rst | 3 + doc/release-notes/whats-new.en.rst | 1 + include/proxy/hdrs/HTTP.h | 1 + include/proxy/hdrs/MIME.h | 2 + include/ts/apidefs.h.in | 2 + include/tscore/ink_config.h.cmake.in | 2 + plugins/compress/CMakeLists.txt | 5 + plugins/compress/README | 2 +- plugins/compress/compress.cc | 183 +++- plugins/compress/configuration.cc | 8 +- plugins/compress/configuration.h | 3 +- plugins/compress/misc.cc | 11 +- plugins/compress/misc.h | 22 +- plugins/compress/sample.compress.config | 2 +- src/api/InkAPIInternal.cc | 3 + src/proxy/hdrs/HTTP.cc | 2 + src/proxy/hdrs/HdrToken.cc | 5 +- src/proxy/hdrs/MIME.cc | 2 + src/proxy/http/HttpTransactHeaders.cc | 47 + src/records/RecordsConfig.cc | 2 +- src/traffic_layout/info.cc | 5 + tests/gold_tests/headers/normalize_ae.gold | 332 +++++++ tests/gold_tests/headers/normalize_ae.test.py | 53 ++ .../normalized_ae_match_vary_cache.test.py | 6 + ...malized_ae_varied_transactions.replay.yaml | 830 ++++++++++++++++++ .../pluginTest/compress/compress.gold | 224 ++++- .../pluginTest/compress/compress.test.py | 72 +- .../pluginTest/compress/compress3.config | 7 + .../pluginTest/compress/compress_userver.gold | 24 +- 36 files changed, 1881 insertions(+), 87 deletions(-) create mode 100644 cmake/Findzstd.cmake create mode 100644 tests/gold_tests/pluginTest/compress/compress3.config diff --git a/CMakeLists.txt b/CMakeLists.txt index 010d74fc708..df0f61910f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,11 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) +find_package(ZSTD) +if(ZSTD_FOUND) + set(HAVE_ZSTD_H TRUE) +endif() + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 337356ca8c3..4e1398d15b7 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -55,7 +55,8 @@ RUN apt-get update; apt-get -y dist-upgrade; \ apt-get -y install libssl-dev libexpat1-dev libpcre3-dev libcap-dev \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ - libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev; \ + libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ + libzstd-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 85e9a7add64..5a160fb24ff 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \ diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake new file mode 100644 index 00000000000..d2a86d0132e --- /dev/null +++ b/cmake/Findzstd.cmake @@ -0,0 +1,53 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +# Findzstd.cmake +# +# This will define the following variables +# +# ZSTD_FOUND +# ZSTD_LIBRARY +# ZSTD_INCLUDE_DIRS +# +# and the following imported target +# +# zstd::zstd +# + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) + +mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +include(SelectLibraryConfigurations) +select_library_configurations(ZSTD) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +if(ZSTD_FOUND) + set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") +endif() + +if(ZSTD_FOUND AND NOT TARGET zstd::zstd) + add_library(zstd::zstd INTERFACE IMPORTED) + target_include_directories(zstd::zstd INTERFACE ${ZSTD_INCLUDE_DIRS}) + target_link_libraries(zstd::zstd INTERFACE "${ZSTD_LIBRARY}") +endif() diff --git a/contrib/docker/ubuntu/noble/Dockerfile b/contrib/docker/ubuntu/noble/Dockerfile index 3325e86da4c..b9181dc998f 100644 --- a/contrib/docker/ubuntu/noble/Dockerfile +++ b/contrib/docker/ubuntu/noble/Dockerfile @@ -48,6 +48,7 @@ RUN apt update \ libpcre3-dev \ hwloc \ libbrotli-dev \ + libzstd-dev \ luajit \ libcap-dev \ libmagick++-dev \ diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index ed5f987388f..bf8e8ae5fa2 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2076,10 +2076,14 @@ Proxy User Variables normalize as for value ``1`` ``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE** normalize as for value ``2`` + ``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE** + normalize as for value ``2`` + ``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE** + normalize as for value ``4`` ===== ====================================================================== This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``). - Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli). + Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard). Security ======== diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index 81dec81dc8f..686445008fc 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -192,12 +192,24 @@ supported-algorithms Provides the compression algorithms that are supported, a comma separate list of values. This will allow |TS| to selectively support ``gzip``, ``deflate``, -and brotli (``br``) compression. The default is ``gzip``. Multiple algorithms can -be selected using ',' delimiter, for instance, ``supported-algorithms -deflate,gzip,br``. Note that this list must **not** contain any white-spaces! +brotli (``br``), and zstd (``zstd``) compression. The default is ``gzip``. +Multiple algorithms can be selected using ',' delimiter, for instance, +``supported-algorithms deflate,gzip,br,zstd``. Note that this list must **not** +contain any white-spaces! + +============== ================================================================= +Algorithm Description +============== ================================================================= +gzip Standard gzip compression (default, widely supported) +deflate Deflate compression (RFC 1951) +br Brotli compression (modern, efficient) +zstd Zstandard compression (fast, high compression ratio) +============== ================================================================= Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will -be considered, and if it is ``2``, only br or gzip will be considered. +be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``, +only zstd, br, or gzip will be considered, and if it is ``5``, all combinations +of zstd, br, and gzip will be considered. Examples ======== @@ -239,6 +251,23 @@ might create a configuration with the following options:: flush true supported-algorithms br,gzip + # Supports zstd compression for high efficiency + [zstd.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + compressible-content-type application/javascript + flush true + supported-algorithms zstd,gzip + + # Supports all compression algorithms + [all.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + flush true + supported-algorithms zstd,br,gzip,deflate + # This origin does it all [bar.example.com] enabled false diff --git a/doc/developer-guide/plugins/http-headers/header-functions.en.rst b/doc/developer-guide/plugins/http-headers/header-functions.en.rst index 9a27115434f..47d07459a2c 100644 --- a/doc/developer-guide/plugins/http-headers/header-functions.en.rst +++ b/doc/developer-guide/plugins/http-headers/header-functions.en.rst @@ -92,6 +92,9 @@ headers. ``TS_HTTP_VALUE_GZIP`` "gzip" +``TS_HTTP_VALUE_ZSTD`` + "zstd" + ``TS_HTTP_VALUE_IDENTITY`` "identity" diff --git a/doc/release-notes/whats-new.en.rst b/doc/release-notes/whats-new.en.rst index c2a41318129..ac52126e24c 100644 --- a/doc/release-notes/whats-new.en.rst +++ b/doc/release-notes/whats-new.en.rst @@ -81,6 +81,7 @@ Plugins * xdebug - ``--enable`` option to selectively enable features has been added * system_stats - Stats about memory have been added * slice plugin - This plugin was promoted to stable. +* compress plugin - Added support for Zstandard (zstd) compression algorithm. JSON-RPC ^^^^^^^^ diff --git a/include/proxy/hdrs/HTTP.h b/include/proxy/hdrs/HTTP.h index eeecba31d74..8759fcc09ab 100644 --- a/include/proxy/hdrs/HTTP.h +++ b/include/proxy/hdrs/HTTP.h @@ -368,6 +368,7 @@ extern c_str_view HTTP_VALUE_COMPRESS; extern c_str_view HTTP_VALUE_DEFLATE; extern c_str_view HTTP_VALUE_GZIP; extern c_str_view HTTP_VALUE_BROTLI; +extern c_str_view HTTP_VALUE_ZSTD; extern c_str_view HTTP_VALUE_IDENTITY; extern c_str_view HTTP_VALUE_KEEP_ALIVE; extern c_str_view HTTP_VALUE_MAX_AGE; diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index dcea4893594..4bed80dc13c 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -602,6 +602,8 @@ extern c_str_view MIME_VALUE_COMPRESS; extern c_str_view MIME_VALUE_DEFLATE; extern c_str_view MIME_VALUE_GZIP; extern c_str_view MIME_VALUE_BROTLI; +extern c_str_view MIME_VALUE_ZSTD; + extern c_str_view MIME_VALUE_IDENTITY; extern c_str_view MIME_VALUE_KEEP_ALIVE; extern c_str_view MIME_VALUE_MAX_AGE; diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index bd37127970d..05049a6d95b 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -1351,6 +1351,7 @@ extern const char *TS_HTTP_VALUE_COMPRESS; extern const char *TS_HTTP_VALUE_DEFLATE; extern const char *TS_HTTP_VALUE_GZIP; extern const char *TS_HTTP_VALUE_BROTLI; +extern const char *TS_HTTP_VALUE_ZSTD; extern const char *TS_HTTP_VALUE_IDENTITY; extern const char *TS_HTTP_VALUE_KEEP_ALIVE; extern const char *TS_HTTP_VALUE_MAX_AGE; @@ -1375,6 +1376,7 @@ extern int TS_HTTP_LEN_COMPRESS; extern int TS_HTTP_LEN_DEFLATE; extern int TS_HTTP_LEN_GZIP; extern int TS_HTTP_LEN_BROTLI; +extern int TS_HTTP_LEN_ZSTD; extern int TS_HTTP_LEN_IDENTITY; extern int TS_HTTP_LEN_KEEP_ALIVE; extern int TS_HTTP_LEN_MAX_AGE; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index fcbd98a5822..2d515338c8a 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -187,3 +187,5 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine YAMLCPP_LIB_VERSION "@YAMLCPP_LIB_VERSION@" #cmakedefine01 TS_HAS_CRIPTS + +#cmakedefine HAVE_ZSTD_H 1 diff --git a/plugins/compress/CMakeLists.txt b/plugins/compress/CMakeLists.txt index f630bf0d1c9..65b24ce9558 100644 --- a/plugins/compress/CMakeLists.txt +++ b/plugins/compress/CMakeLists.txt @@ -20,5 +20,10 @@ target_link_libraries(compress PRIVATE libswoc::libswoc) if(HAVE_BROTLI_ENCODE_H) target_link_libraries(compress PRIVATE brotli::brotlienc) endif() + +if(HAVE_ZSTD_H) + target_link_libraries(compress PRIVATE zstd::zstd) +endif() + verify_global_plugin(compress) verify_remap_plugin(compress) diff --git a/plugins/compress/README b/plugins/compress/README index 759add28e03..e89070f9e54 100644 --- a/plugins/compress/README +++ b/plugins/compress/README @@ -1,7 +1,7 @@ What this plugin does: ===================== -This plugin compresses responses, via gzip or brotli, whichever is applicable +This plugin compresses responses, via gzip, deflate, brotli, or zstd (Zstandard), whichever is applicable it can compress origin responses as well as cached responses installation: diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 44e8235a0d0..b181d675722 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -23,6 +23,9 @@ #include #include +#if HAVE_ZSTD_H +#include +#endif #include "ts/apidefs.h" #include "tscore/ink_config.h" @@ -71,6 +74,10 @@ const int BROTLI_COMPRESSION_LEVEL = 6; const int BROTLI_LGW = 16; #endif +#if HAVE_ZSTD_H +const int ZSTD_COMPRESSION_LEVEL = 6; +#endif + static const char *global_hidden_header_name = nullptr; static TSMutex compress_config_mutex = nullptr; @@ -136,6 +143,13 @@ handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) } } // namespace +// Forward declarations for ZSTD compression functions +#if HAVE_ZSTD_H +static void zstd_compress_init(Data *data); +static void zstd_compress_finish(Data *data); +static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length); +#endif + static Data * data_alloc(int compression_type, int compression_algorithms) { @@ -195,6 +209,22 @@ data_alloc(int compression_type, int compression_algorithms) data->bstrm.avail_out = 0; data->bstrm.total_out = 0; } +#endif +#if HAVE_ZSTD_H + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.next_in = nullptr; + data->zstrm_zstd.avail_in = 0; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.next_out = nullptr; + data->zstrm_zstd.avail_out = 0; + data->zstrm_zstd.total_out = 0; + if (compression_type & COMPRESSION_TYPE_ZSTD) { + debug("zstd compression. Create Zstd Compression Context."); + data->zstrm_zstd.cctx = ZSTD_createCCtx(); + if (!data->zstrm_zstd.cctx) { + fatal("Zstd Compression Context Creation Failed"); + } + } #endif return data; } @@ -216,6 +246,11 @@ data_destroy(Data *data) #if HAVE_BROTLI_ENCODE_H BrotliEncoderDestroyInstance(data->bstrm.br); #endif +#if HAVE_ZSTD_H + if (data->zstrm_zstd.cctx) { + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + } +#endif TSfree(data); } @@ -228,7 +263,10 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty const char *value = nullptr; int value_len = 0; // Delete Content-Encoding if present??? - if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { + if (compression_type & COMPRESSION_TYPE_ZSTD && (algorithm & ALGORITHM_ZSTD)) { + value = TS_HTTP_VALUE_ZSTD; + value_len = TS_HTTP_LEN_ZSTD; + } else if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { value = TS_HTTP_VALUE_BROTLI; value_len = TS_HTTP_LEN_BROTLI; } else if (compression_type & COMPRESSION_TYPE_GZIP && (algorithm & ALGORITHM_GZIP)) { @@ -240,7 +278,6 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty } if (value_len == 0) { - error("no need to add Content-Encoding header"); return TS_SUCCESS; } @@ -361,6 +398,16 @@ compress_transform_init(TSCont contp, Data *data) data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX); } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD) { + zstd_compress_init(data); + if (!data->zstrm_zstd.cctx) { + TSError("Failed to create Zstandard compression context"); + return; + } + } +#endif + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); } @@ -466,6 +513,112 @@ brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_l } #endif +#if HAVE_ZSTD_H +static void +zstd_compress_init(Data *data) +{ + if (!data->zstrm_zstd.cctx) { + error("Failed to initialize Zstd compression context"); + return; + } + + // Set compression level + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, ZSTD_COMPRESSION_LEVEL); + if (ZSTD_isError(result)) { + error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); + return; + } + + // Enable checksum for data integrity + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(result)) { + error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); + return; + } + + debug("zstd compression context initialized with level %d", ZSTD_COMPRESSION_LEVEL); +} + +static void +zstd_compress_finish(Data *data) +{ + if (data->state == transform_state_output) { + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->state = transform_state_finished; + + // Finalize the zstd stream + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); + + if (ZSTD_isError(remaining)) { + error("zstd compression finish failed: %s", ZSTD_getErrorName(remaining)); + break; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (remaining == 0) { /* compression finished */ + break; + } + } + + debug("zstd-transform: Finished zstd compression"); + log_compression_ratio(data->zstrm_zstd.total_in, data->downstream_length); + } +} + +static void +zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + // Set up input buffer for zstd streaming + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; + data->zstrm_zstd.total_in += upstream_length; + + while (input.pos < input.size) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + // Set up output buffer for zstd streaming + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + // Compress the data using streaming API + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, ZSTD_e_continue); + + if (ZSTD_isError(result)) { + error("Zstd compression failed: %s", ZSTD_getErrorName(result)); + return; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + // If we have output space but no more input was consumed, break to avoid infinite loop + if (output.pos == 0 && input.pos < input.size) { + error("zstd-transform: no progress made in compression"); + break; + } + } +} +#endif + static void compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) { @@ -488,8 +641,13 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) upstream_length = amount; } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { + zstd_compress_one(data, upstream_buffer, upstream_length); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { brotli_transform_one(data, upstream_buffer, upstream_length); } else #endif @@ -575,8 +733,14 @@ brotli_transform_finish(Data *data) static void compress_transform_finish(Data *data) { +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { + zstd_compress_finish(data); + debug("compress_transform_finish: zstd compression finish"); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { brotli_transform_finish(data); debug("compress_transform_finish: brotli compression finish"); } else @@ -787,7 +951,14 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration continue; } - if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { + info("Accept-Encoding value [%.*s]", len, value); + + if (strncasecmp(value, "zstd", sizeof("zstd") - 1) == 0) { + if (*algorithms & ALGORITHM_ZSTD) { + compression_acceptable = 1; + } + *compress_type |= COMPRESSION_TYPE_ZSTD; + } else if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { if (*algorithms & ALGORITHM_BROTLI) { compression_acceptable = 1; } diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index dd95952f97e..41b35964eac 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -229,6 +229,12 @@ HostConfiguration::add_compression_algorithms(string &line) string token = extractFirstToken(line, isCommaOrSpace); if (token.empty()) { break; + } else if (token == "zstd") { +#ifdef HAVE_ZSTD_H + compression_algorithms_ |= ALGORITHM_ZSTD; +#else + error("supported-algorithms: zstd support not compiled in."); +#endif } else if (token == "br") { #ifdef HAVE_BROTLI_ENCODE_H compression_algorithms_ |= ALGORITHM_BROTLI; @@ -240,7 +246,7 @@ HostConfiguration::add_compression_algorithms(string &line) } else if (token == "deflate") { compression_algorithms_ |= ALGORITHM_DEFLATE; } else { - error("Unknown compression type. Supported compression-algorithms ."); + error("Unknown compression type. Supported compression-algorithms ."); } } } diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index 22b2173c590..2140d5a3580 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -38,7 +38,8 @@ enum CompressionAlgorithm { ALGORITHM_DEFAULT = 0, ALGORITHM_DEFLATE = 1, ALGORITHM_GZIP = 2, - ALGORITHM_BROTLI = 4 // For bit manipulations + ALGORITHM_BROTLI = 4, + ALGORITHM_ZSTD = 8 }; enum class RangeRequestCtrl : int { diff --git a/plugins/compress/misc.cc b/plugins/compress/misc.cc index a1b33b79cce..13d47e7aee0 100644 --- a/plugins/compress/misc.cc +++ b/plugins/compress/misc.cc @@ -84,8 +84,9 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo bool deflate = false; bool gzip = false; bool br = false; + bool zstd = false; // remove the accept encoding field(s), - // while finding out if gzip or deflate is supported. + // while finding out if gzip, brotli, deflate, or zstandard are supported. while (field) { int val_len; const char *values_ = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, -1, &val_len); @@ -100,6 +101,8 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo br = true; } else if (strcasecmp("deflate", next) == 0) { deflate = true; + } else if (strcasecmp("zstd", next) == 0) { + zstd = true; } } } @@ -111,9 +114,13 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo } // append a new accept-encoding field in the header - if (deflate || gzip || br) { + if (deflate || gzip || br || zstd) { TSMimeHdrFieldCreate(reqp, hdr_loc, &field); TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + if (zstd) { + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "zstd", strlen("zstd")); + info("normalized accept encoding to zstd"); + } if (br) { TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "br", strlen("br")); info("normalized accept encoding to br"); diff --git a/plugins/compress/misc.h b/plugins/compress/misc.h index 9491271df36..4a3da5d99a5 100644 --- a/plugins/compress/misc.h +++ b/plugins/compress/misc.h @@ -32,6 +32,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + #include "configuration.h" // zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html @@ -44,7 +48,8 @@ enum CompressionType { COMPRESSION_TYPE_DEFAULT = 0, COMPRESSION_TYPE_DEFLATE = 1, COMPRESSION_TYPE_GZIP = 2, - COMPRESSION_TYPE_BROTLI = 4 + COMPRESSION_TYPE_BROTLI = 4, + COMPRESSION_TYPE_ZSTD = 8, }; // this one is used to rename the accept encoding header @@ -70,6 +75,18 @@ using b_stream = struct { }; #endif +#if HAVE_ZSTD_H +using zstd_stream = struct { + ZSTD_CCtx *cctx; + const void *next_in; + size_t avail_in; + void *next_out; + size_t avail_out; + size_t total_in; + size_t total_out; +}; +#endif + using Data = struct { TSHttpTxn txn; Gzip::HostConfiguration *hc; @@ -84,6 +101,9 @@ using Data = struct { #if HAVE_BROTLI_ENCODE_H b_stream bstrm; #endif +#if HAVE_ZSTD_H + zstd_stream zstrm_zstd; +#endif }; voidpf gzip_alloc(voidpf opaque, uInt items, uInt size); diff --git a/plugins/compress/sample.compress.config b/plugins/compress/sample.compress.config index b1431a97794..8b0eaaf5be8 100644 --- a/plugins/compress/sample.compress.config +++ b/plugins/compress/sample.compress.config @@ -56,7 +56,7 @@ allow !*/bla* minimum-content-length 1024 #supported algorithms -supported-algorithms br,gzip +supported-algorithms br,gzip,zstd #override the global configuration for a host. #www.foo.nl does NOT inherit anything diff --git a/src/api/InkAPIInternal.cc b/src/api/InkAPIInternal.cc index 85f00287c52..9a4bb620ab1 100644 --- a/src/api/InkAPIInternal.cc +++ b/src/api/InkAPIInternal.cc @@ -232,6 +232,7 @@ const char *TS_HTTP_VALUE_COMPRESS; const char *TS_HTTP_VALUE_DEFLATE; const char *TS_HTTP_VALUE_GZIP; const char *TS_HTTP_VALUE_BROTLI; +const char *TS_HTTP_VALUE_ZSTD; const char *TS_HTTP_VALUE_IDENTITY; const char *TS_HTTP_VALUE_KEEP_ALIVE; const char *TS_HTTP_VALUE_MAX_AGE; @@ -256,6 +257,7 @@ int TS_HTTP_LEN_COMPRESS; int TS_HTTP_LEN_DEFLATE; int TS_HTTP_LEN_GZIP; int TS_HTTP_LEN_BROTLI; +int TS_HTTP_LEN_ZSTD; int TS_HTTP_LEN_IDENTITY; int TS_HTTP_LEN_KEEP_ALIVE; int TS_HTTP_LEN_MAX_AGE; @@ -748,6 +750,7 @@ api_init() TS_HTTP_VALUE_DEFLATE = HTTP_VALUE_DEFLATE.c_str(); TS_HTTP_VALUE_GZIP = HTTP_VALUE_GZIP.c_str(); TS_HTTP_VALUE_BROTLI = HTTP_VALUE_BROTLI.c_str(); + TS_HTTP_VALUE_ZSTD = HTTP_VALUE_ZSTD.c_str(); TS_HTTP_VALUE_IDENTITY = HTTP_VALUE_IDENTITY.c_str(); TS_HTTP_VALUE_KEEP_ALIVE = HTTP_VALUE_KEEP_ALIVE.c_str(); TS_HTTP_VALUE_MAX_AGE = HTTP_VALUE_MAX_AGE.c_str(); diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index a6c41c5f4a3..5f80ceab54f 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc @@ -78,6 +78,7 @@ c_str_view HTTP_VALUE_COMPRESS; c_str_view HTTP_VALUE_DEFLATE; c_str_view HTTP_VALUE_GZIP; c_str_view HTTP_VALUE_BROTLI; +c_str_view HTTP_VALUE_ZSTD; c_str_view HTTP_VALUE_IDENTITY; c_str_view HTTP_VALUE_KEEP_ALIVE; c_str_view HTTP_VALUE_MAX_AGE; @@ -183,6 +184,7 @@ http_init() HTTP_VALUE_DEFLATE = hdrtoken_string_to_wks_sv("deflate"); HTTP_VALUE_GZIP = hdrtoken_string_to_wks_sv("gzip"); HTTP_VALUE_BROTLI = hdrtoken_string_to_wks_sv("br"); + HTTP_VALUE_ZSTD = hdrtoken_string_to_wks_sv("zstd"); HTTP_VALUE_IDENTITY = hdrtoken_string_to_wks_sv("identity"); HTTP_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks_sv("keep-alive"); HTTP_VALUE_MAX_AGE = hdrtoken_string_to_wks_sv("max-age"); diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index 3fde664e1c3..afb81e82ad9 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc @@ -122,7 +122,10 @@ const char *const _hdrtoken_strs[] = { "Early-Data", // RFC-7932 - "br"}; + "br", + + // RFC-8878 + "zstd"}; HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {"file", HdrTokenType::SCHEME }, diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index f46fcdbd317..6a6234fdd86 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -164,6 +164,7 @@ c_str_view MIME_VALUE_COMPRESS; c_str_view MIME_VALUE_DEFLATE; c_str_view MIME_VALUE_GZIP; c_str_view MIME_VALUE_BROTLI; +c_str_view MIME_VALUE_ZSTD; c_str_view MIME_VALUE_IDENTITY; c_str_view MIME_VALUE_KEEP_ALIVE; c_str_view MIME_VALUE_MAX_AGE; @@ -766,6 +767,7 @@ mime_init() MIME_VALUE_DEFLATE = hdrtoken_string_to_wks_sv("deflate"); MIME_VALUE_GZIP = hdrtoken_string_to_wks_sv("gzip"); MIME_VALUE_BROTLI = hdrtoken_string_to_wks_sv("br"); + MIME_VALUE_ZSTD = hdrtoken_string_to_wks_sv("zstd"); MIME_VALUE_IDENTITY = hdrtoken_string_to_wks_sv("identity"); MIME_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks_sv("keep-alive"); MIME_VALUE_MAX_AGE = hdrtoken_string_to_wks_sv("max-age"); diff --git a/src/proxy/http/HttpTransactHeaders.cc b/src/proxy/http/HttpTransactHeaders.cc index 4a26790675e..a7ecee2eb32 100644 --- a/src/proxy/http/HttpTransactHeaders.cc +++ b/src/proxy/http/HttpTransactHeaders.cc @@ -1241,6 +1241,53 @@ HttpTransactHeaders::normalize_accept_encoding(const OverridableHttpConfigParams header->field_delete(ae_field); Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-br non-gzip Accept-Encoding"); } + } else if (normalize_ae == 4) { + // Force Accept-Encoding header to zstd or fallback to br/gzip or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } + } else if (normalize_ae == 5) { + // Force Accept-Encoding header to zstd,br,gzip combinations or individual algorithms or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, br, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "zstd, br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "br, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } } else { static bool logged = false; diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index 2772f23fc49..554eb951a65 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -535,7 +535,7 @@ static const RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.http.allow_multi_range", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , // This defaults to a special invalid value so the HTTP transaction handling code can tell that it was not explicitly set. - {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-3]", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-5]", RECA_NULL} , // #################################################### diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 62732e09473..541985112cc 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -91,6 +91,11 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif +#if HAVE_ZSTD_H + print_feature("TS_HAS_ZSTD", 1, json); +#else + print_feature("TS_HAS_ZSTD", 0, json); +#endif #ifdef F_GETPIPE_SZ print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json); #else diff --git a/tests/gold_tests/headers/normalize_ae.gold b/tests/gold_tests/headers/normalize_ae.gold index 5e44f966741..f9b2c675ae9 100644 --- a/tests/gold_tests/headers/normalize_ae.gold +++ b/tests/gold_tests/headers/normalize_ae.gold @@ -11,6 +11,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -24,6 +44,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -37,6 +77,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -50,6 +110,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -63,6 +143,92 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- X-Au-Test: www.no-oride.com ACCEPT-ENCODING MISSING - @@ -76,6 +242,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -89,6 +275,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -102,6 +308,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -115,6 +341,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -128,3 +374,89 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- diff --git a/tests/gold_tests/headers/normalize_ae.test.py b/tests/gold_tests/headers/normalize_ae.test.py index 313023150d6..ca38755bfe9 100644 --- a/tests/gold_tests/headers/normalize_ae.test.py +++ b/tests/gold_tests/headers/normalize_ae.test.py @@ -40,6 +40,10 @@ server.addResponse("sessionlog.json", request_header, response_header) request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-2.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-4.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-5.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) # Define first ATS. Disable the cache to make sure each request is sent to the # origin server. @@ -65,6 +69,12 @@ def baselineTsSetup(ts): ts.Disk.remap_config.AddLine( 'map http://www.ae-3.com http://127.0.0.1:{0}'.format(server.Variables.Port) + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-4.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-5.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') baselineTsSetup(ts) @@ -124,6 +134,47 @@ def curlTail(hdrValue): tr.MakeCurlCommand(baseCurl + curlTail('gzip;q=0.3, whatever;q=0.666, br;q=0.7'), ts=ts) tr.Processes.Default.ReturnCode = 0 + # ZSTD-related tests for normalize_ae modes 4 and 5 + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, gzip')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, br')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, br, gzip')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('gzip, zstd, br')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('br, zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd;q=0.8, br;q=0.7, gzip;q=0.6')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('deflate, zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('identity, zstd, compress')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('br, compress')) + tr.Processes.Default.ReturnCode = 0 + def perTsTest(shouldWaitForUServer, ts): allAEHdrs(shouldWaitForUServer, True, ts, 'www.no-oride.com') @@ -131,6 +182,8 @@ def perTsTest(shouldWaitForUServer, ts): allAEHdrs(False, False, ts, 'www.ae-1.com') allAEHdrs(False, False, ts, 'www.ae-2.com') allAEHdrs(False, False, ts, 'www.ae-3.com') + allAEHdrs(False, False, ts, 'www.ae-4.com') + allAEHdrs(False, False, ts, 'www.ae-5.com') perTsTest(True, ts) diff --git a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py index 991d2fda32b..90441cb54f5 100644 --- a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py +++ b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py @@ -45,6 +45,12 @@ ts.Disk.remap_config.AddLine( f"map http://www.ae-3.com http://127.0.0.1:{server.Variables.http_port}" + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-4.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-5.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache') ts.Disk.records_config.update( { diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 57842341a59..4982e8cffd9 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -811,3 +811,833 @@ sessions: # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 5 proxy.config.http.normalize_ae:4 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 41 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 42 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 43 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 44 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 45 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 46 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Accept-Encoding header zstd, br, compress, gzip would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 47 ] + - [ Accept-Encoding, "zstd, br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br, compress, gzip would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 48 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 49 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 410 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 6 proxy.config.http.normalize_ae:5 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 51 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 52 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 53 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 54 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 55 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 56 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 57 ] + - [ Accept-Encoding, "zstd, compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Br-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # NOTICE: This case should load an alternate of zstd, gzip Accept-Encoding header. + # However, due to the implementation of calculate_quality_of_match(), + # ATS matches the alternate of gzip Accept-Encoding header. + # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 58 ] + - [ Accept-Encoding, "zstd, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + # - [ X-Cache, { value: miss, as: equal } ] + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. + # However, due to the implementation of calculate_quality_of_match(), + # ATS matches the alternate of gzip Accept-Encoding header. + # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 59 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "br, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Br, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + # - [ X-Cache, { value: miss, as: equal } ] + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 510 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, br would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 511 ] + - [ Accept-Encoding, "compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 512 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header zstd;q=1.1 would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 513 ] + - [ Accept-Encoding, "zstd;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br;q=1.1 would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 514 ] + - [ Accept-Encoding, "br;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 515 ] + - [ Accept-Encoding, "zstd, br;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # NOTICE: This case should make Accept-Encoding header zstd, gzip;q=0.8 match + # the alternate of zstd, gzip Accept-Encoding header. + # However, due to the implementation of calculate_quality_of_match(), + # ATS matches the alternate of gzip Accept-Encoding header. + # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 516 ] + - [ Accept-Encoding, "zstd, gzip;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 6745b65f0ae..8b5bd830f04 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -1,22 +1,22 @@ -> GET ``/obj0 HTTP/1.1 -> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br > Accept-Encoding: br < HTTP/1.1 200 OK @@ -25,64 +25,78 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj1 HTTP/1.1 -> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-1/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br > Accept-Encoding: br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj2 HTTP/1.1 -> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-1/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-2/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br > Accept-Encoding: br < HTTP/1.1 200 OK @@ -91,75 +105,205 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 71 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 71 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 71 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666 > Accept-Encoding: gzip;q=0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x > Accept-Encoding: gzip;q=0.666x < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 > Accept-Encoding: gzip;q=#0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 > Accept-Encoding: gzip; Q = 0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 > Accept-Encoding: gzip;q=0.0 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=-0.1 > Accept-Encoding: gzip;q=-0.1 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb > Accept-Encoding: aaa, gzip;q=0.666, bbb < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb > Accept-Encoding: br ; q=0.666, bbb < HTTP/1.1 200 OK @@ -168,21 +312,21 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666 , > Accept-Encoding: aaa, gzip;q=0.666 , < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> POST ``/obj3 HTTP/1.1 +> POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === diff --git a/tests/gold_tests/pluginTest/compress/compress.test.py b/tests/gold_tests/pluginTest/compress/compress.test.py index fd34c6f7524..4359b64a5cd 100644 --- a/tests/gold_tests/pluginTest/compress/compress.test.py +++ b/tests/gold_tests/pluginTest/compress/compress.test.py @@ -25,7 +25,8 @@ # Skip if plugins not present. # Test.SkipUnless( - Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI')) + Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI'), + Condition.HasATSFeature('TS_HAS_ZSTD')) server = Test.MakeOriginServer("server", options={'--load': f'{Test.TestDirectory}/compress_observer.py'}) @@ -43,7 +44,7 @@ "timestamp": "1469733493.993", "body": body } -for i in range(3): +for i in range(6): # add request/response to the server dictionary request_header = {"headers": f"GET /obj{i} HTTP/1.1\r\nHost: just.any.thing\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header, response_header) @@ -89,6 +90,7 @@ def curl_post(ts, idx, encodingList, out_path): ts.Setup.Copy("compress.config") ts.Setup.Copy("compress2.config") +ts.Setup.Copy("compress3.config") ts.Disk.remap_config.AddLine( f'map http://ae-0/ http://127.0.0.1:{server.Variables.Port}/' + @@ -103,7 +105,16 @@ def curl_post(ts, idx, encodingList, out_path): f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') ts.Disk.remap_config.AddLine( f'map http://ae-3/ http://127.0.0.1:{server.Variables.Port}/' + - f' @plugin=compress.so @pparam={Test.RunDirectory}/compress.config') + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-4/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-5/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') out_path_counter = 0 @@ -119,12 +130,12 @@ def get_out_path(): def get_verify_command(out_path, decrompressor): - return f"{decrompressor} -c {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" + return f"{decrompressor} {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" -for i in range(3): +for i in range(6): - tr = Test.AddTestRun(f'gzip, deflate, sdch, br: {i}') + tr = Test.AddTestRun(f'gzip, deflate, sdch, br, zstd: {i}') if (waitForTs): tr.Processes.Default.StartBefore(ts) waitForTs = False @@ -133,15 +144,21 @@ def get_verify_command(out_path, decrompressor): waitForServer = False tr.Processes.Default.ReturnCode = 0 out_path = get_out_path() - tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br', out_path), ts=ts) - tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br: {i}') + tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br, zstd', out_path), ts=ts) + tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br, zstd: {i}') tr.ReturnCode = 0 if i == 0: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") elif i == 1: - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") elif i == 2: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 3: + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 4: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + elif i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") tr = Test.AddTestRun(f'gzip: {i}') tr.Processes.Default.ReturnCode = 0 @@ -149,7 +166,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, i, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip: {i}') tr.ReturnCode = 0 - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun(f'br: {i}') tr.Processes.Default.ReturnCode = 0 @@ -160,7 +177,7 @@ def get_verify_command(out_path, decrompressor): if i == 1: tr.Processes.Default.Command = f"diff {out_path} {orig_path}" else: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun(f'deflate: {i}') tr.Processes.Default.ReturnCode = 0 @@ -170,6 +187,17 @@ def get_verify_command(out_path, decrompressor): tr.ReturnCode = 0 tr.Processes.Default.Command = f"diff {out_path} {orig_path}" + tr = Test.AddTestRun(f'zstd: {i}') + tr.Processes.Default.ReturnCode = 0 + out_path = get_out_path() + tr.MakeCurlCommand(curl(ts, i, "zstd", out_path)) + tr = Test.AddTestRun(f'verify zstd: {i}') + tr.ReturnCode = 0 + if i == 4 or i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + else: + tr.Processes.Default.Command = get_verify_command(out_path, "cat") + # Test Accept-Encoding normalization. tr = Test.AddTestRun() @@ -178,7 +206,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -186,7 +214,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666x", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666x') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -194,7 +222,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=#0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=#0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -202,7 +230,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip; Q = 0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip; Q = 0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -218,7 +246,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=-0.1", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=-0.1') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -226,7 +254,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -234,7 +262,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, " br ; q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify br ; q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") +tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -242,7 +270,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666 , ", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666 , ') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # post tr = Test.AddTestRun() @@ -251,7 +279,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl_post(ts, 3, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip post') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # compress_long.log contains all the output from the curl commands. The tr removes the carriage returns for easier # readability. Curl seems to have a bug, where it will neglect to output an end of line before outputting an HTTP diff --git a/tests/gold_tests/pluginTest/compress/compress3.config b/tests/gold_tests/pluginTest/compress/compress3.config new file mode 100644 index 00000000000..e3674f0e4cb --- /dev/null +++ b/tests/gold_tests/pluginTest/compress/compress3.config @@ -0,0 +1,7 @@ +cache true +remove-accept-encoding true +compressible-content-type text/* +compressible-content-type application/x-javascript* +compressible-content-type application/javascript* +compressible-content-type application/json* +supported-algorithms zstd, br, gzip diff --git a/tests/gold_tests/pluginTest/compress/compress_userver.gold b/tests/gold_tests/pluginTest/compress/compress_userver.gold index 1ddc6a4f3b5..693bd03905e 100644 --- a/tests/gold_tests/pluginTest/compress/compress_userver.gold +++ b/tests/gold_tests/pluginTest/compress/compress_userver.gold @@ -1,15 +1,33 @@ -0/gzip, deflate, sdch, br +0/gzip, deflate, sdch, br, zstd 0/gzip 0/br 0/deflate -1/gzip, deflate, sdch, br +0/zstd +1/gzip, deflate, sdch, br, zstd 1/gzip 1/br 1/deflate -2/gzip, deflate, sdch, br +1/zstd +2/gzip, deflate, sdch, br, zstd 2/gzip 2/br 2/deflate +2/zstd +3/gzip, deflate, sdch, br, zstd +3/gzip +3/br +3/deflate +3/zstd +4/gzip, deflate, sdch, br, zstd +4/gzip +4/br +4/deflate +4/zstd +5/gzip, deflate, sdch, br, zstd +5/gzip +5/br +5/deflate +5/zstd 0/gzip;q=0.666 0/gzip;q=0.666x 0/gzip;q=#0.666 From facf384cd557737e5e1bf268c29bb4a86ef9db1d Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 01:32:52 +0100 Subject: [PATCH 02/29] Fix Accept-Encoding cache matching and update test expectations This patch fixes the Accept-Encoding quality calculation logic in the HTTP transaction cache and updates test files to reflect the corrected behavior, removing workarounds for previously broken cache matching. Cache quality calculation improvements: - Replace multiplicative quality combining with minimum quality selection for multiple content encodings - Simplify wildcard matching to return quality directly - Remove gzip-specific fallback logic that bypassed normal quality calculation and caused inconsistent cache behavior Test updates for corrected behavior: - Remove NOTICE comments describing broken cache matching behavior - Update test expectations to match correct cache responses instead of workaround responses - Fix cache hit/miss expectations for multi-encoding scenarios - Add missing Content-Encoding headers in server responses - Update response identifiers to match the actual cached alternates Specific test corrections: - "br, compress, gzip" now correctly matches "Br-Gzip-Accept-Encoding" instead of falling back to "Gzip-Accept-Encoding" - "zstd, compress, gzip" now correctly matches "Zstd-Gzip-Accept-Encoding" instead of falling back to "Gzip-Accept-Encoding" - "zstd, gzip;q=0.8" now correctly matches "Zstd-Gzip-Accept-Encoding" instead of falling back to "Gzip-Accept-Encoding" - Individual encoding requests now create proper cache alternates instead of incorrectly matching empty encoding alternates The previous multiplicative quality calculation (q_a * q_b) was causing unexpectedly low quality scores and incorrect cache alternate selection. The new minimum quality approach ensures that multi-encoding responses are properly cached and matched according to HTTP content negotiation standards. All test cases now pass without workarounds and demonstrate correct cache behavior for all compression algorithms (gzip, brotli, zstd) across various Accept-Encoding --- src/iocore/cache/HttpTransactCache.cc | 58 +++------ .../normalized_ae_match_vary_cache.test.py | 2 +- ...malized_ae_varied_transactions.replay.yaml | 115 ++++++++++-------- 3 files changed, 83 insertions(+), 92 deletions(-) diff --git a/src/iocore/cache/HttpTransactCache.cc b/src/iocore/cache/HttpTransactCache.cc index 021985fff47..215b7a49ef1 100644 --- a/src/iocore/cache/HttpTransactCache.cc +++ b/src/iocore/cache/HttpTransactCache.cc @@ -979,60 +979,38 @@ HttpTransactCache::calculate_quality_of_accept_encoding_match(MIMEField *accept_ if (!content_field) { if (!match_accept_content_encoding("identity", accept_field, &wildcard_present, &wildcard_q, &q)) { // CE was not returned, and AE does not have identity - if (match_content_encoding(accept_field, "gzip") and match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } goto encoding_wildcard; } - // use q from identity match - } else { - // "Accept-encoding must correctly handle multiple content encoding" - // The combined quality factor is the product of all quality factors. - // (Note that there may be other possible choice, eg, min(), - // but I think multiplication is the best.) - // For example, if "content-encoding: a, b", and quality factors - // of a and b (in accept-encoding header) are q_a and q_b, resp, - // then the combined quality factor is (q_a * q_b). - // If any one of the content-encoding is not matched, - // then the q value will not be changed. - float combined_q = 1.0; + // Handle multiple content encodings - use minimum quality + float min_q = 1.0; // Start with maximum quality + bool found_match = false; + for (c_value = c_values_list.head; c_value; c_value = c_value->next) { float this_q = -1.0; if (!match_accept_content_encoding(c_value->str, accept_field, &wildcard_present, &wildcard_q, &this_q)) { goto encoding_wildcard; } - combined_q *= this_q; + if (this_q >= 0.0) { + found_match = false; + if (this_q < min_q) { + min_q = this_q; + } + } + } + if (found_match) { + q = min_q; + } else { + q = -1.0; } - q = combined_q; } encoding_wildcard: - // match the wildcard now // if ((q == -1.0) && (wildcard_present == true)) { - q = wildcard_q; - } - ///////////////////////////////////////////////////////////////////////// - // there was an Accept-Encoding, but it didn't match anything, at // - // any quality level --- if this is an identity-coded document, that's // - // still okay, but otherwise, this is just not a match at all. // - ///////////////////////////////////////////////////////////////////////// - if ((q == -1.0) && is_identity_encoding) { - if (match_content_encoding(accept_field, "gzip")) { - if (match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } else { - // always try to fetch GZIP content if we have not tried sending AE before - return -1.0f; - } - } else if (cached_accept_field && !match_content_encoding(cached_accept_field, "gzip")) { - return 0.001f; - } else { - return -1.0f; - } + return wildcard_q; } - // q = (float)-1.0; - return (q); + + return q; } /** diff --git a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py index 90441cb54f5..f83ae463fb9 100644 --- a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py +++ b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py @@ -59,7 +59,7 @@ 'proxy.config.http.response_via_str': 3, # the following variables could affect the results of alternate cache matching, # define them with their default values explicitly - 'proxy.config.cache.limits.http.max_alts': 5, + 'proxy.config.cache.limits.http.max_alts': 6, 'proxy.config.http.cache.ignore_accept_mismatch': 2, 'proxy.config.http.cache.ignore_accept_language_mismatch': 2, 'proxy.config.http.cache.ignore_accept_encoding_mismatch': 2, diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 4982e8cffd9..4910af81027 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -96,7 +96,7 @@ sessions: - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header deflate would match the alternate of empty Accept-Encoding header + # load an alternate of deflate Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -110,16 +110,25 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, deflate ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Deflate-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Deflate-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress would match the alternate of empty Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -133,14 +142,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # load an alternate of gzip Accept-Encoding header - client-request: @@ -162,6 +180,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -176,7 +195,7 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress, gzip would match the alternate of gzip Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -190,14 +209,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Case 2 proxy.config.http.normalize_ae:1 @@ -322,6 +350,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -458,6 +487,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -492,6 +522,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -651,6 +682,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -686,6 +718,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, gzip ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] content: @@ -699,10 +732,6 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -722,6 +751,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, "br, gzip" ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] @@ -733,10 +763,8 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header - client-request: @@ -784,11 +812,6 @@ sessions: - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] - # NOTICE: This case should make Accept-Encoding header br, gzip;q=0.8 match - # the alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -808,8 +831,7 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] # Case 5 proxy.config.http.normalize_ae:4 @@ -915,6 +937,7 @@ sessions: headers: fields: - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, zstd ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] @@ -954,6 +977,7 @@ sessions: headers: fields: - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, br ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] @@ -994,6 +1018,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -1224,6 +1249,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, zstd ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Zstd-Accept-Encoding ] @@ -1264,6 +1290,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, br ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] content: @@ -1304,6 +1331,7 @@ sessions: - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] + - [ Content-Encoding, gzip ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] content: encoding: plain @@ -1341,6 +1369,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, zstd ] - [ Connection, close ] - [ X-Response-Identifier, Zstd-Br-Accept-Encoding ] content: @@ -1354,10 +1383,6 @@ sessions: - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of zstd, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 - client-request: method: "GET" version: "1.1" @@ -1384,6 +1409,7 @@ sessions: - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] + - [ Content-Encoding, zstd ] - [ X-Response-Identifier, Zstd-Gzip-Accept-Encoding ] content: encoding: plain @@ -1393,15 +1419,9 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 - client-request: method: "GET" version: "1.1" @@ -1428,6 +1448,7 @@ sessions: - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] + - [ Content-Encoding, br ] - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] content: encoding: plain @@ -1437,10 +1458,8 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header - client-request: @@ -1609,11 +1628,6 @@ sessions: - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] - # NOTICE: This case should make Accept-Encoding header zstd, gzip;q=0.8 match - # the alternate of zstd, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 - client-request: method: "GET" version: "1.1" @@ -1638,6 +1652,5 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] \ No newline at end of file + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] From eda7c29e722c13248e8d56865d101c8be5f73067 Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 13:42:50 +0100 Subject: [PATCH 03/29] Fix case sensitivity in ZSTD package find command --- CMakeLists.txt | 4 +-- cmake/Findzstd.cmake | 28 ++++++++--------- .../pluginTest/compress/compress.gold | 30 +++++++++---------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index df0f61910f9..0defeeb05e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,8 +341,8 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(ZSTD) -if(ZSTD_FOUND) +find_package(zstd) +if(zstd_FOUND) set(HAVE_ZSTD_H TRUE) endif() diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake index d2a86d0132e..17587dca39a 100644 --- a/cmake/Findzstd.cmake +++ b/cmake/Findzstd.cmake @@ -20,34 +20,34 @@ # # This will define the following variables # -# ZSTD_FOUND -# ZSTD_LIBRARY -# ZSTD_INCLUDE_DIRS +# zstd_FOUND +# zstd_LIBRARY +# zstd_INCLUDE_DIRS # # and the following imported target # # zstd::zstd # -find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) +find_path(zstd_INCLUDE_DIR NAMES zstd.h) -find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) -find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) +find_library(zstd_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(zstd_LIBRARY_RELEASE NAMES zstd zstd_static) -mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) +mark_as_advanced(zstd_LIBRARY zstd_INCLUDE_DIR) include(SelectLibraryConfigurations) -select_library_configurations(ZSTD) +select_library_configurations(zstd) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) +find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARY zstd_INCLUDE_DIR) -if(ZSTD_FOUND) - set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") +if(zstd_FOUND) + set(zstd_INCLUDE_DIRS "${zstd_INCLUDE_DIR}") endif() -if(ZSTD_FOUND AND NOT TARGET zstd::zstd) +if(zstd_FOUND AND NOT TARGET zstd::zstd) add_library(zstd::zstd INTERFACE IMPORTED) - target_include_directories(zstd::zstd INTERFACE ${ZSTD_INCLUDE_DIRS}) - target_link_libraries(zstd::zstd INTERFACE "${ZSTD_LIBRARY}") + target_include_directories(zstd::zstd INTERFACE ${zstd_INCLUDE_DIRS}) + target_link_libraries(zstd::zstd INTERFACE "${zstd_LIBRARY}") endif() diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 8b5bd830f04..1a7014bc9bc 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -175,7 +175,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -216,7 +216,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -249,7 +249,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -258,7 +258,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -267,7 +267,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -276,7 +276,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -292,7 +292,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -301,7 +301,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -319,7 +319,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -328,5 +328,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === From 21e27bffa5e9522f6314185b8b80a8d467c46c9f Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 22:20:01 +0100 Subject: [PATCH 04/29] Add ZSTD version output to compile-time features --- src/traffic_layout/CMakeLists.txt | 4 ++++ src/traffic_layout/info.cc | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index 56766955c62..b86d4f2573a 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -31,6 +31,10 @@ if(HAVE_BROTLI_ENCODE_H) target_link_libraries(traffic_layout PRIVATE brotli::brotlienc) endif() +if(HAVE_ZSTD_H) + target_link_libraries(traffic_layout PRIVATE zstd::zstd) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 541985112cc..6e5abd71046 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -49,6 +49,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + // Produce output about compile time features, useful for checking how things were built static void print_feature(std::string_view name, int value, bool json, bool last = false) @@ -208,6 +212,11 @@ produce_versions(bool json) #else print_var("brotli", undef, json); #endif +#if HAVE_ZSTD_H + print_var("zstd", LBW().print("{}", ZSTD_versionString()).view(), json); +#else + print_var("zstd", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); From 2a2acf66d667251deb5520e73c1740bcc0d30c6b Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 23:52:16 +0100 Subject: [PATCH 05/29] Add ZSTD support to Accept-Encoding and Content-Encoding headers --- src/proxy/http3/QPACK.cc | 3 +- .../pluginTest/compress/compress.gold | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/proxy/http3/QPACK.cc b/src/proxy/http3/QPACK.cc index bcfa4c70b8d..dfdd2d278b3 100644 --- a/src/proxy/http3/QPACK.cc +++ b/src/proxy/http3/QPACK.cc @@ -69,7 +69,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {":status", "503" }, {"accept", "*/*" }, {"accept", "application/dns-message" }, - {"accept-encoding", "gzip, deflate, br" }, + {"accept-encoding", "gzip, deflate, br, zstd" }, {"accept-ranges", "bytes" }, {"access-control-allow-headers", "cache-control" }, {"access-control-allow-headers", "content-type" }, @@ -80,6 +80,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {"cache-control", "no-cache" }, {"cache-control", "no-store" }, {"cache-control", "public, max-age=31536000" }, + {"content-encoding", "zstd" }, {"content-encoding", "br" }, {"content-encoding", "gzip" }, {"content-type", "application/dns-message" }, diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 1a7014bc9bc..6501a0e7318 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -165,6 +165,7 @@ > Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -175,7 +176,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -198,6 +199,7 @@ > Accept-Encoding: zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -206,6 +208,7 @@ > Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -216,7 +219,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -239,6 +242,7 @@ > Accept-Encoding: zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -249,7 +253,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -258,7 +262,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -267,7 +271,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -276,7 +280,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -292,7 +296,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -301,7 +305,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -319,7 +323,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -328,5 +332,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === From 536f920731b6c8e7b5a29a5b4843a9218f98b4cf Mon Sep 17 00:00:00 2001 From: jake champion Date: Wed, 16 Jul 2025 00:09:55 +0100 Subject: [PATCH 06/29] Add ZSTD length definition to HTTP value initialization --- src/api/InkAPIInternal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/InkAPIInternal.cc b/src/api/InkAPIInternal.cc index 9a4bb620ab1..8bb20a2b38a 100644 --- a/src/api/InkAPIInternal.cc +++ b/src/api/InkAPIInternal.cc @@ -774,6 +774,7 @@ api_init() TS_HTTP_LEN_DEFLATE = static_cast(HTTP_VALUE_DEFLATE.length()); TS_HTTP_LEN_GZIP = static_cast(HTTP_VALUE_GZIP.length()); TS_HTTP_LEN_BROTLI = static_cast(HTTP_VALUE_BROTLI.length()); + TS_HTTP_LEN_ZSTD = static_cast(HTTP_VALUE_ZSTD.length()); TS_HTTP_LEN_IDENTITY = static_cast(HTTP_VALUE_IDENTITY.length()); TS_HTTP_LEN_KEEP_ALIVE = static_cast(HTTP_VALUE_KEEP_ALIVE.length()); TS_HTTP_LEN_MAX_AGE = static_cast(HTTP_VALUE_MAX_AGE.length()); From 9a5432d04b969463f321207bbd8f096ad5e9c757 Mon Sep 17 00:00:00 2001 From: jake champion Date: Wed, 16 Jul 2025 00:57:41 +0100 Subject: [PATCH 07/29] Update ZSTD compression level from 6 to 10 for improved performance --- plugins/compress/compress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index b181d675722..04a4620989e 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -75,7 +75,7 @@ const int BROTLI_LGW = 16; #endif #if HAVE_ZSTD_H -const int ZSTD_COMPRESSION_LEVEL = 6; +const int ZSTD_COMPRESSION_LEVEL = 12; #endif static const char *global_hidden_header_name = nullptr; From ab79771971fadb4365efc8910490b746b0b09e1b Mon Sep 17 00:00:00 2001 From: jake champion Date: Wed, 16 Jul 2025 09:39:08 +0100 Subject: [PATCH 08/29] compress: Add configurable compression levels - Add support for configurable compression levels per host: * gzip-compression-level (1-9, default 6) * brotli-compression-level (0-11, default 6) * brotli-lgwin (10-24, default 16) * zstd-compression-level (1-22, default 12) This enables fine-tuning compression performance vs. speed trade-offs on a per-host basis --- doc/admin-guide/plugins/compress.en.rst | 50 ++++++++++++++++- plugins/compress/README | 2 +- plugins/compress/compress.cc | 29 +++------- plugins/compress/configuration.cc | 54 +++++++++++++++++- plugins/compress/configuration.h | 55 ++++++++++++++++++- plugins/compress/sample.compress.config | 28 ++++++++++ .../pluginTest/compress/compress.gold | 4 +- .../pluginTest/compress/compress2.config | 3 + .../pluginTest/compress/compress3.config | 4 ++ 9 files changed, 202 insertions(+), 27 deletions(-) diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index 686445008fc..338e6101c11 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -211,6 +211,41 @@ be considered, if it is ``2``, only br or gzip will be considered, if it is ``4` only zstd, br, or gzip will be considered, and if it is ``5``, all combinations of zstd, br, and gzip will be considered. +gzip-compression-level +----------------------- + +Sets the compression level for gzip compression. Valid values are 1-9, where +1 is fastest compression (lowest compression ratio) and 9 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-compression-level +------------------------- + +Sets the compression level for Brotli compression. Valid values are 0-11, where +0 is fastest compression (lowest compression ratio) and 11 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-lgwin +------------ + +Sets the window size for Brotli compression. Valid values are 10-24, where +larger values provide better compression but use more memory. The default is 16. +This parameter controls the sliding window size used during compression: + +- 10: 1KB window (fastest, least memory) +- 16: 64KB window (default, good balance) +- 24: 16MB window (slowest, most memory, best compression) + +zstd-compression-level +---------------------- + +Sets the compression level for Zstandard compression. Valid values are 1-22, where +1 is fastest compression (lowest compression ratio) and 22 is slowest compression +(highest compression ratio). The default is 12, which provides an excellent +balance between compression speed and ratio for web content. + Examples ======== @@ -226,6 +261,10 @@ might create a configuration with the following options:: compressible-status-code 200, 206 minimum-content-length 860 flush false + gzip-compression-level 6 + brotli-compression-level 6 + brotli-lgwin 16 + zstd-compression-level 12 # Now set a configuration for www.example.com [www.example.com] @@ -243,13 +282,15 @@ might create a configuration with the following options:: flush true supported-algorithms gzip,deflate - # Supports brotli compression + # Supports brotli compression with custom settings [brotli.compress.com] enabled true compressible-content-type text/* compressible-content-type application/json flush true supported-algorithms br,gzip + brotli-compression-level 8 + brotli-lgwin 20 # Supports zstd compression for high efficiency [zstd.compress.com] @@ -259,14 +300,19 @@ might create a configuration with the following options:: compressible-content-type application/javascript flush true supported-algorithms zstd,gzip + zstd-compression-level 15 - # Supports all compression algorithms + # Supports all compression algorithms with optimized settings [all.compress.com] enabled true compressible-content-type text/* compressible-content-type application/json flush true supported-algorithms zstd,br,gzip,deflate + gzip-compression-level 7 + brotli-compression-level 9 + brotli-lgwin 18 + zstd-compression-level 10 # This origin does it all [bar.example.com] diff --git a/plugins/compress/README b/plugins/compress/README index e89070f9e54..f963a8e05e9 100644 --- a/plugins/compress/README +++ b/plugins/compress/README @@ -24,4 +24,4 @@ compress.so /sample.compress.config After modifying plugin.config, restart traffic server (sudo traffic_ctl server restart) the configuration is re-read when a management update is given (sudo traffic_ctl config reload) -See sample.config.compress for an example configuration and the options that are available +See sample.compress.config for an example configuration and the options that are available diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 04a4620989e..ffb3d793d76 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -65,18 +65,7 @@ namespace compress_ns DbgCtl dbg_ctl{TAG}; } -const int ZLIB_COMPRESSION_LEVEL = 6; -const char *dictionary = nullptr; - -// brotli compression quality 1-11. Testing proved level '6' -#if HAVE_BROTLI_ENCODE_H -const int BROTLI_COMPRESSION_LEVEL = 6; -const int BROTLI_LGW = 16; -#endif - -#if HAVE_ZSTD_H -const int ZSTD_COMPRESSION_LEVEL = 12; -#endif +const char *dictionary = nullptr; static const char *global_hidden_header_name = nullptr; @@ -151,7 +140,7 @@ static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t u #endif static Data * -data_alloc(int compression_type, int compression_algorithms) +data_alloc(int compression_type, int compression_algorithms, HostConfiguration *hc) { Data *data; int err; @@ -164,6 +153,7 @@ data_alloc(int compression_type, int compression_algorithms) data->state = transform_state_initialized; data->compression_type = compression_type; data->compression_algorithms = compression_algorithms; + data->hc = hc; data->zstrm.next_in = Z_NULL; data->zstrm.avail_in = 0; data->zstrm.total_in = 0; @@ -180,7 +170,7 @@ data_alloc(int compression_type, int compression_algorithms) window_bits = WINDOW_BITS_DEFLATE; } - err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); + err = deflateInit2(&data->zstrm, data->hc->zlib_compression_level(), Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); if (err != Z_OK) { fatal("gzip-transform: ERROR: deflateInit (%d)!", err); @@ -200,8 +190,8 @@ data_alloc(int compression_type, int compression_algorithms) if (!data->bstrm.br) { fatal("Brotli Encoder Instance Failed"); } - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, BROTLI_COMPRESSION_LEVEL); - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, BROTLI_LGW); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, data->hc->brotli_compression_level()); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, data->hc->brotli_lgw_size()); data->bstrm.next_in = nullptr; data->bstrm.avail_in = 0; data->bstrm.total_in = 0; @@ -523,7 +513,7 @@ zstd_compress_init(Data *data) } // Set compression level - size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, ZSTD_COMPRESSION_LEVEL); + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); if (ZSTD_isError(result)) { error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); return; @@ -536,7 +526,7 @@ zstd_compress_init(Data *data) return; } - debug("zstd compression context initialized with level %d", ZSTD_COMPRESSION_LEVEL); + debug("zstd compression context initialized with level %d", data->hc->zstd_compression_level()); } static void @@ -1058,9 +1048,8 @@ compress_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type, } connp = TSTransformCreate(compress_transform, txnp); - data = data_alloc(compress_type, algorithms); + data = data_alloc(compress_type, algorithms, hc); data->txn = txnp; - data->hc = hc; TSContDataSet(connp, data); TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index 41b35964eac..d4311ec286c 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -108,7 +108,11 @@ enum ParserState { kParseRangeRequest, kParseFlush, kParseAllow, - kParseMinimumContentLength + kParseMinimumContentLength, + kParseGzipCompressionLevel, + kPaseBrotliCompressionLevel, + kParseBrotliLGWSize, + kParseZstdCompressionLevel, }; void @@ -389,6 +393,14 @@ Configuration::Parse(const char *path) state = kParseStart; } else if (token == "minimum-content-length") { state = kParseMinimumContentLength; + } else if (token == "gzip-compression-level") { + state = kParseGzipCompressionLevel; + } else if (token == "brotli-compression-level") { + state = kPaseBrotliCompressionLevel; + } else if (token == "brotli-lgwin") { + state = kParseBrotliLGWSize; + } else if (token == "zstd-compression-level") { + state = kParseZstdCompressionLevel; } else { warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno); } @@ -425,6 +437,46 @@ Configuration::Parse(const char *path) current_host_configuration->set_minimum_content_length(strtoul(token.c_str(), nullptr, 10)); state = kParseStart; break; + case kParseGzipCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 1 || level > 9) { + error("gzip-compression-level must be between 1 and 9, got %d", level); + } else { + current_host_configuration->set_gzip_compression_level(level); + } + state = kParseStart; + break; + } + case kPaseBrotliCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 0 || level > 11) { + error("brotli-compression-level must be between 0 and 11, got %d", level); + } else { + current_host_configuration->set_brotli_compression_level(level); + } + state = kParseStart; + break; + } + case kParseBrotliLGWSize: { + int lgw = strtol(token.c_str(), nullptr, 10); + if (lgw < 10 || lgw > 24) { + error("brotli-lgwin must be between 10 and 24, got %d", lgw); + } else { + current_host_configuration->set_brotli_lgw_size(lgw); + } + state = kParseStart; + break; + } + case kParseZstdCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 1 || level > 22) { + error("zstd-compression-level must be between 1 and 22, got %d", level); + } else { + current_host_configuration->set_zstd_compression_level(level); + } + state = kParseStart; + break; + } } } } diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index 2140d5a3580..0db81784199 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -59,7 +59,11 @@ class HostConfiguration : private atscppapi::noncopyable remove_accept_encoding_(false), flush_(false), compression_algorithms_(ALGORITHM_GZIP), - minimum_content_length_(1024) + minimum_content_length_(1024), + zlib_compression_level_(6), + brotli_compression_level_(6), + brotli_lgw_size_(16), + zstd_compression_level_(12) { } @@ -130,6 +134,51 @@ class HostConfiguration : private atscppapi::noncopyable minimum_content_length_ = x; } + unsigned int + zlib_compression_level() const + { + return zlib_compression_level_; + } + + void + set_gzip_compression_level(int level) + { + zlib_compression_level_ = level; + } + + unsigned int + brotli_compression_level() const + { + return brotli_compression_level_; + } + void + set_brotli_compression_level(int level) + { + brotli_compression_level_ = level; + } + + unsigned int + brotli_lgw_size() const + { + return brotli_lgw_size_; + } + void + set_brotli_lgw_size(unsigned int lgw) + { + brotli_lgw_size_ = lgw; + } + + int + zstd_compression_level() const + { + return zstd_compression_level_; + } + void + set_zstd_compression_level(int level) + { + zstd_compression_level_ = level; + } + void update_defaults(); void add_allow(const std::string &allow); void add_compressible_content_type(const std::string &content_type); @@ -149,6 +198,10 @@ class HostConfiguration : private atscppapi::noncopyable bool flush_; int compression_algorithms_; unsigned int minimum_content_length_; + unsigned int zlib_compression_level_; + unsigned int brotli_compression_level_; + unsigned int brotli_lgw_size_; + int zstd_compression_level_; RangeRequestCtrl range_request_ctl_ = RangeRequestCtrl::NO_COMPRESSION; StringContainer compressible_content_types_; diff --git a/plugins/compress/sample.compress.config b/plugins/compress/sample.compress.config index 8b0eaaf5be8..451f8958349 100644 --- a/plugins/compress/sample.compress.config +++ b/plugins/compress/sample.compress.config @@ -37,6 +37,22 @@ # minimum-content-length: minimum content length for compression to be enabled (in bytes) # - this setting only applies if the origin response has a Content-Length header # +# gzip-compression-level: compression level for gzip (1-9, default 6) +# - 1 is fastest compression (lowest compression ratio) +# - 9 is slowest compression (highest compression ratio) +# +# brotli-compression-level: compression level for brotli (0-11, default 6) +# - 0 is fastest compression (lowest compression ratio) +# - 11 is slowest compression (highest compression ratio) +# +# brotli-lgwin: window size for brotli compression (10-24, default 16) +# - larger values provide better compression but use more memory +# - 10: 1KB window, 16: 64KB window, 24: 16MB window +# +# zstd-compression-level: compression level for zstandard (1-22, default 12) +# - 1 is fastest compression (lowest compression ratio) +# - 22 is slowest compression (highest compression ratio) +# ###################################################################### #first, we configure the default/global plugin behaviour @@ -58,6 +74,12 @@ minimum-content-length 1024 #supported algorithms supported-algorithms br,gzip,zstd +# Compression level settings (optional) +gzip-compression-level 6 +brotli-compression-level 6 +brotli-lgwin 16 +zstd-compression-level 12 + #override the global configuration for a host. #www.foo.nl does NOT inherit anything [www.foo.nl] @@ -68,6 +90,12 @@ compressible-content-type text/* compressible-status-code 200,206,409 minimum-content-length 1024 +# Custom compression settings for this host +gzip-compression-level 8 +brotli-compression-level 9 +brotli-lgwin 20 +zstd-compression-level 15 + allow /this/*.js allow !/notthis/*.js allow !/notthat* diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 6501a0e7318..575c896e56e 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -185,7 +185,7 @@ < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding -< Content-Length: 46 +< Content-Length: 47 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/deflate @@ -228,7 +228,7 @@ < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding -< Content-Length: 46 +< Content-Length: 47 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/deflate diff --git a/tests/gold_tests/pluginTest/compress/compress2.config b/tests/gold_tests/pluginTest/compress/compress2.config index c808d87fcfd..8fa674c687b 100644 --- a/tests/gold_tests/pluginTest/compress/compress2.config +++ b/tests/gold_tests/pluginTest/compress/compress2.config @@ -5,3 +5,6 @@ compressible-content-type application/x-javascript* compressible-content-type application/javascript* compressible-content-type application/json* supported-algorithms gzip, br +gzip-compression-level 7 +brotli-compression-level 8 +brotli-lgwin 18 diff --git a/tests/gold_tests/pluginTest/compress/compress3.config b/tests/gold_tests/pluginTest/compress/compress3.config index e3674f0e4cb..54ff0eed2d8 100644 --- a/tests/gold_tests/pluginTest/compress/compress3.config +++ b/tests/gold_tests/pluginTest/compress/compress3.config @@ -5,3 +5,7 @@ compressible-content-type application/x-javascript* compressible-content-type application/javascript* compressible-content-type application/json* supported-algorithms zstd, br, gzip +gzip-compression-level 5 +brotli-compression-level 7 +brotli-lgwin 17 +zstd-compression-level 10 From d94321de61b80c323f87693928e6e3223ee288f4 Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Wed, 16 Jul 2025 13:15:00 +0100 Subject: [PATCH 09/29] Update compress.gold --- .../pluginTest/compress/compress.gold | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 575c896e56e..970ba0084d1 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -176,7 +176,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -219,7 +219,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -253,7 +253,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -262,7 +262,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -271,7 +271,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -280,7 +280,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -296,7 +296,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -305,7 +305,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -323,7 +323,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -332,5 +332,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === From 729fbac68a744036bc38c09ff44f11a29bd47dde Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Tue, 22 Jul 2025 12:22:22 +0100 Subject: [PATCH 10/29] Update compress.gold From 17e3c6b49373f5b35977f420719f18d0ec3b3efb Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 18 Aug 2025 09:31:17 +0100 Subject: [PATCH 11/29] Use upstream zstd CMake; remove Findzstd.cmake Switch to the upstream zstd CMake package and drop our custom Find-module. - Use `find_package(zstd CONFIG QUIET)` and set `HAVE_ZSTD_H` when found - Provide a compatibility target `zstd::zstd` that aliases `zstd::libzstd_shared`/`zstd::libzstd_static`/`zstd::libzstd` to keep existing link lines working - Remove `cmake/Findzstd.cmake` and rely on distro-provided configs This reduces maintenance and prefers the canonical package configuration while retaining compatibility with current linkage in the tree. --- CMakeLists.txt | 20 ++++++++++++++++- cmake/Findzstd.cmake | 53 -------------------------------------------- 2 files changed, 19 insertions(+), 54 deletions(-) delete mode 100644 cmake/Findzstd.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0defeeb05e7..bbcc2bc2745 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,9 +341,27 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(zstd) +find_package(zstd CONFIG QUIET) if(zstd_FOUND) set(HAVE_ZSTD_H TRUE) + + # Provide a compatibility target name if the upstream package does not export it + # Our code links against `zstd::zstd`; upstream zstd usually exports + # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed. + if(NOT TARGET zstd::zstd) + if(TARGET zstd::libzstd_shared) + set(_zstd_target zstd::libzstd_shared) + elseif(TARGET zstd::libzstd_static) + set(_zstd_target zstd::libzstd_static) + elseif(TARGET zstd::libzstd) + set(_zstd_target zstd::libzstd) + endif() + if(DEFINED _zstd_target) + add_library(zstd_zstd INTERFACE) + target_link_libraries(zstd_zstd INTERFACE ${_zstd_target}) + add_library(zstd::zstd ALIAS zstd_zstd) + endif() + endif() endif() # ncurses is used in traffic_top diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake deleted file mode 100644 index 17587dca39a..00000000000 --- a/cmake/Findzstd.cmake +++ /dev/null @@ -1,53 +0,0 @@ -####################### -# -# Licensed to the Apache Software Foundation (ASF) under one or more contributor license -# agreements. See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# -####################### - -# Findzstd.cmake -# -# This will define the following variables -# -# zstd_FOUND -# zstd_LIBRARY -# zstd_INCLUDE_DIRS -# -# and the following imported target -# -# zstd::zstd -# - -find_path(zstd_INCLUDE_DIR NAMES zstd.h) - -find_library(zstd_LIBRARY_DEBUG NAMES zstdd zstd_staticd) -find_library(zstd_LIBRARY_RELEASE NAMES zstd zstd_static) - -mark_as_advanced(zstd_LIBRARY zstd_INCLUDE_DIR) - -include(SelectLibraryConfigurations) -select_library_configurations(zstd) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARY zstd_INCLUDE_DIR) - -if(zstd_FOUND) - set(zstd_INCLUDE_DIRS "${zstd_INCLUDE_DIR}") -endif() - -if(zstd_FOUND AND NOT TARGET zstd::zstd) - add_library(zstd::zstd INTERFACE IMPORTED) - target_include_directories(zstd::zstd INTERFACE ${zstd_INCLUDE_DIRS}) - target_link_libraries(zstd::zstd INTERFACE "${zstd_LIBRARY}") -endif() From 91029c557572da3b6c45cc6170924609ed3ee439 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 18 Aug 2025 09:38:44 +0100 Subject: [PATCH 12/29] Ensure HAVE_ZSTD_H is always set to a value --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bbcc2bc2745..8bae90bea49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,6 @@ find_package(ZLIB REQUIRED) find_package(zstd CONFIG QUIET) if(zstd_FOUND) - set(HAVE_ZSTD_H TRUE) # Provide a compatibility target name if the upstream package does not export it # Our code links against `zstd::zstd`; upstream zstd usually exports @@ -360,8 +359,13 @@ if(zstd_FOUND) add_library(zstd_zstd INTERFACE) target_link_libraries(zstd_zstd INTERFACE ${_zstd_target}) add_library(zstd::zstd ALIAS zstd_zstd) + set(HAVE_ZSTD_H TRUE) + else() + set(HAVE_ZSTD_H FALSE) endif() endif() +else() + set(HAVE_ZSTD_H FALSE) endif() # ncurses is used in traffic_top From c7389c59e7afbc4aa662f301741d7d660ac07a61 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 18 Aug 2025 09:58:50 +0100 Subject: [PATCH 13/29] place HAVE_ZSTD_H nearer the other cmakedefine calls --- include/tscore/ink_config.h.cmake.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 2d515338c8a..211174465f1 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -46,6 +46,7 @@ #cmakedefine HAVE_POSIX_FADVISE 1 #cmakedefine HAVE_POSIX_FALLOCATE 1 #cmakedefine HAVE_POSIX_MADVISE 1 +#cmakedefine HAVE_ZSTD_H 1 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1 @@ -187,5 +188,3 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine YAMLCPP_LIB_VERSION "@YAMLCPP_LIB_VERSION@" #cmakedefine01 TS_HAS_CRIPTS - -#cmakedefine HAVE_ZSTD_H 1 From 4608badde2130690bd72e4c08a7b0f5abf4bafe7 Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Tue, 19 Aug 2025 08:33:35 +0100 Subject: [PATCH 14/29] Apply suggestions from code review Co-authored-by: JosiahWI <41302989+JosiahWI@users.noreply.github.com> --- src/traffic_layout/info.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 6e5abd71046..0a4f44491c0 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -95,7 +95,7 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif -#if HAVE_ZSTD_H +#ifdef HAVE_ZSTD_H print_feature("TS_HAS_ZSTD", 1, json); #else print_feature("TS_HAS_ZSTD", 0, json); @@ -212,7 +212,7 @@ produce_versions(bool json) #else print_var("brotli", undef, json); #endif -#if HAVE_ZSTD_H +#ifdef HAVE_ZSTD_H print_var("zstd", LBW().print("{}", ZSTD_versionString()).view(), json); #else print_var("zstd", undef, json); From 03e647a7ebb7c5dbcb6552fe11cd34a527a75e4f Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 19 Aug 2025 13:35:49 +0100 Subject: [PATCH 15/29] Renames ZSTD compression functions for consistency Updates function names from zstd_compress_* to zstd_transform_* to align with naming conventions used by other compression algorithms in the codebase. Improves code consistency and maintainability by standardizing function naming patterns across different compression implementations. --- plugins/compress/compress.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index ffb3d793d76..cfece1660e9 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -134,9 +134,9 @@ handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) // Forward declarations for ZSTD compression functions #if HAVE_ZSTD_H -static void zstd_compress_init(Data *data); -static void zstd_compress_finish(Data *data); -static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length); +static void zstd_transform_init(Data *data); +static void zstd_transform_finish(Data *data); +static void zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); #endif static Data * @@ -390,7 +390,7 @@ compress_transform_init(TSCont contp, Data *data) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD) { - zstd_compress_init(data); + zstd_transform_init(data); if (!data->zstrm_zstd.cctx) { TSError("Failed to create Zstandard compression context"); return; @@ -505,7 +505,7 @@ brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_l #if HAVE_ZSTD_H static void -zstd_compress_init(Data *data) +zstd_transform_init(Data *data) { if (!data->zstrm_zstd.cctx) { error("Failed to initialize Zstd compression context"); @@ -530,7 +530,7 @@ zstd_compress_init(Data *data) } static void -zstd_compress_finish(Data *data) +zstd_transform_finish(Data *data) { if (data->state == transform_state_output) { TSIOBufferBlock downstream_blkp; @@ -570,7 +570,7 @@ zstd_compress_finish(Data *data) } static void -zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) { TSIOBufferBlock downstream_blkp; int64_t downstream_length; @@ -633,7 +633,7 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { - zstd_compress_one(data, upstream_buffer, upstream_length); + zstd_transform_one(data, upstream_buffer, upstream_length); } else #endif #if HAVE_BROTLI_ENCODE_H @@ -725,7 +725,7 @@ compress_transform_finish(Data *data) { #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { - zstd_compress_finish(data); + zstd_transform_finish(data); debug("compress_transform_finish: zstd compression finish"); } else #endif From 0bb4fb55bf508675520e516968e0ff1df64d6567 Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 21 Oct 2025 10:49:42 +0100 Subject: [PATCH 16/29] Add flush handling to zstd compression stream --- plugins/compress/compress.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index a7e15aeda29..c494bf169ef 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -607,6 +607,37 @@ zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_len break; } } + + // Handle flushing if enabled + if (!data->hc->flush()) { + return; + } + + // Flush the compression stream + ZSTD_inBuffer empty_input = {nullptr, 0, 0}; + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &empty_input, ZSTD_e_flush); + + if (ZSTD_isError(result)) { + error("Zstd flush failed: %s", ZSTD_getErrorName(result)); + return; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (result == 0) { /* flush complete */ + break; + } + } } #endif From d31c21b0bf876520bf6d8ac1cdd7e577204e414f Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 21 Oct 2025 10:52:53 +0100 Subject: [PATCH 17/29] refactor zstd compression to reduce code duplication --- plugins/compress/compress.cc | 115 ++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index c494bf169ef..3d13da0e362 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -505,6 +505,55 @@ brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_l #endif #if HAVE_ZSTD_H +static bool +zstd_compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, ZSTD_EndDirective mode) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; + + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, mode); + + if (ZSTD_isError(result)) { + error("Zstd compression failed (%d): %s", mode, ZSTD_getErrorName(result)); + return false; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + // Check completion conditions based on mode + if (mode == ZSTD_e_continue) { + // For continue mode, stop when all input is consumed + if (input.pos >= input.size) { + break; + } + // If we have output space but no more input was consumed, break to avoid infinite loop + if (output.pos == 0 && input.pos < input.size) { + error("zstd-transform: no progress made in compression"); + return false; + } + } else if (mode == ZSTD_e_flush) { + // For flush mode, stop when flush is complete (result == 0) + if (result == 0) { + break; + } + } + } + + return true; +} + static void zstd_transform_init(Data *data) { @@ -573,70 +622,22 @@ zstd_transform_finish(Data *data) static void zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) { - TSIOBufferBlock downstream_blkp; - int64_t downstream_length; + bool ok = zstd_compress_operation(data, upstream_buffer, upstream_length, ZSTD_e_continue); + if (!ok) { + error("Zstd compression (CONTINUE) failed"); + return; + } - // Set up input buffer for zstd streaming - ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; data->zstrm_zstd.total_in += upstream_length; - while (input.pos < input.size) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - - // Set up output buffer for zstd streaming - ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; - - // Compress the data using streaming API - size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, ZSTD_e_continue); - - if (ZSTD_isError(result)) { - error("Zstd compression failed: %s", ZSTD_getErrorName(result)); - return; - } - - if (output.pos > 0) { - TSIOBufferProduce(data->downstream_buffer, output.pos); - data->downstream_length += output.pos; - data->zstrm_zstd.total_out += output.pos; - } - - // If we have output space but no more input was consumed, break to avoid infinite loop - if (output.pos == 0 && input.pos < input.size) { - error("zstd-transform: no progress made in compression"); - break; - } - } - - // Handle flushing if enabled if (!data->hc->flush()) { return; } - // Flush the compression stream - ZSTD_inBuffer empty_input = {nullptr, 0, 0}; - for (;;) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - - ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; - - size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &empty_input, ZSTD_e_flush); - - if (ZSTD_isError(result)) { - error("Zstd flush failed: %s", ZSTD_getErrorName(result)); - return; - } - - if (output.pos > 0) { - TSIOBufferProduce(data->downstream_buffer, output.pos); - data->downstream_length += output.pos; - data->zstrm_zstd.total_out += output.pos; - } - - if (result == 0) { /* flush complete */ - break; - } + ok = zstd_compress_operation(data, nullptr, 0, ZSTD_e_flush); + if (!ok) { + error("Zstd compression (FLUSH) failed"); + return; } } #endif From 3c37f0807a71d4c6a24279b5013ccd1d43274a4a Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Sun, 26 Oct 2025 16:54:56 +0000 Subject: [PATCH 18/29] Update compress.gold --- .../pluginTest/compress/compress.gold | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 970ba0084d1..b4e9333c2d2 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -176,7 +176,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -219,7 +219,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -253,7 +253,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -262,7 +262,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -271,7 +271,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -280,7 +280,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -296,7 +296,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -305,7 +305,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -323,7 +323,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -332,5 +332,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 7`` === From 7668be8594e2a32e9461f9dcb1ffe4af03bb2f17 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 10:18:11 +0000 Subject: [PATCH 19/29] Updates Accept-Encoding variability check logic Adjusts the condition for suppressing Vary header checks on Accept-Encoding to only apply when explicitly configured by the operator --- src/iocore/cache/HttpTransactCache.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/iocore/cache/HttpTransactCache.cc b/src/iocore/cache/HttpTransactCache.cc index 215b7a49ef1..7b3249af0f4 100644 --- a/src/iocore/cache/HttpTransactCache.cc +++ b/src/iocore/cache/HttpTransactCache.cc @@ -1222,7 +1222,11 @@ HttpTransactCache::CalcVariability(const HttpConfigAccessor *http_config_params, // Disable Vary mismatch checking for Accept-Encoding. This is only safe to // set if you are promising to fix any Accept-Encoding/Content-Encoding mismatches. - if (http_config_params->get_ignore_accept_encoding_mismatch() && + // Only suppress variability checks when the operator explicitly set + // proxy.config.http.cache.ignore_accept_encoding_mismatch to 1. The + // documented default value of 2 should continue to enforce Vary header + // semantics whenever the origin sends one. + if ((http_config_params->get_ignore_accept_encoding_mismatch() == 1) && !strcasecmp(const_cast(field->str), "Accept-Encoding")) { continue; } From 88a02b2fdd4f8ba6d7cbe50111d0996aae064b7c Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 10:32:21 +0000 Subject: [PATCH 20/29] Remove duplicate luajit package from Dockerfile dependencies --- contrib/docker/ubuntu/noble/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/docker/ubuntu/noble/Dockerfile b/contrib/docker/ubuntu/noble/Dockerfile index 728f2b184b4..8164fb1d59f 100644 --- a/contrib/docker/ubuntu/noble/Dockerfile +++ b/contrib/docker/ubuntu/noble/Dockerfile @@ -50,7 +50,6 @@ RUN apt update \ libbrotli-dev \ libzstd-dev \ luajit \ - luajit \ libluajit-5.1-dev \ libcap-dev \ libmagick++-dev \ From 8e4f9c844dd4e986f07f16467580266f0bffeaf4 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 10:36:04 +0000 Subject: [PATCH 21/29] only include zstd in error message if zstd exists --- plugins/compress/configuration.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index 9e7e34bf0d3..49150130163 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -218,7 +218,11 @@ HostConfiguration::add_compression_algorithms(swoc::TextView line) } else if (token == "deflate") { compression_algorithms_ |= ALGORITHM_DEFLATE; } else { +#ifdef HAVE_ZSTD_H error("Unknown compression type. Supported compression-algorithms ."); +#else + error("Unknown compression type. Supported compression-algorithms ."); +#endif } } } From 23716b1435a62fa57d3e8cd476597819d1e66833 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 13:13:21 +0000 Subject: [PATCH 22/29] Merge remote-tracking branch 'upstream/master' into zstd --- CMakeLists.txt | 15 +- CONTRIBUTING.md | 11 +- ci/jenkins/bin/clang-format.sh | 9 +- doc/CMakeLists.txt | 5 +- doc/admin-guide/configuration/hrw4u.en.rst | 49 +- .../statistics/core/cache-volume.en.rst | 19 + .../monitoring/statistics/core/cache.en.rst | 23 + doc/admin-guide/plugins/header_rewrite.en.rst | 144 ++++- doc/admin-guide/plugins/lua.en.rst | 1 + include/iocore/eventsystem/IOBuffer.h | 12 +- include/iocore/hostdb/HostDBProcessor.h | 18 - include/proxy/http/HttpSM.h | 3 - include/records/RecCore.h | 6 +- include/records/RecProcess.h | 5 + include/ts/apidefs.h.in | 8 +- include/tscore/ink_inet.h | 12 +- include/tsutil/Metrics.h | 36 ++ include/tsutil/Regex.h | 36 +- plugins/compress/CMakeLists.txt | 6 +- plugins/compress/brotli_compress.cc | 148 +++++ .../compress/brotli_compress.h | 32 +- plugins/compress/compress.cc | 552 +++--------------- plugins/compress/compress_common.cc | 37 ++ plugins/compress/compress_common.h | 96 +++ plugins/compress/configuration.cc | 6 +- plugins/compress/configuration.h | 6 +- plugins/compress/debug_macros.h | 2 +- plugins/compress/gzip_compress.cc | 173 ++++++ plugins/compress/gzip_compress.h | 51 ++ plugins/compress/misc.cc | 24 +- plugins/compress/misc.h | 86 +-- plugins/compress/zstd_compress.cc | 178 ++++++ plugins/compress/zstd_compress.h | 44 ++ plugins/esi/esi.cc | 4 +- plugins/esi/lib/EsiProcessor.cc | 2 + .../access_control/CMakeLists.txt | 2 +- .../experimental/access_control/pattern.cc | 101 +--- plugins/experimental/access_control/pattern.h | 17 +- plugins/experimental/geoip_acl/CMakeLists.txt | 1 - plugins/experimental/geoip_acl/acl.cc | 26 +- plugins/experimental/geoip_acl/acl.h | 39 +- plugins/experimental/url_sig/url_sig.cc | 96 ++- plugins/header_rewrite/factory.cc | 1 + plugins/header_rewrite/header_rewrite.cc | 103 +++- plugins/header_rewrite/operator.h | 15 + plugins/header_rewrite/operators.cc | 130 ++++- plugins/header_rewrite/operators.h | 80 +++ plugins/header_rewrite/parser.cc | 7 + plugins/header_rewrite/parser.h | 14 +- plugins/header_rewrite/ruleset.cc | 54 +- plugins/header_rewrite/ruleset.h | 151 ++--- plugins/lua/ts_lua_client_request.cc | 10 +- plugins/lua/ts_lua_http.cc | 2 + plugins/lua/ts_lua_http_config.cc | 16 +- plugins/lua/ts_lua_server_request.cc | 29 +- plugins/lua/ts_lua_vconn.cc | 10 +- plugins/origin_server_auth/aws_auth_v4.cc | 16 + plugins/regex_remap/regex_remap.cc | 33 +- plugins/regex_revalidate/CMakeLists.txt | 1 - plugins/regex_revalidate/regex_revalidate.cc | 153 ++--- src/api/InkAPITest.cc | 23 +- src/iocore/cache/CacheProcessor.cc | 6 +- src/iocore/cache/CacheVC.cc | 10 +- src/iocore/cache/P_CacheStats.h | 3 + src/iocore/eventsystem/P_IOBuffer.h | 4 + src/iocore/eventsystem/RecProcess.cc | 1 - src/iocore/eventsystem/RecRawStatsImpl.cc | 3 +- src/iocore/eventsystem/UnixEventProcessor.cc | 50 +- src/iocore/hostdb/HostDB.cc | 11 +- src/iocore/net/SSLStats.cc | 2 +- src/mgmt/rpc/handlers/config/Configuration.cc | 2 +- src/proxy/hdrs/MIME.cc | 70 ++- src/proxy/http/HttpSM.cc | 36 +- src/proxy/http/remap/PluginFactory.cc | 14 +- src/proxy/http2/HTTP2.cc | 2 +- src/proxy/logging/LogStandalone.cc | 15 +- src/proxy/logging/LogUtils.cc | 2 - src/records/P_RecCore.cc | 13 +- src/records/RecCore.cc | 44 +- src/records/RecRawStats.cc | 2 +- src/records/RecordsConfigUtils.cc | 4 - src/records/test_RecordsConfig.cc | 2 - src/traffic_cache_tool/CacheDefs.h | 10 +- src/traffic_server/traffic_server.cc | 18 +- src/tscore/ArgParser.cc | 42 +- src/tsutil/Metrics.cc | 30 + src/tsutil/Regex.cc | 31 + src/tsutil/unit_tests/test_Regex.cc | 265 ++++++++- tests/Pipfile | 1 - tests/gold_tests/h2/trickle_client.py | 7 +- tests/gold_tests/h2/trickle_server.py | 8 +- .../headers/gold/accept_webp_cache.gold | 2 +- .../pluginTest/compress/compress.gold | 4 +- .../gold/nested_ifs_definitely.gold | 18 + .../header_rewrite/gold/nested_ifs_else.gold | 17 + .../gold/nested_ifs_else_fie.gold | 19 + .../gold/nested_ifs_foo_bar.gold | 20 + .../gold/nested_ifs_foo_fie.gold | 21 + .../header_rewrite/gold/nested_ifs_maybe.gold | 18 + .../header_rewrite/gold/set_body_empty.gold | 16 + .../header_rewrite/gold/set_body_status.gold | 16 + .../gold/set_body_status_stdout.gold | 1 + .../header_rewrite_bundle.test.py | 78 +++ .../header_rewrite/rules/implicit_hook.conf | 2 - .../header_rewrite/rules/nested_ifs.conf | 44 ++ .../header_rewrite/rules/rule_empty_body.conf | 22 + .../rules/rule_set_body_status.conf | 22 + .../gold/origin_server_auth_parsing_ts.gold | 2 +- .../origin_server_auth_parsing_ts_uds 2.gold | 5 + .../origin_server_auth_parsing_ts_uds.gold | 2 +- .../polite_hook_wait/polite_hook_wait.cc | 20 +- .../pluginTest/prefetch/prefetch_cmcd0.gold | 4 +- .../pluginTest/prefetch/prefetch_cmcd1.gold | 4 +- .../regex_revalidate/regex_revalidate.test.py | 1 - .../slice/gold/slice_crr_ident.gold | 6 +- .../strategies/strategies_plugins.test.py | 169 ++++-- ...s_check_dual_cert_selection_plugin.test.py | 18 +- tests/gold_tests/traffic_ctl/gold/test_2.gold | 1 + tests/gold_tests/traffic_ctl/gold/test_3.gold | 1 + tools/autopep8.sh | 107 ---- tools/benchmark/CMakeLists.txt | 5 +- tools/benchmark/benchmark_EventSystem.cc | 7 +- tools/benchmark/benchmark_FreeList.cc | 3 +- tools/benchmark/benchmark_ProxyAllocator.cc | 1 + tools/benchmark/benchmark_Random.cc | 101 ++++ tools/benchmark/benchmark_SharedMutex.cc | 3 +- tools/hrw4u/grammar/hrw4u.g4 | 6 +- tools/hrw4u/grammar/u4wrh.g4 | 48 +- tools/hrw4u/pyproject.toml | 2 +- tools/hrw4u/scripts/testcase.py | 71 ++- tools/hrw4u/src/common.py | 1 + tools/hrw4u/src/generators.py | 20 +- tools/hrw4u/src/hrw_symbols.py | 28 +- tools/hrw4u/src/hrw_visitor.py | 55 +- tools/hrw4u/src/kg_visitor.py | 33 +- tools/hrw4u/src/lsp/completions.py | 16 +- tools/hrw4u/src/lsp/hover.py | 43 +- tools/hrw4u/src/suggestions.py | 9 +- tools/hrw4u/src/symbols.py | 107 ++-- tools/hrw4u/src/symbols_base.py | 22 +- tools/hrw4u/src/tables.py | 350 +++-------- tools/hrw4u/src/types.py | 82 ++- tools/hrw4u/src/visitor.py | 36 +- .../hrw4u/tests/data/conds/nested-ifs.ast.txt | 1 + .../tests/data/conds/nested-ifs.input.txt | 27 + .../tests/data/conds/nested-ifs.output.txt | 27 + tools/hrw4u/tests/data/hooks/remap.ast.txt | 2 +- tools/hrw4u/tests/data/hooks/remap.input.txt | 2 + tools/hrw4u/tests/data/hooks/remap.output.txt | 2 + tools/hrw4u/tests/data/ops/exceptions.txt | 2 + .../ops/http_cntl_invalid_bool.fail.error.txt | 4 +- .../ops/http_cntl_quoted_bool.fail.error.txt | 4 +- .../data/ops/http_cntl_valid_bools.ast.txt | 2 +- .../data/ops/http_cntl_valid_bools.output.txt | 2 +- tools/hrw4u/tests/data/ops/qsa.output.txt | 2 +- .../ops/skip_remap_quoted_bool.fail.error.txt | 4 +- tools/hrw4u/tests/data/vars/exceptions.txt | 5 + .../tests/data/vars/explicit_slots.ast.txt | 1 + .../tests/data/vars/explicit_slots.input.txt | 15 + .../tests/data/vars/explicit_slots.output.txt | 3 + .../data/vars/slot_conflict.fail.error.txt | 3 + .../data/vars/slot_conflict.fail.input.txt | 8 + .../tests/data/vars/vars_count.fail.error.txt | 2 +- 163 files changed, 3637 insertions(+), 1887 deletions(-) create mode 100644 plugins/compress/brotli_compress.cc rename src/records/P_RecProcess.h => plugins/compress/brotli_compress.h (54%) create mode 100644 plugins/compress/compress_common.cc create mode 100644 plugins/compress/compress_common.h create mode 100644 plugins/compress/gzip_compress.cc create mode 100644 plugins/compress/gzip_compress.h create mode 100644 plugins/compress/zstd_compress.cc create mode 100644 plugins/compress/zstd_compress.h create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_definitely.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else_fie.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_bar.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_fie.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_maybe.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/set_body_empty.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status_stdout.gold create mode 100644 tests/gold_tests/pluginTest/header_rewrite/rules/nested_ifs.conf create mode 100644 tests/gold_tests/pluginTest/header_rewrite/rules/rule_empty_body.conf create mode 100644 tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_status.conf create mode 100644 tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds 2.gold create mode 100644 tests/gold_tests/traffic_ctl/gold/test_2.gold create mode 100644 tests/gold_tests/traffic_ctl/gold/test_3.gold delete mode 100755 tools/autopep8.sh create mode 100644 tools/benchmark/benchmark_Random.cc create mode 100644 tools/hrw4u/tests/data/conds/nested-ifs.ast.txt create mode 100644 tools/hrw4u/tests/data/conds/nested-ifs.input.txt create mode 100644 tools/hrw4u/tests/data/conds/nested-ifs.output.txt create mode 100644 tools/hrw4u/tests/data/vars/exceptions.txt create mode 100644 tools/hrw4u/tests/data/vars/explicit_slots.ast.txt create mode 100644 tools/hrw4u/tests/data/vars/explicit_slots.input.txt create mode 100644 tools/hrw4u/tests/data/vars/explicit_slots.output.txt create mode 100644 tools/hrw4u/tests/data/vars/slot_conflict.fail.error.txt create mode 100644 tools/hrw4u/tests/data/vars/slot_conflict.fail.input.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index e908f6aac0b..692b11985a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -601,6 +601,8 @@ if(ENABLE_DOCS) COMPONENTS Runtime REQUIRED ) + + find_program(GRAPHVIZ_DOT dot REQUIRED) endif() if(ENABLE_AUTEST) @@ -809,17 +811,6 @@ add_custom_target( VERBATIM ) -# Leave autopep8 for historical reasons, but have it call yapf. It will not do -# to have conflicting Python formatting targets. This can be removed when at -# least CI is updated to use yapf instead of autopep8. -add_custom_target( - autopep8 - ${CMAKE_SOURCE_DIR}/tools/yapf.sh ${CMAKE_SOURCE_DIR} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - COMMENT "formatting python files" - VERBATIM -) - add_custom_target( yapf ${CMAKE_SOURCE_DIR}/tools/yapf.sh ${CMAKE_SOURCE_DIR} @@ -839,7 +830,7 @@ add_custom_target( # Add a format target that runs all the formatters. add_custom_target( format - DEPENDS clang-format autopep8 cmake-format + DEPENDS clang-format yapf cmake-format COMMENT "formatting all files" ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2750899e99e..fd23c6235ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,16 +55,13 @@ are a few simple rules to follow: 7. If the _PR_ is a Work-In-Progress, and not ready to commit, mark it with the **WIP** label. -8. Make sure you run **clang-format** before making the _PR_. This is easiest - done with e.g. "make clang-format", which works on macOS and Linux. +8. Make sure to **format** your code before making the _PR_. This is easily + done with e.g. "cmake --build build --target format", which works on macOS and Linux. -9. Make sure you run **autopep8** before making the _PR_. This is easiest - done with e.g. "make autopep8". - -10. When making backports, make sure you mark the _PR_ for the appropriate +9. When making backports, make sure you mark the _PR_ for the appropriate Github branch (e.g. **6.2.x**). -11. If you are making backports to an LTS branch, remember that the job of +10. If you are making backports to an LTS branch, remember that the job of merging such a _PR_ is the duty of the release manager. diff --git a/ci/jenkins/bin/clang-format.sh b/ci/jenkins/bin/clang-format.sh index f9c7fbfa32a..d98fb30e688 100755 --- a/ci/jenkins/bin/clang-format.sh +++ b/ci/jenkins/bin/clang-format.sh @@ -48,13 +48,18 @@ autoreconf -if && ./configure ${ATS_MAKE} clang-format [ "0" != "$?" ] && exit 1 -# Only enforce autopep8 on branches where the pre-commit hook was updated to -# check it. Otherwise, none of the PRs for older branches will pass this check. +# Older branches didn't have either autopep8 or yapf. Only run these checks on +# branches where the pre-commit hook was updated to check them. if grep -q autopep8 tools/git/pre-commit; then ${ATS_MAKE} autopep8 [ "0" != "$?" ] && exit 1 fi +if grep -q yapf tools/git/pre-commit; then + ${ATS_MAKE} yapf + [ "0" != "$?" ] && exit 1 +fi + git diff --exit-code [ "0" != "$?" ] && exit 1 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 4a6c070d47e..f407cb3034d 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -57,6 +57,7 @@ set(UML_FILES uml/extras/config-data.plantuml uml/extras/txn_box_config_schema.plantuml ) + # unfortunately, sphinx can't look else for files than its source directory # so these files must be create in the source tree foreach(UML ${UML_FILES}) @@ -64,8 +65,8 @@ foreach(UML ${UML_FILES}) list(APPEND SVG_FILES ${CMAKE_CURRENT_SOURCE_DIR}/uml/images/${uml_name}.svg) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/uml/images/${uml_name}.svg - COMMAND ${Java_JAVA_EXECUTABLE} -jar ${PLANTUML_JAR} -o ${CMAKE_CURRENT_SOURCE_DIR}/uml/images -tsvg - ${CMAKE_CURRENT_SOURCE_DIR}/${UML} + COMMAND ${Java_JAVA_EXECUTABLE} -jar ${PLANTUML_JAR} -o ${CMAKE_CURRENT_SOURCE_DIR}/uml/images -tsvg -graphvizdot + ${GRAPHVIZ_DOT} ${CMAKE_CURRENT_SOURCE_DIR}/${UML} DEPENDS ${UML} VERBATIM ) diff --git a/doc/admin-guide/configuration/hrw4u.en.rst b/doc/admin-guide/configuration/hrw4u.en.rst index 80beec6afec..11bb275dafd 100644 --- a/doc/admin-guide/configuration/hrw4u.en.rst +++ b/doc/admin-guide/configuration/hrw4u.en.rst @@ -231,6 +231,7 @@ The preference is the assignment style when appropriate. ============================= ================================= ================================================ Header Rewrite HRW4U Description ============================= ================================= ================================================ +add-header X-bar foo inbound.{req,resp}.x-Bar += "bar" Add the header to (possibly) an existing header counter my_stat counter("my_stat") Increment internal counter rm-client-header X-Foo inbound.req.X-Foo = "" Remove a client request header rm-cookie foo {in,out}bound.cookie.foo = "" Remove the cookie named foo @@ -254,6 +255,28 @@ set-status-reason "No" http.status.reason = "no" Set the response set-http-cntl http.cntl. = bool Turn on/off <:ref:`C`> controllers ============================= ================================= ================================================ +Adding Headers with the += Operator +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +HRW4U provides a special ``+=`` operator for adding headers:: + + REMAP { + # Using += to add a header (maps to add-header) + inbound.req.X-Custom-Header += "new-value"; + } + +The ``+=`` operator only works with the following pre-defined symbols: + +- ``inbound.req.
`` - Client request headers +- ``inbound.resp.
`` - Origin response headers +- ``outbound.req.
`` - Outbound request headers (context-restricted) +- ``outbound.resp.
`` - Outbound response headers (context-restricted) + +.. note:: + The ``+=`` operator differs from ``=`` in that ``=`` will replace/set the header value (mapping to + ``set-header``), while ``+=`` will add a new instance of the header (mapping to ``add-header``). + This is important for headers that can have multiple values, such as ``Set-Cookie`` or custom headers. + In addition to those operators above, HRW4U supports the following special operators without arguments: ================= ============================ ================================ @@ -300,9 +323,29 @@ TXN_CLOSE_HOOK TXN_CLOSE End of transaction A special section `VARS` is used to declare variables. There is no equivalent in `header_rewrite`, where you managed the variables manually. -.. note:: - The section name is always required in HRW4U, there are no implicit or default hooks. There - can be several if/else block per section block. +Variables and State Slots +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each variable type has a limited number of slots available: + +- ``bool`` - 16 slots (0-15) +- ``int8`` - 4 slots (0-3) +- ``int16`` - 1 slot (0) + +By default, slots are assigned automatically in declaration order. You can explicitly assign +a slot number using the ``@`` syntax:: + + VARS { + priority: bool @7; # Explicitly use slot 7 + active: bool; # Auto-assigned to slot 0 + config: bool @12; # Explicitly use slot 12 + counter: int8 @2; # Explicitly use int8 slot 2 + } + +Explicit slot assignment is useful when you need predictable slot numbers across configurations +or when integrating with existing header_rewrite rules that reference specific slot numbers. In +addition, a remap configuration can use ``@PPARAM`` to set one of these slot variables explicitly +as part of the configuration. Groups ------ diff --git a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst index 40fd3308b57..c6e39eb7449 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst @@ -125,9 +125,28 @@ a configuration with only one cache volume: :literal:`0`. .. ts:stat:: global proxy.process.cache.volume_0.ram_cache.hits integer :type: counter + Accumulates the number of hits to the LRU RAM cache for this volume. + .. ts:stat:: global proxy.process.cache.volume_0.ram_cache.misses integer :type: counter + Accumulates the number of misses to the LRU RAM cache for this volume. Note that this count includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. + +.. ts:stat:: global proxy.process.cache.volume_0.last_open_read.hits integer + :type: counter + + Accumulates the number of hits to the last open read cache for this volume. This cache stores the most recent read operation for each open cache volume. + +.. ts:stat:: global proxy.process.cache.volume_0.aggregation_buffer.hits integer + :type: counter + + Accumulates the number of hits to the aggregation buffer for this volume. This buffer stores data fragments that are on their way to be written to disk for write aggregation. + +.. ts:stat:: global proxy.process.cache.volume_0.all_memory_caches.misses integer + :type: counter + + Accumulates the number of misses to all memory caches (LRU RAM cache, last open read cache, and aggregation buffer) for this volume. This represents the total number of cache accesses that go to disk for this volume. + .. ts:stat:: global proxy.process.cache.volume_0.ram_cache.total_bytes integer :type: gauge :units: bytes diff --git a/doc/admin-guide/monitoring/statistics/core/cache.en.rst b/doc/admin-guide/monitoring/statistics/core/cache.en.rst index df04a2d1960..bb2f7d17f42 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache.en.rst @@ -86,7 +86,30 @@ Cache .. ts:stat:: global proxy.process.cache.ram_cache.bytes_used integer .. ts:stat:: global proxy.process.cache.ram_cache.hits integer + :type: counter + + Accumulates the number of hits to the LRU RAM cache for all volumes. + .. ts:stat:: global proxy.process.cache.ram_cache.misses integer + :type: counter + + Accumulates the number of misses to the LRU RAM cache for all volumes. Note that this includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. + +.. ts:stat:: global proxy.process.cache.last_open_read.hits integer + :type: counter + + Accumulates the number of hits to the last open read cache for all volumes. This cache stores the most recent read operation for each open cache volume. + +.. ts:stat:: global proxy.process.cache.aggregation_buffer.hits integer + :type: counter + + Accumulates the number of hits to the aggregation buffer for all volumes. This buffer stores data fragments that are on their way to be written to disk for write aggregation. + +.. ts:stat:: global proxy.process.cache.all_memory_caches.misses integer + :type: counter + + Accumulates the number of misses to all memory caches (LRU RAM cache, last open read cache, and aggregation buffer) for all volumes. This represents the total number of cache accesses that go to disk. + .. ts:stat:: global proxy.process.cache.ram_cache.total_bytes integer .. ts:stat:: global proxy.process.cache.read.active integer .. ts:stat:: global proxy.process.cache.read_busy.failure integer diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index e7f8c74769c..5ec2e89f20c 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -153,9 +153,19 @@ like the following:: Which converts any 4xx HTTP status code from the origin server to a 404. A response from the origin with a status of 200 would be unaffected by this rule. +Advanced Conditionals +--------------------- + +The header_rewrite plugin supports advanced conditional logic that allows +for more sophisticated rule construction, including branching logic, nested +conditionals, and complex boolean expressions. + +else and elif Clauses +~~~~~~~~~~~~~~~~~~~~~ + An optional ``else`` clause may be specified, which will be executed if the -conditions are not met. The ``else`` clause is specified by starting a new line -with the word ``else``. The following example illustrates this:: +conditions are not met. The ``else`` clause is specified by starting a new +line with the word ``else``. The following example illustrates this:: cond %{STATUS} >399 [AND] cond %{STATUS} <500 @@ -164,10 +174,12 @@ with the word ``else``. The following example illustrates this:: set-status 503 The ``else`` clause is not a condition, and does not take any flags, it is -of course optional, but when specified must be followed by at least one operator. +of course optional, but when specified must be followed by at least one +operator. -You can also do an ``elif`` (else if) clause, which is specified by starting a new line -with the word ``elif``. The following example illustrates this:: +You can also do an ``elif`` (else if) clause, which is specified by +starting a new line with the word ``elif``. The following example +illustrates this:: cond %{STATUS} >399 [AND] cond %{STATUS} <500 @@ -178,14 +190,105 @@ with the word ``elif``. The following example illustrates this:: else set-status 503 -Keep in mind that nesting the ``else`` and ``elif`` clauses is not allowed, but any -number of ``elif`` clauses can be specified. We can consider these clauses are more -powerful and flexible ``switch`` statement. In an ``if-elif-else`` rule, only one -will evaluate its operators. +Any number of ``elif`` clauses can be specified. We can consider these +clauses are more powerful and flexible ``switch`` statement. In an +``if-elif-else`` rule, only one will evaluate its operators. + +Note that while ``else`` and ``elif`` themselves cannot be directly nested, +you can use ``if``/``endif`` blocks within ``else`` or ``elif`` operator +sections to achieve nested conditional logic (see `Nested Conditionals with +if/endif`_). Similarly, each ``else`` and ``elif`` have the same implied :ref:`Hook Condition ` as the initial condition. +Nested Conditionals with if/endif +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For more complex logic requiring nested conditionals, the ``if`` and +``endif`` pseudo-operators can be used. While ``else`` and ``elif`` +themselves cannot be directly nested, you can use ``if``/``endif`` blocks +within any operator section (including inside ``else`` or ``elif`` blocks) +to achieve arbitrary nesting depth. + +The ``if`` operator starts a new conditional block, and ``endif`` closes +it. Each ``if`` must have a matching ``endif``. Here's an example:: + + cond %{READ_RESPONSE_HDR_HOOK} [AND] + cond %{STATUS} >399 + if + cond %{HEADER:X-Custom-Error} ="true" + set-header X-Error-Handled "yes" + else + set-header X-Error-Handled "no" + endif + set-status 500 + +In this example, the nested ``if``/``endif`` block is only evaluated when +the status is greater than 399. The nested block itself can contain +``else`` or ``elif`` clauses, and you can nest multiple levels deep:: + + cond %{READ_RESPONSE_HDR_HOOK} + if + cond %{STATUS} =404 + if + cond %{CLIENT-HEADER:User-Agent} /mobile/ + set-header X-Error-Type "mobile-404" + else + set-header X-Error-Type "desktop-404" + endif + elif + cond %{STATUS} =500 + set-header X-Error-Type "server-error" + endif + +GROUP Conditions in Advanced Conditionals +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `GROUP`_ condition can be combined with advanced conditionals to +create very sophisticated boolean expressions. ``GROUP`` conditions act as +parentheses in your conditional logic, allowing you to mix AND, OR, and NOT +operators in complex ways. + +Here's an example combining ``GROUP`` with ``if``/``endif``:: + + cond %{READ_RESPONSE_HDR_HOOK} [AND] + cond %{STATUS} >399 + if + cond %{GROUP} [OR] + cond %{CLIENT-HEADER:X-Retry} ="true" [AND] + cond %{METHOD} =GET + cond %{GROUP:END} + cond %{CLIENT-HEADER:X-Force-Cache} ="" [NOT] + set-header X-Can-Retry "yes" + else + set-header X-Can-Retry "no" + endif + set-status 500 + +This creates the logic: if error status, then set retry header when +``((X-Retry=true AND METHOD=GET) OR X-Force-Cache header exists)``. +The GROUP is necessary here to properly combine the two conditions with OR. + +You can also use ``GROUP`` with ``else`` and ``elif`` inside nested conditionals:: + + cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{STATUS} >399 + if + cond %{GROUP} [OR] + cond %{HEADER:X-Custom} ="retry" [AND] + cond %{METHOD} =POST + cond %{GROUP:END} + cond %{HEADER:Content-Type} /json/ + set-header X-Error-Handler "json-retry" + elif + cond %{METHOD} =GET + set-header X-Error-Handler "get-error" + else + set-header X-Error-Handler "standard" + endif + set-status 500 + State variables --------------- @@ -923,6 +1026,29 @@ no facility to increment by other amounts, nor is it possible to initialize the counter with any value other than ``0``. Additionally, the counter will reset whenever |TS| is restarted. +if +~~ +:: + + if + + + endif + +This is a pseudo-operator that enables nested conditional blocks within +the operator section of a rule. While ``else`` and ``elif`` themselves +cannot be directly nested, you can use ``if``/``endif`` blocks within any +operator section (including inside ``else`` or ``elif`` blocks) to create +arbitrary nesting depth for complex conditional logic. + +The ``if`` operator must be preceded by conditions and followed by at +least one condition or operator. Each ``if`` must have a matching +``endif`` to close the block. Within an ``if``/``endif`` block, you can +use regular conditions, operators, and even ``else`` and ``elif`` clauses. + +For detailed usage and examples, see `Nested Conditionals with if/endif`_ +in the `Advanced Conditionals`_ section. + no-op ~~~~~ :: diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index aba1ff7db3d..dae206f1de5 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -1838,6 +1838,7 @@ Socket address family TS_LUA_AF_INET (2) TS_LUA_AF_INET6 (10) + TS_LUA_AF_UNIX (1) :ref:`TOP ` diff --git a/include/iocore/eventsystem/IOBuffer.h b/include/iocore/eventsystem/IOBuffer.h index 33c151ff05c..de01b59a6ad 100644 --- a/include/iocore/eventsystem/IOBuffer.h +++ b/include/iocore/eventsystem/IOBuffer.h @@ -102,8 +102,16 @@ enum AllocType { #define BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(_size_index) (((uint64_t)_size_index) < DEFAULT_BUFFER_SIZES) #define BUFFER_SIZE_INDEX_IS_CONSTANT(_size_index) (_size_index >= DEFAULT_BUFFER_SIZES) -#define BUFFER_SIZE_FOR_XMALLOC(_size) (-(_size)) -#define BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(_size) (-(_size)) +#define BUFFER_SIZE_FOR_XMALLOC(_size) (-(_size)) +[[nodiscard]] constexpr int64_t +BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(int64_t size) +{ + // Positive size indices are interpreted as a BUFFER_SIZE_INDEX_*. + // Negative size indices are interpreted as a malloc size. + // A zero size index is BUFFER_SIZE_INDEX_128, which causes this buffer to be freed incorrectly. + ink_release_assert(size > 0 && "Zero-length xmalloc buffer causes heap corruption!"); + return -size; +} #define BUFFER_SIZE_FOR_CONSTANT(_size) (_size - DEFAULT_BUFFER_SIZES) #define BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(_size) (_size + DEFAULT_BUFFER_SIZES) diff --git a/include/iocore/hostdb/HostDBProcessor.h b/include/iocore/hostdb/HostDBProcessor.h index 57ea62e72a2..350802c0d0e 100644 --- a/include/iocore/hostdb/HostDBProcessor.h +++ b/include/iocore/hostdb/HostDBProcessor.h @@ -152,12 +152,6 @@ struct HostDBInfo { */ bool select(ts_time now, ts_seconds fail_window) const; - /// Check if this info is valid. - bool is_valid() const; - - /// Mark this info as invalid. - void invalidate(); - /** Mark the entry as down. * * @param now Time of the failure. @@ -290,18 +284,6 @@ HostDBInfo::migrate_from(HostDBInfo::self_type const &that) this->http_version = that.http_version; } -inline bool -HostDBInfo::is_valid() const -{ - return type != HostDBType::UNSPEC; -} - -inline void -HostDBInfo::invalidate() -{ - type = HostDBType::UNSPEC; -} - // ---- /** Root item for HostDB. * This is the container for HostDB data. It is always an array of @c HostDBInfo instances plus metadata. diff --git a/include/proxy/http/HttpSM.h b/include/proxy/http/HttpSM.h index 8d604aa3251..84ab9e3654b 100644 --- a/include/proxy/http/HttpSM.h +++ b/include/proxy/http/HttpSM.h @@ -357,11 +357,8 @@ class HttpSM : public Continuation, public PluginUserArgs int state_read_client_request_header(int event, void *data); int state_watch_for_client_abort(int event, void *data); int state_read_push_response_header(int event, void *data); - int state_pre_resolve(int event, void *data); int state_hostdb_lookup(int event, void *data); int state_hostdb_reverse_lookup(int event, void *data); - int state_mark_os_down(int event, void *data); - int state_auth_callback(int event, void *data); int state_add_to_list(int event, void *data); int state_remove_from_list(int event, void *data); diff --git a/include/records/RecCore.h b/include/records/RecCore.h index e4c80c94f36..d0a6ed5b4fb 100644 --- a/include/records/RecCore.h +++ b/include/records/RecCore.h @@ -84,10 +84,6 @@ RecErrT _RecRegisterStatFloat(RecT rec_type, const char *name, RecFloat data_def #define RecRegisterStatFloat(rec_type, name, data_default, persist_type) \ _RecRegisterStatFloat((rec_type), (name), (data_default), REC_PERSISTENCE_TYPE(persist_type)) -RecErrT _RecRegisterStatString(RecT rec_type, const char *name, RecStringConst data_default, RecPersistT persist_type); -#define RecRegisterStatString(rec_type, name, data_default, persist_type) \ - _RecRegisterStatString((rec_type), (name), (data_default), REC_PERSISTENCE_TYPE(persist_type)) - RecErrT _RecRegisterStatCounter(RecT rec_type, const char *name, RecCounter data_default, RecPersistT persist_type); #define RecRegisterStatCounter(rec_type, name, data_default, persist_type) \ _RecRegisterStatCounter((rec_type), (name), (data_default), REC_PERSISTENCE_TYPE(persist_type)) @@ -161,7 +157,7 @@ void Enable_Config_Var(std::string_view const &name, RecContextCb record_cb, Rec RecErrT RecSetRecordInt(const char *name, RecInt rec_int, RecSourceT source, bool lock = true); RecErrT RecSetRecordFloat(const char *name, RecFloat rec_float, RecSourceT source, bool lock = true); -RecErrT RecSetRecordString(const char *name, const RecString rec_string, RecSourceT source, bool lock = true); +RecErrT RecSetRecordString(const char *name, RecStringConst rec_string, RecSourceT source, bool lock = true); RecErrT RecSetRecordCounter(const char *name, RecCounter rec_counter, RecSourceT source, bool lock = true); std::optional RecGetRecordInt(const char *name, bool lock = true); diff --git a/include/records/RecProcess.h b/include/records/RecProcess.h index 8f6ed4d2ae9..6bae4553f8c 100644 --- a/include/records/RecProcess.h +++ b/include/records/RecProcess.h @@ -62,6 +62,11 @@ int RecRawStatSyncIntMsecsToFloatSeconds(const char *name, RecDataT data_type, R int RecRegisterRawStatSyncCb(const char *name, RecRawStatSyncCb sync_cb, RecRawStatBlock *rsb, int id); int RecRawStatUpdateSum(RecRawStatBlock *rsb, int id); +int RecExecRawStatSyncCbs(); + +using RecCallbackFunction = std::function; +void RecRegNewSyncStatSync(RecCallbackFunction callback); + //------------------------------------------------------------------------- // RawStat Setting/Getting //------------------------------------------------------------------------- diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index a0f3300d623..078ce0eb69c 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -777,7 +777,6 @@ enum TSOverridableConfigKey { TS_CONFIG_HTTP_CHUNKING_ENABLED, TS_CONFIG_HTTP_NEGATIVE_CACHING_ENABLED, TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME, - TS_CONFIG_HTTP_NEGATIVE_CACHING_LIST, TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE, TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN, TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT, @@ -821,7 +820,6 @@ enum TSOverridableConfigKey { TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DOWN_SERVER, TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES, TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT, - TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE, TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME, TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS, TS_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT, @@ -842,7 +840,6 @@ enum TSOverridableConfigKey { TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE, TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED, TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME, - TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST, TS_CONFIG_SSL_HSTS_MAX_AGE, TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS, TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, @@ -906,8 +903,11 @@ enum TSOverridableConfigKey { TS_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT, TS_CONFIG_HTTP_CACHE_IGNORE_QUERY, TS_CONFIG_HTTP_DROP_CHUNKED_TRAILERS, - TS_CONFIG_HTTP_CACHE_POST_METHOD, TS_CONFIG_HTTP_STRICT_CHUNK_PARSING, + TS_CONFIG_HTTP_NEGATIVE_CACHING_LIST, + TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE, + TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST, + TS_CONFIG_HTTP_CACHE_POST_METHOD, TS_CONFIG_LAST_ENTRY, }; diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index cc9a07a0e84..18319b13535 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -324,8 +324,16 @@ ats_unix_append_id(sockaddr_un *s, int id) { char tmp[16]; int cnt = snprintf(tmp, sizeof(tmp), "-%d", id); - if (static_cast(ats_unix_path_len(s) + cnt) < TS_UNIX_SIZE) { - strncat(s->sun_path, tmp, cnt); + + // Defensive check: snprintf can return negative on error or >= sizeof(tmp) if truncated + if (cnt < 0 || cnt >= static_cast(sizeof(tmp))) { + ink_assert(!"snprintf failed or truncated in ats_unix_append_id"); + return; + } + + int old_len = ats_unix_path_len(s); + if (static_cast(old_len + cnt) < TS_UNIX_SIZE) { + memcpy(s->sun_path + old_len, tmp, cnt + 1); // +1 to include the null terminator #if HAVE_STRUCT_SOCKADDR_UN_SUN_LEN s->sun_len = SUN_LEN(s); #endif diff --git a/include/tsutil/Metrics.h b/include/tsutil/Metrics.h index bbfb7ab1828..4a847f397a1 100644 --- a/include/tsutil/Metrics.h +++ b/include/tsutil/Metrics.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "swoc/MemSpan.h" @@ -535,6 +536,41 @@ class Metrics }; // class Counter + class StaticString + { + public: + using StringStorage = std::unordered_map; + using iterator = StringStorage::iterator; + + static void + createString(const std::string &name, const std::string_view value) + { + auto &instance = Metrics::StaticString::instance(); + return instance._createString(name, value); + } + + static StaticString &instance(); + + iterator + begin() + { + return _strings.begin(); + } + iterator + end() + { + return _strings.end(); + }; + + std::optional lookup(const std::string &name); + + private: + void _createString(const std::string &name, const std::string_view value); + + StringStorage _strings; + mutable std::mutex _mutex; + }; + /** * Derive metrics by summing a set of other metrics. * diff --git a/include/tsutil/Regex.h b/include/tsutil/Regex.h index 9ca0608f760..cd8d7c1cb49 100644 --- a/include/tsutil/Regex.h +++ b/include/tsutil/Regex.h @@ -33,10 +33,18 @@ /// @internal These values are copied from pcre2.h, to avoid having to include it. The values are checked (with /// static_assert) in Regex.cc against PCRE2 named constants, in case they change in future PCRE2 releases. enum REFlags { - RE_CASE_INSENSITIVE = 0x00000008u, ///< Ignore case (default: case sensitive). - RE_UNANCHORED = 0x00000400u, ///< Unanchored (DFA defaults to anchored). - RE_ANCHORED = 0x80000000u, ///< Anchored (Regex defaults to unanchored). - RE_NOTEMPTY = 0x00000004u ///< Not empty (default: may match empty string). + RE_CASE_INSENSITIVE = 0x00000008u, ///< Ignore case (by default, matches are case sensitive). + RE_UNANCHORED = 0x00000400u, ///< Unanchored (@a DFA defaults to anchored). + RE_ANCHORED = 0x80000000u, ///< Anchored (@a Regex defaults to unanchored). + RE_NOTEMPTY = 0x00000004u ///< Not empty (by default, matches may match empty string). +}; + +/// @brief Error codes returned by regular expression operations. +/// +/// @internal As with REFlags, these values are copied from pcre2.h, to avoid having to include it. +enum REErrors { + RE_ERROR_NOMATCH = -1, ///< No match found. + RE_ERROR_NULL = -51 ///< NULL code or subject was passed. }; /// @brief Wrapper for PCRE2 match data. @@ -90,8 +98,24 @@ class RegexMatches class Regex { public: - Regex() = default; - Regex(Regex const &) = delete; // No copying. + Regex() = default; + /** Deep copy constructor. + * + * Creates a new Regex object with a deep copy of the compiled pattern. + * Uses pcre2_code_copy() to duplicate the compiled pattern without + * requiring the original pattern string. + * + * @param other The Regex object to copy from. + */ + Regex(Regex const &other); + /** Deep copy assignment operator. + * + * Replaces the current compiled pattern with a deep copy of the other's pattern. + * + * @param other The Regex object to copy from. + * @return Reference to this object. + */ + Regex &operator=(Regex const &other); Regex(Regex &&that) noexcept; Regex &operator=(Regex &&other); ~Regex(); diff --git a/plugins/compress/CMakeLists.txt b/plugins/compress/CMakeLists.txt index 65b24ce9558..6cd700c10ca 100644 --- a/plugins/compress/CMakeLists.txt +++ b/plugins/compress/CMakeLists.txt @@ -15,13 +15,17 @@ # ####################### -add_atsplugin(compress compress.cc configuration.cc misc.cc) +add_atsplugin(compress compress.cc configuration.cc misc.cc compress_common.cc gzip_compress.cc) target_link_libraries(compress PRIVATE libswoc::libswoc) + if(HAVE_BROTLI_ENCODE_H) + target_sources(compress PRIVATE brotli_compress.cc) target_link_libraries(compress PRIVATE brotli::brotlienc) + target_compile_definitions(compress PRIVATE HAVE_BROTLI_ENCODE_H=1) endif() if(HAVE_ZSTD_H) + target_sources(compress PRIVATE zstd_compress.cc) target_link_libraries(compress PRIVATE zstd::zstd) endif() diff --git a/plugins/compress/brotli_compress.cc b/plugins/compress/brotli_compress.cc new file mode 100644 index 00000000000..da0eb299971 --- /dev/null +++ b/plugins/compress/brotli_compress.cc @@ -0,0 +1,148 @@ +/** @file + + Brotli compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "brotli_compress.h" + +#if HAVE_BROTLI_ENCODE_H + +#include "debug_macros.h" + +#include +#include + +namespace Brotli +{ +const int BROTLI_COMPRESSION_LEVEL = 6; +const int BROTLI_LGW = 16; + +static bool +compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, BrotliEncoderOperation op) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->bstrm.next_in = (uint8_t *)upstream_buffer; + data->bstrm.avail_in = upstream_length; + + bool ok = true; + while (ok) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + data->bstrm.next_out = reinterpret_cast(downstream_buffer); + data->bstrm.avail_out = downstream_length; + data->bstrm.total_out = 0; + + ok = + !!BrotliEncoderCompressStream(data->bstrm.br, op, &data->bstrm.avail_in, &const_cast(data->bstrm.next_in), + &data->bstrm.avail_out, &data->bstrm.next_out, &data->bstrm.total_out); + + if (!ok) { + error("BrotliEncoderCompressStream(%d) call failed", op); + return false; + } + + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out); + data->downstream_length += (downstream_length - data->bstrm.avail_out); + if (data->bstrm.avail_in || BrotliEncoderHasMoreOutput(data->bstrm.br)) { + continue; + } + + break; + } + + return ok; +} + +void +data_alloc(Data *data) +{ + debug("brotli compression. Create Brotli Encoder Instance."); + data->bstrm.br = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!data->bstrm.br) { + fatal("Brotli Encoder Instance Failed"); + } + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, BROTLI_COMPRESSION_LEVEL); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, BROTLI_LGW); + data->bstrm.next_in = nullptr; + data->bstrm.avail_in = 0; + data->bstrm.total_in = 0; + data->bstrm.next_out = nullptr; + data->bstrm.avail_out = 0; + data->bstrm.total_out = 0; +} + +void +data_destroy(Data *data) +{ + BrotliEncoderDestroyInstance(data->bstrm.br); +} + +void +transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + bool ok = compress_operation(data, upstream_buffer, upstream_length, BROTLI_OPERATION_PROCESS); + if (!ok) { + error("BrotliEncoderCompressStream(PROCESS) call failed"); + return; + } + + data->bstrm.total_in += upstream_length; + + if (!data->hc->flush()) { + return; + } + + ok = compress_operation(data, nullptr, 0, BROTLI_OPERATION_FLUSH); + if (!ok) { + error("BrotliEncoderCompressStream(FLUSH) call failed"); + return; + } +} + +void +transform_finish(Data *data) +{ + if (data->state != transform_state_output) { + return; + } + + data->state = transform_state_finished; + + bool ok = compress_operation(data, nullptr, 0, BROTLI_OPERATION_FINISH); + if (!ok) { + error("BrotliEncoderCompressStream(PROCESS) call failed"); + return; + } + + if (data->downstream_length != static_cast(data->bstrm.total_out)) { + error("brotli-transform: output lengths don't match (%" PRId64 ", %zu)", data->downstream_length, data->bstrm.total_out); + } + + debug("brotli-transform: Finished brotli"); + log_compression_ratio(data->bstrm.total_in, data->downstream_length); +} + +} // namespace Brotli + +#endif // HAVE_BROTLI_ENCODE_H diff --git a/src/records/P_RecProcess.h b/plugins/compress/brotli_compress.h similarity index 54% rename from src/records/P_RecProcess.h rename to plugins/compress/brotli_compress.h index 40f34c67385..40be1b90363 100644 --- a/src/records/P_RecProcess.h +++ b/plugins/compress/brotli_compress.h @@ -1,6 +1,6 @@ /** @file - Private record process declarations + Brotli compression implementation @section license License @@ -23,22 +23,24 @@ #pragma once -// Must include 'P_EventSystem.h' before 'I_EventSystem.h' (which is -// included in 'I_RecProcess.h') to prevent multiple-symbol-definition -// complaints if the caller uses both 'P_EventSystem.h' and this 'P_' -// file. -#include "../iocore/eventsystem/P_EventSystem.h" +#include "compress_common.h" -#include "records/RecProcess.h" -#include "P_RecDefs.h" +#if HAVE_BROTLI_ENCODE_H -#include +namespace Brotli +{ +// Initialize brotli compression context +void data_alloc(Data *data); -//------------------------------------------------------------------------- -// Protected Interface -//------------------------------------------------------------------------- +// Destroy brotli compression context +void data_destroy(Data *data); -int RecExecRawStatSyncCbs(); +// Compress one chunk of data +void transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); -using RecCallbackFunction = std::function; -void RecRegNewSyncStatSync(RecCallbackFunction callback); +// Finish compression and flush remaining data +void transform_finish(Data *data); + +} // namespace Brotli + +#endif // HAVE_BROTLI_ENCODE_H diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 3d13da0e362..c2767167fbe 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -23,31 +23,26 @@ #include #include -#include -#if HAVE_ZSTD_H -#include -#endif #include "ts/apidefs.h" #include "tscore/ink_config.h" #include -#if HAVE_BROTLI_ENCODE_H -#include -#endif - #include "ts/ts.h" #include "tscore/ink_defs.h" #include "debug_macros.h" +#include "compress_common.h" #include "misc.h" #include "configuration.h" +#include "gzip_compress.h" +#include "brotli_compress.h" +#include "zstd_compress.h" #include "ts/remap.h" #include "ts/remap_version.h" using namespace std; -using namespace Gzip; // FIXME: custom dictionaries would be nice. configurable/content-type? // a GPRS device might benefit from a higher compression ratio, whereas a desktop w. high bandwidth @@ -66,6 +61,9 @@ namespace compress_ns DbgCtl dbg_ctl{TAG}; } +namespace Compress +{ + const char *dictionary = nullptr; static const char *global_hidden_header_name = nullptr; @@ -78,75 +76,66 @@ Configuration *prev_config = nullptr; namespace { -/** - If client request has both of Range and Accept-Encoding header, follow range-request config. - */ -void -handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) -{ - TSMLoc accept_encoding_hdr_field = - TSMimeHdrFieldFind(req_buf, req_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); - ts::PostScript accept_encoding_defer([&]() -> void { TSHandleMLocRelease(req_buf, req_loc, accept_encoding_hdr_field); }); - if (accept_encoding_hdr_field == TS_NULL_MLOC) { - return; - } + /** + If client request has both of Range and Accept-Encoding header, follow range-request config. + */ + void + handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) + { + TSMLoc accept_encoding_hdr_field = + TSMimeHdrFieldFind(req_buf, req_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + ts::PostScript accept_encoding_defer([&]() -> void { TSHandleMLocRelease(req_buf, req_loc, accept_encoding_hdr_field); }); + if (accept_encoding_hdr_field == TS_NULL_MLOC) { + return; + } - TSMLoc range_hdr_field = TSMimeHdrFieldFind(req_buf, req_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); - ts::PostScript range_defer([&]() -> void { TSHandleMLocRelease(req_buf, req_loc, range_hdr_field); }); - if (range_hdr_field == TS_NULL_MLOC) { - return; - } + TSMLoc range_hdr_field = TSMimeHdrFieldFind(req_buf, req_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); + ts::PostScript range_defer([&]() -> void { TSHandleMLocRelease(req_buf, req_loc, range_hdr_field); }); + if (range_hdr_field == TS_NULL_MLOC) { + return; + } - debug("Both of Accept-Encoding and Range header are found in the request"); + debug("Both of Accept-Encoding and Range header are found in the request"); - switch (hc->range_request_ctl()) { - case RangeRequestCtrl::REMOVE_RANGE: { - debug("Remove the Range header by remove-range config"); - while (range_hdr_field) { - TSMLoc next_dup = TSMimeHdrFieldNextDup(req_buf, req_loc, range_hdr_field); - TSMimeHdrFieldDestroy(req_buf, req_loc, range_hdr_field); - TSHandleMLocRelease(req_buf, req_loc, range_hdr_field); - range_hdr_field = next_dup; + switch (hc->range_request_ctl()) { + case RangeRequestCtrl::REMOVE_RANGE: { + debug("Remove the Range header by remove-range config"); + while (range_hdr_field) { + TSMLoc next_dup = TSMimeHdrFieldNextDup(req_buf, req_loc, range_hdr_field); + TSMimeHdrFieldDestroy(req_buf, req_loc, range_hdr_field); + TSHandleMLocRelease(req_buf, req_loc, range_hdr_field); + range_hdr_field = next_dup; + } + break; } - break; - } - case RangeRequestCtrl::REMOVE_ACCEPT_ENCODING: { - debug("Remove the Accept-Encoding header by remove-accept-encoding config"); - while (accept_encoding_hdr_field) { - TSMLoc next_dup = TSMimeHdrFieldNextDup(req_buf, req_loc, accept_encoding_hdr_field); - TSMimeHdrFieldDestroy(req_buf, req_loc, accept_encoding_hdr_field); - TSHandleMLocRelease(req_buf, req_loc, accept_encoding_hdr_field); - accept_encoding_hdr_field = next_dup; + case RangeRequestCtrl::REMOVE_ACCEPT_ENCODING: { + debug("Remove the Accept-Encoding header by remove-accept-encoding config"); + while (accept_encoding_hdr_field) { + TSMLoc next_dup = TSMimeHdrFieldNextDup(req_buf, req_loc, accept_encoding_hdr_field); + TSMimeHdrFieldDestroy(req_buf, req_loc, accept_encoding_hdr_field); + TSHandleMLocRelease(req_buf, req_loc, accept_encoding_hdr_field); + accept_encoding_hdr_field = next_dup; + } + break; + } + case RangeRequestCtrl::NO_COMPRESSION: + // Do NOT touch header - this config is referred by `transformable()` function + debug("no header modification by no-compression config"); + break; + case RangeRequestCtrl::NONE: + [[fallthrough]]; + default: + debug("Do nothing by none config"); + break; } - break; - } - case RangeRequestCtrl::NO_COMPRESSION: - // Do NOT touch header - this config is referred by `transformable()` function - debug("no header modification by no-compression config"); - break; - case RangeRequestCtrl::NONE: - [[fallthrough]]; - default: - debug("Do nothing by none config"); - break; } -} } // namespace -// Forward declarations for ZSTD compression functions -#if HAVE_ZSTD_H -static void zstd_transform_init(Data *data); -static void zstd_transform_finish(Data *data); -static void zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); -#endif - static Data * data_alloc(int compression_type, int compression_algorithms, HostConfiguration *hc) { - Data *data; - int err; + Data *data = static_cast(TSmalloc(sizeof(Data))); - data = static_cast(TSmalloc(sizeof(Data))); data->downstream_vio = nullptr; data->downstream_buffer = nullptr; data->downstream_reader = nullptr; @@ -155,68 +144,24 @@ data_alloc(int compression_type, int compression_algorithms, HostConfiguration * data->compression_type = compression_type; data->compression_algorithms = compression_algorithms; data->hc = hc; - data->zstrm.next_in = Z_NULL; - data->zstrm.avail_in = 0; - data->zstrm.total_in = 0; - data->zstrm.next_out = Z_NULL; - data->zstrm.avail_out = 0; - data->zstrm.total_out = 0; - data->zstrm.zalloc = gzip_alloc; - data->zstrm.zfree = gzip_free; - data->zstrm.opaque = (voidpf) nullptr; - data->zstrm.data_type = Z_ASCII; - - int window_bits = WINDOW_BITS_GZIP; - if (compression_type & COMPRESSION_TYPE_DEFLATE) { - window_bits = WINDOW_BITS_DEFLATE; - } - err = deflateInit2(&data->zstrm, data->hc->zlib_compression_level(), Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); - - if (err != Z_OK) { - fatal("gzip-transform: ERROR: deflateInit (%d)!", err); + // Initialize algorithm-specific compression contexts + if ((compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE)) && + (compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { + Gzip::data_alloc(data); } - if (dictionary) { - err = deflateSetDictionary(&data->zstrm, reinterpret_cast(dictionary), strlen(dictionary)); - if (err != Z_OK) { - fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err); - } - } #if HAVE_BROTLI_ENCODE_H - data->bstrm.br = nullptr; - if (compression_type & COMPRESSION_TYPE_BROTLI) { - debug("brotli compression. Create Brotli Encoder Instance."); - data->bstrm.br = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); - if (!data->bstrm.br) { - fatal("Brotli Encoder Instance Failed"); - } - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, data->hc->brotli_compression_level()); - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, data->hc->brotli_lgw_size()); - data->bstrm.next_in = nullptr; - data->bstrm.avail_in = 0; - data->bstrm.total_in = 0; - data->bstrm.next_out = nullptr; - data->bstrm.avail_out = 0; - data->bstrm.total_out = 0; + if (compression_type & COMPRESSION_TYPE_BROTLI && compression_algorithms & ALGORITHM_BROTLI) { + Brotli::data_alloc(data); } #endif #if HAVE_ZSTD_H - data->zstrm_zstd.cctx = nullptr; - data->zstrm_zstd.next_in = nullptr; - data->zstrm_zstd.avail_in = 0; - data->zstrm_zstd.total_in = 0; - data->zstrm_zstd.next_out = nullptr; - data->zstrm_zstd.avail_out = 0; - data->zstrm_zstd.total_out = 0; - if (compression_type & COMPRESSION_TYPE_ZSTD) { - debug("zstd compression. Create Zstd Compression Context."); - data->zstrm_zstd.cctx = ZSTD_createCCtx(); - if (!data->zstrm_zstd.cctx) { - fatal("Zstd Compression Context Creation Failed"); - } + if ((compression_type & COMPRESSION_TYPE_ZSTD) && (compression_algorithms & ALGORITHM_ZSTD)) { + Zstd::data_alloc(data); } #endif + return data; } @@ -225,21 +170,24 @@ data_destroy(Data *data) { TSReleaseAssert(data); - // deflateEnd return value ignore is intentional - // it would spew log on every client abort - deflateEnd(&data->zstrm); - if (data->downstream_buffer) { TSIOBufferDestroy(data->downstream_buffer); } -// brotlidestory + // Destroy algorithm-specific compression contexts + if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE)) && + (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { + Gzip::data_destroy(data); + } + #if HAVE_BROTLI_ENCODE_H - BrotliEncoderDestroyInstance(data->bstrm.br); + if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { + Brotli::data_destroy(data); + } #endif #if HAVE_ZSTD_H - if (data->zstrm_zstd.cctx) { - ZSTD_freeCCtx(data->zstrm_zstd.cctx); + if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { + Zstd::data_destroy(data); } #endif @@ -390,8 +338,8 @@ compress_transform_init(TSCont contp, Data *data) } #if HAVE_ZSTD_H - if (data->compression_type & COMPRESSION_TYPE_ZSTD) { - zstd_transform_init(data); + if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { + Zstd::transform_init(data); if (!data->zstrm_zstd.cctx) { TSError("Failed to create Zstandard compression context"); return; @@ -402,246 +350,6 @@ compress_transform_init(TSCont contp, Data *data) TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); } -static void -gzip_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) -{ - TSIOBufferBlock downstream_blkp; - int64_t downstream_length; - int err; - data->zstrm.next_in = (unsigned char *)upstream_buffer; - data->zstrm.avail_in = upstream_length; - - while (data->zstrm.avail_in > 0) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - - data->zstrm.next_out = reinterpret_cast(downstream_buffer); - data->zstrm.avail_out = downstream_length; - - if (!data->hc->flush()) { - err = deflate(&data->zstrm, Z_NO_FLUSH); - } else { - err = deflate(&data->zstrm, Z_SYNC_FLUSH); - } - - if (err != Z_OK) { - warning("deflate() call failed: %d", err); - } - - if (downstream_length > data->zstrm.avail_out) { - TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out); - data->downstream_length += (downstream_length - data->zstrm.avail_out); - } - - if (data->zstrm.avail_out > 0) { - if (data->zstrm.avail_in != 0) { - error("gzip-transform: avail_in is (%d): should be 0", data->zstrm.avail_in); - } - } - } -} - -#if HAVE_BROTLI_ENCODE_H -static bool -brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, BrotliEncoderOperation op) -{ - TSIOBufferBlock downstream_blkp; - int64_t downstream_length; - - data->bstrm.next_in = (uint8_t *)upstream_buffer; - data->bstrm.avail_in = upstream_length; - - bool ok = true; - while (ok) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - - data->bstrm.next_out = reinterpret_cast(downstream_buffer); - data->bstrm.avail_out = downstream_length; - data->bstrm.total_out = 0; - - ok = - !!BrotliEncoderCompressStream(data->bstrm.br, op, &data->bstrm.avail_in, &const_cast(data->bstrm.next_in), - &data->bstrm.avail_out, &data->bstrm.next_out, &data->bstrm.total_out); - - if (!ok) { - error("BrotliEncoderCompressStream(%d) call failed", op); - return false; - } - - TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out); - data->downstream_length += (downstream_length - data->bstrm.avail_out); - if (data->bstrm.avail_in || BrotliEncoderHasMoreOutput(data->bstrm.br)) { - continue; - } - - break; - } - - return ok; -} - -static void -brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) -{ - bool ok = brotli_compress_operation(data, upstream_buffer, upstream_length, BROTLI_OPERATION_PROCESS); - if (!ok) { - error("BrotliEncoderCompressStream(PROCESS) call failed"); - return; - } - - data->bstrm.total_in += upstream_length; - - if (!data->hc->flush()) { - return; - } - - ok = brotli_compress_operation(data, nullptr, 0, BROTLI_OPERATION_FLUSH); - if (!ok) { - error("BrotliEncoderCompressStream(FLUSH) call failed"); - return; - } -} -#endif - -#if HAVE_ZSTD_H -static bool -zstd_compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, ZSTD_EndDirective mode) -{ - TSIOBufferBlock downstream_blkp; - int64_t downstream_length; - - ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; - - for (;;) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - - ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; - - size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, mode); - - if (ZSTD_isError(result)) { - error("Zstd compression failed (%d): %s", mode, ZSTD_getErrorName(result)); - return false; - } - - if (output.pos > 0) { - TSIOBufferProduce(data->downstream_buffer, output.pos); - data->downstream_length += output.pos; - data->zstrm_zstd.total_out += output.pos; - } - - // Check completion conditions based on mode - if (mode == ZSTD_e_continue) { - // For continue mode, stop when all input is consumed - if (input.pos >= input.size) { - break; - } - // If we have output space but no more input was consumed, break to avoid infinite loop - if (output.pos == 0 && input.pos < input.size) { - error("zstd-transform: no progress made in compression"); - return false; - } - } else if (mode == ZSTD_e_flush) { - // For flush mode, stop when flush is complete (result == 0) - if (result == 0) { - break; - } - } - } - - return true; -} - -static void -zstd_transform_init(Data *data) -{ - if (!data->zstrm_zstd.cctx) { - error("Failed to initialize Zstd compression context"); - return; - } - - // Set compression level - size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); - if (ZSTD_isError(result)) { - error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); - return; - } - - // Enable checksum for data integrity - result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); - if (ZSTD_isError(result)) { - error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); - return; - } - - debug("zstd compression context initialized with level %d", data->hc->zstd_compression_level()); -} - -static void -zstd_transform_finish(Data *data) -{ - if (data->state == transform_state_output) { - TSIOBufferBlock downstream_blkp; - int64_t downstream_length; - - data->state = transform_state_finished; - - // Finalize the zstd stream - for (;;) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - - ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; - - size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); - - if (ZSTD_isError(remaining)) { - error("zstd compression finish failed: %s", ZSTD_getErrorName(remaining)); - break; - } - - if (output.pos > 0) { - TSIOBufferProduce(data->downstream_buffer, output.pos); - data->downstream_length += output.pos; - data->zstrm_zstd.total_out += output.pos; - } - - if (remaining == 0) { /* compression finished */ - break; - } - } - - debug("zstd-transform: Finished zstd compression"); - log_compression_ratio(data->zstrm_zstd.total_in, data->downstream_length); - } -} - -static void -zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) -{ - bool ok = zstd_compress_operation(data, upstream_buffer, upstream_length, ZSTD_e_continue); - if (!ok) { - error("Zstd compression (CONTINUE) failed"); - return; - } - - data->zstrm_zstd.total_in += upstream_length; - - if (!data->hc->flush()) { - return; - } - - ok = zstd_compress_operation(data, nullptr, 0, ZSTD_e_flush); - if (!ok) { - error("Zstd compression (FLUSH) failed"); - return; - } -} -#endif - static void compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) { @@ -666,17 +374,17 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { - zstd_transform_one(data, upstream_buffer, upstream_length); + Zstd::transform_one(data, upstream_buffer, upstream_length); } else #endif #if HAVE_BROTLI_ENCODE_H if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { - brotli_transform_one(data, upstream_buffer, upstream_length); + Brotli::transform_one(data, upstream_buffer, upstream_length); } else #endif if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE)) && (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { - gzip_transform_one(data, upstream_buffer, upstream_length); + Gzip::transform_one(data, upstream_buffer, upstream_length); } else { warning("No compression supported. Shouldn't come here."); } @@ -686,91 +394,24 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) } } -static void -gzip_transform_finish(Data *data) -{ - if (data->state == transform_state_output) { - TSIOBufferBlock downstream_blkp; - int64_t downstream_length; - - data->state = transform_state_finished; - - for (;;) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - - char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - data->zstrm.next_out = reinterpret_cast(downstream_buffer); - data->zstrm.avail_out = downstream_length; - - int err = deflate(&data->zstrm, Z_FINISH); - - if (downstream_length > static_cast(data->zstrm.avail_out)) { - TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out); - data->downstream_length += (downstream_length - data->zstrm.avail_out); - } - - if (err == Z_OK) { /* some more data to encode */ - continue; - } - - if (err != Z_STREAM_END) { - warning("deflate should report Z_STREAM_END"); - } - break; - } - - if (data->downstream_length != static_cast(data->zstrm.total_out)) { - error("gzip-transform: output lengths don't match (%" PRId64 ", %lu)", data->downstream_length, data->zstrm.total_out); - } - - debug("gzip-transform: Finished gzip"); - log_compression_ratio(data->zstrm.total_in, data->downstream_length); - } -} - -#if HAVE_BROTLI_ENCODE_H -static void -brotli_transform_finish(Data *data) -{ - if (data->state != transform_state_output) { - return; - } - - data->state = transform_state_finished; - - bool ok = brotli_compress_operation(data, nullptr, 0, BROTLI_OPERATION_FINISH); - if (!ok) { - error("BrotliEncoderCompressStream(PROCESS) call failed"); - return; - } - - if (data->downstream_length != static_cast(data->bstrm.total_out)) { - error("brotli-transform: output lengths don't match (%" PRId64 ", %zu)", data->downstream_length, data->bstrm.total_out); - } - - debug("brotli-transform: Finished brotli"); - log_compression_ratio(data->bstrm.total_in, data->downstream_length); -} -#endif - static void compress_transform_finish(Data *data) { #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { - zstd_transform_finish(data); + Zstd::transform_finish(data); debug("compress_transform_finish: zstd compression finish"); } else #endif #if HAVE_BROTLI_ENCODE_H if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { - brotli_transform_finish(data); + Brotli::transform_finish(data); debug("compress_transform_finish: brotli compression finish"); } else #endif if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE)) && (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { - gzip_transform_finish(data); + Gzip::transform_finish(data); debug("compress_transform_finish: gzip compression finish"); } else { error("No Compression matched, shouldn't come here"); @@ -1283,12 +924,13 @@ management_update(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) return 0; } +} // namespace Compress void TSPluginInit(int argc, const char *argv[]) { - const char *config_path = nullptr; - compress_config_mutex = TSMutexCreate(); + const char *config_path = nullptr; + Compress::compress_config_mutex = TSMutexCreate(); if (argc > 2) { fatal("the compress plugin does not accept more than 1 plugin argument"); @@ -1302,19 +944,19 @@ TSPluginInit(int argc, const char *argv[]) info("TSPluginInit %s", argv[0]); - if (!global_hidden_header_name) { - global_hidden_header_name = init_hidden_header_name(); + if (!Compress::global_hidden_header_name) { + Compress::global_hidden_header_name = init_hidden_header_name(); } - TSCont management_contp = TSContCreate(management_update, nullptr); + TSCont management_contp = TSContCreate(Compress::management_update, nullptr); // Make sure the global configuration is properly loaded and reloaded on changes TSContDataSet(management_contp, (void *)config_path); TSMgmtUpdateRegister(management_contp, TAG); - load_global_configuration(management_contp); + Compress::load_global_configuration(management_contp); // Setup the global hook, main entry point for kicking off the plugin - TSCont transform_global_contp = TSContCreate(transform_global_plugin, nullptr); + TSCont transform_global_contp = TSContCreate(Compress::transform_global_plugin, nullptr); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_global_contp); info("loaded"); @@ -1344,12 +986,12 @@ TSRemapNewInstance(int argc, char *argv[], void **instance, char * /* errbuf ATS } else { config_path = TSstrdup(3 == argc ? argv[2] : ""); } - if (!global_hidden_header_name) { - global_hidden_header_name = init_hidden_header_name(); + if (!Compress::global_hidden_header_name) { + Compress::global_hidden_header_name = init_hidden_header_name(); } - Configuration *config = Configuration::Parse(config_path); - *instance = config; + Compress::Configuration *config = Compress::Configuration::Parse(config_path); + *instance = config; free((void *)config_path); info("Configuration loaded"); @@ -1360,7 +1002,7 @@ void TSRemapDeleteInstance(void *instance) { debug("Cleanup configs read from remap"); - auto c = static_cast(instance); + auto c = static_cast(instance); delete c; } @@ -1371,7 +1013,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo * /* rri ATS_U info("No Rules configured, falling back to default"); } else { info("Remap Rules configured for compress"); - Configuration *config = static_cast(instance); + Compress::Configuration *config = static_cast(instance); // Handle compress request and use the configs populated from remap instance handle_request(txnp, config); } diff --git a/plugins/compress/compress_common.cc b/plugins/compress/compress_common.cc new file mode 100644 index 00000000000..f5b3a2d2302 --- /dev/null +++ b/plugins/compress/compress_common.cc @@ -0,0 +1,37 @@ +/** @file + + Common types and utilities for compression plugin + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "compress_common.h" +#include "debug_macros.h" + +#include + +void +log_compression_ratio(int64_t in, int64_t out) +{ + if (in) { + info("Compressed size %" PRId64 " (bytes), Original size %" PRId64 ", ratio: %f", out, in, ((float)(in - out) / in)); + } else { + debug("Compressed size %" PRId64 " (bytes), Original size %" PRId64 ", ratio: %f", out, in, 0.0F); + } +} diff --git a/plugins/compress/compress_common.h b/plugins/compress/compress_common.h new file mode 100644 index 00000000000..ecb5939dfa4 --- /dev/null +++ b/plugins/compress/compress_common.h @@ -0,0 +1,96 @@ +/** @file + + Common types and structures for compression plugin + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "tscore/ink_config.h" + +#include +#include + +#if HAVE_BROTLI_ENCODE_H +#include +#endif + +#if HAVE_ZSTD_H +#include +#endif + +#include "configuration.h" + +enum CompressionType { + COMPRESSION_TYPE_DEFAULT = 0, + COMPRESSION_TYPE_DEFLATE = 1, + COMPRESSION_TYPE_GZIP = 2, + COMPRESSION_TYPE_BROTLI = 4, + COMPRESSION_TYPE_ZSTD = 8 +}; + +enum transform_state { + transform_state_initialized, + transform_state_output, + transform_state_finished, +}; + +#if HAVE_BROTLI_ENCODE_H +struct BrotliStream { + BrotliEncoderState *br; + uint8_t *next_in; + size_t avail_in; + uint8_t *next_out; + size_t avail_out; + size_t total_in; + size_t total_out; +}; +#endif + +#if HAVE_ZSTD_H +struct ZstdStream { + ZSTD_CCtx *cctx; + int64_t total_in; + int64_t total_out; +}; +#endif + +struct Data { + TSHttpTxn txn; + Compress::HostConfiguration *hc; + TSVIO downstream_vio; + TSIOBuffer downstream_buffer; + TSIOBufferReader downstream_reader; + int64_t downstream_length; + z_stream zstrm; + enum transform_state state; + int compression_type; + int compression_algorithms; +#if HAVE_BROTLI_ENCODE_H + BrotliStream bstrm; +#endif +#if HAVE_ZSTD_H + ZstdStream zstrm_zstd; +#endif +}; + +void log_compression_ratio(int64_t in, int64_t out); diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index 49150130163..3ba265d2845 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -38,7 +38,7 @@ #include -namespace Gzip +namespace Compress { swoc::TextView extractFirstToken(swoc::TextView &view, int (*fp)(int)) @@ -472,4 +472,4 @@ Configuration::Parse(const char *path) return c; } // Configuration::Parse -} // namespace Gzip +} // namespace Compress diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index e67f92f35a9..bc54aae63de 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -31,7 +31,7 @@ #include "tscpp/api/noncopyable.h" #include "swoc/TextView.h" -namespace Gzip +namespace Compress { using StringContainer = std::vector; @@ -243,4 +243,4 @@ class Configuration : private atscppapi::noncopyable }; // class Configuration -} // namespace Gzip +} // namespace Compress diff --git a/plugins/compress/debug_macros.h b/plugins/compress/debug_macros.h index d5ce643a757..8ea10f2f117 100644 --- a/plugins/compress/debug_macros.h +++ b/plugins/compress/debug_macros.h @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License diff --git a/plugins/compress/gzip_compress.cc b/plugins/compress/gzip_compress.cc new file mode 100644 index 00000000000..04111a47489 --- /dev/null +++ b/plugins/compress/gzip_compress.cc @@ -0,0 +1,173 @@ +/** @file + + Gzip/Deflate compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "gzip_compress.h" +#include "debug_macros.h" + +#include +#include +#include + +namespace Compress +{ +extern const char *dictionary; +} + +namespace Gzip +{ +const int ZLIB_COMPRESSION_LEVEL = 6; +voidpf +gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size) +{ + return static_cast(TSmalloc(items * size)); +} + +void +gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address) +{ + TSfree(address); +} + +void +data_alloc(Data *data) +{ + int window_bits = WINDOW_BITS_GZIP; + if (data->compression_type & COMPRESSION_TYPE_DEFLATE) { + window_bits = WINDOW_BITS_DEFLATE; + } + + data->zstrm.next_in = Z_NULL; + data->zstrm.avail_in = 0; + data->zstrm.total_in = 0; + data->zstrm.next_out = Z_NULL; + data->zstrm.avail_out = 0; + data->zstrm.total_out = 0; + data->zstrm.zalloc = Gzip::gzip_alloc; + data->zstrm.zfree = Gzip::gzip_free; + data->zstrm.opaque = (voidpf) nullptr; + data->zstrm.data_type = Z_ASCII; + + int err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); + + if (err != Z_OK) { + fatal("gzip-transform: ERROR: deflateInit (%d)!", err); + } + + if (Compress::dictionary) { + err = deflateSetDictionary(&data->zstrm, reinterpret_cast(Compress::dictionary), strlen(Compress::dictionary)); + if (err != Z_OK) { + fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err); + } + } +} + +void +data_destroy(Data *data) +{ + // deflateEnd return value ignore is intentional + // it would spew log on every client abort + deflateEnd(&data->zstrm); +} + +void +transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + int err; + data->zstrm.next_in = (unsigned char *)upstream_buffer; + data->zstrm.avail_in = upstream_length; + + while (data->zstrm.avail_in > 0) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + data->zstrm.next_out = reinterpret_cast(downstream_buffer); + data->zstrm.avail_out = downstream_length; + + if (!data->hc->flush()) { + err = deflate(&data->zstrm, Z_NO_FLUSH); + } else { + err = deflate(&data->zstrm, Z_SYNC_FLUSH); + } + + if (err != Z_OK) { + warning("deflate() call failed: %d", err); + } + + if (downstream_length > data->zstrm.avail_out) { + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out); + data->downstream_length += (downstream_length - data->zstrm.avail_out); + } + + if (data->zstrm.avail_out > 0) { + if (data->zstrm.avail_in != 0) { + error("gzip-transform: avail_in is (%d): should be 0", data->zstrm.avail_in); + } + } + } +} + +void +transform_finish(Data *data) +{ + if (data->state == transform_state_output) { + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->state = transform_state_finished; + + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + data->zstrm.next_out = reinterpret_cast(downstream_buffer); + data->zstrm.avail_out = downstream_length; + + int err = deflate(&data->zstrm, Z_FINISH); + + if (downstream_length > static_cast(data->zstrm.avail_out)) { + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out); + data->downstream_length += (downstream_length - data->zstrm.avail_out); + } + + if (err == Z_OK) { /* some more data to encode */ + continue; + } + + if (err != Z_STREAM_END) { + warning("deflate should report Z_STREAM_END"); + } + break; + } + + if (data->downstream_length != static_cast(data->zstrm.total_out)) { + error("gzip-transform: output lengths don't match (%" PRId64 ", %lu)", data->downstream_length, data->zstrm.total_out); + } + + debug("gzip-transform: Finished gzip"); + log_compression_ratio(data->zstrm.total_in, data->downstream_length); + } +} + +} // namespace Gzip diff --git a/plugins/compress/gzip_compress.h b/plugins/compress/gzip_compress.h new file mode 100644 index 00000000000..5257bb4bb49 --- /dev/null +++ b/plugins/compress/gzip_compress.h @@ -0,0 +1,51 @@ +/** @file + + Gzip/Deflate compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "compress_common.h" +#include + +// zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html +static const int ZLIB_MEMLEVEL = 9; // min=1 (optimize for memory),max=9 (optimized for speed) +static const int WINDOW_BITS_DEFLATE = -15; +static const int WINDOW_BITS_GZIP = 31; + +namespace Gzip +{ +// Initialize gzip/deflate compression context +void data_alloc(Data *data); + +// Destroy gzip/deflate compression context +void data_destroy(Data *data); + +// Compress one chunk of data +void transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); + +// Finish compression and flush remaining data +void transform_finish(Data *data); + +voidpf gzip_alloc(voidpf opaque, uInt items, uInt size); +void gzip_free(voidpf opaque, voidpf address); + +} // namespace Gzip diff --git a/plugins/compress/misc.cc b/plugins/compress/misc.cc index 13d47e7aee0..762efade03f 100644 --- a/plugins/compress/misc.cc +++ b/plugins/compress/misc.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -30,18 +30,6 @@ #include #include "debug_macros.h" -voidpf -gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size) -{ - return static_cast(TSmalloc(items * size)); -} - -void -gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address) -{ - TSfree(address); -} - namespace { // Strips parameters from value. Returns cleared TextView if a q=f parameter present, where f is less than or equal to @@ -197,13 +185,3 @@ register_plugin() } return 1; } - -void -log_compression_ratio(int64_t in, int64_t out) -{ - if (in) { - info("Compressed size %" PRId64 " (bytes), Original size %" PRId64 ", ratio: %f", out, in, ((float)(in - out) / in)); - } else { - debug("Compressed size %" PRId64 " (bytes), Original size %" PRId64 ", ratio: %f", out, in, 0.0F); - } -} diff --git a/plugins/compress/misc.h b/plugins/compress/misc.h index a69649da24e..6c5f4513322 100644 --- a/plugins/compress/misc.h +++ b/plugins/compress/misc.h @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -23,94 +23,10 @@ #pragma once -#include #include -#include -#include -#if HAVE_BROTLI_ENCODE_H -#include -#endif - -#if HAVE_ZSTD_H -#include -#endif - -#include "configuration.h" - -// zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html -static const int ZLIB_MEMLEVEL = 9; // min=1 (optimize for memory),max=9 (optimized for speed) -static const int WINDOW_BITS_DEFLATE = -15; -static const int WINDOW_BITS_GZIP = 31; - -// misc -enum CompressionType { - COMPRESSION_TYPE_DEFAULT = 0, - COMPRESSION_TYPE_DEFLATE = 1, - COMPRESSION_TYPE_GZIP = 2, - COMPRESSION_TYPE_BROTLI = 4, - COMPRESSION_TYPE_ZSTD = 8, -}; - -// this one is used to rename the accept encoding header -// it will be restored later on -// to make it work, the name must be different then downstream proxies though -// otherwise the downstream will restore the accept encoding header - -enum transform_state { - transform_state_initialized, - transform_state_output, - transform_state_finished, -}; - -#if HAVE_BROTLI_ENCODE_H -using b_stream = struct { - BrotliEncoderState *br; - uint8_t *next_in; - size_t avail_in; - uint8_t *next_out; - size_t avail_out; - size_t total_in; - size_t total_out; -}; -#endif - -#if HAVE_ZSTD_H -using zstd_stream = struct { - ZSTD_CCtx *cctx; - const void *next_in; - size_t avail_in; - void *next_out; - size_t avail_out; - size_t total_in; - size_t total_out; -}; -#endif - -using Data = struct { - TSHttpTxn txn; - Gzip::HostConfiguration *hc; - TSVIO downstream_vio; - TSIOBuffer downstream_buffer; - TSIOBufferReader downstream_reader; - int64_t downstream_length; - z_stream zstrm; - enum transform_state state; - int compression_type; - int compression_algorithms; -#if HAVE_BROTLI_ENCODE_H - b_stream bstrm; -#endif -#if HAVE_ZSTD_H - zstd_stream zstrm_zstd; -#endif -}; - -voidpf gzip_alloc(voidpf opaque, uInt items, uInt size); -void gzip_free(voidpf opaque, voidpf address); void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc); void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char *hidden_header_name); void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char *hidden_header_name); const char *init_hidden_header_name(); int register_plugin(); -void log_compression_ratio(int64_t in, int64_t out); diff --git a/plugins/compress/zstd_compress.cc b/plugins/compress/zstd_compress.cc new file mode 100644 index 00000000000..cbc34431fe9 --- /dev/null +++ b/plugins/compress/zstd_compress.cc @@ -0,0 +1,178 @@ +/** @file + + Zstd compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file + distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under + the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may + obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS + IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. + */ + +#include "zstd_compress.h" + +#if HAVE_ZSTD_H + +#include "debug_macros.h" + +#include + +namespace +{ +bool +compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, ZSTD_EndDirective mode) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; + + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, mode); + + if (ZSTD_isError(result)) { + error("Zstd compression failed (%d): %s", mode, ZSTD_getErrorName(result)); + return false; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (mode == ZSTD_e_continue) { + if (input.pos >= input.size) { + break; + } + if (output.pos == 0 && input.pos < input.size) { + error("zstd-transform: no progress made in compression"); + return false; + } + } else if (result == 0) { + break; + } + } + + return true; +} +} // namespace + +namespace Zstd +{ +void +data_alloc(Data *data) +{ + std::memset(&data->zstrm_zstd, 0, sizeof(data->zstrm_zstd)); + + data->zstrm_zstd.cctx = ZSTD_createCCtx(); + if (!data->zstrm_zstd.cctx) { + fatal("Zstd Compression Context Creation Failed"); + } +} + +void +data_destroy(Data *data) +{ + if (data->zstrm_zstd.cctx) { + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + data->zstrm_zstd.cctx = nullptr; + } +} + +void +transform_init(Data *data) +{ + if (!data->zstrm_zstd.cctx) { + error("Failed to initialize Zstd compression context"); + return; + } + + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); + if (ZSTD_isError(result)) { + error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); + return; + } + + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(result)) { + error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); + return; + } + + debug("zstd compression context initialized with level %d", data->hc->zstd_compression_level()); +} + +void +transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + if (!compress_operation(data, upstream_buffer, upstream_length, ZSTD_e_continue)) { + error("Zstd compression (CONTINUE) failed"); + return; + } + + data->zstrm_zstd.total_in += upstream_length; + + if (!data->hc->flush()) { + return; + } + + if (!compress_operation(data, nullptr, 0, ZSTD_e_flush)) { + error("Zstd compression (FLUSH) failed"); + } +} + +void +transform_finish(Data *data) +{ + if (data->state != transform_state_output) { + return; + } + + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->state = transform_state_finished; + + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); + + if (ZSTD_isError(remaining)) { + error("zstd compression finish failed: %s", ZSTD_getErrorName(remaining)); + break; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (remaining == 0) { + break; + } + } + + debug("zstd-transform: Finished zstd compression"); + log_compression_ratio(data->zstrm_zstd.total_in, data->downstream_length); +} +} // namespace Zstd + +#endif // HAVE_ZSTD_H diff --git a/plugins/compress/zstd_compress.h b/plugins/compress/zstd_compress.h new file mode 100644 index 00000000000..f589adf97d8 --- /dev/null +++ b/plugins/compress/zstd_compress.h @@ -0,0 +1,44 @@ +/** @file + + Zstd compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file + distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under + the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may + obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS + IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. + */ + +#pragma once + +#include "tscore/ink_config.h" +#include "compress_common.h" + +#if HAVE_ZSTD_H + +namespace Zstd +{ +// Initialize Zstd compression context +void data_alloc(Data *data); + +// Destroy Zstd compression context +void data_destroy(Data *data); + +// Configure the context just before streaming starts +void transform_init(Data *data); + +// Compress one upstream chunk +void transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); + +// Finish compression and flush remaining data +void transform_finish(Data *data); +} // namespace Zstd + +#endif // HAVE_ZSTD_H diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index 1f41ff603c3..04c49431f8c 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -837,8 +837,8 @@ transformData(TSCont contp) CONT_DATA_DBG(cont_data, "[%s] trying to process doc", __FUNCTION__); string out_data; string cdata; - int overall_len; - EsiProcessor::ReturnCode retval = cont_data->esi_proc->flush(out_data, overall_len); + int overall_len = 0; + EsiProcessor::ReturnCode retval = cont_data->esi_proc->flush(out_data, overall_len); if ((cont_data->curr_state == ContData::FETCHING_DATA) && cont_data->data_fetcher->isFetchComplete()) { CONT_DATA_DBG(cont_data, "[%s] data ready; last process() will have finished the entire processing", __FUNCTION__); diff --git a/plugins/esi/lib/EsiProcessor.cc b/plugins/esi/lib/EsiProcessor.cc index 89f03efc7ad..b4687fd655b 100644 --- a/plugins/esi/lib/EsiProcessor.cc +++ b/plugins/esi/lib/EsiProcessor.cc @@ -352,6 +352,8 @@ EsiProcessor::ReturnCode EsiProcessor::flush(string &data, int &overall_len) { if (_curr_state == ERRORED) { + overall_len = 0; + data.assign(""); return FAILURE; } if (_curr_state == PROCESSED) { diff --git a/plugins/experimental/access_control/CMakeLists.txt b/plugins/experimental/access_control/CMakeLists.txt index f6ef84f0398..4c465db7a5c 100644 --- a/plugins/experimental/access_control/CMakeLists.txt +++ b/plugins/experimental/access_control/CMakeLists.txt @@ -28,7 +28,7 @@ add_atsplugin( utils.cc ) -target_link_libraries(access_control PRIVATE OpenSSL::SSL OpenSSL::Crypto PCRE::PCRE) +target_link_libraries(access_control PRIVATE OpenSSL::SSL OpenSSL::Crypto) verify_remap_plugin(access_control) diff --git a/plugins/experimental/access_control/pattern.cc b/plugins/experimental/access_control/pattern.cc index 8a579128933..179131653a5 100644 --- a/plugins/experimental/access_control/pattern.cc +++ b/plugins/experimental/access_control/pattern.cc @@ -24,6 +24,8 @@ #include "pattern.h" +#include + static void replaceString(String &str, const String &from, const String &to) { @@ -40,6 +42,8 @@ replaceString(String &str, const String &from, const String &to) Pattern::Pattern() : _pattern(""), _replacement("") {} +Pattern::~Pattern() = default; + /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. * @param pattern PCRE pattern, a string containing PCRE patterns, capturing groups. @@ -49,8 +53,6 @@ Pattern::Pattern() : _pattern(""), _replacement("") {} bool Pattern::init(const String &pattern, const String &replacement, bool replace) { - pcreFree(); - _pattern.assign(pattern); _replacement.assign(replacement); _replace = replace; @@ -59,7 +61,7 @@ Pattern::init(const String &pattern, const String &replacement, bool replace) if (!compile()) { AccessControlDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); - pcreFree(); + _re.reset(); return false; } @@ -139,33 +141,12 @@ Pattern::getPattern() bool Pattern::empty() const { - return _pattern.empty() || nullptr == _re; -} - -/** - * @brief Frees PCRE library related resources. - */ -void -Pattern::pcreFree() -{ - if (_re) { - pcre_free(_re); - _re = nullptr; - } - - if (_extra) { - pcre_free(_extra); - _extra = nullptr; - } + return _pattern.empty() || !_re || _re->empty(); } /** * @brief Destructor, frees PCRE related resources. */ -Pattern::~Pattern() -{ - pcreFree(); -} /** * @brief Capture or capture-and-replace depending on whether a replacement string is specified. @@ -214,16 +195,16 @@ Pattern::process(const String &subject, StringVector &result) bool Pattern::match(const String &subject) { - int matchCount; AccessControlDebug("matching '%s' to '%s'", _pattern.c_str(), subject.c_str()); - if (!_re) { + if (!_re || _re->empty()) { return false; } - matchCount = pcre_exec(_re, _extra, subject.c_str(), subject.length(), 0, PCRE_NOTEMPTY, nullptr, 0); + RegexMatches matches(TOKENCOUNT); + int matchCount = _re->exec(subject, matches, RE_NOTEMPTY); if (matchCount < 0) { - if (matchCount != PCRE_ERROR_NOMATCH) { + if (matchCount != RE_ERROR_NOMATCH) { AccessControlError("matching error %d", matchCount); } return false; @@ -240,31 +221,27 @@ Pattern::match(const String &subject) bool Pattern::capture(const String &subject, StringVector &result) { - int matchCount; - int ovector[OVECOUNT]; - AccessControlDebug("capturing '%s' from '%s'", _pattern.c_str(), subject.c_str()); - if (!_re) { + if (!_re || _re->empty()) { AccessControlError("regular expression not initialized"); return false; } - matchCount = pcre_exec(_re, nullptr, subject.c_str(), subject.length(), 0, PCRE_NOTEMPTY, ovector, OVECOUNT); + RegexMatches matches(TOKENCOUNT); + int matchCount = _re->exec(subject, matches, RE_NOTEMPTY); if (matchCount < 0) { - if (matchCount != PCRE_ERROR_NOMATCH) { + if (matchCount != RE_ERROR_NOMATCH) { AccessControlError("matching error %d", matchCount); } return false; } for (int i = 0; i < matchCount; i++) { - int start = ovector[2 * i]; - int length = ovector[2 * i + 1] - ovector[2 * i]; + std::string_view match_view = matches[i]; + String dst(match_view.data(), match_view.size()); - String dst(subject, start, length); - - AccessControlDebug("capturing '%s' %d[%d,%d]", dst.c_str(), i, ovector[2 * i], ovector[2 * i + 1]); + AccessControlDebug("capturing '%s' %d", dst.c_str(), i); result.push_back(dst); } @@ -280,19 +257,17 @@ Pattern::capture(const String &subject, StringVector &result) bool Pattern::replace(const String &subject, String &result) { - int matchCount; - int ovector[OVECOUNT]; - AccessControlDebug("replacing:'%s' in pattern:'%s', subject:'%s'", _replacement.c_str(), _pattern.c_str(), subject.c_str()); - if (!_re || !_replace) { + if (!_re || _re->empty() || !_replace) { AccessControlError("regular expression not initialized or not configured to replace"); return false; } - matchCount = pcre_exec(_re, nullptr, subject.c_str(), subject.length(), 0, PCRE_NOTEMPTY, ovector, OVECOUNT); + RegexMatches matches(TOKENCOUNT); + int matchCount = _re->exec(subject, matches, RE_NOTEMPTY); if (matchCount < 0) { - if (matchCount != PCRE_ERROR_NOMATCH) { + if (matchCount != RE_ERROR_NOMATCH) { AccessControlError("matching error %d", matchCount); } return false; @@ -308,12 +283,11 @@ Pattern::replace(const String &subject, String &result) int previous = 0; for (int i = 0; i < _tokenCount; i++) { - int replIndex = _tokens[i]; - int start = ovector[2 * replIndex]; - int length = ovector[2 * replIndex + 1] - ovector[2 * replIndex]; + int replIndex = _tokens[i]; + std::string_view match_view = matches[replIndex]; String src(_replacement, _tokenOffset[i], 2); - String dst(subject, start, length); + String dst(match_view.data(), match_view.size()); AccessControlDebug("replacing '%s' with '%s'", src.c_str(), dst.c_str()); @@ -337,31 +311,16 @@ Pattern::replace(const String &subject, String &result) bool Pattern::compile() { - const char *errPtr; /* PCRE error */ + std::string error; /* PCRE error description */ int errOffset; /* PCRE error offset */ AccessControlDebug("compiling pattern:'%s', replace: %s, replacement:'%s'", _pattern.c_str(), _replace ? "true" : "false", _replacement.c_str()); - _re = pcre_compile(_pattern.c_str(), /* the pattern */ - 0, /* options */ - &errPtr, /* for error message */ - &errOffset, /* for error offset */ - nullptr); /* use default character tables */ - - if (nullptr == _re) { - AccessControlError("compile of regex '%s' at char %d: %s", _pattern.c_str(), errOffset, errPtr); - - return false; - } - - _extra = pcre_study(_re, 0, &errPtr); - - if ((nullptr == _extra) && (nullptr != errPtr) && (0 != *errPtr)) { - AccessControlError("failed to study regex '%s': %s", _pattern.c_str(), errPtr); - - pcre_free(_re); - _re = nullptr; + _re = std::make_unique(); + if (!_re->compile(_pattern, error, errOffset, 0)) { + AccessControlError("compile of regex '%s' at char %d: %s", _pattern.c_str(), errOffset, error.c_str()); + _re.reset(); return false; } @@ -398,7 +357,7 @@ Pattern::compile() } if (!success) { - pcreFree(); + _re.reset(); } return success; diff --git a/plugins/experimental/access_control/pattern.h b/plugins/experimental/access_control/pattern.h index 8576d6221fa..4dc28f97b83 100644 --- a/plugins/experimental/access_control/pattern.h +++ b/plugins/experimental/access_control/pattern.h @@ -23,25 +23,20 @@ #pragma once -#ifdef HAVE_PCRE_PCRE_H -#include -#else -#include -#endif - #include "common.h" +class Regex; + /** * @brief PCRE matching, capturing and replacing */ class Pattern { public: - static const int TOKENCOUNT = 10; /**< @brief Capturing groups $0..$9 */ - static const int OVECOUNT = TOKENCOUNT * 3; /**< @brief pcre_exec() array count, handle 10 capture groups */ + static const int TOKENCOUNT = 10; /**< @brief Capturing groups $0..$9 */ Pattern(); - virtual ~Pattern(); + ~Pattern(); // Keep in pattern.cc for pimpl use of Regex. bool init(const String &pattern, const String &replacement, bool replace); bool init(const String &config); @@ -54,10 +49,8 @@ class Pattern private: bool compile(); - void pcreFree(); - pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ + std::unique_ptr _re; /**< @brief Regex compiled object, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ diff --git a/plugins/experimental/geoip_acl/CMakeLists.txt b/plugins/experimental/geoip_acl/CMakeLists.txt index 830e03ada2e..b9bb30e677d 100644 --- a/plugins/experimental/geoip_acl/CMakeLists.txt +++ b/plugins/experimental/geoip_acl/CMakeLists.txt @@ -16,5 +16,4 @@ ####################### add_atsplugin(geoip_acl geoip_acl.cc acl.cc) -target_link_libraries(geoip_acl PRIVATE PCRE::PCRE) verify_remap_plugin(geoip_acl) diff --git a/plugins/experimental/geoip_acl/acl.cc b/plugins/experimental/geoip_acl/acl.cc index 56c533ad6bd..ee775eb695d 100644 --- a/plugins/experimental/geoip_acl/acl.cc +++ b/plugins/experimental/geoip_acl/acl.cc @@ -28,7 +28,6 @@ namespace geoip_acl_ns { DbgCtl dbg_ctl{PLUGIN_NAME}; } - // Implementation of the ACL base class. This wraps the underlying Geo library // that we've found and used. GeoDBHandle Acl::_geoip; @@ -175,25 +174,22 @@ RegexAcl::parse_line(const char *filename, const std::string &line, int lineno, bool RegexAcl::compile(const std::string &str, const char *filename, int lineno) { - const char *error; + bool success{true}; + std::string error; int erroffset; _regex_s = str; - _rex = pcre_compile(_regex_s.c_str(), 0, &error, &erroffset, nullptr); - - if (nullptr != _rex) { - _extra = pcre_study(_rex, 0, &error); - if ((nullptr == _extra) && error && (*error != 0)) { - TSError("[%s] Failed to study regular expression in %s:line %d at offset %d: %s", PLUGIN_NAME, filename, lineno, erroffset, - error); - return false; - } - } else { - TSError("[%s] Failed to compile regular expression in %s:line %d: %s", PLUGIN_NAME, filename, lineno, error); - return false; + _regex = new Regex(); + + if (!_regex->compile(_regex_s, error, erroffset, 0)) { + TSError("[%s] Regex compilation failed in %s:line %d with error (%s) at character %d", PLUGIN_NAME, filename, lineno, + error.c_str(), erroffset); + delete _regex; + _regex = nullptr; + success = false; } - return true; + return success; } void diff --git a/plugins/experimental/geoip_acl/acl.h b/plugins/experimental/geoip_acl/acl.h index 7c089024f76..c07cb170b3e 100644 --- a/plugins/experimental/geoip_acl/acl.h +++ b/plugins/experimental/geoip_acl/acl.h @@ -24,16 +24,14 @@ #include #include - +#include +#include "tsutil/DbgCtl.h" #include "tscore/ink_defs.h" -#ifdef HAVE_PCRE_PCRE_H -#include -#else -#include -#endif +#include "tsutil/Regex.h" #include +#include #include "lulu.h" namespace geoip_acl_ns @@ -102,7 +100,16 @@ class Acl class RegexAcl { public: - RegexAcl(Acl *acl) : _rex(nullptr), _extra(nullptr), _next(nullptr), _acl(acl) {} + RegexAcl(Acl *acl) : _regex(nullptr), _next(nullptr), _acl(acl) {} + ~RegexAcl() + { + if (_regex) { + delete _regex; + } + if (_acl) { + delete _acl; + } + } const std::string & get_regex() const { @@ -125,8 +132,10 @@ class RegexAcl if (0 == len) { return false; } - - return (pcre_exec(_rex, _extra, str, len, 0, PCRE_NOTEMPTY, nullptr, 0) != -1); + if (_regex) { + return _regex->exec(std::string_view(str, len)); + } + return false; } void append(RegexAcl *ra); @@ -135,8 +144,7 @@ class RegexAcl private: bool compile(const std::string &str, const char *filename, int lineno); std::string _regex_s; - pcre *_rex; - pcre_extra *_extra; + Regex *_regex{nullptr}; RegexAcl *_next; Acl *_acl; }; @@ -146,6 +154,15 @@ class CountryAcl : public Acl { public: CountryAcl() { memset(_iso_country_codes, 0, sizeof(_iso_country_codes)); } + ~CountryAcl() + { + RegexAcl *cur = _regexes; + while (cur) { + RegexAcl *next = cur->next(); + delete cur; + cur = next; + } + } void read_regex(const char *fn, int &tokens) override; int process_args(int argc, char *argv[]) override; bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp) const override; diff --git a/plugins/experimental/url_sig/url_sig.cc b/plugins/experimental/url_sig/url_sig.cc index 5e81b571c7d..7e55d90b30a 100644 --- a/plugins/experimental/url_sig/url_sig.cc +++ b/plugins/experimental/url_sig/url_sig.cc @@ -33,6 +33,9 @@ #include #include +#include +#include + #include "tsutil/Regex.h" #include @@ -44,27 +47,26 @@ static const char PLUGIN_NAME[] = "url_sig"; static DbgCtl dbg_ctl{PLUGIN_NAME}; struct config { - TSHttpStatus err_status; - char *err_url; - char keys[MAX_KEY_NUM][MAX_KEY_LEN]; - Regex *excl_regex; - int pristine_url_flag; - char *sig_anchor; - bool ignore_expiry; + config() = default; + config(const config &) = delete; + config(config &&) = delete; + config &operator=(const config &) = delete; + config &operator=(config &&) = delete; + + ~config(); + + TSHttpStatus err_status = TS_HTTP_STATUS_NONE; + std::string err_url; + char keys[MAX_KEY_NUM][MAX_KEY_LEN]; + std::unique_ptr excl_regex; + bool pristine_url_flag = false; + std::string sig_anchor; + bool ignore_expiry = false; }; -static void -free_cfg(struct config *cfg) +config::~config() { Dbg(dbg_ctl, "Cleaning up"); - TSfree(cfg->err_url); - TSfree(cfg->sig_anchor); - - if (cfg->excl_regex) { - delete cfg->excl_regex; - } - - TSfree(cfg); } TSReturnCode @@ -79,8 +81,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) { - char config_filepath_buf[PATH_MAX], *config_file; - struct config *cfg; + char config_filepath_buf[PATH_MAX], *config_file; if ((argc < 3) || (argc > 4)) { snprintf(errbuf, errbuf_size, @@ -109,8 +110,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s int keynum; bool eat_comment = false; - cfg = TSRalloc(); - memset(cfg, 0, sizeof(struct config)); + auto cfg = std::make_unique(); while (fgets(line, sizeof(line), file) != nullptr) { Dbg(dbg_ctl, "LINE: %s (%d)", line, (int)strlen(line)); @@ -147,7 +147,6 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s if (pos == nullptr || strlen(value) >= MAX_KEY_LEN) { snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Maximum key length (%d) exceeded on line %d", MAX_KEY_LEN - 1, line_no); fclose(file); - free_cfg(cfg); return TS_ERROR; } if (strncmp(line, "key", 3) == 0) { @@ -164,7 +163,6 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s if (keynum >= MAX_KEY_NUM || keynum < 0) { snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Key number (%d) >= MAX_KEY_NUM (%d) or NaN", keynum, MAX_KEY_NUM); fclose(file); - free_cfg(cfg); return TS_ERROR; } snprintf(&cfg->keys[keynum][0], MAX_KEY_LEN, "%s", value); @@ -177,12 +175,12 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s value++; } if (cfg->err_status == TS_HTTP_STATUS_MOVED_TEMPORARILY) { - cfg->err_url = TSstrndup(value, strlen(value)); + cfg->err_url = value; } else { - cfg->err_url = nullptr; + cfg->err_url.clear(); } } else if (strncmp(line, "sig_anchor", 10) == 0) { - cfg->sig_anchor = TSstrndup(value, strlen(value)); + cfg->sig_anchor = value; } else if (strncmp(line, "excl_regex", 10) == 0) { // Compile regex. std::string error; @@ -193,11 +191,10 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s continue; } - cfg->excl_regex = new Regex(); + cfg->excl_regex = std::make_unique(); if (!cfg->excl_regex->compile(value, error, erroffset, 0)) { Dbg(dbg_ctl, "Regex compilation failed with error (%s) at character %d", error.c_str(), erroffset); - delete cfg->excl_regex; - cfg->excl_regex = nullptr; + cfg->excl_regex.reset(); } } else if (strncmp(line, "ignore_expiry", 13) == 0) { if (strncmp(value, "true", 4) == 0) { @@ -206,7 +203,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s } } else if (strncmp(line, "url_type", 8) == 0) { if (strncmp(value, "pristine", 8) == 0) { - cfg->pristine_url_flag = 1; + cfg->pristine_url_flag = true; Dbg(dbg_ctl, "Pristine URLs (from config) will be used"); } } else { @@ -218,45 +215,43 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s if (argc > 3) { if (strcasecmp(argv[3], "pristineurl") == 0) { - cfg->pristine_url_flag = 1; + cfg->pristine_url_flag = true; Dbg(dbg_ctl, "Pristine URLs (from args) will be used"); } else { snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - second pparam (if present) must be pristineurl"); - free_cfg(cfg); return TS_ERROR; } } switch (cfg->err_status) { case TS_HTTP_STATUS_MOVED_TEMPORARILY: - if (cfg->err_url == nullptr) { - snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Invalid config, err_status == 302, but err_url == nullptr"); - free_cfg(cfg); + if (cfg->err_url.empty()) { + snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Invalid config, err_status == 302, but err_url is empty"); return TS_ERROR; } break; case TS_HTTP_STATUS_FORBIDDEN: - if (cfg->err_url != nullptr) { - snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Invalid config, err_status == 403, but err_url != nullptr"); - free_cfg(cfg); + if (!cfg->err_url.empty()) { + snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Invalid config, err_status == 403, but err_url is not empty"); return TS_ERROR; } break; default: snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - Return code %d not supported", cfg->err_status); - free_cfg(cfg); return TS_ERROR; } - *ih = (void *)cfg; + // Transfer ownership to ih which will later be deleted in TSRemapDeleteInstance. + *ih = (void *)cfg.release(); return TS_SUCCESS; } void TSRemapDeleteInstance(void *ih) { - free_cfg(static_cast(ih)); + auto *cfg = static_cast(ih); + delete cfg; } static void @@ -344,7 +339,7 @@ fixedBufferWrite(char **dest_end, int *dest_len, const char *src, int src_len) } static char * -urlParse(char const *const url_in, char *anchor, char *new_path_seg, int new_path_seg_len, char *signed_seg, +urlParse(char const *const url_in, char const *anchor, char *new_path_seg, int new_path_seg_len, char *signed_seg, unsigned int signed_seg_len) { char *segment[MAX_SEGMENTS]; @@ -582,7 +577,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) // check for path params. if (query == nullptr || strstr(query, "E=") == nullptr) { - char *const parsed = urlParse(url, cfg->sig_anchor, new_path, 8192, path_params, 8192); + char *const parsed = + urlParse(url, cfg->sig_anchor.empty() ? nullptr : cfg->sig_anchor.c_str(), new_path, 8192, path_params, 8192); if (parsed == nullptr) { err_log(url, url_len, "Unable to parse/decode new url path parameters"); goto deny; @@ -835,16 +831,16 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) TSfree((void *)current_url); switch (cfg->err_status) { - case TS_HTTP_STATUS_MOVED_TEMPORARILY: - Dbg(dbg_ctl, "Redirecting to %s", cfg->err_url); - char *start, *end; - start = cfg->err_url; - end = start + strlen(cfg->err_url); - if (TSUrlParse(rri->requestBufp, rri->requestUrl, (const char **)&start, end) != TS_PARSE_DONE) { - err_log("url", 3, "Error inn TSUrlParse!"); + case TS_HTTP_STATUS_MOVED_TEMPORARILY: { + Dbg(dbg_ctl, "Redirecting to %s", cfg->err_url.c_str()); + char const *start = cfg->err_url.c_str(); + char const *end = start + cfg->err_url.size(); + if (TSUrlParse(rri->requestBufp, rri->requestUrl, &start, end) != TS_PARSE_DONE) { + err_log("url", 3, "Error in TSUrlParse!"); } rri->redirect = 1; break; + } default: TSHttpTxnErrorBodySet(txnp, TSstrdup("Authorization Denied"), sizeof("Authorization Denied") - 1, TSstrdup("text/plain")); break; diff --git a/plugins/header_rewrite/factory.cc b/plugins/header_rewrite/factory.cc index 9e0499f060e..9f5f86083cd 100644 --- a/plugins/header_rewrite/factory.cc +++ b/plugins/header_rewrite/factory.cc @@ -92,6 +92,7 @@ operator_factory(const std::string &op) } else if (op == "set-next-hop-strategy") { o = new OperatorSetNextHopStrategy(); } else { + // Note that we don't support the OperatorIf() pseudo-operator here! TSError("[%s] Unknown operator: %s", PLUGIN_NAME, op.c_str()); return nullptr; } diff --git a/plugins/header_rewrite/header_rewrite.cc b/plugins/header_rewrite/header_rewrite.cc index 89fe252e992..61895124f69 100644 --- a/plugins/header_rewrite/header_rewrite.cc +++ b/plugins/header_rewrite/header_rewrite.cc @@ -36,6 +36,7 @@ #include "resources.h" #include "conditions.h" #include "conditions_geo.h" +#include "operators.h" // Debugs namespace header_rewrite_ns @@ -145,15 +146,14 @@ validate_rule_completion(RuleSet *rule, const std::string &fname, int lineno) switch (rule->get_clause()) { case Parser::CondClause::ELIF: - if (!rule->cur_section()->group.has_conditions() || !rule->cur_section()->has_operator()) { - TSError("[%s] ELIF clause must have both conditions and operators in file: %s, lineno: %d", PLUGIN_NAME, fname.c_str(), - lineno); + if (!rule->section_has_condition() || !rule->section_has_operator()) { + TSError("[%s] ELIF conditions without operators are not allowed in file: %s, lineno: %d", PLUGIN_NAME, fname.c_str(), lineno); return false; } break; case Parser::CondClause::ELSE: - if (rule->cur_section()->group.has_conditions()) { + if (rule->section_has_condition()) { TSError("[%s] conditions not allowed in ELSE clause in file: %s, lineno: %d", PLUGIN_NAME, fname.c_str(), lineno); return false; } @@ -168,6 +168,11 @@ validate_rule_completion(RuleSet *rule, const std::string &fname, int lineno) case Parser::CondClause::COND: break; + + case Parser::CondClause::IF: + case Parser::CondClause::ENDIF: + // IF and ENDIF are handled separately in the main parsing loop + break; } return true; @@ -185,8 +190,11 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook, c std::unique_ptr rule(nullptr); std::string filename; int lineno = 0; + ConditionGroup *group = nullptr; std::stack group_stack; - ConditionGroup *group = nullptr; + std::stack if_stack; + + constexpr int MAX_IF_NESTING_DEPTH = 10; if (0 == fname.size()) { TSError("[%s] no config filename provided", PLUGIN_NAME); @@ -246,7 +254,11 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook, c // Deal with the elif / else special keywords, these are neither conditions nor operators. if (p.is_else() || p.is_elif()) { Dbg(pi_dbg_ctl, "Entering elif/else, CondClause=%d", static_cast(p.get_clause())); - if (rule) { + + if (!if_stack.empty()) { + group = if_stack.top()->new_section(p.get_clause()); + continue; + } else if (rule) { group = rule->new_section(p.get_clause()); continue; } else { @@ -256,8 +268,7 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook, c } // If we are at the beginning of a new condition, save away the previous rule (but only if it has operators). - // This also has to deal with the fact that we allow implicit hooks to end / start a new rule. - if (p.is_cond() && rule && (is_hook || rule->cur_section()->has_operator())) { + if (p.is_cond() && rule && if_stack.empty() && (is_hook || rule->section_has_operator())) { if (!validate_rule_completion(rule.get(), fname, lineno)) { return false; } else { @@ -299,7 +310,13 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook, c // Long term, maybe we need to percolate all this up through add_condition() / add_operator() rather than this big ugly try. try { if (p.is_cond()) { - Condition *cond = rule->make_condition(p, filename.c_str(), lineno); + Condition *cond = nullptr; + + if (!if_stack.empty()) { + cond = if_stack.top()->make_condition(p, filename.c_str(), lineno); + } else { + cond = rule->make_condition(p, filename.c_str(), lineno); + } if (!cond) { throw std::runtime_error("add_condition() failed"); @@ -327,9 +344,54 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook, c group->add_condition(cond); } } - } else { // Operator - if (!rule->add_operator(p, filename.c_str(), lineno)) { - throw std::runtime_error("add_operator() failed"); + } else { + if (p.is_if()) { + if (if_stack.size() >= MAX_IF_NESTING_DEPTH) { + throw std::runtime_error("maximum if nesting depth exceeded"); + } + + auto *op_if = new OperatorIf(); + + if_stack.push(op_if); + group = op_if->get_group(); // Set group to the new OperatorIf's group + Dbg(dbg_ctl, "Started nested OperatorIf, depth: %zu", if_stack.size()); + + } else if (p.is_endif()) { + if (if_stack.empty()) { + throw std::runtime_error("endif without matching if"); + } + + OperatorIf *op_if = if_stack.top(); + + if_stack.pop(); + if (!if_stack.empty()) { + auto *parent_sec = if_stack.top()->cur_section(); + + if (parent_sec->ops.oper) { + parent_sec->ops.oper->append(op_if); + } else { + parent_sec->ops.oper.reset(op_if); + } + group = if_stack.top()->get_group(); + } else { + if (!rule->add_operator(op_if)) { + delete op_if; + throw std::runtime_error("Failed to add nested OperatorIf to RuleSet"); + } + group = rule->get_group(); + } + Dbg(dbg_ctl, "Completed nested OperatorIf, depth now: %zu", if_stack.size()); + + } else { + if (!if_stack.empty()) { + if (!if_stack.top()->add_operator(p, filename.c_str(), lineno)) { + throw std::runtime_error("add_operator() failed in nested OperatorIf"); + } + } else { + if (!rule->add_operator(p, filename.c_str(), lineno)) { + throw std::runtime_error("add_operator() failed"); + } + } } } } catch (std::runtime_error &e) { @@ -353,6 +415,16 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook, c return false; } + // Check for unmatched if statements + if (!if_stack.empty()) { + TSError("[%s] %zu unmatched 'if' statement(s) without 'endif' in file: %s", PLUGIN_NAME, if_stack.size(), fname.c_str()); + while (!if_stack.empty()) { + delete if_stack.top(); + if_stack.pop(); + } + return false; + } + // Add the last rule (possibly the only rule) if (rule) { if (!validate_rule_completion(rule.get(), fname, lineno)) { @@ -434,10 +506,8 @@ cont_rewrite_headers(TSCont contp, TSEvent event, void *edata) // Get the resources necessary to process this event res.gather(conf->resid(hook), hook); - // Evaluation of all rules. This code is sort of duplicate in DoRemap as well. while (rule) { - const RuleSet::OperatorAndMods &ops = rule->eval(res); - const OperModifiers rt = rule->exec(ops, res); + const OperModifiers rt = rule->exec(res); if (rt & OPER_NO_REENABLE) { reenable = false; @@ -703,8 +773,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) res.gather(conf->resid(TS_REMAP_PSEUDO_HOOK), TS_REMAP_PSEUDO_HOOK); do { - const RuleSet::OperatorAndMods &ops = rule->eval(res); - const OperModifiers rt = rule->exec(ops, res); + const OperModifiers rt = rule->exec(res); ink_assert((rt & OPER_NO_REENABLE) == 0); diff --git a/plugins/header_rewrite/operator.h b/plugins/header_rewrite/operator.h index 2d54f6fad42..bc23c93aabb 100644 --- a/plugins/header_rewrite/operator.h +++ b/plugins/header_rewrite/operator.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include "ts/ts.h" @@ -39,6 +40,20 @@ enum OperModifiers { OPER_NO_REENABLE = 16, }; +// Forward declaration +class Operator; + +// Holding the operator and mods - used by both RuleSet and OperatorIf +struct OperatorAndMods { + OperatorAndMods() = default; + + OperatorAndMods(const OperatorAndMods &) = delete; + OperatorAndMods &operator=(const OperatorAndMods &) = delete; + + std::unique_ptr oper; + OperModifiers oper_mods = OPER_NONE; +}; + /////////////////////////////////////////////////////////////////////////////// // Base class for all Operators (this is also the interface) // diff --git a/plugins/header_rewrite/operators.cc b/plugins/header_rewrite/operators.cc index e1a9683d3cd..c8c4f66b953 100644 --- a/plugins/header_rewrite/operators.cc +++ b/plugins/header_rewrite/operators.cc @@ -30,6 +30,9 @@ #include "operators.h" #include "ts/apidefs.h" +#include "conditions.h" +#include "factory.h" +#include "ruleset.h" namespace { @@ -816,8 +819,11 @@ OperatorSetBody::exec(const Resources &res) const std::string value; _value.append_value(value, res); - char *msg = TSstrdup(_value.get_value().c_str()); - TSHttpTxnErrorBodySet(res.state.txnp, msg, _value.size(), nullptr); + char *msg = nullptr; + if (!value.empty()) { + msg = TSstrdup(value.c_str()); + } + TSHttpTxnErrorBodySet(res.state.txnp, msg, value.size(), nullptr); return true; } @@ -1674,3 +1680,123 @@ OperatorSetNextHopStrategy::exec(const Resources &res) const return true; } + +/////////////////////////////////////////////////////////////////////////////// +// OperatorIf class implementations +// Keep this at the end of the files, since this is not really an Operator. +// +ConditionGroup * +OperatorIf::new_section(Parser::CondClause clause) +{ + TSAssert(_cur_section && !_cur_section->next); + + _clause = clause; + _cur_section->next = std::make_unique(); + _cur_section = _cur_section->next.get(); + + return &_cur_section->group; +} + +bool +OperatorIf::add_operator(Parser &p, const char *filename, int lineno) +{ + Operator *op = operator_factory(p.get_op()); + + if (!op) { + TSError("[%s] Unknown operator: %s, file: %s, line: %d", PLUGIN_NAME, p.get_op().c_str(), filename, lineno); + return false; + } + + Dbg(pi_dbg_ctl, " Adding operator: %s(%s)=\"%s\"", p.get_op().c_str(), p.get_arg().c_str(), p.get_value().c_str()); + + try { + op->initialize(p); + } catch (std::exception const &ex) { + delete op; + TSError("[%s] Failed to initialize operator: %s, file: %s, line: %d, error: %s", PLUGIN_NAME, p.get_op().c_str(), filename, + lineno, ex.what()); + return false; + } + + // Add to current section + if (_cur_section->ops.oper) { + _cur_section->ops.oper->append(op); + } else { + _cur_section->ops.oper.reset(op); + _cur_section->ops.oper_mods = op->get_oper_modifiers(); + } + + return true; +} + +Condition * +OperatorIf::make_condition(Parser &p, const char *filename, int lineno) +{ + Condition *cond = condition_factory(p.get_op()); + + if (!cond) { + TSError("[%s] Unknown condition: %s, file: %s, line: %d", PLUGIN_NAME, p.get_op().c_str(), filename, lineno); + return nullptr; + } + + Dbg(pi_dbg_ctl, " Creating condition: %%{%s} with arg: %s", p.get_op().c_str(), p.get_arg().c_str()); + + try { + cond->initialize(p); + } catch (std::exception const &ex) { + delete cond; + TSError("[%s] Failed to initialize condition: %s, file: %s, line: %d, error: %s", PLUGIN_NAME, p.get_op().c_str(), filename, + lineno, ex.what()); + return nullptr; + } + + return cond; +} + +bool +OperatorIf::has_operator() const +{ + const CondOpSection *section = &_sections; + + while (section != nullptr) { + if (section->has_operator()) { + return true; + } + section = section->next.get(); + } + return false; +} + +OperModifiers +OperatorIf::exec_and_return_mods(const Resources &res) const +{ + Dbg(dbg_ctl, "Executing OperatorIf"); + + // Go through each section (if/elif/else) until one matches + for (auto *section = const_cast(&_sections); section != nullptr; section = section->next.get()) { + if (section->group.eval(res)) { + Dbg(dbg_ctl, "OperatorIf section condition matched, executing operators"); + return exec_section(section, res); + } + } + + Dbg(dbg_ctl, "OperatorIf: no section matched"); + return OPER_NONE; +} + +OperModifiers +OperatorIf::exec_section(const CondOpSection *section, const Resources &res) const +{ + if (nullptr == section->ops.oper) { + return section->ops.oper_mods; + } + + auto no_reenable_count = section->ops.oper->do_exec(res); + + ink_assert(no_reenable_count < 2); + if (no_reenable_count) { + return static_cast(section->ops.oper_mods | OPER_NO_REENABLE); + } + + return section->ops.oper_mods; +} diff --git a/plugins/header_rewrite/operators.h b/plugins/header_rewrite/operators.h index a0b825d3d05..179bd799a05 100644 --- a/plugins/header_rewrite/operators.h +++ b/plugins/header_rewrite/operators.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include "ts/ts.h" @@ -29,6 +30,12 @@ #include "resources.h" #include "value.h" +// Forward declarations +class Parser; + +// Full includes needed for member variables +#include "conditions.h" + /////////////////////////////////////////////////////////////////////////////// // Operator declarations. // @@ -670,3 +677,76 @@ class OperatorSetNextHopStrategy : public Operator private: Value _value; }; + +/////////////////////////////////////////////////////////////////////////////// +// OperatorIf class - implements nested if/elif/else as a pseudo-operator. +// Keep this at the end of the files, since this is not really an Operator. +// +class OperatorIf : public Operator +{ +public: + struct CondOpSection { + CondOpSection() = default; + + ~CondOpSection() = default; + + CondOpSection(const CondOpSection &) = delete; + CondOpSection &operator=(const CondOpSection &) = delete; + + bool + has_operator() const + { + return ops.oper != nullptr; + } + + ConditionGroup group; + OperatorAndMods ops; + std::unique_ptr next; // For elif/else sections + }; + + OperatorIf() { Dbg(dbg_ctl, "Calling CTOR for OperatorIf"); } + + // noncopyable + OperatorIf(const OperatorIf &) = delete; + void operator=(const OperatorIf &) = delete; + + ConditionGroup *new_section(Parser::CondClause clause); + bool add_operator(Parser &p, const char *filename, int lineno); + Condition *make_condition(Parser &p, const char *filename, int lineno); + bool has_operator() const; + + ConditionGroup * + get_group() + { + return &_cur_section->group; + } + + Parser::CondClause + get_clause() const + { + return _clause; + } + + CondOpSection * + cur_section() const + { + return _cur_section; + } + + OperModifiers exec_and_return_mods(const Resources &res) const; + +protected: + bool + exec(const Resources &res) const override + { + OperModifiers mods = exec_and_return_mods(res); + return !(mods & OPER_NO_REENABLE); + } + +private: + OperModifiers exec_section(const CondOpSection *section, const Resources &res) const; + + CondOpSection _sections; + CondOpSection *_cur_section = &_sections; + Parser::CondClause _clause = Parser::CondClause::COND; +}; diff --git a/plugins/header_rewrite/parser.cc b/plugins/header_rewrite/parser.cc index a0f93afb774..12f0c15d851 100644 --- a/plugins/header_rewrite/parser.cc +++ b/plugins/header_rewrite/parser.cc @@ -15,6 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ + ////////////////////////////////////////////////////////////////////////////////////////////// // parser.cc: implementation of the config parser // @@ -205,6 +206,12 @@ Parser::preprocess(std::vector tokens) } else if (tokens[0] == "elif") { _clause = CondClause::ELIF; return true; + } else if (tokens[0] == "if") { + _clause = CondClause::IF; + return true; + } else if (tokens[0] == "endif") { + _clause = CondClause::ENDIF; + return true; } // Is it a condition or operator? diff --git a/plugins/header_rewrite/parser.h b/plugins/header_rewrite/parser.h index 9844ad0899f..634df4694be 100644 --- a/plugins/header_rewrite/parser.h +++ b/plugins/header_rewrite/parser.h @@ -113,7 +113,7 @@ std::optional openConfig(const std::string &filename); class Parser { public: - enum class CondClause { OPER, COND, ELIF, ELSE }; + enum class CondClause { OPER, COND, ELIF, ELSE, IF, ENDIF }; Parser() = default; // No from/to URLs for this parser Parser(char *from_url, char *to_url) : _from_url(from_url), _to_url(to_url) {} @@ -165,6 +165,18 @@ class Parser return _clause == CondClause::ELIF; } + bool + is_if() const + { + return _clause == CondClause::IF; + } + + bool + is_endif() const + { + return _clause == CondClause::ENDIF; + } + const std::string & get_op() const { diff --git a/plugins/header_rewrite/ruleset.cc b/plugins/header_rewrite/ruleset.cc index 50d16ac4c47..28fda37a511 100644 --- a/plugins/header_rewrite/ruleset.cc +++ b/plugins/header_rewrite/ruleset.cc @@ -23,10 +23,24 @@ #include "ruleset.h" #include "factory.h" +#include "operators.h" + +RuleSet::RuleSet() +{ + Dbg(dbg_ctl, "RuleSet CTOR"); +} + +RuleSet::~RuleSet() +{ + Dbg(dbg_ctl, "RulesSet DTOR"); +} + +OperModifiers +RuleSet::exec(const Resources &res) const +{ + return _op_if.exec_and_return_mods(res); +} -/////////////////////////////////////////////////////////////////////////////// -// Class implementation (no reason to have these inline) -// void RuleSet::append(std::unique_ptr rule) { @@ -47,7 +61,7 @@ RuleSet::make_condition(Parser &p, const char *filename, int lineno) Condition *c = condition_factory(p.get_op()); if (nullptr == c) { - return nullptr; // Complete failure in the factory + return nullptr; } Dbg(pi_dbg_ctl, " Creating condition: %%{%s} with arg: %s", p.get_op().c_str(), p.get_arg().c_str()); @@ -65,7 +79,6 @@ RuleSet::make_condition(Parser &p, const char *filename, int lineno) return nullptr; } - // Update some ruleset state based on this new condition; _last |= c->last(); _ids = static_cast(_ids | c->get_resource_ids()); @@ -89,17 +102,16 @@ RuleSet::add_operator(Parser &p, const char *filename, int lineno) return false; } - OperatorAndMods &ops = _cur_section->ops; + auto *cur_sec = _op_if.cur_section(); - if (!ops.oper) { - ops.oper = op; + if (!cur_sec->ops.oper) { + cur_sec->ops.oper.reset(op); } else { - ops.oper->append(op); + cur_sec->ops.oper->append(op); } - // Update some ruleset state based on this new operator - ops.oper_mods = static_cast(ops.oper_mods | ops.oper->get_oper_modifiers()); - _ids = static_cast(_ids | ops.oper->get_resource_ids()); + cur_sec->ops.oper_mods = static_cast(cur_sec->ops.oper_mods | cur_sec->ops.oper->get_oper_modifiers()); + _ids = static_cast(_ids | cur_sec->ops.oper->get_resource_ids()); return true; } @@ -120,3 +132,21 @@ RuleSet::get_all_resource_ids() const return ids; } + +bool +RuleSet::add_operator(Operator *op) +{ + auto *cur_sec = _op_if.cur_section(); + + if (!cur_sec->ops.oper) { + cur_sec->ops.oper.reset(op); + } else { + cur_sec->ops.oper->append(op); + } + + // Update some ruleset state based on this new operator + cur_sec->ops.oper_mods = static_cast(cur_sec->ops.oper_mods | cur_sec->ops.oper->get_oper_modifiers()); + _ids = static_cast(_ids | cur_sec->ops.oper->get_resource_ids()); + + return true; +} diff --git a/plugins/header_rewrite/ruleset.h b/plugins/header_rewrite/ruleset.h index 4a5887aab9f..78680bc9d7e 100644 --- a/plugins/header_rewrite/ruleset.h +++ b/plugins/header_rewrite/ruleset.h @@ -30,115 +30,75 @@ #include "resources.h" #include "parser.h" #include "conditions.h" +#include "operators.h" /////////////////////////////////////////////////////////////////////////////// -// Class holding one ruleset. A ruleset is one (or more) pre-conditions, and -// one (or more) operators. +// RuleSet: Represents a complete wrapping a single OperatorIf. // class RuleSet { public: - // Holding the IF and ELSE operators and mods, in two separate linked lists. - struct OperatorAndMods { - OperatorAndMods() = default; - - OperatorAndMods(const OperatorAndMods &) = delete; - OperatorAndMods &operator=(const OperatorAndMods &) = delete; - - Operator *oper = nullptr; - OperModifiers oper_mods = OPER_NONE; - }; - - struct CondOpSection { - CondOpSection() = default; - - ~CondOpSection() - { - delete ops.oper; - delete next; - } - - CondOpSection(const CondOpSection &) = delete; - CondOpSection &operator=(const CondOpSection &) = delete; - - bool - has_operator() const - { - return ops.oper != nullptr; - } - - ConditionGroup group; - OperatorAndMods ops; - CondOpSection *next = nullptr; // For elif / else sections. - }; - - RuleSet() { Dbg(dbg_ctl, "RuleSet CTOR"); } - - ~RuleSet() { Dbg(dbg_ctl, "RulesSet DTOR"); } + RuleSet(); + ~RuleSet(); // noncopyable RuleSet(const RuleSet &) = delete; void operator=(const RuleSet &) = delete; - // No reason to inline these void append(std::unique_ptr rule); Condition *make_condition(Parser &p, const char *filename, int lineno); - bool add_operator(Parser &p, const char *filename, int lineno); ResourceIDs get_all_resource_ids() const; + bool add_operator(Parser &p, const char *filename, int lineno); + bool add_operator(Operator *op); - bool - has_operator() const + ConditionGroup * + get_group() { - const CondOpSection *section = &_sections; - - while (section != nullptr) { - if (section->has_operator()) { - return true; - } - section = section->next; - } - return false; + return _op_if.get_group(); } - void - set_hook(TSHttpHookID hook) + Parser::CondClause + get_clause() const { - _hook = hook; + return _op_if.get_clause(); } ConditionGroup * - get_group() + new_section(Parser::CondClause clause) { - return &_cur_section->group; + return _op_if.new_section(clause); } - TSHttpHookID - get_hook() const + bool + has_operator() const { - return _hook; + return _op_if.has_operator(); } - Parser::CondClause - get_clause() const + bool + section_has_condition() const { - return _clause; + auto *sec = _op_if.cur_section(); + return sec ? sec->group.has_conditions() : false; } - CondOpSection * - cur_section() + bool + section_has_operator() const { - return _cur_section; + auto *sec = _op_if.cur_section(); + return sec ? sec->has_operator() : false; } - ConditionGroup * - new_section(Parser::CondClause clause) + void + set_hook(TSHttpHookID hook) { - TSAssert(_cur_section && !_cur_section->next); - _clause = clause; - _cur_section->next = new CondOpSection(); - _cur_section = _cur_section->next; + _hook = hook; + } - return &_cur_section->group; + TSHttpHookID + get_hook() const + { + return _hook; } ResourceIDs @@ -153,49 +113,14 @@ class RuleSet return _last; } - OperModifiers - exec(const OperatorAndMods &ops, const Resources &res) const - { - if (nullptr == ops.oper) { - return ops.oper_mods; - } - - auto no_reenable_count = ops.oper->do_exec(res); - - ink_assert(no_reenable_count < 2); - if (no_reenable_count) { - return static_cast(ops.oper_mods | OPER_NO_REENABLE); - } - - return ops.oper_mods; - } - - const RuleSet::OperatorAndMods & - eval(const Resources &res) - { - for (CondOpSection *sec = &_sections; sec != nullptr; sec = sec->next) { - if (sec->group.eval(res)) { - return sec->ops; - } - } - - // No matching condition found, return empty operator set. - static OperatorAndMods empty_ops; - return empty_ops; - } + OperModifiers exec(const Resources &res) const; // Linked list of RuleSets std::unique_ptr next; private: - // This holds one condition group, and the ops and optional else_ops, there's - // aways at least one of these in the vector (no "elif" sections). - CondOpSection _sections; - CondOpSection *_cur_section = &_sections; - - // State values (updated when conds / operators are added) - TSHttpHookID _hook = TS_HTTP_READ_RESPONSE_HDR_HOOK; // Which hook is this rule for - ResourceIDs _ids = RSRC_NONE; - bool _last = false; - Parser::CondClause _clause = Parser::CondClause::OPER; + OperatorIf _op_if; + TSHttpHookID _hook = TS_HTTP_READ_RESPONSE_HDR_HOOK; // Which hook is this rule for + ResourceIDs _ids = RSRC_NONE; + bool _last = false; }; diff --git a/plugins/lua/ts_lua_client_request.cc b/plugins/lua/ts_lua_client_request.cc index ac2671cf10c..0fc9e534ba8 100644 --- a/plugins/lua/ts_lua_client_request.cc +++ b/plugins/lua/ts_lua_client_request.cc @@ -790,7 +790,7 @@ ts_lua_client_request_client_addr_get_port(lua_State *L) { struct sockaddr const *client_ip; ts_lua_http_ctx *http_ctx; - int port; + int port = 0; GET_HTTP_CONTEXT(http_ctx, L); @@ -802,7 +802,7 @@ ts_lua_client_request_client_addr_get_port(lua_State *L) } else { if (client_ip->sa_family == AF_INET) { port = ((struct sockaddr_in *)client_ip)->sin_port; - } else { + } else if (client_ip->sa_family == AF_INET6) { port = ((struct sockaddr_in6 *)client_ip)->sin6_port; } @@ -817,7 +817,7 @@ ts_lua_client_request_client_addr_get_incoming_port(lua_State *L) { struct sockaddr const *incoming_addr; ts_lua_http_ctx *http_ctx; - int port; + int port = 0; GET_HTTP_CONTEXT(http_ctx, L); @@ -829,7 +829,7 @@ ts_lua_client_request_client_addr_get_incoming_port(lua_State *L) } else { if (incoming_addr->sa_family == AF_INET) { port = ((struct sockaddr_in *)incoming_addr)->sin_port; - } else { + } else if (incoming_addr->sa_family == AF_INET6) { port = ((struct sockaddr_in6 *)incoming_addr)->sin6_port; } @@ -866,6 +866,8 @@ ts_lua_client_request_client_addr_get_addr(lua_State *L) port = ntohs(((struct sockaddr_in6 *)client_ip)->sin6_port); inet_ntop(AF_INET6, (const void *)&((struct sockaddr_in6 *)client_ip)->sin6_addr, cip, sizeof(cip)); family = AF_INET6; + } else if (client_ip->sa_family == AF_UNIX) { + family = AF_UNIX; } lua_pushstring(L, cip); diff --git a/plugins/lua/ts_lua_http.cc b/plugins/lua/ts_lua_http.cc index b1bc7c104a9..cac64dc44cf 100644 --- a/plugins/lua/ts_lua_http.cc +++ b/plugins/lua/ts_lua_http.cc @@ -1151,6 +1151,8 @@ ts_lua_http_get_ssn_remote_addr(lua_State *L) port = ntohs(((struct sockaddr_in6 *)client_ip)->sin6_port); inet_ntop(AF_INET6, (const void *)&((struct sockaddr_in6 *)client_ip)->sin6_addr, cip, sizeof(cip)); family = AF_INET6; + } else if (client_ip->sa_family == AF_UNIX) { + family = AF_UNIX; } lua_pushstring(L, cip); diff --git a/plugins/lua/ts_lua_http_config.cc b/plugins/lua/ts_lua_http_config.cc index 938538b41be..1543cf25092 100644 --- a/plugins/lua/ts_lua_http_config.cc +++ b/plugins/lua/ts_lua_http_config.cc @@ -24,7 +24,6 @@ typedef enum { TS_LUA_CONFIG_HTTP_CHUNKING_ENABLED = TS_CONFIG_HTTP_CHUNKING_ENABLED, TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_ENABLED = TS_CONFIG_HTTP_NEGATIVE_CACHING_ENABLED, TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME = TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME, - TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_LIST = TS_CONFIG_HTTP_NEGATIVE_CACHING_LIST, TS_LUA_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE = TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE, TS_LUA_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN = TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN, TS_LUA_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT = TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT, @@ -74,7 +73,6 @@ typedef enum { TS_LUA_CONFIG_HTTP_CONNECT_DOWN_POLICY = TS_CONFIG_HTTP_CONNECT_DOWN_POLICY, TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES, TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT, - TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE, TS_LUA_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME = TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME, TS_LUA_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS = TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS, TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT = TS_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT, @@ -97,7 +95,6 @@ typedef enum { TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE = TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE, TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED, TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME, - TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST, TS_LUA_CONFIG_SSL_HSTS_MAX_AGE = TS_CONFIG_SSL_HSTS_MAX_AGE, TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS = TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS, TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME = TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, @@ -152,8 +149,11 @@ typedef enum { TS_LUA_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT = TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT, TS_LUA_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT = TS_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT, TS_LUA_CONFIG_HTTP_CACHE_IGNORE_QUERY = TS_CONFIG_HTTP_CACHE_IGNORE_QUERY, - TS_LUA_CONFIG_HTTP_CACHE_POST_METHOD = TS_CONFIG_HTTP_CACHE_POST_METHOD, TS_LUA_CONFIG_HTTP_STRICT_CHUNK_PARSING = TS_CONFIG_HTTP_STRICT_CHUNK_PARSING, + TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_LIST = TS_CONFIG_HTTP_NEGATIVE_CACHING_LIST, + TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE, + TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST, + TS_LUA_CONFIG_HTTP_CACHE_POST_METHOD = TS_CONFIG_HTTP_CACHE_POST_METHOD, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -169,7 +169,6 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CHUNKING_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME), - TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_LIST), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT), @@ -216,7 +215,6 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_DOWN_POLICY), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT), - TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT), @@ -239,7 +237,6 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME), - TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_SSL_HSTS_MAX_AGE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME), @@ -297,8 +294,11 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IGNORE_QUERY), - TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_POST_METHOD), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_STRICT_CHUNK_PARSING), + TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_CACHING_LIST), + TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE), + TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST), + TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_POST_METHOD), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY), }; diff --git a/plugins/lua/ts_lua_server_request.cc b/plugins/lua/ts_lua_server_request.cc index 26c28974360..913b7fba478 100644 --- a/plugins/lua/ts_lua_server_request.cc +++ b/plugins/lua/ts_lua_server_request.cc @@ -149,6 +149,9 @@ ts_lua_inject_server_request_server_addr_api(lua_State *L) lua_pushinteger(L, AF_INET6); lua_setglobal(L, "TS_LUA_AF_INET6"); + + lua_pushinteger(L, AF_UNIX); + lua_setglobal(L, "TS_LUA_AF_UNIX"); } static void @@ -768,7 +771,11 @@ ts_lua_server_request_server_addr_get_ip(lua_State *L) inet_ntop(AF_INET6, (const void *)&((struct sockaddr_in6 *)server_ip)->sin6_addr, sip, sizeof(sip)); } - lua_pushstring(L, sip); + if (sip[0] == '\0') { + lua_pushnil(L); + } else { + lua_pushstring(L, sip); + } } return 1; @@ -779,7 +786,7 @@ ts_lua_server_request_server_addr_get_port(lua_State *L) { struct sockaddr const *server_ip; ts_lua_http_ctx *http_ctx; - int port; + int port = 0; GET_HTTP_CONTEXT(http_ctx, L); @@ -791,7 +798,7 @@ ts_lua_server_request_server_addr_get_port(lua_State *L) } else { if (server_ip->sa_family == AF_INET) { port = ((struct sockaddr_in *)server_ip)->sin_port; - } else { + } else if (server_ip->sa_family == AF_INET6) { port = ((struct sockaddr_in6 *)server_ip)->sin6_port; } @@ -806,7 +813,7 @@ ts_lua_server_request_server_addr_get_outgoing_port(lua_State *L) { struct sockaddr const *outgoing_addr; ts_lua_http_ctx *http_ctx; - int port; + int port = 0; GET_HTTP_CONTEXT(http_ctx, L); @@ -818,7 +825,7 @@ ts_lua_server_request_server_addr_get_outgoing_port(lua_State *L) } else { if (outgoing_addr->sa_family == AF_INET) { port = ((struct sockaddr_in *)outgoing_addr)->sin_port; - } else { + } else if (outgoing_addr->sa_family == AF_INET6) { port = ((struct sockaddr_in6 *)outgoing_addr)->sin6_port; } @@ -855,6 +862,8 @@ ts_lua_server_request_server_addr_get_addr(lua_State *L) port = ntohs(((struct sockaddr_in6 *)server_ip)->sin6_port); inet_ntop(AF_INET6, (const void *)&((struct sockaddr_in6 *)server_ip)->sin6_addr, sip, sizeof(sip)); family = AF_INET6; + } else if (server_ip->sa_family == AF_UNIX) { + family = AF_UNIX; } lua_pushstring(L, sip); @@ -892,6 +901,8 @@ ts_lua_server_request_server_addr_get_nexthop_addr(lua_State *L) port = ntohs(((struct sockaddr_in6 *)server_ip)->sin6_port); inet_ntop(AF_INET6, (const void *)&((struct sockaddr_in6 *)server_ip)->sin6_addr, sip, sizeof(sip)); family = AF_INET6; + } else if (server_ip->sa_family == AF_UNIX) { + family = AF_UNIX; } lua_pushstring(L, sip); @@ -963,12 +974,14 @@ ts_lua_server_request_server_addr_set_addr(lua_State *L) if (!inet_pton(family, sip, &addr.sin4.sin_addr)) { return luaL_error(L, "invalid ipv4 address"); } - } else { + } else if (family == AF_INET6) { addr.sin6.sin6_family = AF_INET6; addr.sin6.sin6_port = htons(port); if (!inet_pton(family, sip, &addr.sin6.sin6_addr)) { return luaL_error(L, "invalid ipv6 address"); } + } else { + return luaL_error(L, "invalid address family"); } TSHttpTxnServerAddrSet(http_ctx->txnp, &addr.sa); @@ -1009,12 +1022,14 @@ ts_lua_server_request_server_addr_set_outgoing_addr(lua_State *L) if (!inet_pton(family, sip, &addr.sin4.sin_addr)) { return luaL_error(L, "invalid ipv4 address"); } - } else { + } else if (family == AF_INET6) { addr.sin6.sin6_family = AF_INET6; addr.sin6.sin6_port = htons(port); if (!inet_pton(family, sip, &addr.sin6.sin6_addr)) { return luaL_error(L, "invalid ipv6 address"); } + } else { + return luaL_error(L, "invalid address family"); } TSHttpTxnOutgoingAddrSet(http_ctx->txnp, &addr.sa); diff --git a/plugins/lua/ts_lua_vconn.cc b/plugins/lua/ts_lua_vconn.cc index 0c7dd4d34f2..62781be4303 100644 --- a/plugins/lua/ts_lua_vconn.cc +++ b/plugins/lua/ts_lua_vconn.cc @@ -56,9 +56,9 @@ static int ts_lua_vconn_get_remote_addr(lua_State *L) { ts_lua_vconn_ctx *vconn_ctx; - int port; - int family; - char sip[128]; + int port = 0; + int family = AF_UNSPEC; + char sip[128] = ""; GET_VCONN_CONTEXT(vconn_ctx, L); @@ -73,10 +73,12 @@ ts_lua_vconn_get_remote_addr(lua_State *L) port = ntohs(((struct sockaddr_in *)addr)->sin_port); inet_ntop(AF_INET, (const void *)&((struct sockaddr_in *)addr)->sin_addr, sip, sizeof(sip)); family = AF_INET; - } else { + } else if (addr->sa_family == AF_INET6) { port = ntohs(((struct sockaddr_in6 *)addr)->sin6_port); inet_ntop(AF_INET6, (const void *)&((struct sockaddr_in6 *)addr)->sin6_addr, sip, sizeof(sip)); family = AF_INET6; + } else if (addr->sa_family == AF_UNIX) { + family = AF_UNIX; } lua_pushstring(L, sip); diff --git a/plugins/origin_server_auth/aws_auth_v4.cc b/plugins/origin_server_auth/aws_auth_v4.cc index 0242d9408ca..940b7d3e2bd 100644 --- a/plugins/origin_server_auth/aws_auth_v4.cc +++ b/plugins/origin_server_auth/aws_auth_v4.cc @@ -534,6 +534,22 @@ createDefaultExcludeHeaders() m.insert("x-forwarded-for"); m.insert("forwarded"); m.insert("via"); + /* exclude hop-by-hop headers per AWS documentation: + * https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html + * "Do not include hop-by-hop headers that are frequently altered during transit across a complex system. + * This includes all volatile transport headers that are mutated by proxies, load balancers, and the nodes + * in a distributed system, including connection, x-amzn-trace-id, user-agent, keep-alive, transfer-encoding, + * TE, trailer, upgrade, proxy-authorization, and proxy-authenticate." */ + m.insert("connection"); + m.insert("x-amzn-trace-id"); + m.insert("user-agent"); + m.insert("keep-alive"); + m.insert("transfer-encoding"); + m.insert("te"); + m.insert("trailer"); + m.insert("upgrade"); + m.insert("proxy-authorization"); + m.insert("proxy-authenticate"); return m; } const StringSet defaultExcludeHeaders = createDefaultExcludeHeaders(); diff --git a/plugins/regex_remap/regex_remap.cc b/plugins/regex_remap/regex_remap.cc index be57f75998a..de38ba23787 100644 --- a/plugins/regex_remap/regex_remap.cc +++ b/plugins/regex_remap/regex_remap.cc @@ -226,6 +226,11 @@ class RemapRegex { return _lowercase_substitutions; } + inline bool + has_strategy() const + { + return _has_strategy; + } inline std::string const & strategy() const { @@ -268,7 +273,8 @@ class RemapRegex int _connect_timeout = -1; int _dns_timeout = -1; - std::string _strategy = {}; + bool _has_strategy = false; + std::string _strategy = {}; Override *_first_override = nullptr; int _sub_pos[MAX_SUBS]; @@ -317,7 +323,8 @@ RemapRegex::initialize(const std::string ®, const std::string &sub, const std } else if (opt.compare(start, 23, "lowercase_substitutions") == 0) { _lowercase_substitutions = true; } else if (opt.compare(start, 8, "strategy") == 0) { - _strategy = opt_val; + _has_strategy = true; + _strategy = opt_val; } else if (opt_val.size() <= 0) { // All other options have a required value TSError("[%s] Malformed options: %s", PLUGIN_NAME, opt.c_str()); @@ -980,17 +987,19 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) Dbg(dbg_ctl, "Setting DNS timeout to %d", re->dns_timeout_option()); TSHttpTxnDNSTimeoutSet(txnp, re->dns_timeout_option()); } - auto const &strat = re->strategy(); - if (strat.empty() || "null" == strat) { - Dbg(dbg_ctl, "Clearing strategy (use parent.config)"); - TSHttpTxnNextHopStrategySet(txnp, nullptr); - } else { - void const *const stratptr = TSHttpTxnNextHopNamedStrategyGet(txnp, strat.c_str()); - if (nullptr == stratptr) { - Dbg(dbg_ctl, "No strategy found with name '%s'", strat.c_str()); + if (re->has_strategy()) { + auto const &strat = re->strategy(); + if (strat.empty() || "null" == strat) { + Dbg(dbg_ctl, "Clearing strategy (use parent.config)"); + TSHttpTxnNextHopStrategySet(txnp, nullptr); } else { - Dbg(dbg_ctl, "Setting strategy to %s", strat.c_str()); - TSHttpTxnNextHopStrategySet(txnp, stratptr); + void const *const stratptr = TSHttpTxnNextHopNamedStrategyGet(txnp, strat.c_str()); + if (nullptr == stratptr) { + Dbg(dbg_ctl, "No strategy found with name '%s'", strat.c_str()); + } else { + Dbg(dbg_ctl, "Setting strategy to %s", strat.c_str()); + TSHttpTxnNextHopStrategySet(txnp, stratptr); + } } } bool lowercase_substitutions = false; diff --git a/plugins/regex_revalidate/CMakeLists.txt b/plugins/regex_revalidate/CMakeLists.txt index e095fd5684d..b4e04fd202d 100644 --- a/plugins/regex_revalidate/CMakeLists.txt +++ b/plugins/regex_revalidate/CMakeLists.txt @@ -17,5 +17,4 @@ add_atsplugin(regex_revalidate regex_revalidate.cc) -target_link_libraries(regex_revalidate PRIVATE PCRE::PCRE) verify_global_plugin(regex_revalidate) diff --git a/plugins/regex_revalidate/regex_revalidate.cc b/plugins/regex_revalidate/regex_revalidate.cc index 470cd95b1b6..5e8d8cf2f7b 100644 --- a/plugins/regex_revalidate/regex_revalidate.cc +++ b/plugins/regex_revalidate/regex_revalidate.cc @@ -33,11 +33,7 @@ #include #include -#ifdef HAVE_PCRE_PCRE_H -#include -#else -#include -#endif +#include "tsutil/Regex.h" #define CONFIG_TMOUT 60000 #define FREE_TMOUT 300000 @@ -115,9 +111,8 @@ strForResult(TSCacheLookupResult const result) } typedef struct invalidate_t { - const char *regex_text; - pcre *regex; - pcre_extra *regex_extra; + char *regex_text; + Regex *regex; time_t epoch; time_t expiry; TSCacheLookupResult new_result; @@ -136,31 +131,25 @@ typedef struct { static invalidate_t * init_invalidate_t(invalidate_t *i) { - i->regex_text = nullptr; - i->regex = nullptr; - i->regex_extra = nullptr; - i->epoch = 0; - i->expiry = 0; - i->new_result = TS_CACHE_LOOKUP_HIT_STALE; - i->next = nullptr; + i->regex_text = nullptr; + i->regex = nullptr; + i->epoch = 0; + i->expiry = 0; + i->new_result = TS_CACHE_LOOKUP_HIT_STALE; + i->next = nullptr; return i; } static void free_invalidate_t(invalidate_t *i) { - if (i->regex_extra) { -#ifndef PCRE_STUDY_JIT_COMPILE - pcre_free(i->regex_extra); -#else - pcre_free_study(i->regex_extra); -#endif + if (nullptr != i->regex) { + delete i->regex; + i->regex = nullptr; } - if (i->regex) { - pcre_free(i->regex); - } - if (i->regex_text) { - pcre_free_substring(i->regex_text); + if (nullptr != i->regex_text) { + TSfree(i->regex_text); + i->regex_text = nullptr; } TSfree(i); } @@ -212,17 +201,18 @@ static invalidate_t * copy_invalidate_t(invalidate_t *i) { invalidate_t *iptr; - const char *errptr; - int erroffset; - - iptr = (invalidate_t *)TSmalloc(sizeof(invalidate_t)); - iptr->regex_text = TSstrdup(i->regex_text); - iptr->regex = pcre_compile(iptr->regex_text, 0, &errptr, &erroffset, nullptr); // There is no pcre_copy :-( - iptr->regex_extra = pcre_study(iptr->regex, 0, &errptr); // Assuming no errors since this worked before :-/ - iptr->epoch = i->epoch; - iptr->expiry = i->expiry; - iptr->new_result = i->new_result; - iptr->next = nullptr; + + iptr = (invalidate_t *)TSmalloc(sizeof(invalidate_t)); + iptr->regex_text = TSstrdup(i->regex_text); + + // assume this works since the source exists. + iptr->regex = new Regex; + iptr->regex->compile(iptr->regex_text); + + iptr->epoch = i->epoch; + iptr->expiry = i->expiry; + iptr->new_result = i->new_result; + iptr->next = nullptr; return iptr; } @@ -296,26 +286,30 @@ load_state(plugin_state_t *pstate, invalidate_t **ilist) time_t const now = time(nullptr); - const char *errptr; - int erroffset; - int ovector[OVECTOR_SIZE]; - pcre *const config_re = pcre_compile("^([^#].+?)\\s+(\\d+)\\s+(\\d+)\\s+(\\w+)\\s*$", 0, &errptr, &erroffset, nullptr); - TSReleaseAssert(nullptr != config_re); + Regex config_re; + bool const re_stat = config_re.compile("^([^#].+?)\\s+(\\d+)\\s+(\\d+)\\s+(\\w+)\\s*$"); + TSReleaseAssert(true == re_stat); char line[LINE_MAX]; int ln = 0; while (fgets(line, LINE_MAX, fs) != nullptr) { Dbg(dbg_ctl, "state: processing: %d %s", ln, line); ++ln; - int const rc = pcre_exec(config_re, nullptr, line, strlen(line), 0, 0, ovector, OVECTOR_SIZE); + + RegexMatches matches; + int const rc = config_re.exec(line, matches); if (5 == rc) { invalidate_t *const inv = (invalidate_t *)TSmalloc(sizeof(invalidate_t)); init_invalidate_t(inv); - pcre_get_substring(line, ovector, rc, 1, &(inv->regex_text)); - inv->epoch = atoi(line + ovector[4]); - inv->expiry = atoi(line + ovector[6]); + auto const regv = matches[1]; + inv->regex_text = TSstrndup(regv.data(), regv.length()); + Dbg(dbg_ctl, "regex_text: %s", inv->regex_text); + + // atoi will terminate when whitespace/eol is reached + inv->epoch = atoi(matches[2].data()); + inv->expiry = atoi(matches[3].data()); if (inv->expiry < now) { Dbg(dbg_ctl, "state: skipping expired : '%s'", inv->regex_text); @@ -323,16 +317,15 @@ load_state(plugin_state_t *pstate, invalidate_t **ilist) continue; } - int const len = ovector[9] - ovector[8]; - char const *const type = line + ovector[8]; - - if (0 == strncasecmp(type, RESULT_STALE, len)) { + auto const type = matches[4]; + if (0 == strncasecmp(type.data(), RESULT_STALE, type.length())) { Dbg(dbg_ctl, "state: regex line set to result type %s: '%s'", RESULT_STALE, inv->regex_text); - } else if (0 == strncasecmp(type, RESULT_MISS, len)) { + } else if (0 == strncasecmp(type.data(), RESULT_MISS, type.length())) { Dbg(dbg_ctl, "state: regex line set to result type %s: '%s'", RESULT_MISS, inv->regex_text); inv->new_result = TS_CACHE_LOOKUP_MISS; } else { - Dbg(dbg_ctl, "state: unknown regex line result type '%.*s', skipping '%s'", len, type, inv->regex_text); + Dbg(dbg_ctl, "state: unknown regex line result type '%.*s', skipping '%s'", (int)type.length(), type.data(), + inv->regex_text); } // iterate through the loaded config and try to merge @@ -358,7 +351,6 @@ load_state(plugin_state_t *pstate, invalidate_t **ilist) } } - pcre_free(config_re); fclose(fs); return true; } @@ -402,11 +394,10 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) } Dbg(dbg_ctl, "Attempting to load rules from: '%s'", path); - const char *errptr; - int erroffset; - int ovector[OVECTOR_SIZE]; - pcre *const config_re = pcre_compile("^([^#].+?)\\s+(\\d+)(\\s+(\\w+))?\\s*$", 0, &errptr, &erroffset, nullptr); - TSReleaseAssert(nullptr != config_re); + + Regex config_re; + bool const regstat = config_re.compile("^([^#].+?)\\s+(\\d+)(\\s+(\\w+))?\\s*$"); + TSReleaseAssert(true == regstat); char line[LINE_MAX]; int ln = 0; @@ -415,25 +406,40 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) while (fgets(line, LINE_MAX, fs) != nullptr) { Dbg(dbg_ctl, "Processing: %d %s", ln, line); ++ln; - int const rc = pcre_exec(config_re, nullptr, line, strlen(line), 0, 0, ovector, OVECTOR_SIZE); + RegexMatches matches; + int const rc = config_re.exec(line, matches); if (3 <= rc) { i = (invalidate_t *)TSmalloc(sizeof(invalidate_t)); init_invalidate_t(i); - pcre_get_substring(line, ovector, rc, 1, &i->regex_text); - i->regex = pcre_compile(i->regex_text, 0, &errptr, &erroffset, nullptr); - i->epoch = now; - i->expiry = atoi(line + ovector[4]); + auto const regv = matches[1]; + + i->regex = new Regex; + std::string error; + int erroff = 0; + bool rstat = i->regex->compile(regv, error, erroff); + if (!rstat) { + Dbg(dbg_ctl, "Invalid rule regex!, message: %s, offset: %d", error.c_str(), erroff); + free_invalidate_t(i); + i = nullptr; + continue; + } + + i->regex_text = TSstrndup(regv.data(), regv.length()); + Dbg(dbg_ctl, "regex_tex: %s", i->regex_text); + i->epoch = now; + // atoi will terminate when whitespace/eol is reached + i->expiry = atoi(matches[2].data()); if (5 == rc) { - int const len = ovector[9] - ovector[8]; - char const *const type = line + ovector[8]; - if (0 == strncasecmp(type, RESULT_MISS, len)) { + auto const type = matches[4]; + if (0 == strncasecmp(type.data(), RESULT_MISS, type.length())) { Dbg(dbg_ctl, "Regex line set to result type %s: '%s'", RESULT_MISS, i->regex_text); i->new_result = TS_CACHE_LOOKUP_MISS; - } else if (0 != strncasecmp(type, RESULT_STALE, len)) { - Dbg(dbg_ctl, "Unknown regex line result type '%s', using default '%s' '%s'", type, RESULT_STALE, i->regex_text); + } else if (0 != strncasecmp(type.data(), RESULT_STALE, type.length())) { + Dbg(dbg_ctl, "Unknown regex line result type '%.*s', using default '%s' '%s'", (int)type.length(), type.data(), + RESULT_STALE, i->regex_text); } } @@ -446,7 +452,6 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) free_invalidate_t(i); i = nullptr; } else { - i->regex_extra = pcre_study(i->regex, 0, &errptr); if (!*ilist) { *ilist = i; Dbg(dbg_ctl, "Created new list and Loaded %s %jd %jd %s", i->regex_text, (intmax_t)i->epoch, (intmax_t)i->expiry, @@ -485,7 +490,6 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) Dbg(dbg_ctl, "Skipping line %d, too few fields", ln); } } - pcre_free(config_re); fclose(fs); pstate->last_load = s.st_mtime; return true; @@ -695,11 +699,14 @@ main_handler(TSCont cont, TSEvent event, void *edata) now = time(nullptr); } if (date <= iptr->epoch && now < iptr->expiry) { - if (!url) { + if (nullptr == url) { url = TSHttpTxnEffectiveUrlStringGet(txn, &url_len); Dbg(dbg_ctl, "Effective url is is '%.*s'", url_len, url); } - if (pcre_exec(iptr->regex, iptr->regex_extra, url, url_len, 0, 0, nullptr, 0) >= 0) { + Dbg(dbg_ctl, "checking: %.*s, %s", url_len, url, iptr->regex_text); + + std::string_view const urlv(url, url_len); + if (iptr->regex->exec(urlv)) { Dbg(dbg_ctl, "Forced revalidate, Match with rule regex: '%s' epoch: %jd, expiry: %jd, result: '%s'", iptr->regex_text, intmax_t(iptr->epoch), intmax_t(iptr->expiry), strForResult(iptr->new_result)); TSHttpTxnCacheLookupStatusSet(txn, iptr->new_result); @@ -715,7 +722,7 @@ main_handler(TSCont cont, TSEvent event, void *edata) iptr = iptr->next; } } - if (url) { + if (nullptr != url) { TSfree(url); } } diff --git a/src/api/InkAPITest.cc b/src/api/InkAPITest.cc index 521474bedda..21d80fc57f3 100644 --- a/src/api/InkAPITest.cc +++ b/src/api/InkAPITest.cc @@ -5236,6 +5236,14 @@ REGRESSION_TEST(SDK_API_TSMimeHdrField)(RegressionTest *test, int /* atype ATS_U field1Value4Get = TSMimeHdrFieldValueStringGet(bufp1, mime_loc1, field_loc11, 3, &lengthField1Value4); field1Value5Get = TSMimeHdrFieldValueStringGet(bufp1, mime_loc1, field_loc11, 4, &lengthField1Value5); field1ValueAllGet = TSMimeHdrFieldValueStringGet(bufp1, mime_loc1, field_loc11, -1, &lengthField1ValueAll); + + std::string_view sv1{field1Value1Get, static_cast(lengthField1Value1)}; + std::string_view sv2{field1Value2Get, static_cast(lengthField1Value2)}; + std::string_view sv3{field1Value3Get, static_cast(lengthField1Value3)}; + std::string_view sv4{field1Value4Get, static_cast(lengthField1Value4)}; + std::string_view sv5{field1Value5Get, static_cast(lengthField1Value5)}; + std::string_view svall{field1ValueAllGet, static_cast(lengthField1ValueAll)}; + if (((strncmp(field1Value1Get, field1Value1, lengthField1Value1) == 0) && lengthField1Value1 == static_cast(strlen(field1Value1))) && ((strncmp(field1Value2Get, field1Value2, lengthField1Value2) == 0) && @@ -5246,11 +5254,8 @@ REGRESSION_TEST(SDK_API_TSMimeHdrField)(RegressionTest *test, int /* atype ATS_U lengthField1Value4 == static_cast(strlen(field1Value4))) && ((strncmp(field1Value5Get, field1Value5, lengthField1Value5) == 0) && lengthField1Value5 == static_cast(strlen(field1Value5))) && - (strstr(field1ValueAllGet, field1Value1Get) == field1Value1Get) && - (strstr(field1ValueAllGet, field1Value2Get) == field1Value2Get) && - (strstr(field1ValueAllGet, field1Value3Get) == field1Value3Get) && - (strstr(field1ValueAllGet, field1Value4Get) == field1Value4Get) && - (strstr(field1ValueAllGet, field1Value5Get) == field1Value5Get)) { + (svall.find(sv1) != svall.npos) && (svall.find(sv2) != svall.npos) && (svall.find(sv3) != svall.npos) && + (svall.find(sv4) != svall.npos) && (svall.find(sv5) != svall.npos)) { SDK_RPRINT(test, "TSMimeHdrFieldValueStringInsert", "TestCase1&2&3&4&5", TC_PASS, "ok"); SDK_RPRINT(test, "TSMimeHdrFieldValueStringGet", "TestCase1&2&3&4&5", TC_PASS, "ok"); SDK_RPRINT(test, "TSMimeHdrFieldValueStringGet with IDX=-1", "TestCase1&2&3&4&5", TC_PASS, "ok"); @@ -8692,7 +8697,6 @@ std::array SDK_Overridable_Configs = { "proxy.config.http.chunking_enabled", "proxy.config.http.negative_caching_enabled", "proxy.config.http.negative_caching_lifetime", - "proxy.config.http.negative_caching_list", "proxy.config.http.cache.when_to_revalidate", "proxy.config.http.keep_alive_enabled_in", "proxy.config.http.keep_alive_enabled_out", @@ -8736,7 +8740,6 @@ std::array SDK_Overridable_Configs = { "proxy.config.http.connect_attempts_max_retries_down_server", "proxy.config.http.connect_attempts_rr_retries", "proxy.config.http.connect_attempts_timeout", - "proxy.config.http.connect_attempts_retry_backoff_base", "proxy.config.http.down_server.cache_time", "proxy.config.http.doc_in_cache_skip_dns", "proxy.config.http.background_fill_active_timeout", @@ -8757,7 +8760,6 @@ std::array SDK_Overridable_Configs = { "proxy.config.http.response_header_max_size", "proxy.config.http.negative_revalidating_enabled", "proxy.config.http.negative_revalidating_lifetime", - "proxy.config.http.negative_revalidating_list", "proxy.config.ssl.hsts_max_age", "proxy.config.ssl.hsts_include_subdomains", "proxy.config.http.cache.open_read_retry_time", @@ -8820,8 +8822,11 @@ std::array SDK_Overridable_Configs = { "proxy.config.http.no_dns_just_forward_to_parent", "proxy.config.http.cache.ignore_query", "proxy.config.http.drop_chunked_trailers", - "proxy.config.http.cache.post_method", "proxy.config.http.strict_chunk_parsing", + "proxy.config.http.negative_caching_list", + "proxy.config.http.connect_attempts_retry_backoff_base", + "proxy.config.http.negative_revalidating_list", + "proxy.config.http.cache.post_method", } }; // clang-format on diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index 67e2804cecd..e3d217d647d 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -35,8 +35,7 @@ #include "iocore/eventsystem/Action.h" #include "iocore/eventsystem/Continuation.h" -#include "../../records/P_RecProcess.h" - +#include "iocore/eventsystem/Freer.h" #include "tscore/Diags.h" #include "tscore/Filenames.h" #include "tscore/ink_assert.h" @@ -1126,7 +1125,10 @@ register_cache_stats(CacheStatsBlock *rsb, const std::string &prefix) rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); + rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); + rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); + rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); diff --git a/src/iocore/cache/CacheVC.cc b/src/iocore/cache/CacheVC.cc index dad18599390..d230463e1b1 100644 --- a/src/iocore/cache/CacheVC.cc +++ b/src/iocore/cache/CacheVC.cc @@ -459,11 +459,15 @@ CacheVC::handleRead(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) } else if (load_from_last_open_read_call()) { goto LmemHit; } else if (load_from_aggregation_buffer()) { - io.aio_result = io.aiocb.aio_nbytes; + f.doc_from_ram_cache = true; + io.aio_result = io.aiocb.aio_nbytes; SET_HANDLER(&CacheVC::handleReadDone); return EVENT_RETURN; } + ts::Metrics::Counter::increment(cache_rsb.all_mem_misses); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.all_mem_misses); + io.aiocb.aio_fildes = stripe->fd; io.aiocb.aio_offset = stripe->vol_offset(&dir); if (static_cast(io.aiocb.aio_offset + io.aiocb.aio_nbytes) > static_cast(stripe->skip + stripe->len)) { @@ -514,6 +518,8 @@ CacheVC::load_from_last_open_read_call() { if (*this->read_key == this->stripe->first_fragment_key && dir_offset(&this->dir) == this->stripe->first_fragment_offset) { this->buf = this->stripe->first_fragment_data; + ts::Metrics::Counter::increment(cache_rsb.last_open_read_hits); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.last_open_read_hits); return true; } return false; @@ -531,6 +537,8 @@ CacheVC::load_from_aggregation_buffer() [[maybe_unused]] bool success = this->stripe->copy_from_aggregate_write_buffer(doc, dir, this->io.aiocb.aio_nbytes); // We already confirmed that the copy was valid, so it should not fail. ink_assert(success); + ts::Metrics::Counter::increment(cache_rsb.agg_buffer_hits); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.agg_buffer_hits); return true; } diff --git a/src/iocore/cache/P_CacheStats.h b/src/iocore/cache/P_CacheStats.h index 1ac4ba7dea1..4a713220ae1 100644 --- a/src/iocore/cache/P_CacheStats.h +++ b/src/iocore/cache/P_CacheStats.h @@ -45,7 +45,10 @@ struct CacheStatsBlock { ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; + ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; + ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; + ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; ts::Metrics::Counter::AtomicType *pread_count = nullptr; ts::Metrics::Gauge::AtomicType *percent_full = nullptr; ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; diff --git a/src/iocore/eventsystem/P_IOBuffer.h b/src/iocore/eventsystem/P_IOBuffer.h index a72ff11556b..e159cbafc57 100644 --- a/src/iocore/eventsystem/P_IOBuffer.h +++ b/src/iocore/eventsystem/P_IOBuffer.h @@ -942,6 +942,10 @@ MIOBuffer::set(void *b, int64_t len) TS_INLINE void MIOBuffer::append_xmalloced(void *b, int64_t len) { + if (len == 0) { + return; + } + IOBufferBlock *x = new_IOBufferBlock_internal(_location); x->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(len)); append_block_internal(x); diff --git a/src/iocore/eventsystem/RecProcess.cc b/src/iocore/eventsystem/RecProcess.cc index 1940783efff..81d698bab5c 100644 --- a/src/iocore/eventsystem/RecProcess.cc +++ b/src/iocore/eventsystem/RecProcess.cc @@ -29,7 +29,6 @@ #include "P_EventSystem.h" #include "../../records/P_RecCore.h" -#include "../../records/P_RecProcess.h" #include "../../records/P_RecMessage.h" #include "../../records/P_RecUtils.h" #include "../../records/P_RecFile.h" diff --git a/src/iocore/eventsystem/RecRawStatsImpl.cc b/src/iocore/eventsystem/RecRawStatsImpl.cc index 26fe493dbf5..fe96d6deed4 100644 --- a/src/iocore/eventsystem/RecRawStatsImpl.cc +++ b/src/iocore/eventsystem/RecRawStatsImpl.cc @@ -21,9 +21,10 @@ Record statistics support (EThread implementation). limitations under the License. */ +#include "iocore/eventsystem/EventProcessor.h" #include "records/RecDefs.h" +#include "records/RecProcess.h" #include "../../records/P_RecCore.h" -#include "../../records/P_RecProcess.h" #include //------------------------------------------------------------------------- diff --git a/src/iocore/eventsystem/UnixEventProcessor.cc b/src/iocore/eventsystem/UnixEventProcessor.cc index 4a834d74b31..4ac2c7f7bb9 100644 --- a/src/iocore/eventsystem/UnixEventProcessor.cc +++ b/src/iocore/eventsystem/UnixEventProcessor.cc @@ -86,8 +86,14 @@ ThreadAffinityInitializer Thread_Affinity_Initializer; namespace { -int -EventMetricStatSync(const char *, RecDataT, RecData *, RecRawStatBlock *rsb, int) +struct EventStatsBlock { + static constexpr size_t STAT_COUNT = + EThread::Metrics::Graph::N_BUCKETS * 2 + EThread::Metrics::Slice::N_STAT_ID * EThread::Metrics::N_TIMESCALES; + std::array stats; +} events_rsb; + +void +EventMetricStatSync() { using Graph = EThread::Metrics::Graph; @@ -99,15 +105,10 @@ EventMetricStatSync(const char *, RecDataT, RecData *, RecRawStatBlock *rsb, int t->metrics.summarize(summary); } - ink_mutex_acquire(&(rsb->mutex)); - // Update a specific enumerated stat. auto slice_stat_update = [=](EThread::Metrics::Slice::STAT_ID stat_id, int stat_idx, size_t value) { - auto idx = stat_idx + static_cast(stat_id); - auto stat = rsb->global[idx]; - stat->sum = value; - stat->count = 1; - RecRawStatUpdateSum(rsb, idx); + auto idx = stat_idx + static_cast(stat_id); + ts::Metrics::Gauge::store(events_rsb.stats[idx], value); }; // Enumerated stats are first - one set for each time scale. @@ -129,16 +130,12 @@ EventMetricStatSync(const char *, RecDataT, RecData *, RecRawStatBlock *rsb, int // Next are the event loop histogram buckets. for (Graph::raw_type idx = 0; idx < Graph::N_BUCKETS; ++idx, ++id) { - rsb->global[id]->sum = summary._loop_timing[idx]; - rsb->global[id]->count = 1; - RecRawStatUpdateSum(rsb, id); + ts::Metrics::Gauge::store(events_rsb.stats[id], summary._loop_timing[idx]); } // Last are the plugin API histogram buckets. for (Graph::raw_type idx = 0; idx < Graph::N_BUCKETS; ++idx, ++id) { - rsb->global[id]->sum = summary._api_timing[idx]; - rsb->global[id]->count = 1; - RecRawStatUpdateSum(rsb, id); + ts::Metrics::Gauge::store(events_rsb.stats[id], summary._api_timing[idx]); } // Check if it's time to schedule a decay of the histogram data. @@ -150,9 +147,6 @@ EventMetricStatSync(const char *, RecDataT, RecData *, RecRawStatBlock *rsb, int ++(t->metrics._decay_count); } } - - ink_mutex_release(&(rsb->mutex)); - return REC_ERR_OKAY; } /// This is a wrapper used to convert a static function into a continuation. The function pointer is @@ -524,16 +518,15 @@ EventProcessor::start(int n_event_threads, size_t stacksize) thread_group[ET_CALL]._spawnQueue.push(make_event_for_scheduling(&Thread_Affinity_Initializer, EVENT_IMMEDIATE, nullptr)); // Get our statistics set up - RecRawStatBlock *rsb = RecAllocateRawStatBlock(EThread::Metrics::N_STATS); - unsigned stat_idx = 0; - char name[256]; + unsigned stat_idx = 0; + char name[256]; // Enumerated statistics, one set per time scale. for (unsigned ts_idx = 0; ts_idx < EThread::Metrics::N_TIMESCALES; ++ts_idx) { auto sample_count = EThread::Metrics::SLICE_SAMPLE_COUNT[ts_idx]; - for (unsigned id = 0; id < EThread::Metrics::Slice::N_STAT_ID; ++id) { - snprintf(name, sizeof(name), "%s.%ds", EThread::Metrics::Slice::STAT_NAME[id], sample_count); - RecRegisterRawStat(rsb, RECT_PROCESS, name, RECD_INT, RECP_NON_PERSISTENT, stat_idx++, NULL); + for (auto id : EThread::Metrics::Slice::STAT_NAME) { + snprintf(name, sizeof(name), "%s.%ds", id, sample_count); + events_rsb.stats[stat_idx++] = ts::Metrics::Gauge::createPtr(name); } } @@ -541,18 +534,19 @@ EventProcessor::start(int n_event_threads, size_t stacksize) for (Graph::raw_type id = 0; id < Graph::N_BUCKETS; ++id) { snprintf(name, sizeof(name), "%s%zums", EThread::Metrics::LOOP_HISTOGRAM_STAT_STEM.data(), static_cast(EThread::Metrics::LOOP_HISTOGRAM_BUCKET_SIZE.count() * Graph::min_for_bucket(id))); - RecRegisterRawStat(rsb, RECT_PROCESS, name, RECD_INT, RECP_NON_PERSISTENT, stat_idx++, NULL); + events_rsb.stats[stat_idx++] = ts::Metrics::Gauge::createPtr(name); } // plugin API timings for (Graph::raw_type id = 0; id < Graph::N_BUCKETS; ++id) { snprintf(name, sizeof(name), "%s%zums", EThread::Metrics::API_HISTOGRAM_STAT_STEM.data(), static_cast(EThread::Metrics::API_HISTOGRAM_BUCKET_SIZE.count() * Graph::min_for_bucket(id))); - RecRegisterRawStat(rsb, RECT_PROCESS, name, RECD_INT, RECP_NON_PERSISTENT, stat_idx++, NULL); + events_rsb.stats[stat_idx++] = ts::Metrics::Gauge::createPtr(name); } - // Name must be that of a stat, pick one at random since we do all of them in one pass/callback. - RecRegisterRawStatSyncCb(name, EventMetricStatSync, rsb, 0); + debug_assert_message(stat_idx == events_rsb.stats.size(), "events_rsp stats overrun!"); + + RecRegNewSyncStatSync(EventMetricStatSync); this->spawn_event_threads(ET_CALL, n_event_threads, stacksize); diff --git a/src/iocore/hostdb/HostDB.cc b/src/iocore/hostdb/HostDB.cc index 0902aebfa26..4db02ef94cc 100644 --- a/src/iocore/hostdb/HostDB.cc +++ b/src/iocore/hostdb/HostDB.cc @@ -21,6 +21,7 @@ limitations under the License. */ +#include "iocore/hostdb/HostDBProcessor.h" #include "swoc/swoc_file.h" #include "tscore/Regression.h" #include "tsutil/ts_bw_format.h" @@ -740,21 +741,18 @@ HostDBContinuation::lookup_done(TextView query_name, ts_seconds answer_ttl, SRVH if (query_name.empty()) { if (hash.is_byname()) { Dbg(dbg_ctl_hostdb, "lookup_done() failed for '%.*s'", int(hash.host_name.size()), hash.host_name.data()); + record->record_type = HostDBType::ADDR; } else if (hash.is_srv()) { Dbg(dbg_ctl_dns_srv, "SRV failed for '%.*s'", int(hash.host_name.size()), hash.host_name.data()); + record->record_type = HostDBType::SRV; } else { ip_text_buffer b; Dbg(dbg_ctl_hostdb, "failed for %s", hash.ip.toString(b, sizeof b)); + record->record_type = HostDBType::HOST; } record->ip_timestamp = hostdb_current_timestamp; record->ip_timeout_interval = ts_seconds(std::clamp(hostdb_ip_fail_timeout_interval, 1u, HOST_DB_MAX_TTL)); - if (hash.is_srv()) { - record->record_type = HostDBType::SRV; - } else if (!hash.is_byname()) { - record->record_type = HostDBType::HOST; - } - record->set_failed(); } else { @@ -785,6 +783,7 @@ HostDBContinuation::lookup_done(TextView query_name, ts_seconds answer_ttl, SRVH if (hash.is_byname()) { Dbg_bw(dbg_ctl_hostdb, "done {} TTL {}", hash.host_name, answer_ttl); + record->record_type = HostDBType::ADDR; } else if (hash.is_srv()) { ink_assert(srv && srv->hosts.size() && srv->hosts.size() <= hostdb_round_robin_max_count); diff --git a/src/iocore/net/SSLStats.cc b/src/iocore/net/SSLStats.cc index ee8c6cb629e..5d0b90c74c7 100644 --- a/src/iocore/net/SSLStats.cc +++ b/src/iocore/net/SSLStats.cc @@ -24,8 +24,8 @@ #include "SSLStats.h" #include "P_SSLConfig.h" #include "P_SSLUtils.h" -#include "../../records/P_RecProcess.h" #include "iocore/net/SSLMultiCertConfigLoader.h" +#include "records/RecProcess.h" #include diff --git a/src/mgmt/rpc/handlers/config/Configuration.cc b/src/mgmt/rpc/handlers/config/Configuration.cc index 66141280acd..cf3f2fc60d5 100644 --- a/src/mgmt/rpc/handlers/config/Configuration.cc +++ b/src/mgmt/rpc/handlers/config/Configuration.cc @@ -95,7 +95,7 @@ namespace return false; } } else if constexpr (std::is_same_v) { - if (RecSetRecordString(info.name.c_str(), const_cast(info.value.c_str()), REC_SOURCE_DEFAULT) != REC_ERR_OKAY) { + if (RecSetRecordString(info.name.c_str(), info.value.c_str(), REC_SOURCE_DEFAULT) != REC_ERR_OKAY) { return false; } } diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index 7efeeb4b660..8eae5d78529 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -25,10 +25,12 @@ #include "tscore/ink_platform.h" #include "tscore/ink_memory.h" #include +#include #include #include #include #include +#include #include "proxy/hdrs/MIME.h" #include "proxy/hdrs/HdrHeap.h" #include "proxy/hdrs/HdrToken.h" @@ -51,17 +53,61 @@ using swoc::TextView; * C O N S T A N T S * * * ***********************************************************************/ -static DFA *day_names_dfa = nullptr; -static DFA *month_names_dfa = nullptr; -static constexpr const char *day_names[] = { +namespace +{ +constexpr std::array day_names = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", }; -static constexpr const char *month_names[] = { +constexpr std::array month_names = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", }; +template +consteval std::array +make_packed(const std::array &names) +{ + std::array packed{}; + + auto tl = [](char c) -> char { return (c >= 'A' && c <= 'Z') ? (c + 32) : c; }; + + for (size_t i = 0; i < count; ++i) { + const auto &sv = names[i]; + const uint32_t c0 = tl(static_cast(sv[0])); + const uint32_t c1 = tl(static_cast(sv[1])); + const uint32_t c2 = tl(static_cast(sv[2])); + packed[i] = (c0 << 16) | (c1 << 8) | c2; + } + return packed; +} + +constexpr std::array day_names_packed = make_packed(day_names); +constexpr std::array month_names_packed = make_packed(month_names); + +// Case-insensitive match of first 3 characters of input string against array of names +// Longer strings will match if their first 3 characters match - this is intentional for +// matching non-standard day/month names like "Thursday" or "September". +template +__attribute__((always_inline)) constexpr int +match_3char_ci(const std::string_view s, const std::array &names_packed) +{ + if (s.size() < 3) { + return -1; + } + + auto tl = [](char c) -> char { return (c >= 'A' && c <= 'Z') ? (c + 32) : c; }; + const uint32_t packed = (tl(s[0]) << 16) | (tl(s[1]) << 8) | tl(s[2]); + + for (size_t i = 0; i < count; i++) { + if (packed == names_packed[i]) { + return i; + } + } + return -1; +} +} // namespace + struct MDY { uint8_t m; uint8_t d; @@ -595,11 +641,6 @@ mime_init() init = 0; hdrtoken_init(); - day_names_dfa = new DFA; - day_names_dfa->compile(day_names, SIZEOF(day_names), RE_CASE_INSENSITIVE); - - month_names_dfa = new DFA; - month_names_dfa->compile(month_names, SIZEOF(month_names), RE_CASE_INSENSITIVE); MIME_FIELD_ACCEPT = hdrtoken_string_to_wks_sv("Accept"); MIME_FIELD_ACCEPT_CHARSET = hdrtoken_string_to_wks_sv("Accept-Charset"); @@ -3125,8 +3166,7 @@ mime_parse_int64(const char *buf, const char *end) int mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) { - unsigned int three_char_wday, three_char_mon; - std::string_view view{buf, size_t(length)}; + unsigned int three_char_wday, three_char_mon; ink_assert(length >= 29); ink_assert(!is_ws(buf[0])); @@ -3157,7 +3197,7 @@ mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) } } if (tp->tm_wday < 0) { - tp->tm_wday = day_names_dfa->match(view); + tp->tm_wday = match_3char_ci({buf, 3}, day_names_packed); if (tp->tm_wday < 0) { return 0; } @@ -3210,7 +3250,7 @@ mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) } } if (tp->tm_mon < 0) { - tp->tm_mon = month_names_dfa->match(view); + tp->tm_mon = match_3char_ci({buf + 8, 3}, month_names_packed); if (tp->tm_mon < 0) { return 0; } @@ -3355,7 +3395,7 @@ mime_parse_day(const char *&buf, const char *end, int *day) e += 1; } - *day = day_names_dfa->match({buf, size_t(e - buf)}); + *day = match_3char_ci({buf, static_cast(e - buf)}, day_names_packed); if (*day < 0) { return false; } else { @@ -3378,7 +3418,7 @@ mime_parse_month(const char *&buf, const char *end, int *month) e += 1; } - *month = month_names_dfa->match({buf, size_t(e - buf)}); + *month = match_3char_ci({buf, static_cast(e - buf)}, month_names_packed); if (*month < 0) { return false; } else { diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc index 95d677a390c..1dcc5dcca3f 100644 --- a/src/proxy/http/HttpSM.cc +++ b/src/proxy/http/HttpSM.cc @@ -2356,13 +2356,6 @@ HttpSM::process_hostdb_info(HostDBRecord *record) } } -int -HttpSM::state_pre_resolve(int event, void * /* data ATS_UNUSED */) -{ - STATE_ENTER(&HttpSM::state_hostdb_lookup, event); - return 0; -} - ////////////////////////////////////////////////////////////////////////////// // // HttpSM::state_hostdb_lookup() @@ -2433,31 +2426,6 @@ HttpSM::state_hostdb_reverse_lookup(int event, void *data) return 0; } -////////////////////////////////////////////////////////////////////////////// -// -// HttpSM:state_mark_os_down() -// -////////////////////////////////////////////////////////////////////////////// -int -HttpSM::state_mark_os_down(int event, void *data) -{ - STATE_ENTER(&HttpSM::state_mark_os_down, event); - - if (event == EVENT_HOST_DB_LOOKUP && data) { - auto r = static_cast(data); - - // Look for the entry we need mark down in the round robin - ink_assert(t_state.current.server != nullptr); - ink_assert(t_state.dns_info.looking_up == ResolveInfo::ORIGIN_SERVER); - if (auto *info = r->find(&t_state.dns_info.addr.sa); info != nullptr) { - info->mark_down(ts_clock::now()); - } - } - // We either found our entry or we did not. Either way find - // the entry we should use now - return state_hostdb_lookup(event, data); -} - ///////////////////////////////////////////////////////////////////////////////// // HttpSM::state_cache_open_write() // @@ -3426,9 +3394,9 @@ HttpSM::tunnel_handler_100_continue_ua(int event, HttpTunnelConsumer *c) _ua.get_entry()->in_tunnel = false; c->write_success = true; - // remove the buffer reader from the consumer's vc + // Disable any write operation in case there are timeout events. if (c->vc != nullptr) { - c->vc->do_io_write(); + c->vc->do_io_write(nullptr, 0, nullptr); } } diff --git a/src/proxy/http/remap/PluginFactory.cc b/src/proxy/http/remap/PluginFactory.cc index a29b0499b9e..f95e0c91ba0 100644 --- a/src/proxy/http/remap/PluginFactory.cc +++ b/src/proxy/http/remap/PluginFactory.cc @@ -104,16 +104,10 @@ PluginFactory::~PluginFactory() _instList.apply([](RemapPluginInst *pluginInst) -> void { delete pluginInst; }); _instList.clear(); - if (!TSSystemState::is_event_system_shut_down()) { - uint32_t elevate_access = 0; - - elevate_access = RecGetRecordInt("proxy.config.plugin.load_elevated").value_or(0); - ElevateAccess access(elevate_access ? ElevateAccess::FILE_PRIVILEGE : 0); - - fs::remove_all(_runtimeDir, _ec); - } else { - fs::remove_all(_runtimeDir, _ec); // Try anyways - } + // Don't delete _runtimeDir here - plugin DSOs may still be loaded in memory via dlopen handles. + // Deleting the .so files breaks debugging/symbol resolution. Obsolete .so files are cleaned up + // when the old plugin is unloaded (refcount drops to 0), leaving empty directories that are + // removed by cleanup() on next startup. PluginDbg(_dbg_ctl(), "destroyed plugin factory %s", getUuid()); delete _uuid; diff --git a/src/proxy/http2/HTTP2.cc b/src/proxy/http2/HTTP2.cc index a772639dbd4..620ccc8f4ea 100644 --- a/src/proxy/http2/HTTP2.cc +++ b/src/proxy/http2/HTTP2.cc @@ -21,6 +21,7 @@ * limitations under the License. */ +#include "iocore/eventsystem/IOBuffer.h" #include "proxy/hdrs/VersionConverter.h" #include "proxy/hdrs/HeaderValidator.h" #include "proxy/http2/HTTP2.h" @@ -30,7 +31,6 @@ #include "tsutil/LocalBuffer.h" #include "../../records/P_RecCore.h" -#include "../../records/P_RecProcess.h" const char *const HTTP2_CONNECTION_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; diff --git a/src/proxy/logging/LogStandalone.cc b/src/proxy/logging/LogStandalone.cc index 43541d010c3..e4c9399a76d 100644 --- a/src/proxy/logging/LogStandalone.cc +++ b/src/proxy/logging/LogStandalone.cc @@ -33,6 +33,7 @@ #include "tscore/ink_sys_control.h" #include "tscore/signals.h" #include "tscore/Layout.h" +#include "tsutil/Metrics.h" #include "proxy/shared/DiagsConfig.h" // Needs LibRecordsConfigInit() @@ -89,13 +90,13 @@ initialize_records() // Define version info records // auto &version = AppVersionInfo::get_version(); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.short", version.version(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.long", version.full_version(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_number", version.build_number(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_time", version.build_time(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_date", version.build_date(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_machine", version.build_machine(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_person", version.build_person(), RECP_NON_PERSISTENT); + ts::Metrics::StaticString::createString("proxy.process.version.server.short", version.version()); + ts::Metrics::StaticString::createString("proxy.process.version.server.long", version.full_version()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_number", version.build_number()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_time", version.build_time()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_date", version.build_date()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_machine", version.build_machine()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_person", version.build_person()); } /*------------------------------------------------------------------------- diff --git a/src/proxy/logging/LogUtils.cc b/src/proxy/logging/LogUtils.cc index 16bd56dbba9..e88a2002b36 100644 --- a/src/proxy/logging/LogUtils.cc +++ b/src/proxy/logging/LogUtils.cc @@ -59,8 +59,6 @@ using namespace std::literals; #include #include -#include "../../records/P_RecProcess.h" - #include "proxy/logging/LogUtils.h" #include "proxy/logging/LogLimits.h" diff --git a/src/records/P_RecCore.cc b/src/records/P_RecCore.cc index 08af786ab10..0f5df9cf49f 100644 --- a/src/records/P_RecCore.cc +++ b/src/records/P_RecCore.cc @@ -66,15 +66,6 @@ _RecRegisterStatFloat(RecT rec_type, const char *name, RecFloat data_default, Re REC_REGISTER_STAT_XXX(rec_float, RECD_FLOAT); } -RecErrT -_RecRegisterStatString(RecT rec_type, const char *name, RecStringConst data_in, RecPersistT persist_type) -{ - // NOTE(cmcfarlen): RecRegisterState calls RecDataSet which call strdup on the string data. - // therefore, this const cast will not be modified nor escape the stack past here. - char *data_default = const_cast(data_in); - REC_REGISTER_STAT_XXX(rec_string, RECD_STRING); -} - RecErrT _RecRegisterStatCounter(RecT rec_type, const char *name, RecCounter data_default, RecPersistT persist_type) { @@ -230,10 +221,10 @@ RecSetRecordFloat(const char *name, RecFloat rec_float, RecSourceT source, bool } RecErrT -RecSetRecordString(const char *name, const RecString rec_string, RecSourceT source, bool lock) +RecSetRecordString(const char *name, RecStringConst rec_string, RecSourceT source, bool lock) { RecData data; - data.rec_string = rec_string; + data.rec_string = const_cast(rec_string); return RecSetRecord(RECT_NULL, name, RECD_STRING, &data, nullptr, source, lock); } diff --git a/src/records/RecCore.cc b/src/records/RecCore.cc index 9aab580928e..6008ee4fec8 100644 --- a/src/records/RecCore.cc +++ b/src/records/RecCore.cc @@ -547,6 +547,23 @@ RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), v if (lock) { ink_rwlock_unlock(&g_records_rwlock); } + + // Also check for StaticString metrics + if (err == REC_ERR_FAIL) { + auto &strings = ts::Metrics::StaticString::instance(); + + if (auto m = strings.lookup(std::string{name}); m) { + RecRecord r; + r.rec_type = RECT_PLUGIN; + r.data_type = RECD_STRING; + r.name = name; + r.data.rec_string = const_cast(m->data()); + r.data_default.rec_string = const_cast(m->data()); + + callback(&r, data); + err = REC_ERR_OKAY; + } + } } return err; @@ -556,7 +573,6 @@ RecErrT RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)(const RecRecord *, void *), void *data, bool /* lock ATS_UNUSED */) { - int num_records; Regex regex; if (!regex.compile(match, RE_CASE_INSENSITIVE | RE_UNANCHORED)) { @@ -566,22 +582,36 @@ RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)( if ((rec_type & (RECT_PROCESS | RECT_NODE | RECT_PLUGIN))) { // First find the new metrics, this is a bit of a hack, because we still use the old // librecords callback with a "pseudo" record. - RecRecord tmp; - - tmp.rec_type = RECT_PROCESS; - for (auto &&[name, type, val] : ts::Metrics::instance()) { if (regex.exec(name.data())) { + RecRecord tmp; + + tmp.rec_type = RECT_PROCESS; + tmp.name = name.data(); tmp.data_type = type == ts::Metrics::MetricType::COUNTER ? RECD_COUNTER : RECD_INT; tmp.data.rec_int = val; callback(&tmp, data); } } - // Fall through to return any matching string metrics + // Finally check string metrics + for (auto &&[name, value] : ts::Metrics::StaticString::instance()) { + if (regex.exec(name)) { + RecRecord tmp; + + tmp.rec_type = RECT_PROCESS; + + tmp.name = name.data(); + tmp.data_type = RECD_STRING; + // NOTE(cmcfarlen): unfortunate relic here that the callbacks expect a non-const rec_string + // This should be temp until traffic_ctl uses ts::Metrics directly + tmp.data.rec_string = const_cast(value.c_str()); + callback(&tmp, data); + } + } } - num_records = g_num_records; + int num_records = g_num_records; for (int i = 0; i < num_records; i++) { RecRecord *r = &(g_records[i]); diff --git a/src/records/RecRawStats.cc b/src/records/RecRawStats.cc index 850c9fbc3c4..e78667233d4 100644 --- a/src/records/RecRawStats.cc +++ b/src/records/RecRawStats.cc @@ -22,7 +22,7 @@ Record statistics support */ #include "P_RecCore.h" -#include "P_RecProcess.h" +#include "records/RecProcess.h" #include #include diff --git a/src/records/RecordsConfigUtils.cc b/src/records/RecordsConfigUtils.cc index 5b887fc9556..f572a957495 100644 --- a/src/records/RecordsConfigUtils.cc +++ b/src/records/RecordsConfigUtils.cc @@ -102,10 +102,6 @@ initialize_record(const RecordElement *record, void *) RecRegisterStatFloat(type, record->name, tempFloat, RECP_NON_PERSISTENT); break; - case RECD_STRING: - RecRegisterStatString(type, record->name, (RecString)record->value, RECP_NON_PERSISTENT); - break; - case RECD_COUNTER: tempCounter = static_cast(ink_atoi64(record->value)); RecRegisterStatCounter(type, record->name, tempCounter, RECP_NON_PERSISTENT); diff --git a/src/records/test_RecordsConfig.cc b/src/records/test_RecordsConfig.cc index 1bea78e14d0..36b9a354722 100644 --- a/src/records/test_RecordsConfig.cc +++ b/src/records/test_RecordsConfig.cc @@ -47,8 +47,6 @@ RecordsConfigRegister() RecRegisterConfigCounter(RECT_CONFIG, "proxy.config.link_test_3", 0, RECU_DYNAMIC, RECC_NULL, nullptr); // NODE - RecRegisterStatString(RECT_NODE, "proxy.node.cb_test_1", "cb_test_1__original", RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_NODE, "proxy.node.cb_test_2", "cb_test_2__original", RECP_NON_PERSISTENT); RecRegisterStatInt(RECT_NODE, "proxy.node.cb_test_int", 0, RECP_NON_PERSISTENT); RecRegisterStatFloat(RECT_NODE, "proxy.node.cb_test_float", 0.0f, RECP_NON_PERSISTENT); RecRegisterStatCounter(RECT_NODE, "proxy.node.cb_test_count", 0, RECP_NON_PERSISTENT); diff --git a/src/traffic_cache_tool/CacheDefs.h b/src/traffic_cache_tool/CacheDefs.h index db76bb99f05..1d9a8c64644 100644 --- a/src/traffic_cache_tool/CacheDefs.h +++ b/src/traffic_cache_tool/CacheDefs.h @@ -307,7 +307,7 @@ struct url_matcher { std::cout << "Check your regular expression" << std::endl; } - if (!port.compile(R"([0-9]+$)")) { + if (!port.compile(R"(^[0-9]+$)")) { std::cout << "Check your regular expression" << std::endl; return; } @@ -320,7 +320,7 @@ struct url_matcher { std::cout << "Check your regular expression" << std::endl; return; } - if (!port.compile(R"([0-9]+$)")) { + if (!port.compile(R"(^[0-9]+$)")) { std::cout << "Check your regular expression" << std::endl; return; } @@ -336,12 +336,12 @@ struct url_matcher { uint8_t portmatch(const char *hostname, int length) const { - return port.match({hostname, size_t(length)}) ? 1 : 0; + return port.exec({hostname, static_cast(length)}) ? 1 : 0; } private: - DFA port; - DFA regex; + Regex port; + DFA regex; }; using swoc::Errata; diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 7c5c0b2077a..9bf267fb8da 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -48,6 +48,7 @@ #include "ts/ts.h" // This is sadly needed because of us using TSThreadInit() for some reason. #include "swoc/swoc_file.h" +#include "tsutil/Metrics.h" #include #include @@ -712,13 +713,13 @@ initialize_records() // Define version info records // auto &version = AppVersionInfo::get_version(); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.short", version.version(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.long", version.full_version(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_number", version.build_number(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_time", version.build_time(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_date", version.build_date(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_machine", version.build_machine(), RECP_NON_PERSISTENT); - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.build_person", version.build_person(), RECP_NON_PERSISTENT); + ts::Metrics::StaticString::createString("proxy.process.version.server.short", version.version()); + ts::Metrics::StaticString::createString("proxy.process.version.server.long", version.full_version()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_number", version.build_number()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_time", version.build_time()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_date", version.build_date()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_machine", version.build_machine()); + ts::Metrics::StaticString::createString("proxy.process.version.server.build_person", version.build_person()); } void @@ -2030,8 +2031,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) Machine::init(hostname, &machine_addr.sa); } - RecRegisterStatString(RECT_PROCESS, "proxy.process.version.server.uuid", (char *)Machine::instance()->process_uuid.getString(), - RECP_NON_PERSISTENT); + ts::Metrics::StaticString::createString("proxy.process.version.server.uuid", Machine::instance()->process_uuid.getString()); res_track_memory = RecGetRecordInt("proxy.config.res_track_memory").value_or(0); diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index d863a3cc256..c7b474c889e 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -504,26 +504,28 @@ bool ArgParser::Command::parse(Arguments &ret, AP_StrVec &args) { bool command_called = false; - // iterate through all arguments - for (unsigned i = 0; i < args.size(); i++) { - if (_name == args[i]) { - command_called = true; - // handle the option - append_option_data(ret, args, i); - // handle the action - if (_f) { - ret._action = _f; - } - std::string err = handle_args(ret, args, _key, _arg_num, i); - if (!err.empty()) { - help_message(err); - } - // set ENV var - if (!_envvar.empty()) { - const char *const env = getenv(_envvar.c_str()); - ret.set_env(_key, nullptr != env ? env : ""); - } - break; + // Only check the first remaining argument for command name to avoid + // treating arguments as commands (e.g., "metric match host" where "host" is an arg, not a command) + if (!args.empty() && _name == args[0]) { + command_called = true; + // Note: handle_args modifies its index parameter (designed for loop usage), but we + // discard the result. This causes unsigned underflow (0 - 1 = UINT_MAX) which is + // harmless since we don't use index afterward. + unsigned index{0}; + // handle the option + append_option_data(ret, args, index); + // handle the action + if (_f) { + ret._action = _f; + } + const std::string err = handle_args(ret, args, _key, _arg_num, index); + if (!err.empty()) { + help_message(err); + } + // set ENV var + if (!_envvar.empty()) { + const char *const env = getenv(_envvar.c_str()); + ret.set_env(_key, nullptr != env ? env : ""); } } if (command_called) { diff --git a/src/tsutil/Metrics.cc b/src/tsutil/Metrics.cc index 50624cf42bb..b19c61cf1b4 100644 --- a/src/tsutil/Metrics.cc +++ b/src/tsutil/Metrics.cc @@ -24,6 +24,7 @@ #include "tsutil/Assert.h" #include #include +#include #include #include #include "tsutil/Metrics.h" @@ -292,4 +293,33 @@ Metrics::Derived::update_derived() details::DerivativeMetrics::instance().update(); } +Metrics::StaticString & +Metrics::StaticString::instance() +{ + static Metrics::StaticString i{}; + return i; +} + +void +Metrics::StaticString::_createString(const std::string &name, const std::string_view value) +{ + std::lock_guard l(_mutex); + + _strings[name] = value; +} + +std::optional +Metrics::StaticString::lookup(const std::string &name) +{ + std::lock_guard l(_mutex); + auto it = _strings.find(name); + std::optional result{}; + + if (it != _strings.end()) { + result = it->second; + } + + return result; +} + } // namespace ts diff --git a/src/tsutil/Regex.cc b/src/tsutil/Regex.cc index 32604fb6ae5..c40d64491be 100644 --- a/src/tsutil/Regex.cc +++ b/src/tsutil/Regex.cc @@ -30,12 +30,16 @@ #include #include #include +#include static_assert(RE_CASE_INSENSITIVE == PCRE2_CASELESS, "Update RE_CASE_INSENSITIVE for current PCRE2 version."); static_assert(RE_UNANCHORED == PCRE2_MULTILINE, "Update RE_UNANCHORED for current PCRE2 version."); static_assert(RE_ANCHORED == PCRE2_ANCHORED, "Update RE_ANCHORED for current PCRE2 version."); static_assert(RE_NOTEMPTY == PCRE2_NOTEMPTY, "Update RE_NOTEMPTY for current PCRE2 version."); +static_assert(RE_ERROR_NOMATCH == PCRE2_ERROR_NOMATCH, "Update RE_ERROR_NOMATCH for current PCRE2 version."); +static_assert(RE_ERROR_NULL == PCRE2_ERROR_NULL, "Update RE_ERROR_NULL for current PCRE2 version."); + //---------------------------------------------------------------------------- namespace { @@ -151,6 +155,7 @@ RegexMatches::malloc(size_t size, void *caller) return ::malloc(size); } +//---------------------------------------------------------------------------- void RegexMatches::free(void *p, void *caller) { @@ -217,6 +222,32 @@ struct Regex::_Code { } }; +//---------------------------------------------------------------------------- +Regex::Regex(Regex const &other) +{ + auto *other_code = _Code::get(other._code); + if (other_code != nullptr) { + // Use PCRE2's built-in function to deep copy the compiled pattern + auto *copied_code = pcre2_code_copy(other_code); + _Code::set(_code, copied_code); + } +} + +//---------------------------------------------------------------------------- +Regex & +Regex::operator=(Regex const &other) +{ + if (this != &other) { + // Use copy-and-swap idiom: create a temporary copy, then swap with it + Regex temp(other); // Copy constructor does the deep copy + + // Swap the internal pointers + std::swap(_code, temp._code); + // temp's destructor will clean up our old _code + } + return *this; +} + //---------------------------------------------------------------------------- Regex::Regex(Regex &&that) noexcept { diff --git a/src/tsutil/unit_tests/test_Regex.cc b/src/tsutil/unit_tests/test_Regex.cc index c2021322bfd..679117f3153 100644 --- a/src/tsutil/unit_tests/test_Regex.cc +++ b/src/tsutil/unit_tests/test_Regex.cc @@ -171,7 +171,7 @@ TEST_CASE("Regex", "[libts][Regex]") Regex r; RegexMatches matches; REQUIRE(r.exec("foo") == false); - REQUIRE(r.exec("foo", matches) == PCRE2_ERROR_NULL); + REQUIRE(r.exec("foo", matches) == RE_ERROR_NULL); } // test for recompiling the regular expression @@ -217,10 +217,10 @@ TEST_CASE("Regex RE_NOTEMPTY flag behavior", "[libts][Regex][flags][RE_NOTEMPTY] // boolean overload with RE_NOTEMPTY should not match CHECK(r.exec(std::string_view(""), RE_NOTEMPTY) == false); - // matches overload should return a negative value (PCRE2_ERROR_NOMATCH) + // matches overload should return RE_ERROR_NOMATCH RegexMatches matches; int rc = r.exec(std::string_view(""), matches, RE_NOTEMPTY); - CHECK(rc < 0); + CHECK(rc == RE_ERROR_NOMATCH); } SECTION("non-empty subject unaffected by RE_NOTEMPTY for this pattern") @@ -230,3 +230,262 @@ TEST_CASE("Regex RE_NOTEMPTY flag behavior", "[libts][Regex][flags][RE_NOTEMPTY] CHECK(r.exec(std::string_view("a"), RE_NOTEMPTY) == false); } } + +TEST_CASE("Regex error codes", "[libts][Regex][errors]") +{ + SECTION("RE_ERROR_NULL when regex not compiled") + { + Regex r; + RegexMatches matches; + + // exec on uncompiled regex should return RE_ERROR_NULL + CHECK(r.exec("test", matches) == RE_ERROR_NULL); + } + + SECTION("RE_ERROR_NOMATCH when pattern does not match") + { + Regex r; + REQUIRE(r.compile(R"(^foo$)") == true); + + RegexMatches matches; + + // Pattern does not match, should return RE_ERROR_NOMATCH + CHECK(r.exec("bar", matches) == RE_ERROR_NOMATCH); + CHECK(r.exec("foobar", matches) == RE_ERROR_NOMATCH); + CHECK(r.exec("", matches) == RE_ERROR_NOMATCH); + + // The following should match and return 1 (which shouldn't be RE_ERROR_NOMATCH) + CHECK(r.exec("foo", matches) != RE_ERROR_NOMATCH); + CHECK(r.exec("foo", matches) == 1); + } +} + +TEST_CASE("Regex copy constructor", "[libts][Regex][copy]") +{ + SECTION("Copy constructor creates independent copy") + { + Regex original; + REQUIRE(original.compile(R"(^test\d+$)") == true); + + // Test original works + CHECK(original.exec("test123") == true); + CHECK(original.exec("test") == false); + + // Copy using copy constructor + Regex copy(original); + + // Both should work independently + CHECK(copy.exec("test123") == true); + CHECK(copy.exec("test") == false); + CHECK(original.exec("test456") == true); + CHECK(original.exec("test") == false); + } + + SECTION("Copy constructor with capture groups") + { + Regex original; + REQUIRE(original.compile(R"(^(\w+)@(\w+)\.(\w+)$)") == true); + + Regex copy(original); + + // Test both original and copy with captures + RegexMatches original_matches; + REQUIRE(original.exec("user@example.com", original_matches) == 4); + CHECK(original_matches[0] == "user@example.com"); + CHECK(original_matches[1] == "user"); + CHECK(original_matches[2] == "example"); + CHECK(original_matches[3] == "com"); + + RegexMatches copy_matches; + REQUIRE(copy.exec("admin@test.org", copy_matches) == 4); + CHECK(copy_matches[0] == "admin@test.org"); + CHECK(copy_matches[1] == "admin"); + CHECK(copy_matches[2] == "test"); + CHECK(copy_matches[3] == "org"); + } + + SECTION("Copy constructor with empty regex") + { + Regex original; // Not compiled + Regex copy(original); + + // Both should be empty + CHECK(original.empty() == true); + CHECK(copy.empty() == true); + + // Neither should match anything + CHECK(original.exec("test") == false); + CHECK(copy.exec("test") == false); + } + + SECTION("Copy constructor with case insensitive flag") + { + Regex original; + REQUIRE(original.compile(R"(^FOO$)", RE_CASE_INSENSITIVE) == true); + + Regex copy(original); + + // Both should match case-insensitively + CHECK(original.exec("foo") == true); + CHECK(original.exec("FOO") == true); + CHECK(original.exec("FoO") == true); + CHECK(copy.exec("foo") == true); + CHECK(copy.exec("FOO") == true); + CHECK(copy.exec("FoO") == true); + } + + SECTION("Multiple copies can coexist") + { + Regex original; + REQUIRE(original.compile(R"(\d+)") == true); + + Regex copy1(original); + Regex copy2(original); + Regex copy3(copy1); + + // All should work independently + CHECK(original.exec("123") == true); + CHECK(copy1.exec("456") == true); + CHECK(copy2.exec("789") == true); + CHECK(copy3.exec("000") == true); + } + + SECTION("Copy can be stored in vector") + { + Regex pattern; + REQUIRE(pattern.compile(R"(test\d+)") == true); + + std::vector patterns; + patterns.push_back(pattern); + patterns.push_back(pattern); + patterns.push_back(pattern); + + // All copies in vector should work + for (auto &p : patterns) { + CHECK(p.exec("test123") == true); + CHECK(p.exec("test") == false); + } + } +} + +TEST_CASE("Regex copy assignment", "[libts][Regex][copy]") +{ + SECTION("Copy assignment replaces existing pattern") + { + Regex regex1; + Regex regex2; + + REQUIRE(regex1.compile(R"(foo)") == true); + REQUIRE(regex2.compile(R"(bar)") == true); + + CHECK(regex1.exec("foo") == true); + CHECK(regex1.exec("bar") == false); + CHECK(regex2.exec("foo") == false); + CHECK(regex2.exec("bar") == true); + + // Copy assign regex1 to regex2 + regex2 = regex1; + + // Now both should match "foo" + CHECK(regex1.exec("foo") == true); + CHECK(regex1.exec("bar") == false); + CHECK(regex2.exec("foo") == true); + CHECK(regex2.exec("bar") == false); + } + + SECTION("Copy assignment from empty regex") + { + Regex compiled; + Regex empty; + + REQUIRE(compiled.compile(R"(test)") == true); + CHECK(compiled.exec("test") == true); + + // Assign empty to compiled + compiled = empty; + + // Now compiled should be empty + CHECK(compiled.empty() == true); + CHECK(compiled.exec("test") == false); + } + + SECTION("Copy assignment to empty regex") + { + Regex empty; + Regex compiled; + + REQUIRE(compiled.compile(R"(test)") == true); + + // Assign compiled to empty + empty = compiled; + + // Now empty should work + CHECK(empty.exec("test") == true); + CHECK(compiled.exec("test") == true); + } + + SECTION("Self-assignment is safe") + { + Regex regex; + REQUIRE(regex.compile(R"(test)") == true); + + // Self-assign (disable warning for intentional self-assignment test) + // Use a pointer indirection to avoid compiler warnings about self-assignment + Regex *ptr = ®ex; + regex = *ptr; + + // Should still work + CHECK(regex.exec("test") == true); + CHECK(regex.exec("foo") == false); + } + + SECTION("Copy assignment with capture groups") + { + Regex regex1; + Regex regex2; + + REQUIRE(regex1.compile(R"(^(\d{3})-(\d{3})-(\d{4})$)") == true); + REQUIRE(regex2.compile(R"(foo)") == true); + + regex2 = regex1; + + RegexMatches matches; + REQUIRE(regex2.exec("123-456-7890", matches) == 4); + CHECK(matches[0] == "123-456-7890"); + CHECK(matches[1] == "123"); + CHECK(matches[2] == "456"); + CHECK(matches[3] == "7890"); + } + + SECTION("Copy assignment chain") + { + Regex r1, r2, r3; + REQUIRE(r1.compile(R"(test\d+)") == true); + + // Chain assignment + r3 = r2 = r1; + + // All should work + CHECK(r1.exec("test123") == true); + CHECK(r2.exec("test456") == true); + CHECK(r3.exec("test789") == true); + } +} + +TEST_CASE("Regex copy with RE_NOTEMPTY flag", "[libts][Regex][copy][flags]") +{ + SECTION("Copied regex preserves RE_NOTEMPTY behavior") + { + Regex original; + REQUIRE(original.compile("^$") == true); + + Regex copy(original); + + // Both should have same behavior with RE_NOTEMPTY + CHECK(original.exec(std::string_view("")) == true); + CHECK(original.exec(std::string_view(""), RE_NOTEMPTY) == false); + + CHECK(copy.exec(std::string_view("")) == true); + CHECK(copy.exec(std::string_view(""), RE_NOTEMPTY) == false); + } +} diff --git a/tests/Pipfile b/tests/Pipfile index c2cf352c3fd..6119e276564 100644 --- a/tests/Pipfile +++ b/tests/Pipfile @@ -20,7 +20,6 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] -autopep8 = "*" pyflakes = "*" [packages] diff --git a/tests/gold_tests/h2/trickle_client.py b/tests/gold_tests/h2/trickle_client.py index 5b777086ffe..2ef603db811 100644 --- a/tests/gold_tests/h2/trickle_client.py +++ b/tests/gold_tests/h2/trickle_client.py @@ -17,7 +17,6 @@ # limitations under the License. import argparse -from email.message import EmailMessage as HttpHeaders import logging import math import socket @@ -252,11 +251,7 @@ def send_http2_request_to_server(hostname: str, port: int, cert_file: str, write :return: 0 if the request was successful, 1 otherwise. """ - request_headers = HttpHeaders() - request_headers.add_header(':method', 'GET') - request_headers.add_header(':path', '/some/path') - request_headers.add_header(':authority', hostname) - request_headers.add_header(':scheme', 'https') + request_headers = {':method': 'GET', ':path': '/some/path', ':authority': hostname, ':scheme': 'https'} scheme = request_headers[':scheme'] replay_server = f"127.0.0.1:{port}" diff --git a/tests/gold_tests/h2/trickle_server.py b/tests/gold_tests/h2/trickle_server.py index 1ccfa8fc1b8..36f42bd9070 100644 --- a/tests/gold_tests/h2/trickle_server.py +++ b/tests/gold_tests/h2/trickle_server.py @@ -358,6 +358,7 @@ def run_server(listen_port, https_pem, ca_pem) -> List[int]: logging.info(f"Serving HTTP/2 Proxy on 127.0.0.1:{listen_port} with pem '{https_pem}'") pool = eventlet.GreenPool() + manager = None while True: try: new_connection_socket, _ = listening_socket.accept() @@ -367,7 +368,9 @@ def run_server(listen_port, https_pem, ca_pem) -> List[int]: pool.spawn_n(manager.run_forever) except KeyboardInterrupt as e: logging.debug("Handling KeyboardInterrupt") - return manager.get_data_delays() + if manager is not None: + return manager.get_data_delays() + return [] except SystemExit: break @@ -391,6 +394,9 @@ def main() -> int: logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s') data_delays = run_server(args.listen_port, args.cert_key, args.ca_cert) + if not data_delays: + logging.error('No data delays were recorded') + return 1 logging.info(f'Smallest delay: {min(data_delays)} ms') logging.info(f'Largest delay: {max(data_delays)} ms') average = statistics.mean(data_delays) diff --git a/tests/gold_tests/headers/gold/accept_webp_cache.gold b/tests/gold_tests/headers/gold/accept_webp_cache.gold index 5e2cf841d0e..71c016243f1 100644 --- a/tests/gold_tests/headers/gold/accept_webp_cache.gold +++ b/tests/gold_tests/headers/gold/accept_webp_cache.gold @@ -11,6 +11,6 @@ < Date: `` < Age: `` < Connection: keep-alive -< Via: http/1.1 `` (ApacheTrafficServer/`` [uScHs f p eN:t cCHp s ]) +< Via: http/1.1 `` (ApacheTrafficServer/`` [uScRs f p eN:t cCHp s ]) < Server: ATS/`` `` diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index b4e9333c2d2..76ea2da492e 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -185,7 +185,7 @@ < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding -< Content-Length: 47 +< Content-Length: 4`` === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/deflate @@ -228,7 +228,7 @@ < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding -< Content-Length: 47 +< Content-Length: 4`` === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/deflate diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_definitely.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_definitely.gold new file mode 100644 index 00000000000..aaa750a26e0 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_definitely.gold @@ -0,0 +1,18 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +> X-Foo: definitely +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-When-200-Before: Yes +< X-Foo: Definitely +< X-When-200-After: Yes +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else.gold new file mode 100644 index 00000000000..a04b910a7d2 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else.gold @@ -0,0 +1,17 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-When-200-Before: Yes +< X-Foo: Nothing +< X-When-200-After: Yes +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else_fie.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else_fie.gold new file mode 100644 index 00000000000..09d4e47708d --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_else_fie.gold @@ -0,0 +1,19 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +> X-Fie: fie +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-When-200-Before: Yes +< X-Foo: Nothing +< X-Fie-Anywhere: Yes +< X-When-200-After: Yes +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_bar.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_bar.gold new file mode 100644 index 00000000000..b583ca387dd --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_bar.gold @@ -0,0 +1,20 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +> X-Foo: foo +> X-Bar: bar +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-When-200-Before: Yes +< X-Foo: Yes +< X-Foo-And-Bar: Yes +< X-When-200-After: Yes +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_fie.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_fie.gold new file mode 100644 index 00000000000..126f4bdade6 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_foo_fie.gold @@ -0,0 +1,21 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +> X-Foo: foo +> X-Fie: fie +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-When-200-Before: Yes +< X-Foo: Yes +< X-Foo-And-Fie: Yes +< X-Fie-Anywhere: Yes +< X-When-200-After: Yes +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_maybe.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_maybe.gold new file mode 100644 index 00000000000..f975acf4b18 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/nested_ifs_maybe.gold @@ -0,0 +1,18 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +> X-Foo: maybe +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-When-200-Before: Yes +< X-Foo: Maybe +< X-When-200-After: Yes +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_empty.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_empty.gold new file mode 100644 index 00000000000..385c4f5fdb5 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_empty.gold @@ -0,0 +1,16 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +`` +< HTTP/1.1 200 OK +< Date: `` +< Proxy-Connection: keep-alive +< Server: ATS/`` +< Cache-Control: no-store +< Content-Type: text/html; charset=utf-8 +< Content-Language: en +< Content-Length: 0 +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status.gold new file mode 100644 index 00000000000..93988eb9d1b --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status.gold @@ -0,0 +1,16 @@ +`` +> GET `` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +> Proxy-Connection: Keep-Alive +`` +< HTTP/1.1 200 OK +< Date: `` +< Proxy-Connection: keep-alive +< Server: ATS/`` +< Cache-Control: no-store +< Content-Type: text/html +< Content-Language: en +< Content-Length: 3 +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status_stdout.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status_stdout.gold new file mode 100644 index 00000000000..08839f6bb29 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/set_body_status_stdout.gold @@ -0,0 +1 @@ +200 diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.test.py b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.test.py index 71e87518f37..e7ef3b691c0 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.test.py +++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.test.py @@ -86,6 +86,18 @@ "from": f"{url_base}_9/", "to": f"{origin_base}_9/", "plugins": [("header_rewrite", [f"{mgr.run_dir}/regex_tests.conf"])] + }, { + "from": f"{url_base}_10/", + "to": f"{origin_base}_10/", + "plugins": [("header_rewrite", [f"{mgr.run_dir}/rule_empty_body.conf"])] + }, { + "from": f"{url_base}_11/", + "to": f"{origin_base}_11/", + "plugins": [("header_rewrite", [f"{mgr.run_dir}/rule_set_body_status.conf"])] + }, { + "from": f"{url_base}_12/", + "to": f"{origin_base}_12/", + "plugins": [("header_rewrite", [f"{mgr.run_dir}/nested_ifs.conf"])] } ] @@ -177,6 +189,31 @@ "timestamp": "1469733493.993", "body": "" }, def_resp), + ( + { + "headers": "GET /to_10/ HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "ATS should not serve this body" + }), + ( + { + "headers": "GET /to_11/ HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "ATS should not serve this body" + }), + ({ + "headers": "GET /to_12/ HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, def_resp), ] mgr.add_server_responses(origin_rules) @@ -289,6 +326,47 @@ "curl": f'{curl_proxy} "http://{url_base}_9/" -H "X-Test1: none"', "gold": "gold/regex_match2_only.gold", }, + { + "desc": "set-body with empty string (expect empty body with Content-Length: 0)", + "curl": f'{curl_proxy} "http://{url_base}_10/"', + "gold": "gold/set_body_empty.gold", + }, + { + "desc": "set-body with STATUS variable (expect body with '200')", + "curl": f'{curl_proxy} "http://{url_base}_11/"', + "gold": "gold/set_body_status.gold", + "gold_stdout": "gold/set_body_status_stdout.gold", + }, + { + "desc": "Nested if/elif/else - X-Foo=foo + X-Bar=bar path", + "curl": f'{curl_proxy} "http://{url_base}_12/" -H "X-Foo: foo" -H "X-Bar: bar"', + "gold": "gold/nested_ifs_foo_bar.gold", + }, + { + "desc": "Nested if/elif/else - X-Foo=foo + X-Fie=fie path", + "curl": f'{curl_proxy} "http://{url_base}_12/" -H "X-Foo: foo" -H "X-Fie: fie"', + "gold": "gold/nested_ifs_foo_fie.gold", + }, + { + "desc": "Nested if/elif/else - X-Foo=maybe path", + "curl": f'{curl_proxy} "http://{url_base}_12/" -H "X-Foo: maybe"', + "gold": "gold/nested_ifs_maybe.gold", + }, + { + "desc": "Nested if/elif/else - X-Foo=definitely path", + "curl": f'{curl_proxy} "http://{url_base}_12/" -H "X-Foo: definitely"', + "gold": "gold/nested_ifs_definitely.gold", + }, + { + "desc": "Nested if/elif/else - else path (no X-Foo)", + "curl": f'{curl_proxy} "http://{url_base}_12/"', + "gold": "gold/nested_ifs_else.gold", + }, + { + "desc": "Nested if/elif/else - else path with X-Fie (tests second if)", + "curl": f'{curl_proxy} "http://{url_base}_12/" -H "X-Fie: fie"', + "gold": "gold/nested_ifs_else_fie.gold", + }, ] mgr.execute_tests(test_runs) diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/implicit_hook.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/implicit_hook.conf index 0f909f1046c..c957bfa7eb7 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/rules/implicit_hook.conf +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/implicit_hook.conf @@ -27,8 +27,6 @@ elif else set-header X-Response-Foo "No" -# ToDo: This should use the implicit hook of %{REMAP_PSEUDO_HOOK}, needs #12557 -cond %{REMAP_PSEUDO_HOOK} [AND] cond %{CLIENT-HEADER:X-Fie} ="fie" [NOCASE] add-header X-Client-Foo "Yes" elif diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/nested_ifs.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/nested_ifs.conf new file mode 100644 index 00000000000..5453b0b6fa9 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/nested_ifs.conf @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +cond %{SEND_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} =200 + set-header X-When-200-Before "Yes" + if + cond %{CLIENT-HEADER:X-Foo} ="foo" + set-header X-Foo "Yes" + if + cond %{CLIENT-HEADER:X-Bar} ="bar" [NOCASE] + set-header X-Foo-And-Bar "Yes" + elif + cond %{CLIENT-HEADER:X-Fie} ="fie" [NOCASE] + set-header X-Foo-And-Fie "Yes" + endif + elif + cond %{CLIENT-HEADER:X-Foo} ="maybe" + set-header X-Foo "Maybe" + elif + cond %{CLIENT-HEADER:X-Foo} ="definitely" + set-header X-Foo "Definitely" + else + set-header X-Foo "Nothing" + endif + if + cond %{CLIENT-HEADER:X-Fie} ="fie" [NOCASE] + set-header X-Fie-Anywhere "Yes" + endif + set-header X-When-200-After "Yes" diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_empty_body.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_empty_body.conf new file mode 100644 index 00000000000..73a19b5fda1 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_empty_body.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cond %{REMAP_PSEUDO_HOOK} + set-status 200 + +cond %{SEND_RESPONSE_HDR_HOOK} + set-body "" diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_status.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_status.conf new file mode 100644 index 00000000000..58256210b7a --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_status.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cond %{REMAP_PSEUDO_HOOK} + set-status 200 + +cond %{SEND_RESPONSE_HDR_HOOK} + set-body "%{STATUS}" diff --git a/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts.gold b/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts.gold index 7e4ffc02ed0..d4e98d2a6bb 100644 --- a/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts.gold +++ b/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts.gold @@ -2,4 +2,4 @@ ``DIAG: (origin_server_auth) Set the header x-amz-content-sha256: UNSIGNED-PAYLOAD ``DIAG: (origin_server_auth) Set the header x-amz-date: `` ``DIAG: (origin_server_auth) Set the header x-amz-security-token: hkMsi6/bfHyBKrSeM/H0hoXeyx8z1yZ/mJ0c+B/TqYx=tTJDjnQWtul38Z9iVJjeH1HB4VT2c=2o3yE3o=I9kmFs/lJDR85qWjB8e5asY/WbjyRpbAzmDipQpboIcYnUYg55bxrQFidV/q8gZa5A9MpR3n=op1C0lWjeBqcEJxpevNZxteSQTQfeGsi98Cdf+On=/SINVlKrNhMnmMsDOLMGx1YYt9d4UsRg1jtVrwxL4Vd/F7aHCZySAXKv+1rkhACR023wpa3dhp+xirGJxSO9LWwvcrTdM4xJo4RS8B40tGENOJ1NKixUJxwN/6og58Oft/u==uleR89Ja=7zszK2H7tX3DqmEYNvNDYQh/7VBRe5otghQtPwJzWpXAGk+Vme4hPPM5K6axH2LxipXzRiIV=oxNs0upKNu1FvuzbCQmkQdKQVmXl0344vngngrgN7wkEfrYtmKwICmpAS0cbW9jdSClgziVo4NaFc/hsIfok=4UA3hVtxIdw74lFNXD0RR7HKXkFPLIn85M7peOZsqMUCfO4gxr7KCfabszQQf0YcP/mt79XK50=WrSJG7oUyn+clUySPhlegqHAfT9a50uSK5WiQmOnGNGLF4wDO10sqKN1xRgQbYHPtwL+Ye0EMisvmYA3==kScorTSGaQWyibSWXAvxq9+IVGBYShVJ6S7DmTT=u/2d/fGEge+Xmbxlftza=cxJ=Md=k1Q71Lp6Boa56d7wtYRpK6tXHJ9I/2r7rN1E4OtwkFqb7SfWV3UXwyUrXyaaNPTIbqnAHnbgUGtuU6pgICpfREiIxVqvKBf6ErbxHRmMmAuYKxk5E9Mn6nnbxR4WTniweKYeDv2w39zge/tss+36Moeuio9d2eoyRFqXhq=rUGtDwX3fzXV0wV+dUojxOYQ57GQDl7+68PwHPcX794OIXuGOxBk83lNIYIcYz3Vc7qnGy6tFTz7f6S9+EZuSGN7TY5VKkT2eWye46DebrDF9Nwzs/FVpTzbPD/KGDIBtFIbazglhKoWe9txqb1QW8vFNNVOEhYa+cViO3g8ZmY1wG960US2zsnX5Eg8Q5a4h3+sxaJSJ4ONiXZWJuAgKRQzcrszu+M5C0ZVoCOv1goEgfNJeSm/yFc/3rx8wmeWLIJFtq65B7zF72HRKq1nthHAguaxXr20nguHpKkDpNBDVa=WwuJsbeGI -``DIAG: (origin_server_auth) Set the header Authorization: AWS4-HMAC-SHA256 Credential=``/us-east-1/s3/aws4_request,SignedHeaders=accept;client-ip;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=`` +``DIAG: (origin_server_auth) Set the header Authorization: AWS4-HMAC-SHA256 Credential=``/us-east-1/s3/aws4_request,SignedHeaders=accept;client-ip;host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=`` diff --git a/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds 2.gold b/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds 2.gold new file mode 100644 index 00000000000..74604268788 --- /dev/null +++ b/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds 2.gold @@ -0,0 +1,5 @@ +``DIAG: (origin_server_auth) New rule: access_key=1234567, virtual_host=yes, version=awsv4 +``DIAG: (origin_server_auth) Set the header x-amz-content-sha256: UNSIGNED-PAYLOAD +``DIAG: (origin_server_auth) Set the header x-amz-date: `` +``DIAG: (origin_server_auth) Set the header x-amz-security-token: hkMsi6/bfHyBKrSeM/H0hoXeyx8z1yZ/mJ0c+B/TqYx=tTJDjnQWtul38Z9iVJjeH1HB4VT2c=2o3yE3o=I9kmFs/lJDR85qWjB8e5asY/WbjyRpbAzmDipQpboIcYnUYg55bxrQFidV/q8gZa5A9MpR3n=op1C0lWjeBqcEJxpevNZxteSQTQfeGsi98Cdf+On=/SINVlKrNhMnmMsDOLMGx1YYt9d4UsRg1jtVrwxL4Vd/F7aHCZySAXKv+1rkhACR023wpa3dhp+xirGJxSO9LWwvcrTdM4xJo4RS8B40tGENOJ1NKixUJxwN/6og58Oft/u==uleR89Ja=7zszK2H7tX3DqmEYNvNDYQh/7VBRe5otghQtPwJzWpXAGk+Vme4hPPM5K6axH2LxipXzRiIV=oxNs0upKNu1FvuzbCQmkQdKQVmXl0344vngngrgN7wkEfrYtmKwICmpAS0cbW9jdSClgziVo4NaFc/hsIfok=4UA3hVtxIdw74lFNXD0RR7HKXkFPLIn85M7peOZsqMUCfO4gxr7KCfabszQQf0YcP/mt79XK50=WrSJG7oUyn+clUySPhlegqHAfT9a50uSK5WiQmOnGNGLF4wDO10sqKN1xRgQbYHPtwL+Ye0EMisvmYA3==kScorTSGaQWyibSWXAvxq9+IVGBYShVJ6S7DmTT=u/2d/fGEge+Xmbxlftza=cxJ=Md=k1Q71Lp6Boa56d7wtYRpK6tXHJ9I/2r7rN1E4OtwkFqb7SfWV3UXwyUrXyaaNPTIbqnAHnbgUGtuU6pgICpfREiIxVqvKBf6ErbxHRmMmAuYKxk5E9Mn6nnbxR4WTniweKYeDv2w39zge/tss+36Moeuio9d2eoyRFqXhq=rUGtDwX3fzXV0wV+dUojxOYQ57GQDl7+68PwHPcX794OIXuGOxBk83lNIYIcYz3Vc7qnGy6tFTz7f6S9+EZuSGN7TY5VKkT2eWye46DebrDF9Nwzs/FVpTzbPD/KGDIBtFIbazglhKoWe9txqb1QW8vFNNVOEhYa+cViO3g8ZmY1wG960US2zsnX5Eg8Q5a4h3+sxaJSJ4ONiXZWJuAgKRQzcrszu+M5C0ZVoCOv1goEgfNJeSm/yFc/3rx8wmeWLIJFtq65B7zF72HRKq1nthHAguaxXr20nguHpKkDpNBDVa=WwuJsbeGI +``DIAG: (origin_server_auth) Set the header Authorization: AWS4-HMAC-SHA256 Credential=``/us-east-1/s3/aws4_request,SignedHeaders=accept;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=`` diff --git a/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds.gold b/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds.gold index 74604268788..bc07529e3fd 100644 --- a/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds.gold +++ b/tests/gold_tests/pluginTest/origin_server_auth/gold/origin_server_auth_parsing_ts_uds.gold @@ -2,4 +2,4 @@ ``DIAG: (origin_server_auth) Set the header x-amz-content-sha256: UNSIGNED-PAYLOAD ``DIAG: (origin_server_auth) Set the header x-amz-date: `` ``DIAG: (origin_server_auth) Set the header x-amz-security-token: hkMsi6/bfHyBKrSeM/H0hoXeyx8z1yZ/mJ0c+B/TqYx=tTJDjnQWtul38Z9iVJjeH1HB4VT2c=2o3yE3o=I9kmFs/lJDR85qWjB8e5asY/WbjyRpbAzmDipQpboIcYnUYg55bxrQFidV/q8gZa5A9MpR3n=op1C0lWjeBqcEJxpevNZxteSQTQfeGsi98Cdf+On=/SINVlKrNhMnmMsDOLMGx1YYt9d4UsRg1jtVrwxL4Vd/F7aHCZySAXKv+1rkhACR023wpa3dhp+xirGJxSO9LWwvcrTdM4xJo4RS8B40tGENOJ1NKixUJxwN/6og58Oft/u==uleR89Ja=7zszK2H7tX3DqmEYNvNDYQh/7VBRe5otghQtPwJzWpXAGk+Vme4hPPM5K6axH2LxipXzRiIV=oxNs0upKNu1FvuzbCQmkQdKQVmXl0344vngngrgN7wkEfrYtmKwICmpAS0cbW9jdSClgziVo4NaFc/hsIfok=4UA3hVtxIdw74lFNXD0RR7HKXkFPLIn85M7peOZsqMUCfO4gxr7KCfabszQQf0YcP/mt79XK50=WrSJG7oUyn+clUySPhlegqHAfT9a50uSK5WiQmOnGNGLF4wDO10sqKN1xRgQbYHPtwL+Ye0EMisvmYA3==kScorTSGaQWyibSWXAvxq9+IVGBYShVJ6S7DmTT=u/2d/fGEge+Xmbxlftza=cxJ=Md=k1Q71Lp6Boa56d7wtYRpK6tXHJ9I/2r7rN1E4OtwkFqb7SfWV3UXwyUrXyaaNPTIbqnAHnbgUGtuU6pgICpfREiIxVqvKBf6ErbxHRmMmAuYKxk5E9Mn6nnbxR4WTniweKYeDv2w39zge/tss+36Moeuio9d2eoyRFqXhq=rUGtDwX3fzXV0wV+dUojxOYQ57GQDl7+68PwHPcX794OIXuGOxBk83lNIYIcYz3Vc7qnGy6tFTz7f6S9+EZuSGN7TY5VKkT2eWye46DebrDF9Nwzs/FVpTzbPD/KGDIBtFIbazglhKoWe9txqb1QW8vFNNVOEhYa+cViO3g8ZmY1wG960US2zsnX5Eg8Q5a4h3+sxaJSJ4ONiXZWJuAgKRQzcrszu+M5C0ZVoCOv1goEgfNJeSm/yFc/3rx8wmeWLIJFtq65B7zF72HRKq1nthHAguaxXr20nguHpKkDpNBDVa=WwuJsbeGI -``DIAG: (origin_server_auth) Set the header Authorization: AWS4-HMAC-SHA256 Credential=``/us-east-1/s3/aws4_request,SignedHeaders=accept;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=`` +``DIAG: (origin_server_auth) Set the header Authorization: AWS4-HMAC-SHA256 Credential=``/us-east-1/s3/aws4_request,SignedHeaders=accept;host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=`` diff --git a/tests/gold_tests/pluginTest/polite_hook_wait/polite_hook_wait.cc b/tests/gold_tests/pluginTest/polite_hook_wait/polite_hook_wait.cc index 92b86245d60..1e071a0dfcb 100644 --- a/tests/gold_tests/pluginTest/polite_hook_wait/polite_hook_wait.cc +++ b/tests/gold_tests/pluginTest/polite_hook_wait/polite_hook_wait.cc @@ -27,11 +27,11 @@ using atscppapi::TSContUniqPtr; using atscppapi::TSThreadUniqPtr; /* -Test handling a blocking call (in a spawned thread) on a transaction hook without blocking the thread executing the hooks. +Test spawning a thread in one transaction hook that runs in parallel with the transaction, until a transaction +continuation on a later hook waits for the thread results. -It is dependent on continuations hooked globally on a transaction hook running before continations hooked for just the -one transaction. It is dependent on the ability of a global continuation on a txn hook to add a per-txn continuation on -the same hook. +(To block the transaction until the thread completes, but allow the event task to continue processing events not +related to the transaction, follow the simpler example in example/plugins/c-api/thread_1.) */ #define PINAME "polite_hook_wait" @@ -42,7 +42,7 @@ char PIName[] = PINAME; DbgCtl dbg_ctl{PINAME}; -enum Test_step { BEGIN, GLOBAL_CONT_READ_HDRS, THREAD, TXN_CONT_READ_HDRS, END }; +enum Test_step { BEGIN, GLOBAL_CONT_READ_HDRS, THREAD, TXN_CONT, END }; char const * step_cstr(int test_step) @@ -62,8 +62,8 @@ step_cstr(int test_step) result = "THREAD"; break; - case TXN_CONT_READ_HDRS: - result = "TXN_CONT_READ_HDRS"; + case TXN_CONT: + result = "TXN_CONT"; break; default: @@ -163,7 +163,7 @@ Blocking_action::_global_cont_func(TSCont, TSEvent event, void *eventData) return 0; } - TSHttpTxnHookAdd(txn, TS_HTTP_READ_REQUEST_HDR_HOOK, ba._txn_hook_cont.get()); + TSHttpTxnHookAdd(txn, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, ba._txn_hook_cont.get()); while (!ba._cont_mutex_locked.load(std::memory_order_acquire)) { std::this_thread::yield(); @@ -171,7 +171,7 @@ Blocking_action::_global_cont_func(TSCont, TSEvent event, void *eventData) } break; case TS_EVENT_HTTP_SEND_RESPONSE_HDR: - next_step(TXN_CONT_READ_HDRS); + next_step(TXN_CONT); if (!AuxDataMgr::data(txn).txn_valid) { static const char msg[] = "authorization denied\n"; @@ -224,7 +224,7 @@ Blocking_action::_txn_cont_func(TSCont, TSEvent event, void *eventData) next_step(THREAD); TSReleaseAssert(eventData != nullptr); - TSReleaseAssert(TS_EVENT_HTTP_READ_REQUEST_HDR == event); + TSReleaseAssert(TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE == event); TSHttpTxn txn{static_cast(eventData)}; diff --git a/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd0.gold b/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd0.gold index b19633eb21f..b66dbbd36e6 100644 --- a/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd0.gold +++ b/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd0.gold @@ -1,9 +1,9 @@ /tests/request.txt 200 TCP_MISS FIN 33 - -/tests/request.txt 200 TCP_HIT - 33 - +/tests/request.txt 200 ``_HIT - 33 - /tests/prefetch.txt 200 TCP_MISS - 16 tests/request.txt /tests/prefetch.txt 200 TCP_MISS FIN 34 - /tests/request.txt 200 ``_HIT - 33 - -/tests/prefetch.txt 208 TCP_HIT - 20 tests/request.txt +/tests/prefetch.txt 208 ``_HIT - 20 tests/request.txt /tests/prefetch.txt 200 ``_HIT - 34 - /tests/query?this=foo&that 200 TCP_MISS FIN 41 - /tests/query?bar=baz 200 TCP_MISS - 16 tests/query diff --git a/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd1.gold b/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd1.gold index 2aef7cd223d..9c40a29b07e 100644 --- a/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd1.gold +++ b/tests/gold_tests/pluginTest/prefetch/prefetch_cmcd1.gold @@ -1,11 +1,11 @@ /tests/request.txt 200 TCP_MISS FIN 33 - /tests/prefetch.txt 200 TCP_MISS - 16 tests/request.txt /tests/prefetch.txt 200 TCP_MISS FIN 34 - -/tests/prefetch.txt 200 TCP_HIT - 34 - +/tests/prefetch.txt 200 TCP_``HIT - 34 - /tests/query?this=foo&that 200 TCP_MISS FIN 41 - /tests/query?bar=baz 200 TCP_MISS - 16 tests/query /tests/query?bar=baz 200 TCP_MISS FIN 35 - -/tests/query?bar=baz 200 TCP_HIT - 35 - +/tests/query?bar=baz 200 TCP_``HIT - 35 - /root.txt 200 TCP_MISS FIN 30 - /rooted 200 TCP_MISS - 16 root.txt /rooted 200 TCP_MISS FIN 28 - diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py index d497ecfc205..aac1205ca8a 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py @@ -125,7 +125,6 @@ { 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'regex_revalidate', - # 'proxy.config.diags.debug.enabled': 0, 'proxy.config.http.insert_age_in_response': 0, 'proxy.config.http.response_via_str': 3, }) diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold b/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold index 8cb833638f0..9e2c13a786f 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold @@ -2,12 +2,12 @@ cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2:: x-crr-ident=::-:: cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5:: x-crr-ident=::Etag "plain":: uid=::plain 1:: crc=TCP_MISS cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-:: uid=::plain:: crc=TCP_MISS cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2:: x-crr-ident=::-:: uid=::plain 0:: crc=TCP_REFRESH_MISS -cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5:: x-crr-ident=::Etag "plain":: uid=::-:: crc=TCP_HIT +cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5:: x-crr-ident=::Etag "plain":: uid=::-:: crc=TCP_MEM_HIT cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-:: uid=::plain:: crc=TCP_MISS cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2:: x-crr-ident=::-:: uid=::chg 0:: crc=TCP_REFRESH_MISS cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5:: x-crr-ident=::Etag "chg":: uid=::chg 1:: crc=TCP_REFRESH_MISS cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-:: uid=::chg:: crc=TCP_MISS -cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=0-2:: x-crr-ident=::-:: uid=::-:: crc=TCP_HIT -cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5:: x-crr-ident=::Etag "chg":: uid=::-:: crc=TCP_HIT +cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=0-2:: x-crr-ident=::-:: uid=::-:: crc=TCP_MEM_HIT +cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5:: x-crr-ident=::Etag "chg":: uid=::-:: crc=TCP_MEM_HIT cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-:: uid=::chg:: crc=TCP_MISS cpuup=/404.txt sssc=404 pssc=404 phr=DIRECT range=::-:: x-crr-ident=::-:: uid=::-:: crc=TCP_MISS diff --git a/tests/gold_tests/pluginTest/strategies/strategies_plugins.test.py b/tests/gold_tests/pluginTest/strategies/strategies_plugins.test.py index 6a8b1a549b8..d718431994a 100644 --- a/tests/gold_tests/pluginTest/strategies/strategies_plugins.test.py +++ b/tests/gold_tests/pluginTest/strategies/strategies_plugins.test.py @@ -36,9 +36,10 @@ dns = Test.MakeDNServer("dns") origins = [] -num_origins = 3 -for ind in range(num_origins): - name = f"nh{ind}" + +chars = ['0', '1', '2', 'p', 's'] +for char in chars: + name = f"nh{char}" origin = Test.MakeOriginServer(name, options={"--verbose": ""}) request_header = { "headers": f"GET / HTTP/1.1\r\nHost: origin\r\n\r\n", @@ -91,21 +92,31 @@ 'proxy.config.http.parent_proxy.mark_down_hostdb': 0, 'proxy.config.http.parent_proxy.self_detect': 0, 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': "url_rewrite|next_hop|dns|parent|regex_remap|header_rewrite|tslua|http|hostdb", + 'proxy.config.diags.debug.tags': "next_hop|dns|http|parent|regex_remap|header_rewrite|tslua", }) ts.Disk.MakeConfigFile("hdr_rw.config").AddLines( [ - "cond %{REMAP_PSEUDO_HOOK}", - 'cond %{CLIENT-HEADER:Strategy} ="" [NOT]', - "set-next-hop-strategy %{CLIENT-HEADER:Strategy}", + 'cond %{CLIENT-HEADER:Strategy} ="nemo"', + "set-next-hop-strategy nemo", + 'cond %{CLIENT-HEADER:Strategy} ="nh0"', + "set-next-hop-strategy nh0", + 'cond %{CLIENT-HEADER:Strategy} ="nh1"', + "set-next-hop-strategy nh1", + 'cond %{CLIENT-HEADER:Strategy} ="null"', + "set-next-hop-strategy null", + 'cond %{CLIENT-HEADER:Strategy} ="clear"', + 'set-next-hop-strategy ""', ]) ts.Disk.MakeConfigFile("regex_remap.config").AddLines( [ "/nh0 http://origin/path @strategy=nh1", '/nh1 http://origin/path @strategy=', "/nh2 http://origin/path @strategy=nh0", + '/null http://origin/path @strategy=null', "/nemo http://origin/path @strategy=nemo", + "# fallthrough", + "/ http://origin/path", ]) ts.Disk.MakeConfigFile("strategies.lua").AddLines( [ @@ -117,6 +128,8 @@ ' ts.http.set_next_hop_strategy("")', ' elseif uri:find("nh2") then', ' ts.http.set_next_hop_strategy("nh0")', + ' elseif uri:find("null") then', + ' ts.http.set_next_hop_strategy("null")', ' elseif uri:find("nemo") then', ' ts.http.set_next_hop_strategy("nemo")', ' end', @@ -134,43 +147,59 @@ s = ts.Disk.strategies s.AddLine("groups:") -for ind in range(num_origins - 1): - name = f"nh{ind}" +for ind in range(len(origins)): + char = chars[ind] + org = origins[ind] + name = f"nh{chars[ind]}" s.AddLines( [ - f" - &g{ind}", + f" - &g{char}", f" - host: {name}", f" protocol:", f" - scheme: http", - f" port: {origins[ind].Variables.Port}", + f" port: {org.Variables.Port}", f" weight: 1.0", ]) s.AddLine("strategies:") # third ts_nh -for ind in range(num_origins - 1): +for char in chars: s.AddLines( [ - f" - strategy: nh{ind}", + f" - strategy: nh{char}", f" policy: consistent_hash", f" hash_key: path", f" go_direct: false", f" parent_is_proxy: false", f" ignore_self_detect: true", f" groups:", - f" - *g{ind}", + f" - *g{char}", f" scheme: http", ]) ts.Disk.remap_config.AddLines( [ + "# header rewrite", + "map http://nhp_hr http://origin @plugin=header_rewrite.so @pparam=hdr_rw.config", + "map http://nhs_hr http://origin @strategy=nh0 @plugin=header_rewrite.so @pparam=hdr_rw.config", + "# modify strategy/parent", "map http://nh0_hr http://origin @strategy=nh0 @plugin=header_rewrite.so @pparam=hdr_rw.config", "map http://nh1_hr http://origin @strategy=nh1 @plugin=header_rewrite.so @pparam=hdr_rw.config", "map http://nh2_hr http://origin @plugin=header_rewrite.so @pparam=hdr_rw.config", + "", + "# regex_remap", + "map http://nhp_rr http://origin @plugin=regex_remap.so @pparam=regex_remap.config", + "map http://nhs_rr http://origin @strategy=nh0 @plugin=regex_remap.so @pparam=regex_remap.config", + "# modify strategy/parent", "map http://nh0_rr http://origin @strategy=nh0 @plugin=regex_remap.so @pparam=regex_remap.config", "map http://nh1_rr http://origin @strategy=nh1 @plugin=regex_remap.so @pparam=regex_remap.config", "map http://nh2_rr http://origin @plugin=regex_remap.so @pparam=regex_remap.config", + "", + "# tslua", + "map http://nhp_lua http://origin @plugin=tslua.so @pparam=strategies.lua", + "map http://nhs_lua http://origin @strategy=nh0 @plugin=tslua.so @pparam=strategies.lua", + "# modify strategy/parent", "map http://nh0_lua http://origin @strategy=nh0 @plugin=tslua.so @pparam=strategies.lua", "map http://nh1_lua http://origin @strategy=nh1 @plugin=tslua.so @pparam=strategies.lua", "map http://nh2_lua http://origin @plugin=tslua.so @pparam=strategies.lua", @@ -183,22 +212,38 @@ # header rewrite -# 0 - nh0 default request -tr = Test.AddTestRun("nh0_hr straight through request") +# 0 - nhp request to parent.config +tr = Test.AddTestRun("nhp_hr parent.config through request") ps = tr.Processes.Default -for ind in range(num_origins): +for ind in range(len(origins)): origin = origins[ind] ps.StartBefore(origin, ready=When.PortOpen(origin.Variables.Port)) tr.StillRunningAfter = origin ps.StartBefore(dns) ps.StartBefore(Test.Processes.ts) +tr.MakeCurlCommand(curl_and_args + " http://nhp_hr/path", ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = dns + +# 1 - nhs_hr default request +tr = Test.AddTestRun("nhs_hr straight through request") +ps = tr.Processes.Default +tr.MakeCurlCommand(curl_and_args + " http://nhs_hr/path", ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") +tr.StillRunningAfter = ts + +# 2 - nh0_hr default request +tr = Test.AddTestRun("nh0_hr straight through request") +ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + " http://nh0_hr/path", ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns -# 1 - nh1_hr default request +# 3 - nh1_hr default request tr = Test.AddTestRun("nh1_hr straight through request") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + " http://nh1_hr/path", ts=ts) @@ -206,7 +251,7 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("nh1", "expected nh1") tr.StillRunningAfter = ts -# 2 - nh2_hr default request +# 4 - nh2_hr default request tr = Test.AddTestRun("nh2_hr straight through request") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + " http://nh2_hr/path", ts=ts) @@ -214,117 +259,153 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") tr.StillRunningAfter = ts -# 3 switch strategies +# 5 switch strategies tr = Test.AddTestRun("nh0_hr switch to nh1") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh0_hr/path -H "Strategy: nh1"', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh1", "expected nh1") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 4 strategy to parent.config +# 6 strategy to parent.config tr = Test.AddTestRun("nh1_hr switch to parent.config") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh1_hr/path -H "Strategy: null"', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 5 parent.config strategy to strategy +# 7 parent.config strategy to strategy tr = Test.AddTestRun("nh2_hr switch to nh0") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh2_hr/path -H "Strategy: nh0"', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 6 try to switch to non existent strategy +# 8 try to switch to non existent strategy tr = Test.AddTestRun("nh0_hr switch to nemo (fail)") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh0_hr/path -H "Strategy: nemo"', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns # regex_remap -# 7 switch strategies +# 9 use parent.config +tr = Test.AddTestRun("nhp_rr parent.config") +ps = tr.Processes.Default +tr.MakeCurlCommand(curl_and_args + ' http://nhp_rr/nhp', ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = dns + +# 10 use strategies +tr = Test.AddTestRun("nhs_rr strategies.yaml") +ps = tr.Processes.Default +tr.MakeCurlCommand(curl_and_args + ' http://nhs_rr/nh', ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") +tr.StillRunningAfter = ts +tr.StillRunningAfter = dns + +# 11 switch strategies tr = Test.AddTestRun("nh0_rr switch to nh1") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh0_rr/nh0', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh1", "expected nh1") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 8 strategy to parent.config +# 12 strategy to parent.config tr = Test.AddTestRun("nh1_rr switch to parent.config") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh1_rr/nh1', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 9 parent.config strategy to strategy +# 13 parent.config strategy to strategy tr = Test.AddTestRun("nh2_rr switch to nh0") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh2_rr/nh2', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 10 switch strategies (fail) +# 14 switch strategies (fail) tr = Test.AddTestRun("nh0_rr switch to nemo") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh0_rr/nemo', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns # tslua -# 11 switch strategies +# 15 parent.config +tr = Test.AddTestRun("nhp_lua parent.config") +ps = tr.Processes.Default +tr.MakeCurlCommand(curl_and_args + ' http://nhp_lua/nh', ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = dns + +# 16 strategies.yaml +tr = Test.AddTestRun("nhs_lua strategies.yaml") +ps = tr.Processes.Default +tr.MakeCurlCommand(curl_and_args + ' http://nhs_lua/nh', ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") +tr.StillRunningAfter = ts +tr.StillRunningAfter = dns + +# 17 switch strategies tr = Test.AddTestRun("nh0_lua switch to nh1") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh0_lua/nh0', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh1", "expected nh1") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 12 strategy to parent.config +# 18 strategy to parent.config tr = Test.AddTestRun("nh1_lua switch to parent.config") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh1_lua/nh1', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh2", "expected nh2") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 13 parent.config strategy to strategy +# 19 parent.config strategy to strategy tr = Test.AddTestRun("nh2_lua switch to nh0") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh2_lua/nh2', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns -# 14 switch strategies, fail +# 20 switch strategies, fail tr = Test.AddTestRun("nh0_lua switch to nemo") ps = tr.Processes.Default tr.MakeCurlCommand(curl_and_args + ' http://nh0_lua/nemo', ts=ts) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("nh0", "expected nh0") tr.StillRunningAfter = ts -tr.StillRunnerAfter = dns +tr.StillRunningAfter = dns # Overriding the built in ERROR check since we expect some ERROR messages ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "Some tests are failure tests") diff --git a/tests/gold_tests/tls/tls_check_dual_cert_selection_plugin.test.py b/tests/gold_tests/tls/tls_check_dual_cert_selection_plugin.test.py index c2be665da54..14b3b2fe1cc 100644 --- a/tests/gold_tests/tls/tls_check_dual_cert_selection_plugin.test.py +++ b/tests/gold_tests/tls/tls_check_dual_cert_selection_plugin.test.py @@ -90,7 +90,7 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: ECDSA", "Should select EC cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (ECDSA|ecdsa_)", "Should select EC cert") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") # Should receive a RSA cert @@ -100,7 +100,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: RSA-PSS", "Should select RSA cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (RSA-PSS|rsa_pss_)", "Should select RSA cert") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") # Should receive a EC cert @@ -110,7 +110,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: ECDSA", "Should select EC cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (ECDSA|ecdsa_)", "Should select EC cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?group.com", "Should select a group SAN") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") @@ -121,7 +121,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: RSA-PSS", "Should select RSA cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (RSA-PSS|rsa_pss_)", "Should select RSA cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?group.com", "Should select a group SAN") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") @@ -132,7 +132,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: RSA-PSS", "Should select RSA cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (RSA-PSS|rsa_pss_)", "Should select RSA cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?group.com", "Should select a group SAN") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") @@ -143,7 +143,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: ECDSA", "Should select EC cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (ECDSA|ecdsa_)", "Should select EC cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?group.com", "Should select a group SAN") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") @@ -169,7 +169,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: RSA-PSS", "Should select RSA cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (RSA-PSS|rsa_pss_)", "Should select RSA cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?foo.com", "Should select foo.com") tr.Processes.Default.Streams.All += Testers.ContainsExpression("unable to verify the first certificate", "Different signer") @@ -179,7 +179,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: RSA-PSS", "Should select RSA cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (RSA-PSS|rsa_pss_)", "Should select RSA cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?foo.com", "Should select foo.com") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") @@ -190,6 +190,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: ECDSA", "Should select EC cert") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Peer signature type: (ECDSA|ecdsa_)", "Should select EC cert") tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN ?= ?foo.com", "Should select foo.com") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("unable to verify the first certificate", "Correct signer") diff --git a/tests/gold_tests/traffic_ctl/gold/test_2.gold b/tests/gold_tests/traffic_ctl/gold/test_2.gold new file mode 100644 index 00000000000..3c75916cf79 --- /dev/null +++ b/tests/gold_tests/traffic_ctl/gold/test_2.gold @@ -0,0 +1 @@ +proxy.config.diags.debug.enabled: 1 diff --git a/tests/gold_tests/traffic_ctl/gold/test_3.gold b/tests/gold_tests/traffic_ctl/gold/test_3.gold new file mode 100644 index 00000000000..e12f994befe --- /dev/null +++ b/tests/gold_tests/traffic_ctl/gold/test_3.gold @@ -0,0 +1 @@ +proxy.config.diags.debug.tags: rpc # default http|dns diff --git a/tools/autopep8.sh b/tools/autopep8.sh deleted file mode 100755 index 1657ed81234..00000000000 --- a/tools/autopep8.sh +++ /dev/null @@ -1,107 +0,0 @@ -#! /usr/bin/env bash -# -# Simple wrapper to run autopep8 on a directory. -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Update these VERSION variables with the new desired autopep8 tag when a new -# autopep8 version is desired. -# See: -# https://github.com/hhatto/autopep8/tags -AUTOPEP8_VERSION="1.5.3" -VERSION="autopep8 1.5.3 (pycodestyle: 2.6.0)" - -# Tie this to exactly the pycodestyle version that shows up in the setup.py of -# autopep8 so we know we run with the same version each time. -# See: -# https://github.com/hhatto/autopep8/blob/master/setup.py -PYCODESTYLE_TAG="2.6.0" - -function main() { - set -e # exit on error - - if ! type virtualenv >/dev/null 2>/dev/null - then - pip install -q virtualenv - fi - - GIT_DIR=$(git rev-parse --absolute-git-dir) - AUTOPEP8_VENV=${AUTOPEP8_VENV:-${GIT_DIR}/fmt/autopep8_${AUTOPEP8_VERSION}_venv} - if [ ! -e ${AUTOPEP8_VENV} ] - then - virtualenv ${AUTOPEP8_VENV} - fi - source ${AUTOPEP8_VENV}/bin/activate - - pip install -q --upgrade pip - pip install -q "pycodestyle==${PYCODESTYLE_TAG}" - pip install -q "autopep8==${AUTOPEP8_VERSION}" - - ver=$(autopep8 --version 2>&1) - if [ "$ver" != "$VERSION" ] - then - echo "Wrong version of autopep8!" - echo "Expected: \"${VERSION}\", got: \"${ver}\"" - exit 1 - fi - - DIR=${@:-.} - - # Only run autopep8 on tracked files. This saves time and possibly avoids - # formatting files the user doesn't want formatted. - tmp_dir=$(mktemp -d -t tracked-git-files.XXXXXXXXXX) - files=${tmp_dir}/git_files.txt - files_filtered=${tmp_dir}/git_files_filtered.txt - git ls-tree -r HEAD --name-only ${DIR} | grep -vE "lib/yamlcpp" > ${files} - # Add to the above any newly added staged files. - git diff --cached --name-only --diff-filter=A >> ${files} - # Keep this list of Python extensions the same with the list of - # extensions searched for in the tools/git/pre-commit hook. - grep -E '\.py$|\.cli.ext$|\.test.ext$' ${files} > ${files_filtered} - # Prepend the filenames with "./" to make the modified file output consistent - # with the clang-format target output. - sed -i'.bak' 's:^:\./:' ${files_filtered} - rm -f ${files_filtered}.bak - - # Efficiently retrieving modification timestamps in a platform - # independent way is challenging. We use find's -newer argument, which - # seems to be broadly supported. The following file is created and has a - # timestamp just before running autopep8. Any file with a timestamp - # after this we assume was modified by autopep8. - start_time_file=${tmp_dir}/format_start.$$ - touch ${start_time_file} - autopep8 \ - --ignore-local-config \ - -i \ - -j 0 \ - --exclude "${DIR}/lib/yamlcpp" \ - --max-line-length 132 \ - --aggressive \ - --aggressive \ - $(cat ${files_filtered}) - find $(cat ${files_filtered}) -newer ${start_time_file} - - rm -rf ${tmp_dir} - deactivate -} - -if [[ "$(basename -- "$0")" == 'autopep8.sh' ]]; then - main "$@" -else - GIT_DIR=$(git rev-parse --absolute-git-dir) - AUTOPEP8_VENV=${AUTOPEP8_VENV:-${GIT_DIR}/fmt/autopep8_${AUTOPEP8_VERSION}_venv} -fi diff --git a/tools/benchmark/CMakeLists.txt b/tools/benchmark/CMakeLists.txt index 74bf072e08d..6a062bfaad8 100644 --- a/tools/benchmark/CMakeLists.txt +++ b/tools/benchmark/CMakeLists.txt @@ -28,7 +28,10 @@ if(TS_USE_HWLOC) endif() add_executable(benchmark_ProxyAllocator benchmark_ProxyAllocator.cc) -target_link_libraries(benchmark_ProxyAllocator PRIVATE Catch2::Catch2 ts::tscore ts::inkevent libswoc::libswoc) +target_link_libraries(benchmark_ProxyAllocator PRIVATE Catch2::Catch2WithMain ts::tscore ts::inkevent libswoc::libswoc) add_executable(benchmark_SharedMutex benchmark_SharedMutex.cc) target_link_libraries(benchmark_SharedMutex PRIVATE Catch2::Catch2 ts::tscore libswoc::libswoc) + +add_executable(benchmark_Random benchmark_Random.cc) +target_link_libraries(benchmark_Random PRIVATE Catch2::Catch2WithMain ts::tscore) diff --git a/tools/benchmark/benchmark_EventSystem.cc b/tools/benchmark/benchmark_EventSystem.cc index 4410d16691c..faa759c2160 100644 --- a/tools/benchmark/benchmark_EventSystem.cc +++ b/tools/benchmark/benchmark_EventSystem.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include "iocore/eventsystem/Continuation.h" #include "iocore/eventsystem/EventSystem.h" @@ -81,8 +82,8 @@ TEST_CASE("event process benchmark", "") }; } -struct EventProcessorListener : Catch::TestEventListenerBase { - using TestEventListenerBase::TestEventListenerBase; +struct EventProcessorListener : Catch::EventListenerBase { + using EventListenerBase::EventListenerBase; void testRunStarting(Catch::TestRunInfo const & /* testRunInfo ATS_UNUSED */) override @@ -108,7 +109,7 @@ main(int argc, char *argv[]) { Catch::Session session; - using namespace Catch::clara; + using namespace Catch::Clara; auto cli = session.cli() | Opt(nevents, "n")["--ts-nevents"]("number of events (default: 1)\n") | Opt(nthreads, "n")["--ts-nthreads"]("number of ethreads (default: 1)\n"); diff --git a/tools/benchmark/benchmark_FreeList.cc b/tools/benchmark/benchmark_FreeList.cc index 29c3b46d8d5..88eae093e3c 100644 --- a/tools/benchmark/benchmark_FreeList.cc +++ b/tools/benchmark/benchmark_FreeList.cc @@ -23,6 +23,7 @@ #include #include +#include #include "tscore/ink_hw.h" #include "tscore/ink_thread.h" @@ -183,7 +184,7 @@ main(int argc, char *argv[]) { Catch::Session session; - using namespace Catch::clara; + using namespace Catch::Clara; bool opt_enable_hugepage = false; diff --git a/tools/benchmark/benchmark_ProxyAllocator.cc b/tools/benchmark/benchmark_ProxyAllocator.cc index bed12f4729d..232b559a5fc 100644 --- a/tools/benchmark/benchmark_ProxyAllocator.cc +++ b/tools/benchmark/benchmark_ProxyAllocator.cc @@ -23,6 +23,7 @@ limitations under the License. #define CATCH_CONFIG_ENABLE_BENCHMARKING #include +#include #include "iocore/eventsystem/Thread.h" #include "tscore/Allocator.h" diff --git a/tools/benchmark/benchmark_Random.cc b/tools/benchmark/benchmark_Random.cc new file mode 100644 index 00000000000..d1a7e7f7620 --- /dev/null +++ b/tools/benchmark/benchmark_Random.cc @@ -0,0 +1,101 @@ + +/** @file + +Simple benchmark for ProxyAllocator + +@section license License + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "tscore/ink_rand.h" +#include +#include +#define CATCH_CONFIG_ENABLE_BENCHMARKING +#include +#include +#include + +#include "tscore/Random.h" + +TEST_CASE("BenchRandom", "[bench][random]") +{ + InkRand gen(42); + ts::Random::seed(13); + + BENCHMARK("IncRand") + { + return gen.random(); + }; + + BENCHMARK("ts::Random") + { + return ts::Random::random(); + }; + + std::mt19937_64 mt; + BENCHMARK("std::mt19937_64") + { + return mt(); + }; + + std::ranlux48_base rb; + BENCHMARK("std::ranlux48_base") + { + return rb(); + }; + + std::ranlux24_base rb24; + BENCHMARK("std::ranlux24_base") + { + return rb24(); + }; + + std::uniform_int_distribution mtdist{0, UINT64_MAX}; + + BENCHMARK("std::uniform_int_distribution") + { + return mtdist(mt); + }; +} + +TEST_CASE("RandomDistribution", "[random][distribution]") +{ + auto g = Catch::Generators::random(std::numeric_limits::min(), std::numeric_limits::max()); + InkRand gen(g.get()); + int iterations = 1000000; + constexpr int buckets = 100; + int counts[buckets] = {0}; + + for (int i = 0; i < iterations; i++) { + counts[gen.random() % buckets]++; + } + + double expected = static_cast(iterations) / static_cast(buckets); + + double min = std::numeric_limits::max(); + double max = std::numeric_limits::min(); + + for (int count : counts) { + double ratio = static_cast(count) / expected; + + min = std::min(ratio, min); + max = std::max(ratio, max); + } + REQUIRE(0.95 < min); + REQUIRE(max < 1.05); +} diff --git a/tools/benchmark/benchmark_SharedMutex.cc b/tools/benchmark/benchmark_SharedMutex.cc index e6b0f14cf24..33895c71c7d 100644 --- a/tools/benchmark/benchmark_SharedMutex.cc +++ b/tools/benchmark/benchmark_SharedMutex.cc @@ -28,6 +28,7 @@ #include #include +#include #include "tsutil/Bravo.h" @@ -113,7 +114,7 @@ main(int argc, char *argv[]) { Catch::Session session; - using namespace Catch::clara; + using namespace Catch::Clara; // clang-format off auto cli = session.cli() | diff --git a/tools/hrw4u/grammar/hrw4u.g4 b/tools/hrw4u/grammar/hrw4u.g4 index 852d72c0561..5db49f51be8 100644 --- a/tools/hrw4u/grammar/hrw4u.g4 +++ b/tools/hrw4u/grammar/hrw4u.g4 @@ -69,6 +69,7 @@ LBRACKET : '['; RBRACKET : ']'; EQUALS : '=='; EQUAL : '='; +PLUSEQUAL : '+='; NEQ : '!='; GT : '>'; LT : '<'; @@ -79,6 +80,7 @@ NOT_TILDE : '!~'; COLON : ':'; COMMA : ','; SEMICOLON : ';'; +AT : '@'; COMMENT : '#' ~[\r\n]* ; WS : [ \t\r\n]+ -> skip ; @@ -120,13 +122,14 @@ variablesItem ; variableDecl - : name=IDENT COLON typeName=IDENT SEMICOLON + : name=IDENT COLON typeName=IDENT (AT slot=NUMBER)? SEMICOLON ; statement : BREAK SEMICOLON | functionCall SEMICOLON | lhs=IDENT EQUAL value SEMICOLON + | lhs=IDENT PLUSEQUAL value SEMICOLON | op=IDENT SEMICOLON ; @@ -154,6 +157,7 @@ block blockItem : statement + | conditional | commentLine ; diff --git a/tools/hrw4u/grammar/u4wrh.g4 b/tools/hrw4u/grammar/u4wrh.g4 index a88b2c585c9..53972889ee2 100644 --- a/tools/hrw4u/grammar/u4wrh.g4 +++ b/tools/hrw4u/grammar/u4wrh.g4 @@ -21,6 +21,8 @@ grammar u4wrh; // Lexer Rules // ----------------------------- COND : 'cond'; +IF_OP : 'if'; +ENDIF_OP : 'endif'; ELIF : 'elif'; ELSE : 'else'; AND_MOD : 'AND'; @@ -48,24 +50,24 @@ fragment HEXDIGIT : [0-9a-fA-F]; // Percent blocks - treat entire %{...} as one token PERCENT_BLOCK : '%{' ~[}\r\n]* '}' '}'?; -IDENT : [@a-zA-Z_][a-zA-Z0-9_@.-]* ; -COMPLEX_STRING : (~[ \t\r\n[\]{}(),=!><~%])+; -NUMBER : [0-9]+ ; -LPAREN : '('; -RPAREN : ')'; -LBRACE : '{'; -RBRACE : '}'; -LBRACKET : '['; -RBRACKET : ']'; -EQUALS : '='; -NEQ : '!='; -GT : '>'; -LT : '<'; -COMMA : ','; - -EOL : '\r'? '\n'; -COMMENT : '#' ~[\r\n]* ; -WS : [ \t]+ -> skip ; +IDENT : [@a-zA-Z_][a-zA-Z0-9_@.-]* ; +COMPLEX_STRING : (~[ \t\r\n[\]{}(),=!><~%#])+; +NUMBER : [0-9]+ ; +LPAREN : '('; +RPAREN : ')'; +LBRACE : '{'; +RBRACE : '}'; +LBRACKET : '['; +RBRACKET : ']'; +EQUALS : '='; +NEQ : '!='; +GT : '>'; +LT : '<'; +COMMA : ','; + +EOL : '\r'? '\n'; +COMMENT : '#'~[\r\n]*; +WS : [ \t]+ -> skip ; // ----------------------------- // Parser Rules @@ -78,6 +80,8 @@ program line : condLine EOL | opLine EOL + | ifLine EOL + | endifLine EOL | elifLine EOL | elseLine EOL | commentLine EOL @@ -88,6 +92,14 @@ condLine : COND condBody modList? ; +ifLine + : IF_OP + ; + +endifLine + : ENDIF_OP + ; + elifLine : ELIF ; diff --git a/tools/hrw4u/pyproject.toml b/tools/hrw4u/pyproject.toml index 5a85df79818..f66a9ada00b 100644 --- a/tools/hrw4u/pyproject.toml +++ b/tools/hrw4u/pyproject.toml @@ -20,7 +20,7 @@ build-backend = "setuptools.build_meta" [project] name = "hrw4u" -version = "1.3.4" +version = "1.4.0" description = "HRW4U CLI tool for Apache Traffic Server header rewrite rules" authors = [ {name = "Leif Hedstrom", email = "leif@apache.org"} diff --git a/tools/hrw4u/scripts/testcase.py b/tools/hrw4u/scripts/testcase.py index 757af95959e..802e32f2166 100755 --- a/tools/hrw4u/scripts/testcase.py +++ b/tools/hrw4u/scripts/testcase.py @@ -29,6 +29,29 @@ KNOWN_MARKS = {"hooks", "conds", "ops", "vars", "examples", "invalid"} +def load_exceptions(test_dir: Path) -> dict[str, str]: + """Load exceptions from exceptions.txt in the test directory. + Returns a dict mapping test filename to direction (hrw4u or u4wrh).""" + exceptions_file = test_dir / "exceptions.txt" + exceptions = {} + + if not exceptions_file.exists(): + return exceptions + + for line in exceptions_file.read_text().splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + + parts = line.split(':', 1) + if len(parts) == 2: + test_name = parts[0].strip() + direction = parts[1].strip() + exceptions[test_name] = direction + + return exceptions + + def parse_tree(input_text: str) -> tuple[hrw4uParser, any]: stream = InputStream(input_text) lexer = hrw4uLexer(stream) @@ -38,12 +61,30 @@ def parse_tree(input_text: str) -> tuple[hrw4uParser, any]: return parser, tree -def process_file(input_path: Path, update_ast: bool = False, update_output: bool = False, update_error: bool = False) -> bool: +def process_file( + input_path: Path, + update_ast: bool = False, + update_output: bool = False, + update_error: bool = False, + exceptions: dict[str, str] = None) -> bool: base = input_path.with_suffix('') ast_path = base.with_suffix('.ast.txt') output_path = base.with_suffix('.output.txt') error_path = base.with_suffix('.error.txt') + # Check if this test has a direction exception + if exceptions is None: + exceptions = load_exceptions(input_path.parent) + + test_filename = input_path.name.replace('.input.txt', '.input') + if test_filename in exceptions: + exception_direction = exceptions[test_filename] + # Skip updating for hrw4u if test is u4wrh-only (and vice versa) + # Since this script runs hrw4u, skip if marked as u4wrh + if exception_direction == 'u4wrh': + # This test is reverse-only, skip updating + return True + input_text = input_path.read_text() if input_path.name.endswith(".fail.input.txt"): @@ -90,15 +131,33 @@ def run_batch(group: str | None = None, update_ast: bool = False, update_output: print(f"No test files found for pattern: {base_dir}/{pattern}") sys.exit(1) + # Group files by directory to load exceptions once per directory + files_by_dir = {} + for f in input_files: + if f.parent not in files_by_dir: + files_by_dir[f.parent] = [] + files_by_dir[f.parent].append(f) + total = len(input_files) failed = 0 + skipped = 0 - for f in input_files: - ok = process_file(f, update_ast=update_ast, update_output=update_output, update_error=update_error) - if not ok: - failed += 1 + for test_dir, files in sorted(files_by_dir.items()): + exceptions = load_exceptions(test_dir) + + for f in files: + # Check if this test should be skipped + test_filename = f.name.replace('.input.txt', '.input') + if test_filename in exceptions and exceptions[test_filename] == 'u4wrh': + skipped += 1 + continue + + ok = process_file( + f, update_ast=update_ast, update_output=update_output, update_error=update_error, exceptions=exceptions) + if not ok: + failed += 1 - print(f"\nUpdated: {total - failed}, Failed: {failed}") + print(f"\nUpdated: {total - failed - skipped}, Skipped: {skipped}, Failed: {failed}") if failed: sys.exit(1) diff --git a/tools/hrw4u/src/common.py b/tools/hrw4u/src/common.py index c05a4ad40b7..28478933c0f 100644 --- a/tools/hrw4u/src/common.py +++ b/tools/hrw4u/src/common.py @@ -66,6 +66,7 @@ class SystemDefaults: class HeaderOperations: """Operation constants for various resource types""" OPERATIONS: Final = (MagicStrings.RM_HEADER.value, MagicStrings.SET_HEADER.value) + ADD_OPERATION: Final = MagicStrings.ADD_HEADER.value COOKIE_OPERATIONS: Final = (MagicStrings.RM_COOKIE.value, MagicStrings.SET_COOKIE.value) DESTINATION_OPERATIONS: Final = (MagicStrings.RM_DESTINATION.value, MagicStrings.SET_DESTINATION.value) diff --git a/tools/hrw4u/src/generators.py b/tools/hrw4u/src/generators.py index c5924b541e7..31a547cd71a 100644 --- a/tools/hrw4u/src/generators.py +++ b/tools/hrw4u/src/generators.py @@ -40,20 +40,22 @@ def _clean_tag(tag: str) -> str: """Extract clean tag name from %{TAG:payload} format.""" return tag.strip().removeprefix('%{').removesuffix('}').split(':')[0] - def generate_reverse_condition_map(self, condition_map: tuple[tuple[str, tuple], ...]) -> dict[str, str]: + def generate_reverse_condition_map(self, condition_map: tuple[tuple[str, Any], ...]) -> dict[str, str]: """Generate reverse condition mapping from forward condition map.""" reverse_map = {} - for ident_key, (tag, _, _, _, _, _) in condition_map: + for ident_key, params in condition_map: if not ident_key.endswith('.'): - clean_tag = self._clean_tag(tag) - reverse_map[clean_tag] = ident_key + tag = params.target if params else None + if tag: + clean_tag = self._clean_tag(tag) + reverse_map[clean_tag] = ident_key return reverse_map - def generate_reverse_function_map(self, function_map: tuple[tuple[str, tuple], ...]) -> dict[str, str]: + def generate_reverse_function_map(self, function_map: tuple[tuple[str, Any], ...]) -> dict[str, str]: """Generate reverse function mapping from forward function map.""" - return {tag: func_name for func_name, (tag, _) in function_map} + return {params.target: func_name for func_name, params in function_map} @cache def generate_section_hook_mapping(self) -> dict[str, str]: @@ -70,7 +72,9 @@ def generate_ip_mapping(self) -> dict[str, str]: from hrw4u.tables import CONDITION_MAP ip_mapping = {} - for condition_key, (tag, *_, reverse_info) in CONDITION_MAP.items(): + for condition_key, params in CONDITION_MAP.items(): + tag = params.target if params else None + reverse_info = params.rev if params else None if reverse_info and reverse_info.get("reverse_tag") == "IP": payload = reverse_info.get("reverse_payload") if payload: @@ -162,7 +166,7 @@ def get_reverse_condition_map(condition_map: dict[str, tuple]) -> dict[str, str] return _table_generator.generate_reverse_condition_map(tuple(condition_map.items())) -def get_reverse_function_map(function_map: dict[str, tuple]) -> dict[str, str]: +def get_reverse_function_map(function_map: dict[str, Any]) -> dict[str, str]: """Get reverse function mapping.""" return _table_generator.generate_reverse_function_map(tuple(function_map.items())) diff --git a/tools/hrw4u/src/hrw_symbols.py b/tools/hrw4u/src/hrw_symbols.py index b25049cff05..34870279775 100644 --- a/tools/hrw4u/src/hrw_symbols.py +++ b/tools/hrw4u/src/hrw_symbols.py @@ -42,8 +42,9 @@ def _rev_conditions_exact(self) -> dict[str, str]: return reverse_map # Fallback to building from condition map if not available result = {} - for ident_key, (tag, _, uppercase, *_) in self._condition_map.items(): + for ident_key, params in self._condition_map.items(): if not ident_key.endswith("."): + tag = params.target tag_key = tag.strip().removeprefix("%{").removesuffix("}").split(":", 1)[0] result[tag_key] = ident_key return result @@ -55,8 +56,10 @@ def _rev_conditions_prefix(self) -> list[tuple[str, str, bool]]: return reverse_map # Fallback to building from condition map if not available result = [] - for ident_key, (tag, _, uppercase, *_) in self._condition_map.items(): + for ident_key, params in self._condition_map.items(): if ident_key.endswith("."): + tag = params.target + uppercase = params.upper if params else False result.append((tag, ident_key, uppercase)) return result @@ -65,7 +68,7 @@ def _rev_functions(self) -> dict[str, str]: """Cached reverse function mapping.""" if reverse_map := tables.REVERSE_RESOLUTION_MAP.get('FUNCTIONS'): return reverse_map - return {tag: fn_name for fn_name, (tag, _) in self._function_map.items()} + return {params.target: fn_name for fn_name, params in self._function_map.items()} @cached_property def _rev_sections(self) -> dict[str, str]: @@ -138,8 +141,10 @@ def _resolve_ambiguous_exact(self, tag: str, section: SectionType | None) -> str elif tag == "IP": return None - for key, (mapped_tag, _, _, restricted, _, _) in self._condition_map.items(): + for key, params in self._condition_map.items(): + mapped_tag = params.target tag_part = mapped_tag.replace("%{", "").replace("}", "").split(":")[0] + restricted = params.sections if params else None if tag_part == tag: if not restricted or not section or section not in restricted: pass @@ -279,12 +284,15 @@ def _handle_statement_function(self, name: str, args: list[str], section: Sectio qargs = [status_code, self._rewrite_inline_percents(f'"{url_arg}"', section)] elif name == "add-header" and args: + # Convert add-header command to += syntax for reverse mapping header_name = args[0] prefix = self.get_prefix_for_context("header_ops", section) prefixed_header = f"{prefix}{header_name}" - processed_args = [self._rewrite_inline_percents(arg, section) for arg in args[1:]] - qargs = [prefixed_header] + processed_args + if len(args) > 1: + value = self._rewrite_inline_percents(args[1], section) + return f"{prefixed_header} += {value}" + raise SymbolResolutionError("add-header", "Missing value for add-header") elif name == "set-plugin-cntl" and len(args) >= 2: qualifier = args[0] value = args[1] @@ -472,12 +480,14 @@ def op_to_hrw4u(self, cmd: str, args: list[str], section: SectionType | None, op rewritten_value = self._rewrite_inline_percents(value, section) return f"{var_name} = {rewritten_value}" - for lhs_key, (commands, _, uppercase, _) in tables.OPERATOR_MAP.items(): + for lhs_key, params in tables.OPERATOR_MAP.items(): + commands = params.target if params else None if (isinstance(commands, (list, tuple)) and cmd in commands) or (cmd == commands): + uppercase = params.upper if params else False return self._handle_operator_command(cmd, toks, lhs_key, uppercase, section) - for name, (forward_cmd, _) in tables.STATEMENT_FUNCTION_MAP.items(): - if forward_cmd == cmd: + for name, params in tables.STATEMENT_FUNCTION_MAP.items(): + if params.target == cmd: return self._handle_statement_function(name, args, section, op_state) raise SymbolResolutionError(line, f"Unknown operator: {cmd}") diff --git a/tools/hrw4u/src/hrw_visitor.py b/tools/hrw4u/src/hrw_visitor.py index d08c343b40d..149b139dfe5 100644 --- a/tools/hrw4u/src/hrw_visitor.py +++ b/tools/hrw4u/src/hrw_visitor.py @@ -54,8 +54,9 @@ def __init__( self.symbol_resolver = InverseSymbolResolver() self._section_opened = False - self._in_if_block = False + self._if_depth = 0 # Track nesting depth of if blocks self._in_elif_mode = False + self._just_closed_nested = False @lru_cache(maxsize=128) def _cached_percent_parsing(self, pct_text: str) -> tuple[str, str | None]: @@ -87,10 +88,10 @@ def _start_new_section(self, section_type: SectionType) -> None: with self.debug_context(f"start_section {section_type.value}"): if self._section_opened and self._section_label == section_type: self.debug(f"continuing existing section") - if self._in_if_block: + while self._if_depth > 0: self.decrease_indent() self.emit("}") - self._in_if_block = False + self._if_depth -= 1 self._reset_condition_state() if self.output and self.output[-1] != "": self.output.append("") @@ -171,6 +172,20 @@ def visitCommentLine(self, ctx: u4wrhParser.CommentLineContext) -> None: else: self.output.append(comment_text) + def visitIfLine(self, ctx: u4wrhParser.IfLineContext) -> None: + """Handle if operator (starts nested conditional).""" + with self.debug_context("visitIfLine"): + self._flush_pending_condition() + self._just_closed_nested = False + return None + + def visitEndifLine(self, ctx: u4wrhParser.EndifLineContext) -> None: + """Handle endif operator (closes nested conditional).""" + with self.debug_context("visitEndifLine"): + self._close_if_block() + self._just_closed_nested = True + return None + def visitElifLine(self, ctx: u4wrhParser.ElifLineContext) -> None: """Handle elif line transitions.""" with self.debug_context("visitElifLine"): @@ -357,10 +372,10 @@ def visitOpLine(self, ctx: u4wrhParser.OpLineContext) -> None: # Condition block lifecycle methods - specific to inverse visitor def _close_if_block(self) -> None: """Close open if block.""" - if self._in_if_block: + if self._if_depth > 0: self.decrease_indent() self.emit("}") - self._in_if_block = False + self._if_depth -= 1 def _close_section(self) -> None: """Close open section.""" @@ -371,7 +386,8 @@ def _close_section(self) -> None: def _close_if_and_section(self) -> None: """Close open if blocks and sections.""" - self._close_if_block() + while self._if_depth > 0: + self._close_if_block() self._close_section() self._in_elif_mode = False @@ -384,27 +400,22 @@ def _ensure_section_open(self, section_label: SectionType) -> None: def _start_elif_mode(self) -> None: """Handle elif line transitions.""" - if self._in_if_block: + # After endif, we need to close the parent if-statement + if self._if_depth > 0: self.decrease_indent() - self._in_if_block = False + self._if_depth -= 1 self._in_elif_mode = True + self._just_closed_nested = False def _handle_else_transition(self) -> None: """Handle else line transitions.""" - if self._in_if_block: + if self._if_depth > 0: self.decrease_indent() - - if self.output and self.output[-1].strip() == "}": - self.output[-1] = self.format_with_indent("} else {", self.current_indent) - else: - self.emit("} else {") - - self._in_if_block = True - self.increase_indent() - else: - self.emit("else {") - self.increase_indent() - self._in_if_block = True + self._if_depth -= 1 + self.emit("} else {") + self._if_depth += 1 + self.increase_indent() + self._just_closed_nested = False def _start_if_block(self, condition_expr: str) -> None: """Start a new if block.""" @@ -414,5 +425,5 @@ def _start_if_block(self, condition_expr: str) -> None: else: self.emit(f"if {condition_expr} {{") - self._in_if_block = True + self._if_depth += 1 self.increase_indent() diff --git a/tools/hrw4u/src/kg_visitor.py b/tools/hrw4u/src/kg_visitor.py index 78178dadeb5..8b4b88929a8 100644 --- a/tools/hrw4u/src/kg_visitor.py +++ b/tools/hrw4u/src/kg_visitor.py @@ -151,48 +151,55 @@ def _create_semantic_knowledge_nodes(self) -> None: "description": f"Apache Traffic Server hook for {hrw4u_section}" }, f"ats_hook:{ats_hook}") - for op_pattern, (command, validator, uppercase, restricted_sections) in OPERATOR_MAP.items(): + for op_pattern, params in OPERATOR_MAP.items(): + validator = params.validate if params else None + restricted_sections = params.sections if params else None + command = params.target if params else None self._add_node( "SemanticOperator", { "pattern": op_pattern, "hrw_operator": str(command) if isinstance(command, str) else str(command), - "validates_uppercase": uppercase, + "validates_uppercase": params.upper if params else False, "has_validator": validator is not None, "restricted_sections": [s.value for s in restricted_sections] if restricted_sections else None, "description": f"Operator pattern {op_pattern} -> {command}" }, f"sem_op:{op_pattern}") - for cond_pattern, (tag, validator, uppercase, restricted, default_expr, reverse_info) in CONDITION_MAP.items(): + for cond_pattern, params in CONDITION_MAP.items(): + validator = params.validate if params else None + restricted = params.sections if params else None + reverse_info = params.rev if params else None + tag = params.target if params else None self._add_node( "SemanticCondition", { "pattern": cond_pattern, "hrw_condition": tag, - "validates_uppercase": uppercase, + "validates_uppercase": params.upper if params else False, "has_validator": validator is not None, "restricted_sections": [s.value for s in restricted] if restricted else None, - "has_default_expression": default_expr, + "has_default_expression": params.prefix if params else False, "reverse_mapping": reverse_info, "description": f"Condition pattern {cond_pattern} -> {tag}" }, f"sem_cond:{cond_pattern}") - for func_name, (tag, validator) in FUNCTION_MAP.items(): + for func_name, params in FUNCTION_MAP.items(): self._add_node( "SemanticFunction", { "name": func_name, - "hrw_condition": tag, - "has_validator": validator is not None, + "hrw_condition": params.target, + "has_validator": params.validate is not None, "type": "condition_function", - "description": f"Function {func_name} -> %{{{tag}}}" + "description": f"Function {func_name} -> %{{{params.target}}}" }, f"sem_func:{func_name}") - for func_name, (command, validator) in STATEMENT_FUNCTION_MAP.items(): + for func_name, params in STATEMENT_FUNCTION_MAP.items(): self._add_node( "SemanticFunction", { "name": func_name, - "hrw_operator": command, - "has_validator": validator is not None, + "hrw_operator": params.target, + "has_validator": params.validate is not None, "type": "statement_function", - "description": f"Statement function {func_name} -> {command}" + "description": f"Statement function {func_name} -> {params.target}" }, f"sem_stmt_func:{func_name}") for suffix_group in SuffixGroup: diff --git a/tools/hrw4u/src/lsp/completions.py b/tools/hrw4u/src/lsp/completions.py index ab13ad6abcf..1e8cf041974 100644 --- a/tools/hrw4u/src/lsp/completions.py +++ b/tools/hrw4u/src/lsp/completions.py @@ -224,18 +224,22 @@ def get_operator_completions(self, base_prefix: str, current_section: SectionTyp seen_labels = set() # Add condition completions - for key, (tag, _, _, sections, _, _) in tables.CONDITION_MAP.items(): + for key, params in tables.CONDITION_MAP.items(): if key.startswith(base_prefix) and key not in seen_labels: seen_labels.add(key) + sections = params.sections if params else None + tag = params.target if params else None item = self.builder.condition_completion(key, tag, sections, current_section, replacement_range) if item: completions.append(item.to_lsp_dict()) # Add operator completions - for key, (commands, _, _, sections) in tables.OPERATOR_MAP.items(): + for key, params in tables.OPERATOR_MAP.items(): if key.startswith(base_prefix) and key not in seen_labels: seen_labels.add(key) + sections = params.sections if params else None + commands = params.target if params else None item = self.builder.operator_completion(key, commands, sections, current_section, replacement_range) if item: @@ -248,13 +252,13 @@ def get_function_completions(self) -> list[dict[str, Any]]: completions = [] # Regular functions - for func_name, (tag, _) in tables.FUNCTION_MAP.items(): - item = self.builder.function_completion(func_name, tag, "Function") + for func_name, params in tables.FUNCTION_MAP.items(): + item = self.builder.function_completion(func_name, params.target, "Function") completions.append(item.to_lsp_dict()) # Statement functions - for func_name, (tag, _) in tables.STATEMENT_FUNCTION_MAP.items(): - item = self.builder.function_completion(func_name, tag, "Statement") + for func_name, params in tables.STATEMENT_FUNCTION_MAP.items(): + item = self.builder.function_completion(func_name, params.target, "Statement") completions.append(item.to_lsp_dict()) return completions diff --git a/tools/hrw4u/src/lsp/hover.py b/tools/hrw4u/src/lsp/hover.py index 343fc2545d3..6a6f6c92c51 100644 --- a/tools/hrw4u/src/lsp/hover.py +++ b/tools/hrw4u/src/lsp/hover.py @@ -403,8 +403,15 @@ def get_operator_hover_info(operator: str) -> Dict[str, Any]: # Check exact matches first if operator in OPERATOR_MAP: - commands, _, is_prefix, sections = OPERATOR_MAP[operator] - cmd_str = commands if isinstance(commands, str) else ' / '.join(commands) + params = OPERATOR_MAP[operator] + commands = params.target if params else None + if isinstance(commands, str): + cmd_str = commands + elif commands: + cmd_str = ' / '.join(commands) + else: + cmd_str = "unknown" + sections = params.sections if params else None section_info = "" if sections: @@ -415,10 +422,17 @@ def get_operator_hover_info(operator: str) -> Dict[str, Any]: f"**{operator}** - HRW4U Operator\n\n" + f"**Maps to:** `{cmd_str}`{section_info}") # Check prefix matches - for key, (commands, _, is_prefix, sections) in OPERATOR_MAP.items(): - if is_prefix and operator.startswith(key): - cmd_str = commands if isinstance(commands, str) else ' / '.join(commands) + for key, params in OPERATOR_MAP.items(): + if key.endswith('.') and operator.startswith(key): + commands = params.target if params else None + if isinstance(commands, str): + cmd_str = commands + elif commands: + cmd_str = ' / '.join(commands) + else: + cmd_str = "unknown" suffix = operator[len(key):] + sections = params.sections if params else None section_info = "" if sections: @@ -431,7 +445,9 @@ def get_operator_hover_info(operator: str) -> Dict[str, Any]: # Check condition map if operator in CONDITION_MAP: - tag, _, is_prefix, sections, _, _ = CONDITION_MAP[operator] + params = CONDITION_MAP[operator] + tag = params.target if params else None + sections = params.sections if params else None section_info = "" if sections: @@ -442,9 +458,11 @@ def get_operator_hover_info(operator: str) -> Dict[str, Any]: f"**{operator}** - HRW4U Condition\n\n" + f"**Maps to:** `{tag}`{section_info}") # Check condition prefix matches - for key, (tag, _, is_prefix, sections, is_conditional, _) in CONDITION_MAP.items(): - if is_prefix and operator.startswith(key): + for key, params in CONDITION_MAP.items(): + if key.endswith('.') and operator.startswith(key): + tag = params.target if params else None suffix = operator[len(key):] + sections = params.sections if params else None section_info = "" if sections: @@ -581,14 +599,15 @@ def get_function_hover_info(function_name: str) -> Dict[str, Any]: # Fallback to basic documentation if function_name in FUNCTION_MAP: - tag, _ = FUNCTION_MAP[function_name] + params = FUNCTION_MAP[function_name] return HoverInfoProvider.create_hover_info( - f"**{function_name}()** - HRW4U Function\n\n" + f"**Maps to:** `{tag}`\n\n" + f"Used in conditional expressions.") + f"**{function_name}()** - HRW4U Function\n\n" + f"**Maps to:** `{params.target}`\n\n" + + f"Used in conditional expressions.") if function_name in STATEMENT_FUNCTION_MAP: - tag, _ = STATEMENT_FUNCTION_MAP[function_name] + params = STATEMENT_FUNCTION_MAP[function_name] return HoverInfoProvider.create_hover_info( - f"**{function_name}()** - HRW4U Statement Function\n\n" + f"**Maps to:** `{tag}`\n\n" + + f"**{function_name}()** - HRW4U Statement Function\n\n" + f"**Maps to:** `{params.target}`\n\n" + f"Used as statements in code blocks.") return HoverInfoProvider.create_hover_info(f"**{function_name}()** - Unknown HRW4U function") diff --git a/tools/hrw4u/src/suggestions.py b/tools/hrw4u/src/suggestions.py index c1570e847ae..13ce94c30b1 100644 --- a/tools/hrw4u/src/suggestions.py +++ b/tools/hrw4u/src/suggestions.py @@ -85,11 +85,12 @@ def _get_contextual_symbols(self, context_type: str, section: SectionType | None def _is_symbol_valid_in_section(self, symbol: str, section: SectionType, context_type: str) -> bool: table_map = tables.OPERATOR_MAP if context_type == 'assignment' else tables.CONDITION_MAP - tuple_index = 3 if context_type == 'assignment' else 3 - for key, data in table_map.items(): - if (key == symbol or key == f"{symbol}.") and data[tuple_index]: - return section in data[tuple_index] + for key, params in table_map.items(): + if key == symbol or key == f"{symbol}.": + sections = params.sections if params else None + if sections: + return section in sections return True diff --git a/tools/hrw4u/src/symbols.py b/tools/hrw4u/src/symbols.py index 1d88305da89..c40010c64dc 100644 --- a/tools/hrw4u/src/symbols.py +++ b/tools/hrw4u/src/symbols.py @@ -39,11 +39,11 @@ def symbol_for(self, name: str) -> types.Symbol | None: def get_statement_spec(self, name: str) -> tuple[str, Callable[[str], None] | None]: # Use cached lookup from base class - if result := self._lookup_statement_function_cached(name): - return result + if params := self._lookup_statement_function_cached(name): + return params.target, params.validate raise SymbolResolutionError(name, "Unknown operator or invalid standalone use") - def declare_variable(self, name: str, type_name: str) -> str: + def declare_variable(self, name: str, type_name: str, explicit_slot: int | None = None) -> str: try: var_type = types.VarType.from_str(type_name) except ValueError as e: @@ -51,44 +51,54 @@ def declare_variable(self, name: str, type_name: str) -> str: error.add_note(f"Available types: {', '.join([vt.name for vt in types.VarType])}") raise error - if self._var_counter[var_type] >= var_type.limit: - error = SymbolResolutionError(name, f"Too many '{type_name}' variables (max {var_type.limit})") - error.add_note(f"Current count: {self._var_counter[var_type]}") - raise error + # Determine slot number + if explicit_slot is not None: + if explicit_slot < 0 or explicit_slot >= var_type.limit: + raise SymbolResolutionError( + name, f"Slot @{explicit_slot} out of range for type '{type_name}' (valid: 0-{var_type.limit-1})") + for var_name, sym in self._symbols.items(): + if sym.var_type == var_type and sym.slot == explicit_slot: + raise SymbolResolutionError(name, f"Slot @{explicit_slot} already used by variable '{var_name}'") + + slot = explicit_slot + else: + used_slots = {sym.slot for sym in self._symbols.values() if sym.var_type == var_type} + slot = next((i for i in range(var_type.limit) if i not in used_slots), None) - symbol = types.Symbol(var_type, self._var_counter[var_type]) - self._var_counter[var_type] += 1 + if slot is None: + raise SymbolResolutionError(name, f"No available slots for type '{type_name}' (max {var_type.limit})") + + symbol = types.Symbol(var_type, slot) self._symbols[name] = symbol return symbol.as_cond() def resolve_assignment(self, name: str, value: str, section: SectionType | None = None) -> str: with self.debug_context("resolve_assignment", name, value, section): - for op_key, (commands, validator, uppercase, restricted_sections) in self._operator_map.items(): + for op_key, params in self._operator_map.items(): if op_key.endswith("."): if name.startswith(op_key): - self.validate_section_access(name, section, restricted_sections) + self.validate_section_access(name, section, params.sections if params else None) qualifier = name[len(op_key):] - if uppercase: + if params and params.upper: qualifier = qualifier.upper() - if validator: - validator(qualifier) + if params and params.validate: + params.validate(qualifier) - # Add boolean value validation for http.cntl assignments. + # Add boolean value validation for http.cntl assignments if op_key == "http.cntl.": types.SuffixGroup.BOOL_FIELDS.validate(value) + commands = params.target if params else None if isinstance(commands, (list, tuple)): - if value == '""': - return f"{commands[0]} {qualifier}" - else: - return f"{commands[1]} {qualifier} {value}" - else: - return f"{commands} {qualifier} {value}" + return f"{commands[0 if value == '\"\"' else 1]} {qualifier}" + ("" if value == '""' else f" {value}") + return f"{commands} {qualifier} {value}" + elif name == op_key: - self.validate_section_access(name, section, restricted_sections) - if validator: - validator(value) - return f"{commands} {value}" + # Exact match - validate and return + self.validate_section_access(name, section, params.sections if params else None) + if params and params.validate: + params.validate(value) + return f"{params.target if params else None} {value}" if resolved_lhs := self.symbol_for(name): if resolved_rhs := self.symbol_for(value): @@ -105,26 +115,51 @@ def resolve_assignment(self, name: str, value: str, section: SectionType | None error.add_symbol_suggestion(suggestions) raise error + def resolve_add_assignment(self, name: str, value: str, section: SectionType | None = None) -> str: + """Resolve += assignment, if it is supported for the given operator.""" + with self.debug_context("resolve_add_assignment", name, value, section): + for op_key, params in self._operator_map.items(): + if op_key.endswith(".") and name.startswith(op_key) and params and params.add: + self.validate_section_access(name, section, params.sections) + qualifier = name[len(op_key):] + if params.validate: + params.validate(qualifier) + + from hrw4u.common import HeaderOperations + return f"{HeaderOperations.ADD_OPERATION} {qualifier} {value}" + + # += not allowed if no matching operator with 'add' flag found + error = SymbolResolutionError(name, "+= operator is not supported for this assignment") + error.add_note("Only operators with 'add' flag support +=") + raise error + def resolve_condition(self, name: str, section: SectionType | None = None) -> tuple[str, bool]: with self.debug_context("resolve_condition", name, section): if symbol := self.symbol_for(name): return symbol.as_cond(), False - if condition_info := self._lookup_condition_cached(name): - tag, _, _, restricted, default_expr, _ = condition_info + if params := self._lookup_condition_cached(name): + tag = params.target if params else None + restricted = params.sections if params else None self.validate_section_access(name, section, restricted) - return tag, default_expr + # For exact matches, default_expr is determined by whether it's a prefix pattern + return tag, False # Check prefix matches using base class utility prefix_matches = self.find_prefix_matches(name, self._condition_map) - for prefix, (tag, validator, uppercase, restricted, default_expr, _) in prefix_matches: + for prefix, params in prefix_matches: + tag = params.target if params else None + validator = params.validate if params else None + restricted = params.sections if params else None + self.validate_section_access(name, section, restricted) suffix = name[len(prefix):] - suffix_norm = suffix.upper() if uppercase else suffix + suffix_norm = suffix.upper() if (params and params.upper) else suffix if validator: validator(suffix_norm) resolved = f"%{{{tag}:{suffix_norm}}}" - return resolved, default_expr + # For prefix matches, default_expr is True (indicated by prefix flag) + return resolved, (params.prefix if params else False) error = SymbolResolutionError(name, "Unknown condition symbol") declared_vars = list(self._symbols.keys()) @@ -135,8 +170,9 @@ def resolve_condition(self, name: str, section: SectionType | None = None) -> tu def resolve_function(self, func_name: str, args: list[str], strip_quotes: bool = False) -> str: with self.debug_context("resolve_function", func_name, args): - if function_info := self._lookup_function_cached(func_name): - tag, validator = function_info + if params := self._lookup_function_cached(func_name): + tag = params.target + validator = params.validate if validator: validator(args) @@ -156,8 +192,9 @@ def resolve_function(self, func_name: str, args: list[str], strip_quotes: bool = def resolve_statement_func(self, func_name: str, args: list[str]) -> str: with self.debug_context("resolve_statement_func", func_name, args): - if function_info := self._lookup_statement_function_cached(func_name): - command, validator = function_info + if params := self._lookup_statement_function_cached(func_name): + command = params.target + validator = params.validate if validator: validator(args) diff --git a/tools/hrw4u/src/symbols_base.py b/tools/hrw4u/src/symbols_base.py index c0103ea9a0b..57490650744 100644 --- a/tools/hrw4u/src/symbols_base.py +++ b/tools/hrw4u/src/symbols_base.py @@ -38,22 +38,19 @@ def __init__(self, debug: bool = SystemDefaults.DEFAULT_DEBUG) -> None: # Cached table access for performance - Python 3.11+ cached_property @cached_property - def _condition_map( - self) -> dict[str, tuple[str, Callable[[str], None] | None, bool, set[SectionType] | None, bool, dict | None]]: + def _condition_map(self) -> dict[str, types.MapParams]: return tables.CONDITION_MAP @cached_property - def _operator_map( - self - ) -> dict[str, tuple[str | list[str] | tuple[str, ...], Callable[[str], None] | None, bool, set[SectionType] | None]]: + def _operator_map(self) -> dict[str, types.MapParams]: return tables.OPERATOR_MAP @cached_property - def _function_map(self) -> dict[str, tuple[str, Callable[[list[str]], None] | None]]: + def _function_map(self) -> dict[str, types.MapParams]: return tables.FUNCTION_MAP @cached_property - def _statement_function_map(self) -> dict[str, tuple[str, Callable[[list[str]], None] | None]]: + def _statement_function_map(self) -> dict[str, types.MapParams]: return tables.STATEMENT_FUNCTION_MAP @cached_property @@ -65,22 +62,19 @@ def validate_section_access(self, name: str, section: SectionType | None, restri raise SymbolResolutionError(name, f"{name} is not available in the {section.value} section") @lru_cache(maxsize=256) - def _lookup_condition_cached( - self, name: str) -> tuple[str, Callable[[str], None] | None, bool, set[SectionType] | None, bool, dict | None] | None: + def _lookup_condition_cached(self, name: str) -> types.MapParams | None: return self._condition_map.get(name) @lru_cache(maxsize=256) - def _lookup_operator_cached( - self, name: str - ) -> tuple[str | list[str] | tuple[str, ...], Callable[[str], None] | None, bool, set[SectionType] | None] | None: + def _lookup_operator_cached(self, name: str) -> types.MapParams | None: return self._operator_map.get(name) @lru_cache(maxsize=128) - def _lookup_function_cached(self, name: str) -> tuple[str, Callable[[list[str]], None] | None] | None: + def _lookup_function_cached(self, name: str) -> types.MapParams | None: return self._function_map.get(name) @lru_cache(maxsize=128) - def _lookup_statement_function_cached(self, name: str) -> tuple[str, Callable[[list[str]], None] | None] | None: + def _lookup_statement_function_cached(self, name: str) -> types.MapParams | None: return self._statement_function_map.get(name) def _debug_enter(self, method_name: str, *args: Any) -> None: diff --git a/tools/hrw4u/src/tables.py b/tools/hrw4u/src/tables.py index 7454f692761..85d93394ba8 100644 --- a/tools/hrw4u/src/tables.py +++ b/tools/hrw4u/src/tables.py @@ -20,274 +20,102 @@ from dataclasses import dataclass from hrw4u.generators import get_complete_reverse_resolution_map from hrw4u.validation import Validator -import hrw4u.types as types +from hrw4u.types import MapParams, SuffixGroup from hrw4u.states import SectionType from hrw4u.common import HeaderOperations -OPERATOR_MAP: dict[str, tuple[str | list[str] | tuple[str, ...], Callable[[str], None] | None, bool, set[SectionType] | None]] = { - "http.cntl.": ("set-http-cntl", Validator.suffix_group(types.SuffixGroup.HTTP_CNTL_FIELDS), True, None), - "http.status.reason": ("set-status-reason", Validator.quoted_or_simple(), False, None), - "http.status": ("set-status", Validator.range(0, 999), False, None), - "inbound.conn.dscp": ("set-conn-dscp", Validator.nbit_int(6), False, None), - "inbound.conn.mark": ("set-conn-mark", Validator.nbit_int(32), False, None), - "outbound.conn.dscp": - ("set-conn-dscp", Validator.nbit_int(6), False, {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), - "outbound.conn.mark": - ("set-conn-mark", Validator.nbit_int(32), False, {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), - "inbound.cookie.": (HeaderOperations.COOKIE_OPERATIONS, Validator.http_token(), False, None), - "inbound.req.": (HeaderOperations.OPERATIONS, Validator.http_header_name(), False, None), - "inbound.resp.body": ("set-body", Validator.quoted_or_simple(), False, None), - "inbound.resp.": (HeaderOperations.OPERATIONS, Validator.http_header_name(), False, None), - "inbound.status.reason": ("set-status-reason", Validator.quoted_or_simple(), False, None), - "inbound.status": ("set-status", Validator.range(0, 999), False, None), - "inbound.url.": (HeaderOperations.DESTINATION_OPERATIONS, Validator.suffix_group(types.SuffixGroup.URL_FIELDS), True, None), - "outbound.cookie.": - ( - HeaderOperations.COOKIE_OPERATIONS, Validator.http_token(), False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), - "outbound.req.": - ( - HeaderOperations.OPERATIONS, Validator.http_header_name(), False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), - "outbound.resp.": - ( - HeaderOperations.OPERATIONS, Validator.http_header_name(), False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}), - "outbound.status.reason": - ( - "set-status-reason", Validator.quoted_or_simple(), False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}), - "outbound.status": - ( - "set-status", Validator.range(0, 999), False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}), - "outbound.url.": - ( - HeaderOperations.DESTINATION_OPERATIONS, Validator.suffix_group(types.SuffixGroup.URL_FIELDS), True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}) +# yapf: disable +OPERATOR_MAP: dict[str, MapParams] = { + "http.cntl.": MapParams(target="set-http-cntl", upper=True, validate=Validator.suffix_group(SuffixGroup.HTTP_CNTL_FIELDS)), + "http.status.reason": MapParams(target="set-status-reason", validate=Validator.quoted_or_simple()), + "http.status": MapParams(target="set-status", validate=Validator.range(0, 999)), + "inbound.conn.dscp": MapParams(target="set-conn-dscp", validate=Validator.nbit_int(6)), + "inbound.conn.mark": MapParams(target="set-conn-mark", validate=Validator.nbit_int(32)), + "outbound.conn.dscp": MapParams(target="set-conn-dscp", validate=Validator.nbit_int(6), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.mark": MapParams(target="set-conn-mark", validate=Validator.nbit_int(32), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "inbound.cookie.": MapParams(target=HeaderOperations.COOKIE_OPERATIONS, validate=Validator.http_token()), + "inbound.req.": MapParams(target=HeaderOperations.OPERATIONS, add=True, validate=Validator.http_header_name()), + "inbound.resp.body": MapParams(target="set-body", validate=Validator.quoted_or_simple()), + "inbound.resp.": MapParams(target=HeaderOperations.OPERATIONS, add=True, validate=Validator.http_header_name()), + "inbound.status.reason": MapParams(target="set-status-reason", validate=Validator.quoted_or_simple()), + "inbound.status": MapParams(target="set-status", validate=Validator.range(0, 999)), + "inbound.url.": MapParams(target=HeaderOperations.DESTINATION_OPERATIONS, upper=True, validate=Validator.suffix_group(SuffixGroup.URL_FIELDS)), + "outbound.cookie.": MapParams(target=HeaderOperations.COOKIE_OPERATIONS, validate=Validator.http_token(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.req.": MapParams(target=HeaderOperations.OPERATIONS, add=True, validate=Validator.http_header_name(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.resp.": MapParams(target=HeaderOperations.OPERATIONS, add=True, validate=Validator.http_header_name(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}), + "outbound.status.reason": MapParams(target="set-status-reason", validate=Validator.quoted_or_simple(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}), + "outbound.status": MapParams(target="set-status", validate=Validator.range(0, 999), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}), + "outbound.url.": MapParams(target=HeaderOperations.DESTINATION_OPERATIONS, upper=True, validate=Validator.suffix_group(SuffixGroup.URL_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}) } -STATEMENT_FUNCTION_MAP: dict[str, tuple[str, Callable[[list[str]], None] | None]] = { - "add-header": - ("add-header", Validator.arg_count(2).arg_at(0, Validator.http_header_name()).arg_at(1, Validator.quoted_or_simple())), - "counter": ("counter", Validator.arg_count(1).quoted_or_simple()), - "set-debug": ("set-debug", Validator.arg_count(0)), - "no-op": ("no-op", Validator.arg_count(0)), - "remove_query": ("rm-destination QUERY", Validator.arg_count(1).quoted_or_simple()), - "keep_query": ("rm-destination QUERY", Validator.arg_count(1).quoted_or_simple()), - "run-plugin": ("run-plugin", Validator.min_args(1).quoted_or_simple()), - "set-body-from": ("set-body-from", Validator.arg_count(1).quoted_or_simple()), - "set-config": ("set-config", Validator.arg_count(2).quoted_or_simple()), - "set-redirect": - ("set-redirect", Validator.arg_count(2).arg_at(0, Validator.range(300, 399)).arg_at(1, Validator.quoted_or_simple())), - "skip-remap": - ("skip-remap", Validator.arg_count(1).suffix_group(types.SuffixGroup.BOOL_FIELDS)._add(Validator.normalize_arg_at(0))), - "set-plugin-cntl": - ( - "set-plugin-cntl", Validator.arg_count(2)._add(Validator.normalize_arg_at(0)).arg_at( - 0, Validator.suffix_group(types.SuffixGroup.PLUGIN_CNTL_FIELDS))._add(Validator.normalize_arg_at(1))._add( - Validator.conditional_arg_validation(types.SuffixGroup.PLUGIN_CNTL_MAPPING.value))), +STATEMENT_FUNCTION_MAP: dict[str, MapParams] = { + "add-header": MapParams(target="add-header", validate=Validator.arg_count(2).arg_at(0, Validator.http_header_name()).arg_at(1, Validator.quoted_or_simple())), + "counter": MapParams(target="counter", validate=Validator.arg_count(1).quoted_or_simple()), + "set-debug": MapParams(target="set-debug", validate=Validator.arg_count(0)), + "no-op": MapParams(target="no-op", validate=Validator.arg_count(0)), + "remove_query": MapParams(target="rm-destination QUERY", validate=Validator.arg_count(1).quoted_or_simple()), + "keep_query": MapParams(target="rm-destination QUERY", validate=Validator.arg_count(1).quoted_or_simple()), + "run-plugin": MapParams(target="run-plugin", validate=Validator.min_args(1).quoted_or_simple()), + "set-body-from": MapParams(target="set-body-from", validate=Validator.arg_count(1).quoted_or_simple()), + "set-config": MapParams(target="set-config", validate=Validator.arg_count(2).quoted_or_simple()), + "set-redirect": MapParams(target="set-redirect", validate=Validator.arg_count(2).arg_at(0, Validator.range(300, 399)).arg_at(1, Validator.quoted_or_simple())), + "skip-remap": MapParams(target="skip-remap", validate=Validator.arg_count(1).suffix_group(SuffixGroup.BOOL_FIELDS)._add(Validator.normalize_arg_at(0))), + "set-plugin-cntl": MapParams(target="set-plugin-cntl", validate=Validator.arg_count(2)._add(Validator.normalize_arg_at(0)).arg_at(0, Validator.suffix_group(SuffixGroup.PLUGIN_CNTL_FIELDS))._add(Validator.normalize_arg_at(1))._add(Validator.conditional_arg_validation(SuffixGroup.PLUGIN_CNTL_MAPPING.value))), } -FUNCTION_MAP = { - "access": ("ACCESS", Validator.arg_count(1).quoted_or_simple()), - "cache": ("CACHE", Validator.arg_count(0)), - "cidr": ("CIDR", Validator.arg_count(2).arg_at(0, Validator.range(1, 32)).arg_at(1, Validator.range(1, 128))), - "internal": ("INTERNAL-TRANSACTION", Validator.arg_count(0)), - "random": ("RANDOM", Validator.arg_count(1).nbit_int(32)), - "ssn-txn-count": ("SSN-TXN-COUNT", Validator.arg_count(0)), - "txn-count": ("TXN-COUNT", Validator.arg_count(0)), +FUNCTION_MAP: dict[str, MapParams] = { + "access": MapParams(target="ACCESS", validate=Validator.arg_count(1).quoted_or_simple()), + "cache": MapParams(target="CACHE", validate=Validator.arg_count(0)), + "cidr": MapParams(target="CIDR", validate=Validator.arg_count(2).arg_at(0, Validator.range(1, 32)).arg_at(1, Validator.range(1, 128))), + "internal": MapParams(target="INTERNAL-TRANSACTION", validate=Validator.arg_count(0)), + "random": MapParams(target="RANDOM", validate=Validator.arg_count(1).nbit_int(32)), + "ssn-txn-count": MapParams(target="SSN-TXN-COUNT", validate=Validator.arg_count(0)), + "txn-count": MapParams(target="TXN-COUNT", validate=Validator.arg_count(0)), } -CONDITION_MAP: dict[str, tuple[str, Callable[[str], None] | None, bool, set[SectionType] | None, bool, dict | None]] = { +CONDITION_MAP: dict[str, MapParams] = { # Exact matches with reverse mapping info - "inbound.ip": ("%{IP:CLIENT}", None, False, None, False, { - "reverse_tag": "IP", - "reverse_payload": "CLIENT" - }), - "inbound.method": ("%{METHOD}", None, False, None, False, { - "reverse_tag": "METHOD", - "ambiguous": True - }), - "inbound.server": ("%{IP:INBOUND}", None, False, None, False, { - "reverse_tag": "IP", - "reverse_payload": "INBOUND" - }), - "inbound.status": ("%{STATUS}", None, False, None, False, { - "reverse_tag": "STATUS", - "ambiguous": True - }), - "now": ("%{NOW}", None, False, None, False, None), - "outbound.ip": - ( - "%{IP:SERVER}", - None, - False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - False, - { - "reverse_tag": "IP", - "reverse_payload": "SERVER" - }, - ), - "outbound.method": - ( - "%{METHOD}", - None, - False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - False, - { - "reverse_tag": "METHOD", - "ambiguous": True - }, - ), - "outbound.server": - ( - "%{IP:OUTBOUND}", - None, - False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - False, - { - "reverse_tag": "IP", - "reverse_payload": "OUTBOUND" - }, - ), - "outbound.status": - ( - "%{STATUS}", - None, - False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}, - False, - { - "reverse_tag": "STATUS", - "ambiguous": True - }, - ), - "tcp.info": ("%{TCP-INFO}", None, False, None, False, None), + "inbound.ip": MapParams(target="%{IP:CLIENT}", rev={"reverse_tag": "IP", "reverse_payload": "CLIENT"}), + "inbound.method": MapParams(target="%{METHOD}", rev={"reverse_tag": "METHOD", "ambiguous": True}), + "inbound.server": MapParams(target="%{IP:INBOUND}", rev={"reverse_tag": "IP", "reverse_payload": "INBOUND"}), + "inbound.status": MapParams(target="%{STATUS}", rev={"reverse_tag": "STATUS", "ambiguous": True}), + "now": MapParams(target="%{NOW}"), + "outbound.ip": MapParams(target="%{IP:SERVER}", sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, rev={"reverse_tag": "IP", "reverse_payload": "SERVER"}), + "outbound.method": MapParams(target="%{METHOD}", sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, rev={"reverse_tag": "METHOD", "ambiguous": True}), + "outbound.server": MapParams(target="%{IP:OUTBOUND}", sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, rev={"reverse_tag": "IP", "reverse_payload": "OUTBOUND"}), + "outbound.status": MapParams(target="%{STATUS}", sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}, rev={"reverse_tag": "STATUS", "ambiguous": True}), + "tcp.info": MapParams(target="%{TCP-INFO}"), - # Prefix matches with reverse mapping info - "capture.": ("LAST-CAPTURE", Validator.range(0, 9), False, None, True, None), - "from.url.": ("FROM-URL", Validator.suffix_group(types.SuffixGroup.URL_FIELDS), True, None, True, None), - "geo.": ("GEO", Validator.suffix_group(types.SuffixGroup.GEO_FIELDS), True, None, True, None), - "http.cntl.": ("HTTP-CNTL", Validator.suffix_group(types.SuffixGroup.HTTP_CNTL_FIELDS), True, None, False, None), - "id.": ("ID", Validator.suffix_group(types.SuffixGroup.ID_FIELDS), True, None, False, None), - "inbound.conn.client-cert.SAN.": - ("INBOUND:CLIENT-CERT:SAN", Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), True, None, True, None), - "inbound.conn.server-cert.SAN.": - ("INBOUND:SERVER-CERT:SAN", Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), True, None, True, None), - "inbound.conn.client-cert.san.": - ("INBOUND:CLIENT-CERT:SAN", Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), True, None, True, None), - "inbound.conn.server-cert.san.": - ("INBOUND:SERVER-CERT:SAN", Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), True, None, True, None), - "inbound.conn.client-cert.": - ("INBOUND:CLIENT-CERT", Validator.suffix_group(types.SuffixGroup.CERT_FIELDS), True, None, True, None), - "inbound.conn.server-cert.": - ("INBOUND:SERVER-CERT", Validator.suffix_group(types.SuffixGroup.CERT_FIELDS), True, None, True, None), - "inbound.conn.": ("INBOUND", Validator.suffix_group(types.SuffixGroup.CONN_FIELDS), True, None, True, None), - "inbound.cookie.": ("COOKIE", Validator.http_token(), False, None, True, { - "reverse_fallback": "inbound.cookie." - }), - "inbound.req.": ("CLIENT-HEADER", Validator.http_header_name(), False, None, True, { - "reverse_fallback": "inbound.req." - }), - "inbound.resp.": ("HEADER", Validator.http_header_name(), False, None, True, { - "reverse_context": "header_condition" - }), - "inbound.url.": ("CLIENT-URL", Validator.suffix_group(types.SuffixGroup.URL_FIELDS), True, None, True, None), - "now.": ("NOW", Validator.suffix_group(types.SuffixGroup.DATE_FIELDS), True, None, False, None), - "outbound.conn.client-cert.SAN.": - ( - "OUTBOUND:CLIENT-CERT:SAN", - Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "outbound.conn.server-cert.SAN.": - ( - "OUTBOUND:SERVER-CERT:SAN", - Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "outbound.conn.client-cert.san.": - ( - "OUTBOUND:CLIENT-CERT:SAN", - Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "outbound.conn.server-cert.san.": - ( - "OUTBOUND:SERVER-CERT:SAN", - Validator.suffix_group(types.SuffixGroup.SAN_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "outbound.conn.client-cert.": - ( - "OUTBOUND:CLIENT-CERT", - Validator.suffix_group(types.SuffixGroup.CERT_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "outbound.conn.server-cert.": - ( - "OUTBOUND:SERVER-CERT", - Validator.suffix_group(types.SuffixGroup.CERT_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "outbound.conn.": - ( - "OUTBOUND", Validator.suffix_group(types.SuffixGroup.CONN_FIELDS), True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, True, None), - "outbound.cookie.": - ( - "COOKIE", Validator.http_token(), False, {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, True, { - "reverse_fallback": "inbound.cookie." - }), - "outbound.req.": - ( - "HEADER", Validator.http_header_name(), False, {SectionType.PRE_REMAP, SectionType.REMAP, - SectionType.READ_REQUEST}, True, { - "reverse_context": "header_condition" - }), - "outbound.resp.": - ( - "HEADER", - Validator.http_header_name(), - False, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}, - True, - { - "reverse_context": "header_condition" - }, - ), - "outbound.url.": - ( - "NEXT-HOP", - Validator.suffix_group(types.SuffixGroup.URL_FIELDS), - True, - {SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, - True, - None, - ), - "to.url.": ("TO-URL", Validator.suffix_group(types.SuffixGroup.URL_FIELDS), True, None, True, None), + # Prefix matches + "capture.": MapParams(target="LAST-CAPTURE", prefix=True, validate=Validator.range(0, 9)), + "from.url.": MapParams(target="FROM-URL", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.URL_FIELDS)), + "geo.": MapParams(target="GEO", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.GEO_FIELDS)), + "http.cntl.": MapParams(target="HTTP-CNTL", upper=True, validate=Validator.suffix_group(SuffixGroup.HTTP_CNTL_FIELDS)), + "id.": MapParams(target="ID", upper=True, validate=Validator.suffix_group(SuffixGroup.ID_FIELDS)), + "inbound.conn.client-cert.SAN.": MapParams(target="INBOUND:CLIENT-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS)), + "inbound.conn.server-cert.SAN.": MapParams(target="INBOUND:SERVER-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS)), + "inbound.conn.client-cert.san.": MapParams(target="INBOUND:CLIENT-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS)), + "inbound.conn.server-cert.san.": MapParams(target="INBOUND:SERVER-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS)), + "inbound.conn.client-cert.": MapParams(target="INBOUND:CLIENT-CERT", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.CERT_FIELDS)), + "inbound.conn.server-cert.": MapParams(target="INBOUND:SERVER-CERT", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.CERT_FIELDS)), + "inbound.conn.": MapParams(target="INBOUND", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.CONN_FIELDS)), + "inbound.cookie.": MapParams(target="COOKIE", prefix=True, validate=Validator.http_token(), rev={"reverse_fallback": "inbound.cookie."}), + "inbound.req.": MapParams(target="CLIENT-HEADER", prefix=True, validate=Validator.http_header_name(), rev={"reverse_fallback": "inbound.req."}), + "inbound.resp.": MapParams(target="HEADER", prefix=True, validate=Validator.http_header_name(), rev={"reverse_context": "header_condition"}), + "inbound.url.": MapParams(target="CLIENT-URL", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.URL_FIELDS)), + "now.": MapParams(target="NOW", upper=True, validate=Validator.suffix_group(SuffixGroup.DATE_FIELDS)), + "outbound.conn.client-cert.SAN.": MapParams(target="OUTBOUND:CLIENT-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.server-cert.SAN.": MapParams(target="OUTBOUND:SERVER-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.client-cert.san.": MapParams(target="OUTBOUND:CLIENT-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.server-cert.san.": MapParams(target="OUTBOUND:SERVER-CERT:SAN", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.SAN_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.client-cert.": MapParams(target="OUTBOUND:CLIENT-CERT", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.CERT_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.server-cert.": MapParams(target="OUTBOUND:SERVER-CERT", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.CERT_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.conn.": MapParams(target="OUTBOUND", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.CONN_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "outbound.cookie.": MapParams(target="COOKIE", prefix=True, validate=Validator.http_token(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, rev={"reverse_fallback": "inbound.cookie."}), + "outbound.req.": MapParams(target="HEADER", prefix=True, validate=Validator.http_header_name(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}, rev={"reverse_context": "header_condition"}), + "outbound.resp.": MapParams(target="HEADER", prefix=True, validate=Validator.http_header_name(), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST, SectionType.SEND_REQUEST}, rev={"reverse_context": "header_condition"}), + "outbound.url.": MapParams(target="NEXT-HOP", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.URL_FIELDS), sections={SectionType.PRE_REMAP, SectionType.REMAP, SectionType.READ_REQUEST}), + "to.url.": MapParams(target="TO-URL", upper=True, prefix=True, validate=Validator.suffix_group(SuffixGroup.URL_FIELDS)), } FALLBACK_TAG_MAP: dict[str, tuple[str, bool]] = { @@ -314,6 +142,7 @@ # Operator command mappings for reverse resolution OPERATOR_COMMAND_MAP: dict[str, tuple[str, str, Callable, Callable]] = { + "add-header": ("header_ops", "header", lambda toks: toks[1], lambda qual: qual), "set-header": ("header_ops", "header", lambda toks: toks[1], lambda qual: qual), "rm-header": ("header_ops", "header", lambda toks: toks[1], lambda qual: qual), "set-cookie": ("cookie_ops", "cookie", lambda toks: toks[1], lambda qual: qual), @@ -321,6 +150,7 @@ "set-destination": ("destination_ops", "destination", lambda toks: toks[1].lower(), lambda qual: qual), "rm-destination": ("destination_ops", "destination", lambda toks: toks[1].lower(), lambda qual: qual) } +# yapf: enable REVERSE_RESOLUTION_MAP = get_complete_reverse_resolution_map() @@ -412,23 +242,19 @@ def match_connection_pattern(cls, expression: str) -> PatternMatch | None: @classmethod def match_any_pattern(cls, expression: str) -> PatternMatch | None: """Try to match expression against all pattern types.""" - # Try field patterns first (most specific) + if match := cls.match_field_pattern(expression): return match - # Try certificate patterns if match := cls.match_certificate_pattern(expression): return match - # Try connection patterns if match := cls.match_connection_pattern(expression): return match - # Try header patterns if match := cls.match_header_pattern(expression): return match - # Try cookie patterns if match := cls.match_cookie_pattern(expression): return match diff --git a/tools/hrw4u/src/types.py b/tools/hrw4u/src/types.py index 213e084a59f..0cacafe673b 100644 --- a/tools/hrw4u/src/types.py +++ b/tools/hrw4u/src/types.py @@ -19,10 +19,14 @@ from enum import Enum from dataclasses import dataclass -from typing import Self +from typing import Self, Callable, TYPE_CHECKING + +if TYPE_CHECKING: + from hrw4u.states import SectionType class MagicStrings(str, Enum): + ADD_HEADER = "add-header" RM_HEADER = "rm-header" SET_HEADER = "set-header" RM_COOKIE = "rm-cookie" @@ -159,10 +163,80 @@ def from_str(cls, type_str: str) -> Self: @dataclass(slots=True, frozen=True) class Symbol: var_type: VarType - index: int + slot: int def as_cond(self) -> str: - return f"%{{STATE-{self.var_type.cond_tag}:{self.index}}}" + return f"%{{STATE-{self.var_type.cond_tag}:{self.slot}}}" def as_operator(self, value: str) -> str: - return f"{self.var_type.op_tag} {self.index} {value}" + return f"{self.var_type.op_tag} {self.slot} {value}" + + +class MapParams: + """Map parameters for table entries combining flags and metadata. + """ + + def __init__( + self, + upper: bool = False, + add: bool = False, + prefix: bool = False, + validate: Callable[[str], None] | None = None, + sections: set[SectionType] | None = None, + rev: dict | None = None, + target: str | list[str] | tuple[str, ...] | None = None) -> None: + object.__setattr__( + self, '_params', { + 'upper': upper, + 'add': add, + 'prefix': prefix, + 'validate': validate, + 'sections': sections, + 'rev': rev, + 'target': target + }) + + def __getattr__(self, name: str): + if name.startswith('_'): + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") + return self._params.get(name, False if name in ('upper', 'add', 'prefix') else None) + + def __setattr__(self, name: str, value: object) -> None: + """Prevent modification after initialization (immutable).""" + raise AttributeError(f"'{type(self).__name__}' object is immutable") + + def __repr__(self) -> str: + non_defaults = [] + for k, v in self._params.items(): + if k in ('upper', 'add', 'prefix'): + if v: + non_defaults.append(f"{k}=True") + elif v is not None: + if isinstance(v, set): + non_defaults.append(f"{k}={{{', '.join(str(s) for s in v)}}}") + elif k == 'validate': + non_defaults.append(f"{k}=") + else: + non_defaults.append(f"{k}=...") + + if not non_defaults: + return "MapParams()" + return f"MapParams({', '.join(non_defaults)})" + + def __hash__(self) -> int: + hashable_items = [] + for k, v in self._params.items(): + if isinstance(v, set): + hashable_items.append((k, frozenset(v))) + elif isinstance(v, dict): + hashable_items.append((k, frozenset(v.items()) if v else None)) + elif callable(v): + hashable_items.append((k, id(v))) + else: + hashable_items.append((k, v)) + return hash(frozenset(hashable_items)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MapParams): + return NotImplemented + return self._params == other._params diff --git a/tools/hrw4u/src/visitor.py b/tools/hrw4u/src/visitor.py index 5d35cb4ca6e..a3cbc285da8 100644 --- a/tools/hrw4u/src/visitor.py +++ b/tools/hrw4u/src/visitor.py @@ -356,6 +356,18 @@ def visitStatement(self, ctx) -> None: self.emit_statement(out) return + case _ if ctx.PLUSEQUAL(): + if ctx.lhs is None: + raise SymbolResolutionError("assignment", "Missing left-hand side in += assignment") + lhs = ctx.lhs.text + rhs = ctx.value().getText() + if rhs.startswith('"') and rhs.endswith('"'): + rhs = self._substitute_strings(rhs, ctx) + self._dbg(f"add assignment: {lhs} += {rhs}") + out = self.symbol_resolver.resolve_add_assignment(lhs, rhs, self.current_section) + self.emit_statement(out) + return + case _: if ctx.op is None: raise SymbolResolutionError("operator", "Missing operator in statement") @@ -387,17 +399,21 @@ def visitVariableDecl(self, ctx) -> None: if ctx.typeName is None: raise SymbolResolutionError("variable", "Missing type name in declaration") name = ctx.name.text - type = ctx.typeName.text + type_name = ctx.typeName.text + explicit_slot = int(ctx.slot.text) if ctx.slot else None if '.' in name or ':' in name: raise SymbolResolutionError("variable", f"Variable name '{name}' cannot contain '.' or ':' characters") - symbol = self.symbol_resolver.declare_variable(name, type) - self._dbg(f"bind `{name}' to {symbol}") + symbol = self.symbol_resolver.declare_variable(name, type_name, explicit_slot) + slot_info = f" @{explicit_slot}" if explicit_slot is not None else "" + self._dbg(f"bind `{name}' to {symbol}{slot_info}") except Exception as e: name = getattr(ctx, 'name', None) type_name = getattr(ctx, 'typeName', None) - note = f"Variable declaration: {name.text}:{type_name.text}" if name and type_name else None + slot = getattr(ctx, 'slot', None) + note = f"Variable declaration: {name.text}:{type_name.text}" + \ + (f" @{slot.text}" if slot else "") if name and type_name else None with self.trap(ctx, note=note): raise e return @@ -433,6 +449,15 @@ def visitBlock(self, ctx) -> None: for item in ctx.blockItem(): if item.statement(): self.visit(item.statement()) + elif item.conditional(): + # Nested conditional - emit if/endif operators with saved state + self.emit_statement("if") + saved_indents = self.stmt_indent, self.cond_indent + self.stmt_indent += 1 + self.cond_indent = self.stmt_indent + self.visit(item.conditional()) + self.stmt_indent, self.cond_indent = saved_indents + self.emit_statement("endif") elif item.commentLine() and self.preserve_comments: self.visit(item.commentLine()) @@ -452,7 +477,7 @@ def visitComparison(self, ctx, *, last: bool = False) -> None: else: lhs = self.visitFunctionCall(comp.functionCall()) if not lhs: - return # Skip on error + return operator = ctx.getChild(1) negate = operator.symbol.type in (hrw4uParser.NEQ, hrw4uParser.NOT_TILDE) @@ -484,7 +509,6 @@ def visitComparison(self, ctx, *, last: bool = False) -> None: case _ if ctx.set_(): inner = ctx.set_().getText()[1:-1] # We no longer strip the quotes here for sets, fixed in #12256 - # parts = [s.strip().strip("'") for s in inner.split(",")] cond_txt = f"{lhs} ({inner})" case _: diff --git a/tools/hrw4u/tests/data/conds/nested-ifs.ast.txt b/tools/hrw4u/tests/data/conds/nested-ifs.ast.txt new file mode 100644 index 00000000000..0a10091aebf --- /dev/null +++ b/tools/hrw4u/tests/data/conds/nested-ifs.ast.txt @@ -0,0 +1 @@ +(program (programItem (section (varSection VARS { (variables (variablesItem (variableDecl bool_0 : bool ;)) (variablesItem (variableDecl bool_1 : bool ;)) (variablesItem (variableDecl bool_2 : bool ;))) }))) (programItem (section REMAP { (sectionBody (conditional (ifStatement if (condition (expression (term (factor (comparison (comparable inbound.req.X-Foo) == (value "bar")))))) (block { (blockItem (statement inbound.req.X-Hello = (value "there") ;)) (blockItem (conditional (ifStatement if (condition (expression (term (factor (comparison (comparable inbound.req.X-Fie) == (value "fie")))))) (block { (blockItem (statement inbound.req.X-first = (value "1") ;)) (blockItem (conditional (ifStatement if (condition (expression (expression (term (factor bool_0))) || (term (factor ( (expression (term (term (factor bool_1)) && (factor bool_2))) ))))) (block { (blockItem (statement inbound.req.X-Parsed = (value "more") ;)) })) (elseClause else (block { (blockItem (statement inbound.req.X-Parsed = (value "yes") ;)) })))) })) (elifClause elif (condition (expression (term (factor (comparison (comparable inbound.req.X-Fum) == (value "bar")))))) (block { (blockItem (statement inbound.req.X-Parsed = (value "no") ;)) })) (elseClause else (block { (blockItem (statement inbound.req.X-More = (value "yes") ;)) })))) })) (elifClause elif (condition (expression (term (factor (comparison (comparable inbound.req.X-Foo) == (value "foo") (modifier with (modifierList NOCASE , PRE))))))) (block { (blockItem (statement inbound.req.X-Nocase = (value "foo") ;)) })) (elseClause else (block { (blockItem (statement inbound.req.X-Something = (value "no-bar") ;)) })))) })) ) diff --git a/tools/hrw4u/tests/data/conds/nested-ifs.input.txt b/tools/hrw4u/tests/data/conds/nested-ifs.input.txt new file mode 100644 index 00000000000..27164785311 --- /dev/null +++ b/tools/hrw4u/tests/data/conds/nested-ifs.input.txt @@ -0,0 +1,27 @@ +VARS { + bool_0: bool; + bool_1: bool; + bool_2: bool; +} + +REMAP { + if inbound.req.X-Foo == "bar" { + inbound.req.X-Hello = "there"; + if inbound.req.X-Fie == "fie" { + inbound.req.X-first = "1"; + if bool_0 || (bool_1 && bool_2) { + inbound.req.X-Parsed = "more"; + } else { + inbound.req.X-Parsed = "yes"; + } + } elif inbound.req.X-Fum == "bar" { + inbound.req.X-Parsed = "no"; + } else { + inbound.req.X-More = "yes"; + } + } elif inbound.req.X-Foo == "foo" with NOCASE,PRE { + inbound.req.X-Nocase = "foo"; + } else { + inbound.req.X-Something = "no-bar"; + } +} diff --git a/tools/hrw4u/tests/data/conds/nested-ifs.output.txt b/tools/hrw4u/tests/data/conds/nested-ifs.output.txt new file mode 100644 index 00000000000..f7b802f5af8 --- /dev/null +++ b/tools/hrw4u/tests/data/conds/nested-ifs.output.txt @@ -0,0 +1,27 @@ +cond %{REMAP_PSEUDO_HOOK} [AND] +cond %{CLIENT-HEADER:X-Foo} ="bar" + set-header X-Hello "there" + if + cond %{CLIENT-HEADER:X-Fie} ="fie" + set-header X-first "1" + if + cond %{STATE-FLAG:0} [OR] + cond %{GROUP} + cond %{STATE-FLAG:1} [AND] + cond %{STATE-FLAG:2} + cond %{GROUP:END} + set-header X-Parsed "more" + else + set-header X-Parsed "yes" + endif + elif + cond %{CLIENT-HEADER:X-Fum} ="bar" + set-header X-Parsed "no" + else + set-header X-More "yes" + endif +elif + cond %{CLIENT-HEADER:X-Foo} ="foo" [NOCASE,PRE] + set-header X-Nocase "foo" +else + set-header X-Something "no-bar" diff --git a/tools/hrw4u/tests/data/hooks/remap.ast.txt b/tools/hrw4u/tests/data/hooks/remap.ast.txt index 0b229e4e488..c8f2a6d6321 100644 --- a/tools/hrw4u/tests/data/hooks/remap.ast.txt +++ b/tools/hrw4u/tests/data/hooks/remap.ast.txt @@ -1 +1 @@ -(program (programItem (section REMAP { (sectionBody (conditional (ifStatement if (condition (expression (term (factor (comparison (comparable inbound.req.X-Remap) == (value "yes")))))) (block { (blockItem (statement inbound.req.X-Remap = (value "") ;)) })) (elseClause else (block { (blockItem (statement inbound.req.X-Remap = (value "It was not yes") ;)) })))) })) ) +(program (programItem (section REMAP { (sectionBody (conditional (ifStatement if (condition (expression (term (factor (comparison (comparable inbound.req.X-Remap) == (value "yes")))))) (block { (blockItem (statement inbound.req.X-Remap = (value "") ;)) (blockItem (statement inbound.req.X-Appended += (value "HRW4U") ;)) })) (elseClause else (block { (blockItem (statement inbound.req.X-Remap = (value "It was not yes") ;)) (blockItem (statement inbound.req.X-Appended = (value "") ;)) })))) })) ) diff --git a/tools/hrw4u/tests/data/hooks/remap.input.txt b/tools/hrw4u/tests/data/hooks/remap.input.txt index 0b31ed8c1cf..1dd3c4d14be 100644 --- a/tools/hrw4u/tests/data/hooks/remap.input.txt +++ b/tools/hrw4u/tests/data/hooks/remap.input.txt @@ -1,7 +1,9 @@ REMAP { if inbound.req.X-Remap == "yes" { inbound.req.X-Remap = ""; + inbound.req.X-Appended += "HRW4U"; } else { inbound.req.X-Remap = "It was not yes"; + inbound.req.X-Appended = ""; } } diff --git a/tools/hrw4u/tests/data/hooks/remap.output.txt b/tools/hrw4u/tests/data/hooks/remap.output.txt index 243fbb65c73..d151e910282 100644 --- a/tools/hrw4u/tests/data/hooks/remap.output.txt +++ b/tools/hrw4u/tests/data/hooks/remap.output.txt @@ -1,5 +1,7 @@ cond %{REMAP_PSEUDO_HOOK} [AND] cond %{CLIENT-HEADER:X-Remap} ="yes" rm-header X-Remap + add-header X-Appended "HRW4U" else set-header X-Remap "It was not yes" + rm-header X-Appended diff --git a/tools/hrw4u/tests/data/ops/exceptions.txt b/tools/hrw4u/tests/data/ops/exceptions.txt index b96ebf70922..9628625f810 100644 --- a/tools/hrw4u/tests/data/ops/exceptions.txt +++ b/tools/hrw4u/tests/data/ops/exceptions.txt @@ -3,3 +3,5 @@ # QSA (Query String Append) is a reverse-only test qsa.input: u4wrh +# HTTP-CNTL valid bools can not reverse back to the original input +http_cntl_valid_bools.input: hrw4u diff --git a/tools/hrw4u/tests/data/ops/http_cntl_invalid_bool.fail.error.txt b/tools/hrw4u/tests/data/ops/http_cntl_invalid_bool.fail.error.txt index c05550731ae..8bc6dbdf7d4 100644 --- a/tools/hrw4u/tests/data/ops/http_cntl_invalid_bool.fail.error.txt +++ b/tools/hrw4u/tests/data/ops/http_cntl_invalid_bool.fail.error.txt @@ -1 +1,3 @@ -Invalid boolean value 'invalid_value'. Must be one of: 0, 1, FALSE, NO, OFF, ON, TRUE, YES \ No newline at end of file +tests/data/ops/http_cntl_invalid_bool.fail.input.txt:2:4: error: Invalid boolean value 'invalid_value'. Must be one of: 0, 1, FALSE, NO, OFF, ON, TRUE, YES + 2 | http.cntl.LOGGING = invalid_value; + | ^ diff --git a/tools/hrw4u/tests/data/ops/http_cntl_quoted_bool.fail.error.txt b/tools/hrw4u/tests/data/ops/http_cntl_quoted_bool.fail.error.txt index 4a4000faccc..26e842fd6a6 100644 --- a/tools/hrw4u/tests/data/ops/http_cntl_quoted_bool.fail.error.txt +++ b/tools/hrw4u/tests/data/ops/http_cntl_quoted_bool.fail.error.txt @@ -1 +1,3 @@ -Invalid boolean value '"true"'. Must be one of: 0, 1, FALSE, NO, OFF, ON, TRUE, YES and must not be quoted \ No newline at end of file +tests/data/ops/http_cntl_quoted_bool.fail.input.txt:2:4: error: Invalid boolean value '"true"'. Must be one of: 0, 1, FALSE, NO, OFF, ON, TRUE, YES and must not be quoted + 2 | http.cntl.LOGGING = "true"; + | ^ diff --git a/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.ast.txt b/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.ast.txt index 6029280f569..870d41622da 100644 --- a/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.ast.txt +++ b/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.ast.txt @@ -1 +1 @@ -(program (section SEND_RESPONSE { (sectionBody (statement http.cntl.LOGGING = (value TRUE) ;)) (sectionBody (statement http.cntl.TXN_DEBUG = (value FALSE) ;)) (sectionBody (statement http.cntl.REQ_CACHEABLE = (value YES) ;)) (sectionBody (statement http.cntl.RESP_CACHEABLE = (value NO) ;)) (sectionBody (statement http.cntl.SERVER_NO_STORE = (value ON) ;)) (sectionBody (statement http.cntl.SKIP_REMAP = (value OFF) ;)) (sectionBody (statement http.cntl.INTERCEPT_RETRY = (value 1) ;)) (sectionBody (statement http.cntl.LOGGING = (value 0) ;)) (sectionBody (statement http.cntl.TXN_DEBUG = (value true) ;)) (sectionBody (statement http.cntl.REQ_CACHEABLE = (value false) ;)) (sectionBody (statement http.cntl.RESP_CACHEABLE = (value yes) ;)) (sectionBody (statement http.cntl.SERVER_NO_STORE = (value no) ;)) (sectionBody (statement http.cntl.SKIP_REMAP = (value on) ;)) (sectionBody (statement http.cntl.INTERCEPT_RETRY = (value off) ;)) (sectionBody (statement http.cntl.LOGGING = (value True) ;)) (sectionBody (statement http.cntl.TXN_DEBUG = (value False) ;)) (sectionBody (statement http.cntl.LOGGING = (value TRue) ;)) }) ) +(program (programItem (section SEND_RESPONSE { (sectionBody (statement http.cntl.LOGGING = (value TRUE) ;)) (sectionBody (statement http.cntl.TXN_DEBUG = (value FALSE) ;)) (sectionBody (statement http.cntl.REQ_CACHEABLE = (value YES) ;)) (sectionBody (statement http.cntl.RESP_CACHEABLE = (value NO) ;)) (sectionBody (statement http.cntl.SERVER_NO_STORE = (value ON) ;)) (sectionBody (statement http.cntl.SKIP_REMAP = (value OFF) ;)) (sectionBody (statement http.cntl.INTERCEPT_RETRY = (value 1) ;)) (sectionBody (statement http.cntl.LOGGING = (value 0) ;)) (sectionBody (statement http.cntl.TXN_DEBUG = (value true) ;)) (sectionBody (statement http.cntl.REQ_CACHEABLE = (value false) ;)) (sectionBody (statement http.cntl.RESP_CACHEABLE = (value yes) ;)) (sectionBody (statement http.cntl.SERVER_NO_STORE = (value no) ;)) (sectionBody (statement http.cntl.SKIP_REMAP = (value on) ;)) (sectionBody (statement http.cntl.INTERCEPT_RETRY = (value off) ;)) (sectionBody (statement http.cntl.LOGGING = (value True) ;)) (sectionBody (statement http.cntl.TXN_DEBUG = (value False) ;)) (sectionBody (statement http.cntl.LOGGING = (value TRue) ;)) })) ) diff --git a/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.output.txt b/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.output.txt index d90fac00d6e..9fbb2661e29 100644 --- a/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.output.txt +++ b/tools/hrw4u/tests/data/ops/http_cntl_valid_bools.output.txt @@ -15,4 +15,4 @@ cond %{SEND_RESPONSE_HDR_HOOK} [AND] set-http-cntl INTERCEPT_RETRY off set-http-cntl LOGGING True set-http-cntl TXN_DEBUG False - set-http-cntl LOGGING TRue \ No newline at end of file + set-http-cntl LOGGING TRue diff --git a/tools/hrw4u/tests/data/ops/qsa.output.txt b/tools/hrw4u/tests/data/ops/qsa.output.txt index 6ea35877983..03e002907a3 100644 --- a/tools/hrw4u/tests/data/ops/qsa.output.txt +++ b/tools/hrw4u/tests/data/ops/qsa.output.txt @@ -2,4 +2,4 @@ # test, because in hrw4u, we don't use QSA. cond %{REMAP_PSEUDO_HOOK} [AND] cond %{GEO:COUNTRY} =SE - set-redirect 302 https://www.example.com/SE/%{CLIENT-URL:PATH} [QSA] + set-redirect 302 "https://www.example.com/SE/%{CLIENT-URL:PATH}?%{CLIENT-URL:QUERY}" diff --git a/tools/hrw4u/tests/data/ops/skip_remap_quoted_bool.fail.error.txt b/tools/hrw4u/tests/data/ops/skip_remap_quoted_bool.fail.error.txt index 4a4000faccc..8ba0760bb48 100644 --- a/tools/hrw4u/tests/data/ops/skip_remap_quoted_bool.fail.error.txt +++ b/tools/hrw4u/tests/data/ops/skip_remap_quoted_bool.fail.error.txt @@ -1 +1,3 @@ -Invalid boolean value '"true"'. Must be one of: 0, 1, FALSE, NO, OFF, ON, TRUE, YES and must not be quoted \ No newline at end of file +tests/data/ops/skip_remap_quoted_bool.fail.input.txt:2:4: error: Invalid boolean value '"true"'. Must be one of: 0, 1, FALSE, NO, OFF, ON, TRUE, YES and must not be quoted + 2 | skip-remap("true"); + | ^ diff --git a/tools/hrw4u/tests/data/vars/exceptions.txt b/tools/hrw4u/tests/data/vars/exceptions.txt new file mode 100644 index 00000000000..64e57bf9bb0 --- /dev/null +++ b/tools/hrw4u/tests/data/vars/exceptions.txt @@ -0,0 +1,5 @@ +# Operations tests direction exceptions +# Format: test_name: direction +# +# Explicit slot assignment syntax cannot be reversed +explicit_slots.input: hrw4u diff --git a/tools/hrw4u/tests/data/vars/explicit_slots.ast.txt b/tools/hrw4u/tests/data/vars/explicit_slots.ast.txt new file mode 100644 index 00000000000..1d0b442daee --- /dev/null +++ b/tools/hrw4u/tests/data/vars/explicit_slots.ast.txt @@ -0,0 +1 @@ +(program (programItem (section (varSection VARS { (variables (variablesItem (variableDecl parent_config : bool @ 7 ;)) (variablesItem (variableDecl parent_child : bool @ 12 ;)) (variablesItem (variableDecl match : bool ;)) (variablesItem (variableDecl active_flag : bool @ 3 ;)) (variablesItem (variableDecl counter : int8 @ 2 ;)) (variablesItem (variableDecl priority : int8 ;)) (variablesItem (variableDecl status : int16 ;))) }))) (programItem (section SEND_RESPONSE { (sectionBody (conditional (ifStatement if (condition (expression (term (factor parent_config)))) (block { (blockItem (statement inbound.resp.X-Parent = (value true) ;)) })))) })) ) diff --git a/tools/hrw4u/tests/data/vars/explicit_slots.input.txt b/tools/hrw4u/tests/data/vars/explicit_slots.input.txt new file mode 100644 index 00000000000..e8ff9f5a6a9 --- /dev/null +++ b/tools/hrw4u/tests/data/vars/explicit_slots.input.txt @@ -0,0 +1,15 @@ +VARS { + parent_config: bool @7; + parent_child: bool @12; + match: bool; + active_flag: bool @3; + counter: int8 @2; + priority: int8; + status: int16; +} + +SEND_RESPONSE { + if parent_config { + inbound.resp.X-Parent = true; + } +} diff --git a/tools/hrw4u/tests/data/vars/explicit_slots.output.txt b/tools/hrw4u/tests/data/vars/explicit_slots.output.txt new file mode 100644 index 00000000000..adaa5f08a57 --- /dev/null +++ b/tools/hrw4u/tests/data/vars/explicit_slots.output.txt @@ -0,0 +1,3 @@ +cond %{SEND_RESPONSE_HDR_HOOK} [AND] +cond %{STATE-FLAG:7} + set-header X-Parent true diff --git a/tools/hrw4u/tests/data/vars/slot_conflict.fail.error.txt b/tools/hrw4u/tests/data/vars/slot_conflict.fail.error.txt new file mode 100644 index 00000000000..721dbcddf0c --- /dev/null +++ b/tools/hrw4u/tests/data/vars/slot_conflict.fail.error.txt @@ -0,0 +1,3 @@ +tests/data/vars/slot_conflict.fail.input.txt:3:4: error: Slot @5 already used by variable 'first' + 3 | second: bool @5; # Error: slot already used + | ^ diff --git a/tools/hrw4u/tests/data/vars/slot_conflict.fail.input.txt b/tools/hrw4u/tests/data/vars/slot_conflict.fail.input.txt new file mode 100644 index 00000000000..f60cabb213a --- /dev/null +++ b/tools/hrw4u/tests/data/vars/slot_conflict.fail.input.txt @@ -0,0 +1,8 @@ +VARS { + first: bool @5; + second: bool @5; # Error: slot already used +} + +SEND_RESPONSE { + set_header("X-Test", "value"); +} diff --git a/tools/hrw4u/tests/data/vars/vars_count.fail.error.txt b/tools/hrw4u/tests/data/vars/vars_count.fail.error.txt index 5846224ebd3..513c1418594 100644 --- a/tools/hrw4u/tests/data/vars/vars_count.fail.error.txt +++ b/tools/hrw4u/tests/data/vars/vars_count.fail.error.txt @@ -1,3 +1,3 @@ -tests/data/vars/vars_count.fail.input.txt:7:3: error: Too many 'int8' variables (max 4) +tests/data/vars/vars_count.fail.input.txt:7:3: error: No available slots for type 'int8' (max 4) 7 | Five: int8; | ^ From e03fa5cf656c1e5230ce0071ed24e34cb92d3d2a Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 13:34:13 +0000 Subject: [PATCH 23/29] update zstd compression context initialization to return a success status --- plugins/compress/compress.cc | 15 ++++++++++----- plugins/compress/zstd_compress.cc | 17 +++++++++++++---- plugins/compress/zstd_compress.h | 4 ++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index c2767167fbe..181ca0eec04 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -339,9 +339,8 @@ compress_transform_init(TSCont contp, Data *data) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { - Zstd::transform_init(data); - if (!data->zstrm_zstd.cctx) { - TSError("Failed to create Zstandard compression context"); + if (!Zstd::transform_init(data)) { + TSError("Failed to configure Zstandard compression context"); return; } } @@ -386,7 +385,13 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { Gzip::transform_one(data, upstream_buffer, upstream_length); } else { - warning("No compression supported. Shouldn't come here."); + warning("No compression supported. Passing data through without transformation."); + int64_t written = TSIOBufferWrite(data->downstream_buffer, upstream_buffer, upstream_length); + if (written == TS_ERROR || written != upstream_length) { + error("Failed to copy upstream data to downstream buffer"); + return; + } + data->downstream_length += written; } TSIOBufferReaderConsume(upstream_reader, upstream_length); @@ -414,7 +419,7 @@ compress_transform_finish(Data *data) Gzip::transform_finish(data); debug("compress_transform_finish: gzip compression finish"); } else { - error("No Compression matched, shouldn't come here"); + debug("compress_transform_finish: no compression active, passthrough mode"); } } diff --git a/plugins/compress/zstd_compress.cc b/plugins/compress/zstd_compress.cc index cbc34431fe9..135900d5a25 100644 --- a/plugins/compress/zstd_compress.cc +++ b/plugins/compress/zstd_compress.cc @@ -92,27 +92,36 @@ data_destroy(Data *data) } } -void +bool transform_init(Data *data) { if (!data->zstrm_zstd.cctx) { error("Failed to initialize Zstd compression context"); - return; + return false; } size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); if (ZSTD_isError(result)) { error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); - return; + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.total_out = 0; + return false; } result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); if (ZSTD_isError(result)) { error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); - return; + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.total_out = 0; + return false; } debug("zstd compression context initialized with level %d", data->hc->zstd_compression_level()); + return true; } void diff --git a/plugins/compress/zstd_compress.h b/plugins/compress/zstd_compress.h index f589adf97d8..3386450a4d0 100644 --- a/plugins/compress/zstd_compress.h +++ b/plugins/compress/zstd_compress.h @@ -31,8 +31,8 @@ void data_alloc(Data *data); // Destroy Zstd compression context void data_destroy(Data *data); -// Configure the context just before streaming starts -void transform_init(Data *data); +// Configure the context just before streaming starts. Returns true when ready. +bool transform_init(Data *data); // Compress one upstream chunk void transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); From b87c8645cdba70aff9f104c97b38ebaacf924b27 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 13:36:29 +0000 Subject: [PATCH 24/29] input validation for zstd compression --- plugins/compress/zstd_compress.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugins/compress/zstd_compress.cc b/plugins/compress/zstd_compress.cc index 135900d5a25..16e0e3a62f9 100644 --- a/plugins/compress/zstd_compress.cc +++ b/plugins/compress/zstd_compress.cc @@ -23,6 +23,7 @@ #include "debug_macros.h" #include +#include namespace { @@ -32,6 +33,16 @@ compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_len TSIOBufferBlock downstream_blkp; int64_t downstream_length; + if (upstream_length < 0) { + error("zstd-transform: negative upstream length (%" PRId64 ")", upstream_length); + return false; + } + + if (upstream_buffer == nullptr && upstream_length > 0) { + error("upstream_buffer is NULL with non-zero length"); + return false; + } + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; for (;;) { @@ -127,6 +138,11 @@ transform_init(Data *data) void transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) { + if (upstream_length < 0) { + error("Zstd compression received negative upstream length (%" PRId64 ")", upstream_length); + return; + } + if (!compress_operation(data, upstream_buffer, upstream_length, ZSTD_e_continue)) { error("Zstd compression (CONTINUE) failed"); return; From 1b1ed1c54c599818ab38ee6ee043988868ce01e1 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 13:36:58 +0000 Subject: [PATCH 25/29] input validation for zstd compression --- plugins/compress/zstd_compress.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/compress/zstd_compress.cc b/plugins/compress/zstd_compress.cc index 16e0e3a62f9..45c6beb7924 100644 --- a/plugins/compress/zstd_compress.cc +++ b/plugins/compress/zstd_compress.cc @@ -49,6 +49,11 @@ compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_len downstream_blkp = TSIOBufferStart(data->downstream_buffer); char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + if (downstream_length <= 0) { + error("zstd-transform: downstream block has non-positive length (%" PRId64 ")", downstream_length); + return false; + } + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, mode); @@ -175,6 +180,11 @@ transform_finish(Data *data) downstream_blkp = TSIOBufferStart(data->downstream_buffer); char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + if (downstream_length <= 0) { + error("zstd-transform: downstream block has non-positive length (%" PRId64 ")", downstream_length); + break; + } + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); From 6fa64e3e9705180732dbaf27a5d395cf8a7490d5 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 3 Nov 2025 13:37:45 +0000 Subject: [PATCH 26/29] replace TSError with error for Zstandard compression context failure --- plugins/compress/compress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 181ca0eec04..71dced8e38e 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -340,7 +340,7 @@ compress_transform_init(TSCont contp, Data *data) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { if (!Zstd::transform_init(data)) { - TSError("Failed to configure Zstandard compression context"); + error("Failed to configure Zstandard compression context"); return; } } From 6d440f1c27f02887ce9920659c09d93eedb1829b Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Sat, 8 Nov 2025 08:08:34 +0000 Subject: [PATCH 27/29] revert content encoding quality factor calculation the original works fine --- src/iocore/cache/HttpTransactCache.cc | 58 ++++++++++++++++++--------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/src/iocore/cache/HttpTransactCache.cc b/src/iocore/cache/HttpTransactCache.cc index 7b3249af0f4..47117ad90ee 100644 --- a/src/iocore/cache/HttpTransactCache.cc +++ b/src/iocore/cache/HttpTransactCache.cc @@ -979,38 +979,60 @@ HttpTransactCache::calculate_quality_of_accept_encoding_match(MIMEField *accept_ if (!content_field) { if (!match_accept_content_encoding("identity", accept_field, &wildcard_present, &wildcard_q, &q)) { // CE was not returned, and AE does not have identity + if (match_content_encoding(accept_field, "gzip") and match_content_encoding(cached_accept_field, "gzip")) { + return 1.0f; + } goto encoding_wildcard; } - } else { - // Handle multiple content encodings - use minimum quality - float min_q = 1.0; // Start with maximum quality - bool found_match = false; + // use q from identity match + } else { + // "Accept-encoding must correctly handle multiple content encoding" + // The combined quality factor is the product of all quality factors. + // (Note that there may be other possible choice, eg, min(), + // but I think multiplication is the best.) + // For example, if "content-encoding: a, b", and quality factors + // of a and b (in accept-encoding header) are q_a and q_b, resp, + // then the combined quality factor is (q_a * q_b). + // If any one of the content-encoding is not matched, + // then the q value will not be changed. + float combined_q = 1.0; for (c_value = c_values_list.head; c_value; c_value = c_value->next) { float this_q = -1.0; if (!match_accept_content_encoding(c_value->str, accept_field, &wildcard_present, &wildcard_q, &this_q)) { goto encoding_wildcard; } - if (this_q >= 0.0) { - found_match = false; - if (this_q < min_q) { - min_q = this_q; - } - } - } - if (found_match) { - q = min_q; - } else { - q = -1.0; + combined_q *= this_q; } + q = combined_q; } encoding_wildcard: + // match the wildcard now // if ((q == -1.0) && (wildcard_present == true)) { - return wildcard_q; + q = wildcard_q; } - - return q; + ///////////////////////////////////////////////////////////////////////// + // there was an Accept-Encoding, but it didn't match anything, at // + // any quality level --- if this is an identity-coded document, that's // + // still okay, but otherwise, this is just not a match at all. // + ///////////////////////////////////////////////////////////////////////// + if ((q == -1.0) && is_identity_encoding) { + if (match_content_encoding(accept_field, "gzip")) { + if (match_content_encoding(cached_accept_field, "gzip")) { + return 1.0f; + } else { + // always try to fetch GZIP content if we have not tried sending AE before + return -1.0f; + } + } else if (cached_accept_field && !match_content_encoding(cached_accept_field, "gzip")) { + return 0.001f; + } else { + return -1.0f; + } + } + // q = (float)-1.0; + return (q); } /** From 28f53cb2c3d24abfffe3b735ba72c971398d1493 Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Mon, 17 Nov 2025 20:36:36 +0000 Subject: [PATCH 28/29] Update plugins/compress/compress.cc --- plugins/compress/compress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 71dced8e38e..88c52e02860 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -620,7 +620,7 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration continue; } - info("Accept-Encoding value [%.*s]", len, value); + debug("Accept-Encoding value [%.*s]", len, value); if (strncasecmp(value, "zstd", sizeof("zstd") - 1) == 0) { if (*algorithms & ALGORITHM_ZSTD) { From 32b2f0d665cbfc1a59b1788f9a626951e4919620 Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Mon, 17 Nov 2025 22:31:39 +0000 Subject: [PATCH 29/29] Update normalized_ae_varied_transactions.replay.yaml --- .../replays/normalized_ae_varied_transactions.replay.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 25284b21d23..4910af81027 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -122,7 +122,7 @@ sessions: - [ X-Response-Identifier, Deflate-Accept-Encoding ] proxy-response: - status: 404 + status: 200 headers: fields: - [ X-Response-Identifier, { value: Deflate-Accept-Encoding, as: equal } ]