From a8329a928257e3685e1ca61264b19670775c000a Mon Sep 17 00:00:00 2001 From: JosiahWI <41302989+JosiahWI@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:50:34 -0500 Subject: [PATCH 01/11] Support OpenSSL 3.0 APIs for Diffie-Hellman (#13349) * Add unit tests for SSL Diffie-Hellman key configuration * Support OpenSSL 3.0 APIs for Diffie-Hellman * Move DH keygen to SSLKeyUtils.{h,cc} (cherry picked from commit ec93038b7e98d7881f111cb5fc45e71529f73e0a) --- src/iocore/net/CMakeLists.txt | 4 + src/iocore/net/P_SSLUtils.h | 26 +++ src/iocore/net/SSLKeyUtils.cc | 190 ++++++++++++++++++ src/iocore/net/SSLKeyUtils.h | 42 ++++ src/iocore/net/SSLUtils.cc | 76 +------ src/iocore/net/unit_tests/test_SSLDHParams.cc | 186 +++++++++++++++++ 6 files changed, 457 insertions(+), 67 deletions(-) create mode 100644 src/iocore/net/SSLKeyUtils.cc create mode 100644 src/iocore/net/SSLKeyUtils.h create mode 100644 src/iocore/net/unit_tests/test_SSLDHParams.cc diff --git a/src/iocore/net/CMakeLists.txt b/src/iocore/net/CMakeLists.txt index 6dc65f024c3..63e82e66bc8 100644 --- a/src/iocore/net/CMakeLists.txt +++ b/src/iocore/net/CMakeLists.txt @@ -52,6 +52,7 @@ add_library( SSLSessionCache.cc SSLSessionTicket.cc SSLUtils.cc + SSLKeyUtils.cc OCSPStapling.cc TLSBasicSupport.cc TLSEventSupport.cc @@ -146,6 +147,9 @@ if(BUILD_TESTING) unit_tests/test_OCSPStapling.cc unit_tests/unit_test_main.cc ) + if(SSLLIB_IS_OPENSSL3) + target_sources(test_net PRIVATE unit_tests/test_SSLDHParams.cc) + endif() # Use link groups to solve circular dependency set(LINK_GROUP_LIBS ts::logging diff --git a/src/iocore/net/P_SSLUtils.h b/src/iocore/net/P_SSLUtils.h index 8e2cbf57884..af5728e4869 100644 --- a/src/iocore/net/P_SSLUtils.h +++ b/src/iocore/net/P_SSLUtils.h @@ -25,6 +25,10 @@ #include "iocore/net/SSLTypes.h" #include "tscore/Diags.h" +#ifdef OPENSSL_IS_OPENSSL3 +#include +#include +#endif #define OPENSSL_THREAD_DEFINES #if __has_include() #include @@ -110,6 +114,24 @@ namespace detail } }; +#ifdef OPENSSL_IS_OPENSSL3 + struct PKEYCTXDeleter { + void + operator()(EVP_PKEY_CTX *pctx) + { + EVP_PKEY_CTX_free(pctx); + } + }; + + struct DecoderCTXDeleter { + void + operator()(OSSL_DECODER_CTX *dctx) + { + OSSL_DECODER_CTX_free(dctx); + } + }; +#endif + } // namespace detail } // namespace ssl @@ -134,3 +156,7 @@ struct ats_wildcard_matcher { using scoped_X509 = std::unique_ptr; using scoped_BIO = std::unique_ptr; +#ifdef OPENSSL_IS_OPENSSL3 +using scoped_PKEY_CTX = std::unique_ptr; +using scoped_Decoder_CTX = std::unique_ptr; +#endif diff --git a/src/iocore/net/SSLKeyUtils.cc b/src/iocore/net/SSLKeyUtils.cc new file mode 100644 index 00000000000..4bf14c5f473 --- /dev/null +++ b/src/iocore/net/SSLKeyUtils.cc @@ -0,0 +1,190 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLKeyUtils.h" +#include "P_SSLUtils.h" + +#include +#ifdef OPENSSL_IS_OPENSSL3 +#include +#else +#include +#endif + +#ifdef OPENSSL_IS_OPENSSL3 +#include +#include +#include +#include +#else +#include +#include +#include +#include +#endif + +#ifdef OPENSSL_IS_OPENSSL3 + +EVP_PKEY * +gen_dh_2048_256_pkey() +{ + scoped_PKEY_CTX pctx{EVP_PKEY_CTX_new_from_name(NULL, "DH", NULL)}; + if (!pctx) { + Error("failed to create OpenSSL pkey context"); + return nullptr; + } + + if (EVP_PKEY_keygen_init(pctx.get()) <= 0) { + Error("failed to initialize OpenSSL keygen"); + return nullptr; + } + + char prime_group[]{"dh_2048_256"}; + OSSL_PARAM const params[]{OSSL_PARAM_utf8_string("group", prime_group, 0), OSSL_PARAM_END}; + + if (!EVP_PKEY_CTX_set_params(pctx.get(), params)) { + Error("SSL dhparams source returned invalid parameters"); + return nullptr; + } + + EVP_PKEY *pkey{}; + EVP_PKEY_generate(pctx.get(), &pkey); + + return pkey; +} + +EVP_PKEY * +load_dhparams_file(char const *dhparams_file) +{ + EVP_PKEY *pkey{}; + scoped_Decoder_CTX dctx{OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, "DH", OSSL_KEYMGMT_SELECT_ALL_PARAMETERS, NULL, NULL)}; + if (!dctx) { + Error("failed to create OpenSSL decoder context"); + return nullptr; + } + + ink_assert(OSSL_DECODER_CTX_get_num_decoders(dctx.get()) > 0); + scoped_BIO bio{BIO_new_file(dhparams_file, "r")}; + if (!bio) { + Error("failed to open parameters file"); + return nullptr; + } + if (!OSSL_DECODER_from_bio(dctx.get(), bio.get())) { + Error("SSL dhparams source returned invalid parameters"); + return nullptr; + } + + return pkey; +} + +bool +set_ctx_dh(SSL_CTX *ctx, dh_key_t *pkey) +{ + bool result{SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE) && SSL_CTX_set0_tmp_dh_pkey(ctx, pkey)}; + if (!result) { + EVP_PKEY_free(pkey); + } + return result; +} + +#else + +DH * +load_dhparams_file(char const *dhparams_file) +{ + scoped_BIO bio(BIO_new_file(dhparams_file, "r")); + DH *dh{PEM_read_bio_DHparams(bio.get(), nullptr, nullptr, nullptr)}; + if (!dh) { + Error("SSL dhparams source returned invalid parameters"); + return nullptr; + } + + return dh; +} + +#if TS_USE_GET_DH_2048_256 +DH * +gen_dh_2048_256_pkey() +{ + return DH_get_2048_256(); +} +#else +DH * +gen_dh_2048_256_pkey() +{ + /* Build 2048-bit MODP Group with 256-bit Prime Order Subgroup from RFC 5114 */ + static const unsigned char dh2048_p[] = { + 0x87, 0xA8, 0xE6, 0x1D, 0xB4, 0xB6, 0x66, 0x3C, 0xFF, 0xBB, 0xD1, 0x9C, 0x65, 0x19, 0x59, 0x99, 0x8C, 0xEE, 0xF6, 0x08, + 0x66, 0x0D, 0xD0, 0xF2, 0x5D, 0x2C, 0xEE, 0xD4, 0x43, 0x5E, 0x3B, 0x00, 0xE0, 0x0D, 0xF8, 0xF1, 0xD6, 0x19, 0x57, 0xD4, + 0xFA, 0xF7, 0xDF, 0x45, 0x61, 0xB2, 0xAA, 0x30, 0x16, 0xC3, 0xD9, 0x11, 0x34, 0x09, 0x6F, 0xAA, 0x3B, 0xF4, 0x29, 0x6D, + 0x83, 0x0E, 0x9A, 0x7C, 0x20, 0x9E, 0x0C, 0x64, 0x97, 0x51, 0x7A, 0xBD, 0x5A, 0x8A, 0x9D, 0x30, 0x6B, 0xCF, 0x67, 0xED, + 0x91, 0xF9, 0xE6, 0x72, 0x5B, 0x47, 0x58, 0xC0, 0x22, 0xE0, 0xB1, 0xEF, 0x42, 0x75, 0xBF, 0x7B, 0x6C, 0x5B, 0xFC, 0x11, + 0xD4, 0x5F, 0x90, 0x88, 0xB9, 0x41, 0xF5, 0x4E, 0xB1, 0xE5, 0x9B, 0xB8, 0xBC, 0x39, 0xA0, 0xBF, 0x12, 0x30, 0x7F, 0x5C, + 0x4F, 0xDB, 0x70, 0xC5, 0x81, 0xB2, 0x3F, 0x76, 0xB6, 0x3A, 0xCA, 0xE1, 0xCA, 0xA6, 0xB7, 0x90, 0x2D, 0x52, 0x52, 0x67, + 0x35, 0x48, 0x8A, 0x0E, 0xF1, 0x3C, 0x6D, 0x9A, 0x51, 0xBF, 0xA4, 0xAB, 0x3A, 0xD8, 0x34, 0x77, 0x96, 0x52, 0x4D, 0x8E, + 0xF6, 0xA1, 0x67, 0xB5, 0xA4, 0x18, 0x25, 0xD9, 0x67, 0xE1, 0x44, 0xE5, 0x14, 0x05, 0x64, 0x25, 0x1C, 0xCA, 0xCB, 0x83, + 0xE6, 0xB4, 0x86, 0xF6, 0xB3, 0xCA, 0x3F, 0x79, 0x71, 0x50, 0x60, 0x26, 0xC0, 0xB8, 0x57, 0xF6, 0x89, 0x96, 0x28, 0x56, + 0xDE, 0xD4, 0x01, 0x0A, 0xBD, 0x0B, 0xE6, 0x21, 0xC3, 0xA3, 0x96, 0x0A, 0x54, 0xE7, 0x10, 0xC3, 0x75, 0xF2, 0x63, 0x75, + 0xD7, 0x01, 0x41, 0x03, 0xA4, 0xB5, 0x43, 0x30, 0xC1, 0x98, 0xAF, 0x12, 0x61, 0x16, 0xD2, 0x27, 0x6E, 0x11, 0x71, 0x5F, + 0x69, 0x38, 0x77, 0xFA, 0xD7, 0xEF, 0x09, 0xCA, 0xDB, 0x09, 0x4A, 0xE9, 0x1E, 0x1A, 0x15, 0x97}; + static const unsigned char dh2048_g[] = { + 0x3F, 0xB3, 0x2C, 0x9B, 0x73, 0x13, 0x4D, 0x0B, 0x2E, 0x77, 0x50, 0x66, 0x60, 0xED, 0xBD, 0x48, 0x4C, 0xA7, 0xB1, 0x8F, + 0x21, 0xEF, 0x20, 0x54, 0x07, 0xF4, 0x79, 0x3A, 0x1A, 0x0B, 0xA1, 0x25, 0x10, 0xDB, 0xC1, 0x50, 0x77, 0xBE, 0x46, 0x3F, + 0xFF, 0x4F, 0xED, 0x4A, 0xAC, 0x0B, 0xB5, 0x55, 0xBE, 0x3A, 0x6C, 0x1B, 0x0C, 0x6B, 0x47, 0xB1, 0xBC, 0x37, 0x73, 0xBF, + 0x7E, 0x8C, 0x6F, 0x62, 0x90, 0x12, 0x28, 0xF8, 0xC2, 0x8C, 0xBB, 0x18, 0xA5, 0x5A, 0xE3, 0x13, 0x41, 0x00, 0x0A, 0x65, + 0x01, 0x96, 0xF9, 0x31, 0xC7, 0x7A, 0x57, 0xF2, 0xDD, 0xF4, 0x63, 0xE5, 0xE9, 0xEC, 0x14, 0x4B, 0x77, 0x7D, 0xE6, 0x2A, + 0xAA, 0xB8, 0xA8, 0x62, 0x8A, 0xC3, 0x76, 0xD2, 0x82, 0xD6, 0xED, 0x38, 0x64, 0xE6, 0x79, 0x82, 0x42, 0x8E, 0xBC, 0x83, + 0x1D, 0x14, 0x34, 0x8F, 0x6F, 0x2F, 0x91, 0x93, 0xB5, 0x04, 0x5A, 0xF2, 0x76, 0x71, 0x64, 0xE1, 0xDF, 0xC9, 0x67, 0xC1, + 0xFB, 0x3F, 0x2E, 0x55, 0xA4, 0xBD, 0x1B, 0xFF, 0xE8, 0x3B, 0x9C, 0x80, 0xD0, 0x52, 0xB9, 0x85, 0xD1, 0x82, 0xEA, 0x0A, + 0xDB, 0x2A, 0x3B, 0x73, 0x13, 0xD3, 0xFE, 0x14, 0xC8, 0x48, 0x4B, 0x1E, 0x05, 0x25, 0x88, 0xB9, 0xB7, 0xD2, 0xBB, 0xD2, + 0xDF, 0x01, 0x61, 0x99, 0xEC, 0xD0, 0x6E, 0x15, 0x57, 0xCD, 0x09, 0x15, 0xB3, 0x35, 0x3B, 0xBB, 0x64, 0xE0, 0xEC, 0x37, + 0x7F, 0xD0, 0x28, 0x37, 0x0D, 0xF9, 0x2B, 0x52, 0xC7, 0x89, 0x14, 0x28, 0xCD, 0xC6, 0x7E, 0xB6, 0x18, 0x4B, 0x52, 0x3D, + 0x1D, 0xB2, 0x46, 0xC3, 0x2F, 0x63, 0x07, 0x84, 0x90, 0xF0, 0x0E, 0xF8, 0xD6, 0x47, 0xD1, 0x48, 0xD4, 0x79, 0x54, 0x51, + 0x5E, 0x23, 0x27, 0xCF, 0xEF, 0x98, 0xC5, 0x82, 0x66, 0x4B, 0x4C, 0x0F, 0x6C, 0xC4, 0x16, 0x59}; + DH *dh; + BIGNUM *p; + BIGNUM *g; + + if ((dh = DH_new()) == nullptr) { + return nullptr; + } + p = BN_bin2bn(dh2048_p, sizeof(dh2048_p), nullptr); + g = BN_bin2bn(dh2048_g, sizeof(dh2048_g), nullptr); + if (p == nullptr || g == nullptr) { + DH_free(dh); + BN_free(p); + BN_free(g); + return nullptr; + } + DH_set0_pqg(dh, p, nullptr, g); + return (dh); +} +#endif // TS_USE_GET_DH_2048_256 + +bool +set_ctx_dh(SSL_CTX *ctx, dh_key_t *pkey) +{ + bool result{SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE) && SSL_CTX_set_tmp_dh(ctx, pkey)}; + DH_free(pkey); + return result; +} + +#endif // OPENSSL_IS_OPENSSL3 diff --git a/src/iocore/net/SSLKeyUtils.h b/src/iocore/net/SSLKeyUtils.h new file mode 100644 index 00000000000..64bea23e9ea --- /dev/null +++ b/src/iocore/net/SSLKeyUtils.h @@ -0,0 +1,42 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#if OPENSSL_IS_OPENSSL3 +#include +#else +#include +#endif +#include + +#ifdef OPENSSL_IS_OPENSSL3 +using dh_key_t = EVP_PKEY; +#else +using dh_key_t = DH; +#endif + +// Both gen_dh_2048_256_pkey and load_dhparams_file return owning pointers. +dh_key_t *gen_dh_2048_256_pkey(); +dh_key_t *load_dhparams_file(char const *dhparams_file); + +// Takes ownership of pkey. +bool set_ctx_dh(SSL_CTX *ctx, dh_key_t *pkey); diff --git a/src/iocore/net/SSLUtils.cc b/src/iocore/net/SSLUtils.cc index 9783f3f27ab..0a6750b98ae 100644 --- a/src/iocore/net/SSLUtils.cc +++ b/src/iocore/net/SSLUtils.cc @@ -26,6 +26,7 @@ #include "P_SSLConfig.h" #include "P_SSLNetVConnection.h" #include "P_TLSKeyLogger.h" +#include "SSLKeyUtils.h" #include "SSLStats.h" #include "SSLSessionCache.h" #include "SSLSessionTicket.h" @@ -41,6 +42,7 @@ #include "tscore/ink_config.h" #include "tscore/SimpleTokenizer.h" #include "tscore/Layout.h" +#include "tscore/ink_assert.h" #include "tscore/ink_cap.h" #include "tscore/ink_mutex.h" #include "tscore/Filenames.h" @@ -54,8 +56,10 @@ #include #include #include -#include #include +#ifdef OPENSSL_IS_OPENSSL3 +#include +#endif #include #include #if HAVE_ENGINE_LOAD_DYNAMIC @@ -483,59 +487,6 @@ ssl_alpn_select_callback(SSL *ssl, const unsigned char **out, unsigned char *out return SSL_TLSEXT_ERR_NOACK; } -#if TS_USE_GET_DH_2048_256 == 0 -/* Build 2048-bit MODP Group with 256-bit Prime Order Subgroup from RFC 5114 */ -static DH * -DH_get_2048_256() -{ - static const unsigned char dh2048_p[] = { - 0x87, 0xA8, 0xE6, 0x1D, 0xB4, 0xB6, 0x66, 0x3C, 0xFF, 0xBB, 0xD1, 0x9C, 0x65, 0x19, 0x59, 0x99, 0x8C, 0xEE, 0xF6, 0x08, - 0x66, 0x0D, 0xD0, 0xF2, 0x5D, 0x2C, 0xEE, 0xD4, 0x43, 0x5E, 0x3B, 0x00, 0xE0, 0x0D, 0xF8, 0xF1, 0xD6, 0x19, 0x57, 0xD4, - 0xFA, 0xF7, 0xDF, 0x45, 0x61, 0xB2, 0xAA, 0x30, 0x16, 0xC3, 0xD9, 0x11, 0x34, 0x09, 0x6F, 0xAA, 0x3B, 0xF4, 0x29, 0x6D, - 0x83, 0x0E, 0x9A, 0x7C, 0x20, 0x9E, 0x0C, 0x64, 0x97, 0x51, 0x7A, 0xBD, 0x5A, 0x8A, 0x9D, 0x30, 0x6B, 0xCF, 0x67, 0xED, - 0x91, 0xF9, 0xE6, 0x72, 0x5B, 0x47, 0x58, 0xC0, 0x22, 0xE0, 0xB1, 0xEF, 0x42, 0x75, 0xBF, 0x7B, 0x6C, 0x5B, 0xFC, 0x11, - 0xD4, 0x5F, 0x90, 0x88, 0xB9, 0x41, 0xF5, 0x4E, 0xB1, 0xE5, 0x9B, 0xB8, 0xBC, 0x39, 0xA0, 0xBF, 0x12, 0x30, 0x7F, 0x5C, - 0x4F, 0xDB, 0x70, 0xC5, 0x81, 0xB2, 0x3F, 0x76, 0xB6, 0x3A, 0xCA, 0xE1, 0xCA, 0xA6, 0xB7, 0x90, 0x2D, 0x52, 0x52, 0x67, - 0x35, 0x48, 0x8A, 0x0E, 0xF1, 0x3C, 0x6D, 0x9A, 0x51, 0xBF, 0xA4, 0xAB, 0x3A, 0xD8, 0x34, 0x77, 0x96, 0x52, 0x4D, 0x8E, - 0xF6, 0xA1, 0x67, 0xB5, 0xA4, 0x18, 0x25, 0xD9, 0x67, 0xE1, 0x44, 0xE5, 0x14, 0x05, 0x64, 0x25, 0x1C, 0xCA, 0xCB, 0x83, - 0xE6, 0xB4, 0x86, 0xF6, 0xB3, 0xCA, 0x3F, 0x79, 0x71, 0x50, 0x60, 0x26, 0xC0, 0xB8, 0x57, 0xF6, 0x89, 0x96, 0x28, 0x56, - 0xDE, 0xD4, 0x01, 0x0A, 0xBD, 0x0B, 0xE6, 0x21, 0xC3, 0xA3, 0x96, 0x0A, 0x54, 0xE7, 0x10, 0xC3, 0x75, 0xF2, 0x63, 0x75, - 0xD7, 0x01, 0x41, 0x03, 0xA4, 0xB5, 0x43, 0x30, 0xC1, 0x98, 0xAF, 0x12, 0x61, 0x16, 0xD2, 0x27, 0x6E, 0x11, 0x71, 0x5F, - 0x69, 0x38, 0x77, 0xFA, 0xD7, 0xEF, 0x09, 0xCA, 0xDB, 0x09, 0x4A, 0xE9, 0x1E, 0x1A, 0x15, 0x97}; - static const unsigned char dh2048_g[] = { - 0x3F, 0xB3, 0x2C, 0x9B, 0x73, 0x13, 0x4D, 0x0B, 0x2E, 0x77, 0x50, 0x66, 0x60, 0xED, 0xBD, 0x48, 0x4C, 0xA7, 0xB1, 0x8F, - 0x21, 0xEF, 0x20, 0x54, 0x07, 0xF4, 0x79, 0x3A, 0x1A, 0x0B, 0xA1, 0x25, 0x10, 0xDB, 0xC1, 0x50, 0x77, 0xBE, 0x46, 0x3F, - 0xFF, 0x4F, 0xED, 0x4A, 0xAC, 0x0B, 0xB5, 0x55, 0xBE, 0x3A, 0x6C, 0x1B, 0x0C, 0x6B, 0x47, 0xB1, 0xBC, 0x37, 0x73, 0xBF, - 0x7E, 0x8C, 0x6F, 0x62, 0x90, 0x12, 0x28, 0xF8, 0xC2, 0x8C, 0xBB, 0x18, 0xA5, 0x5A, 0xE3, 0x13, 0x41, 0x00, 0x0A, 0x65, - 0x01, 0x96, 0xF9, 0x31, 0xC7, 0x7A, 0x57, 0xF2, 0xDD, 0xF4, 0x63, 0xE5, 0xE9, 0xEC, 0x14, 0x4B, 0x77, 0x7D, 0xE6, 0x2A, - 0xAA, 0xB8, 0xA8, 0x62, 0x8A, 0xC3, 0x76, 0xD2, 0x82, 0xD6, 0xED, 0x38, 0x64, 0xE6, 0x79, 0x82, 0x42, 0x8E, 0xBC, 0x83, - 0x1D, 0x14, 0x34, 0x8F, 0x6F, 0x2F, 0x91, 0x93, 0xB5, 0x04, 0x5A, 0xF2, 0x76, 0x71, 0x64, 0xE1, 0xDF, 0xC9, 0x67, 0xC1, - 0xFB, 0x3F, 0x2E, 0x55, 0xA4, 0xBD, 0x1B, 0xFF, 0xE8, 0x3B, 0x9C, 0x80, 0xD0, 0x52, 0xB9, 0x85, 0xD1, 0x82, 0xEA, 0x0A, - 0xDB, 0x2A, 0x3B, 0x73, 0x13, 0xD3, 0xFE, 0x14, 0xC8, 0x48, 0x4B, 0x1E, 0x05, 0x25, 0x88, 0xB9, 0xB7, 0xD2, 0xBB, 0xD2, - 0xDF, 0x01, 0x61, 0x99, 0xEC, 0xD0, 0x6E, 0x15, 0x57, 0xCD, 0x09, 0x15, 0xB3, 0x35, 0x3B, 0xBB, 0x64, 0xE0, 0xEC, 0x37, - 0x7F, 0xD0, 0x28, 0x37, 0x0D, 0xF9, 0x2B, 0x52, 0xC7, 0x89, 0x14, 0x28, 0xCD, 0xC6, 0x7E, 0xB6, 0x18, 0x4B, 0x52, 0x3D, - 0x1D, 0xB2, 0x46, 0xC3, 0x2F, 0x63, 0x07, 0x84, 0x90, 0xF0, 0x0E, 0xF8, 0xD6, 0x47, 0xD1, 0x48, 0xD4, 0x79, 0x54, 0x51, - 0x5E, 0x23, 0x27, 0xCF, 0xEF, 0x98, 0xC5, 0x82, 0x66, 0x4B, 0x4C, 0x0F, 0x6C, 0xC4, 0x16, 0x59}; - DH *dh; - BIGNUM *p; - BIGNUM *g; - - if ((dh = DH_new()) == nullptr) { - return nullptr; - } - p = BN_bin2bn(dh2048_p, sizeof(dh2048_p), nullptr); - g = BN_bin2bn(dh2048_g, sizeof(dh2048_g), nullptr); - if (p == nullptr || g == nullptr) { - DH_free(dh); - BN_free(p); - BN_free(g); - return nullptr; - } - DH_set0_pqg(dh, p, nullptr, g); - return (dh); -} -#endif - bool SSLMultiCertConfigLoader::_enable_cert_compression(SSL_CTX *ctx) { @@ -590,28 +541,19 @@ SSLMultiCertConfigLoader::_enable_early_data([[maybe_unused]] SSL_CTX *ctx) static SSL_CTX * ssl_context_enable_dhe(const char *dhparams_file, SSL_CTX *ctx) { - DH *server_dh; + dh_key_t *pkey{}; if (dhparams_file) { - scoped_BIO bio(BIO_new_file(dhparams_file, "r")); - server_dh = PEM_read_bio_DHparams(bio.get(), nullptr, nullptr, nullptr); + pkey = load_dhparams_file(dhparams_file); } else { - server_dh = DH_get_2048_256(); - } - - if (!server_dh) { - Error("SSL dhparams source returned invalid parameters"); - return nullptr; + pkey = gen_dh_2048_256_pkey(); } - if (!SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE) || !SSL_CTX_set_tmp_dh(ctx, server_dh)) { - DH_free(server_dh); + if (!pkey || !set_ctx_dh(ctx, pkey)) { Error("failed to configure SSL DH"); return nullptr; } - DH_free(server_dh); - return ctx; } diff --git a/src/iocore/net/unit_tests/test_SSLDHParams.cc b/src/iocore/net/unit_tests/test_SSLDHParams.cc new file mode 100644 index 00000000000..6a32a7458ba --- /dev/null +++ b/src/iocore/net/unit_tests/test_SSLDHParams.cc @@ -0,0 +1,186 @@ +/** @file + + Catch based unit tests for the DH-parameter handling behavior of + SSLMultiCertConfigLoader::init_server_ssl_ctx, which is the inknet + public boundary that transitively invokes ssl_context_enable_dhe + and (when a file is configured) load_dhparams_file. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include "../P_SSLCertLookup.h" +#include "../P_SSLConfig.h" +#include "../P_SSLUtils.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace +{ + +std::string +make_valid_dh_pem() +{ + EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_from_name(nullptr, "DH", nullptr); + REQUIRE(pctx != nullptr); + REQUIRE(EVP_PKEY_paramgen_init(pctx) > 0); + char prime_group[]{"dh_2048_256"}; + OSSL_PARAM const params[2] = { + OSSL_PARAM_construct_utf8_string("group", prime_group, 0), + OSSL_PARAM_construct_end(), + }; + REQUIRE(EVP_PKEY_CTX_set_params(pctx, params) > 0); + EVP_PKEY *pkey = nullptr; + REQUIRE(EVP_PKEY_generate(pctx, &pkey) > 0); + + BIO *bio = BIO_new(BIO_s_mem()); + REQUIRE(PEM_write_bio_Parameters(bio, pkey) == 1); + BUF_MEM *bm = nullptr; + BIO_get_mem_ptr(bio, &bm); + std::string out{bm->data, bm->length}; + BIO_free(bio); + EVP_PKEY_free(pkey); + EVP_PKEY_CTX_free(pctx); + return out; +} + +std::string +make_rsa_pem() +{ + EVP_PKEY *pkey = EVP_RSA_gen(2048); + REQUIRE(pkey != nullptr); + BIO *bio = BIO_new(BIO_s_mem()); + REQUIRE(PEM_write_bio_PrivateKey(bio, pkey, nullptr, nullptr, 0, nullptr, nullptr) == 1); + BUF_MEM *bm = nullptr; + BIO_get_mem_ptr(bio, &bm); + std::string out{bm->data, bm->length}; + BIO_free(bio); + EVP_PKEY_free(pkey); + return out; +} + +class TempFile +{ +public: + explicit TempFile(std::string const &contents) + { + char tmpl[] = "/tmp/ats_dhparams_XXXXXX"; + int fd = mkstemp(tmpl); + REQUIRE(fd != -1); + this->path = tmpl; + if (!contents.empty()) { + REQUIRE(write(fd, contents.data(), contents.size()) == static_cast(contents.size())); + } + close(fd); + } + TempFile(TempFile const &) = delete; + TempFile(TempFile &&) = delete; + TempFile &operator=(TempFile const &) = delete; + TempFile &operator=(TempFile &&) = delete; + ~TempFile() { unlink(this->path.c_str()); } + + char const * + get_path() const + { + return this->path.c_str(); + } + +private: + std::string path; +}; + +// Drives ssl_context_enable_dhe via init_server_ssl_ctx, holding every +// non-DHE input fixed and varying only dhparamsFile. An empty CertLoadData +// selects the "default generated ctx" branch which still traverses +// ssl_context_enable_dhe but skips cert/key loading entirely, so a non-empty +// returned vector with a non-null SSL_CTX is observable iff DHE configuration +// succeeded. +bool +init_with_dhparams(char const *dhparams_file) +{ + SSLConfigParams params; + params.dhparamsFile = dhparams_file ? ats_strdup(dhparams_file) : nullptr; + + SSLMultiCertConfigLoader loader{¶ms}; + SSLMultiCertConfigLoader::CertLoadData data; + auto contexts = loader.init_server_ssl_ctx(data, nullptr); + + bool ok = !contexts.empty() && contexts.front().ctx != nullptr; + for (auto const &lc : contexts) { + SSL_CTX_free(lc.ctx); + } + return ok; +} + +} // namespace + +TEST_CASE("ssl_context_enable_dhe: nullptr dhparams file falls back to built-in DH parameters") +{ + CHECK(init_with_dhparams(nullptr)); +} + +TEST_CASE("ssl_context_enable_dhe: valid dh_2048_256 DH PEM file is accepted") +{ + TempFile dh{make_valid_dh_pem()}; + CHECK(init_with_dhparams(dh.get_path())); +} + +TEST_CASE("ssl_context_enable_dhe: nonexistent dhparams path is rejected") +{ + CHECK_FALSE(init_with_dhparams("/tmp/ats_dhparams_does_not_exist_zzz_xyz")); +} + +TEST_CASE("ssl_context_enable_dhe: empty dhparams file is rejected") +{ + TempFile empty{""}; + CHECK_FALSE(init_with_dhparams(empty.get_path())); +} + +TEST_CASE("ssl_context_enable_dhe: non-PEM garbage in dhparams file is rejected") +{ + TempFile garbage{"this is definitely not a PEM-encoded DH parameter block\n"}; + CHECK_FALSE(init_with_dhparams(garbage.get_path())); +} + +TEST_CASE("ssl_context_enable_dhe: PEM of wrong key type (RSA) is rejected by DH-only decoder") +{ + TempFile rsa{make_rsa_pem()}; + CHECK_FALSE(init_with_dhparams(rsa.get_path())); +} + +TEST_CASE("ssl_context_enable_dhe: truncated DH PEM (missing END marker) is rejected") +{ + std::string pem = make_valid_dh_pem(); + auto end = pem.find("-----END"); + REQUIRE(end != std::string::npos); + TempFile truncated{pem.substr(0, end)}; + CHECK_FALSE(init_with_dhparams(truncated.get_path())); +} From 7b31a6cb02d88ea1b7d8b1167e61debf56eaba88 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Wed, 29 Jul 2026 17:09:38 -0500 Subject: [PATCH 02/11] Fix H2 origin payload handling (#13363) HTTP/2 origin responses can legally carry a non-zero Content-Length when no payload is sent, such as responses to HEAD requests. ATS discarded the outbound request method after encoding the H2 HEADERS frame and could therefore reject a valid no-body response as a payload-length error. An H2 DATA sender also treated every byte visible through its IOBufferReader as eligible for the current write. Reader availability is independent of the finite VIO operation: VIO::ntodo() is the authoritative boundary, and the ordinary network VConnection already caps writes to it. Without that cap, the regression sent 327,675 bytes for a 300,000-byte PUT and the H2 origin returned GOAWAY with PROTOCOL_ERROR. This retains the outbound request method on the H2 stream for response validation. It also caps DATA payloads to the remaining write VIO bytes, still setting END_STREAM when the final authorized bytes are sent, and extends the H2 origin replay coverage with HEAD and large PUT cases. (cherry picked from commit dcb18509fa299a159d9ac511a06ac53bacc17254) --- include/proxy/http2/Http2Stream.h | 21 ++- src/proxy/http2/Http2ConnectionState.cc | 15 ++- .../h2/gold/http-request-method-metrics.gold | 2 +- tests/gold_tests/h2/h2origin.test.py | 6 +- .../h2/replay_h2origin/h2-origin.yaml | 123 ++++++++++++++++++ 5 files changed, 160 insertions(+), 7 deletions(-) diff --git a/include/proxy/http2/Http2Stream.h b/include/proxy/http2/Http2Stream.h index dbf64edb4b9..3fb388670c6 100644 --- a/include/proxy/http2/Http2Stream.h +++ b/include/proxy/http2/Http2Stream.h @@ -164,6 +164,7 @@ class Http2Stream : public ProxyTransaction void increment_data_length(uint64_t length); bool payload_length_is_valid() const; bool is_write_vio_done() const; + int64_t write_vio_ntodo() const; void update_sent_count(unsigned num_bytes); Http2StreamId get_id() const; Http2StreamState get_state() const; @@ -175,6 +176,7 @@ class Http2Stream : public ProxyTransaction void set_receive_headers(HTTPHdr &h2_headers); void reset_receive_headers(); void reset_send_headers(); + void set_sent_request_method(int method); MIOBuffer *read_vio_writer() const; int64_t read_vio_read_avail(); bool is_read_enabled() const; @@ -215,6 +217,7 @@ class Http2Stream : public ProxyTransaction Http2StreamId _id = -1; Http2StreamState _state = Http2StreamState::HTTP2_STREAM_STATE_IDLE; int64_t _http_sm_id = -1; + int _sent_request_method{-1}; HTTPHdr _receive_header; #if TS_USE_MALLOC_ALLOCATOR @@ -316,6 +319,12 @@ Http2Stream::is_write_vio_done() const return this->write_vio.ntodo() == 0; } +inline int64_t +Http2Stream::write_vio_ntodo() const +{ + return this->write_vio.ntodo(); +} + inline void Http2Stream::update_sent_count(unsigned num_bytes) { @@ -391,6 +400,12 @@ Http2Stream::reset_send_headers() this->_send_header.create(HTTPType::RESPONSE); } +inline void +Http2Stream::set_sent_request_method(int method) +{ + _sent_request_method = method; +} + // Check entire DATA payload length if content-length: header exists inline void Http2Stream::increment_data_length(uint64_t length) @@ -407,9 +422,9 @@ Http2Stream::payload_length_is_valid() const // Skip Content-Length check on [RFC 7230] 3.3.2 conditions bool is_payload_precluded = - this->is_outbound_connection() && (_send_header.method_get_wksidx() == HTTP_WKSIDX_HEAD || - (_send_header.method_get_wksidx() == HTTP_WKSIDX_GET && _send_header.presence(mask) && - _receive_header.status_get() == HTTPStatus::NOT_MODIFIED)); + this->is_outbound_connection() && + (_sent_request_method == HTTP_WKSIDX_HEAD || (_sent_request_method == HTTP_WKSIDX_GET && _send_header.presence(mask) && + _receive_header.status_get() == HTTPStatus::NOT_MODIFIED)); if (content_length != 0 && !is_payload_precluded && content_length != data_length) { Warning("Bad payload length content_length=%d data_legnth=%d session_id=%" PRId64, content_length, diff --git a/src/proxy/http2/Http2ConnectionState.cc b/src/proxy/http2/Http2ConnectionState.cc index 3eaa2f464a6..e8119ee9f67 100644 --- a/src/proxy/http2/Http2ConnectionState.cc +++ b/src/proxy/http2/Http2ConnectionState.cc @@ -2332,6 +2332,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len uint8_t flags = 0x00; IOBufferReader *resp_reader = stream->get_data_reader_for_send(); + bool last_write_vio_payload{false}; SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); @@ -2366,6 +2367,16 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len } else { payload_length = resp_reader->read_avail(); } + const int64_t remaining_write = stream->write_vio_ntodo(); + if (remaining_write != INT64_MAX) { + if (remaining_write > 0) { + last_write_vio_payload = payload_length >= static_cast(remaining_write); + payload_length = std::min(payload_length, static_cast(remaining_write)); + } else { + last_write_vio_payload = true; + payload_length = 0; + } + } } else { payload_length = 0; } @@ -2395,7 +2406,8 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len return Http2SendDataFrameResult::NO_PAYLOAD; } - if (stream->is_write_vio_done() && !resp_reader->is_read_avail_more_than(payload_length) && !stream->expect_send_trailer()) { + if (stream->is_write_vio_done() && (last_write_vio_payload || !resp_reader->is_read_avail_more_than(payload_length)) && + !stream->expect_send_trailer()) { Http2StreamDebug(this->session, stream->get_id(), "End of Data Frame"); flags |= HTTP2_FLAGS_DATA_END_STREAM; } @@ -2524,6 +2536,7 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) flags |= HTTP2_FLAGS_HEADERS_END_HEADERS; if (stream->is_outbound_connection()) { // Will be sending a request_header int method = send_hdr->method_get_wksidx(); + stream->set_sent_request_method(method); // Set END_STREAM on request headers for POST, etc. methods combined with // an explicit length 0. Some origins RST on request headers with diff --git a/tests/gold_tests/h2/gold/http-request-method-metrics.gold b/tests/gold_tests/h2/gold/http-request-method-metrics.gold index f949dc4270f..6418e16a565 100644 --- a/tests/gold_tests/h2/gold/http-request-method-metrics.gold +++ b/tests/gold_tests/h2/gold/http-request-method-metrics.gold @@ -1,3 +1,3 @@ proxy.process.http.get_requests 4 proxy.process.http.post_requests 11 -proxy.process.http.put_requests 0 +proxy.process.http.put_requests 1 diff --git a/tests/gold_tests/h2/h2origin.test.py b/tests/gold_tests/h2/h2origin.test.py index 24e92b553e9..4bd98b60005 100644 --- a/tests/gold_tests/h2/h2origin.test.py +++ b/tests/gold_tests/h2/h2origin.test.py @@ -83,7 +83,7 @@ timeout = 30 watcher = tr.Processes.Process("watcher") watcher.Command = f"sleep {timeout}" -watcher.Ready = When.FileContains(ts.Disk.squid_log.Name, r'14 http/1.1 http/2') +watcher.Ready = When.FileContains(ts.Disk.squid_log.Name, r'16 http/2 http/2') watcher.TimeOut = timeout tr.StillRunningAfter = ts tr.StillRunningAfter = server @@ -93,7 +93,7 @@ tr.Processes.Default.ReturnCode = 0 # UUIDs 1-4 should be http/1.1 clients and H2 origin -# UUIDs 5-9 should be http/2 clients and H2 origins +# UUIDs 5-11 and 15-16 should be http/2 clients and H2 origins ts.Disk.squid_log.Content = Testers.ContainsExpression(" [1-4] http/1.1 http/2", "cases 1-4 request http/1.1") ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [1-4] http/2 http/2", "cases 1-4 request http/1.1") ts.Disk.squid_log.Content += Testers.ContainsExpression(" 1[1-4] http/1.1 http/2", "cases 12-14 request http/1.1") @@ -102,6 +102,8 @@ ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [5-9] http/1.1 http/2", "cases 5-11 request http/2") ts.Disk.squid_log.Content += Testers.ContainsExpression(" 1[0-1] http/2 http/2", "cases 5-11 request http/2") ts.Disk.squid_log.Content += Testers.ExcludesExpression(" 1[0-1] http/1.1 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" 1[5-6] http/2 http/2", "cases 15-16 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" 1[5-6] http/1.1 http/2", "cases 15-16 request http/2") tr = Test.AddTestRun("Test HTTP method Metrics") tr.Processes.Default.Command = ( diff --git a/tests/gold_tests/h2/replay_h2origin/h2-origin.yaml b/tests/gold_tests/h2/replay_h2origin/h2-origin.yaml index acf7035fcd6..0310eacb347 100644 --- a/tests/gold_tests/h2/replay_h2origin/h2-origin.yaml +++ b/tests/gold_tests/h2/replay_h2origin/h2-origin.yaml @@ -477,3 +477,126 @@ sessions: content: encoding: plain size: 3200 + + # + # Test 8: HEAD response with a non-zero Content-Length and no body. + # + - all: { headers: { fields: [[ uuid, 15 ]]}} + + client-request: + version: '2' + scheme: https + method: HEAD + url: /some/head + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + stack: http2 + tls: + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + version: '2' + scheme: https + method: HEAD + url: /some/head + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + headers: + encoding: esc_json + fields: + - [ Content-Length, 100 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 200 + headers: + encoding: esc_json + fields: + - [ Content-Length, 100 ] + content: + encoding: plain + size: 0 + + # + # Test 9: large PUT body with a large response. + # + - all: { headers: { fields: [[ uuid, 16 ]]}} + + client-request: + version: '2' + scheme: https + method: PUT + url: /some/large-put + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 300000 ] + content: + encoding: plain + size: 300000 + + proxy-request: + protocol: + stack: http2 + tls: + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + version: '2' + scheme: https + method: PUT + url: /some/large-put + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 300000 ] + content: + encoding: plain + size: 300000 + + server-response: + version: '2' + status: 200 + headers: + encoding: esc_json + fields: + - [ Content-Length, 300000 ] + content: + encoding: plain + size: 300000 + + proxy-response: + version: '2' + status: 200 + headers: + encoding: esc_json + fields: + - [ Content-Length, 300000 ] + content: + encoding: plain + size: 300000 From 71095695a1a9a8b49a835149a4ff61d5a50b7eb9 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Wed, 29 Jul 2026 11:54:16 -0500 Subject: [PATCH 03/11] Allow dynamic TLS record sizing (#13416) The documented -1 value for proxy.config.ssl.max_record_size is incorrectly rejected by records validation, leaving dynamic TLS record sizing unreachable from records.yaml. This widens the accepted range to include the dynamic sentinel, clarifies the documented modes, and extends TLS wire-level coverage to verify the small-to-large record transition. Fixes: #13288 (cherry picked from commit a0ece04992e7c3b3ef8b06f9c135c68b360e0843) --- doc/admin-guide/files/records.yaml.en.rst | 5 +- src/records/RecordsConfig.cc | 2 +- tests/gold_tests/tls/tls_record_size.test.py | 72 ++++++++++--------- .../gold_tests/tls/tls_record_size_client.py | 50 +++++++++++-- 4 files changed, 86 insertions(+), 43 deletions(-) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index 96a2dcad765..00f000a5e82 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -4306,8 +4306,9 @@ SSL Termination This configuration specifies the maximum number of bytes to write into a SSL record when replying over a SSL session. In some circumstances this setting can improve response latency by reducing - buffering at the SSL layer. This setting can have a value between 0 - and 16383 (max TLS record size). + buffering at the SSL layer. This setting accepts ``-1`` for dynamic + sizing, ``0`` for the default behavior, or a fixed maximum between + ``1`` and ``16383`` bytes. The default of ``0`` means to always write all available data into a single SSL record. diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index e3bc28f4e97..2065d87b09c 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -1242,7 +1242,7 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.session_cache.skip_cache_on_bucket_contention", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.max_record_size", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-16383]", RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.max_record_size", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[-1-16383]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.session_cache.timeout", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , diff --git a/tests/gold_tests/tls/tls_record_size.test.py b/tests/gold_tests/tls/tls_record_size.test.py index da1960a30d1..88081abae87 100644 --- a/tests/gold_tests/tls/tls_record_size.test.py +++ b/tests/gold_tests/tls/tls_record_size.test.py @@ -1,7 +1,7 @@ ''' -Exercise the TLS record-size clamp (proxy.config.ssl.max_record_size > 0): on a -large TLS download the body must arrive intact and every application-data record -on the wire must be clamped to the configured size. +Exercise fixed and dynamic TLS record sizing. On a large TLS download the body +must arrive intact and application-data records on the wire must follow the +configured sizing strategy. ''' # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -19,30 +19,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: only the positive (fixed-clamp) branch of the record-sizing logic is -# covered here. The documented dynamic mode (max_record_size == -1) cannot be -# enabled through records.yaml because the record's validity check is [0-16383], -# which rejects -1; that inconsistency is pre-existing, so the dynamic branch -# stays uncovered by design. - import os import sys Test.Summary = __doc__ -class TestRecordSizeClamp: - '''Verify max_record_size clamps every record of a large TLS download.''' - - # Comfortably larger than the clamp so many records pass through it. - _body_len: int = 1024 * 1024 - _max_record: int = 4096 +class TestRecordSize: + '''Verify fixed and dynamic TLS record sizing on large downloads.''' _server_counter: int = 0 _ts_counter: int = 0 - def __init__(self) -> None: + def __init__(self, max_record: int, body_len: int) -> None: '''Declare the test Processes.''' + self._max_record = max_record + self._body_len = body_len self._server = self._configure_server() self._ts = self._configure_trafficserver() @@ -51,15 +43,15 @@ def _configure_server(self) -> 'Process': :return: The origin server Process. ''' - server = Test.MakeOriginServer(f'server-{TestRecordSizeClamp._server_counter}') - TestRecordSizeClamp._server_counter += 1 + server = Test.MakeOriginServer(f'server-{TestRecordSize._server_counter}') + TestRecordSize._server_counter += 1 - body = "x" * TestRecordSizeClamp._body_len + body = "x" * self._body_len request_header = {"headers": "GET /obj HTTP/1.1\r\nHost: ex.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_header = { "headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n" - f"Cache-Control: max-age=3600\r\nContent-Length: {TestRecordSizeClamp._body_len}\r\n\r\n", + f"Cache-Control: max-age=3600\r\nContent-Length: {self._body_len}\r\n\r\n", "timestamp": "1469733493.993", "body": body } @@ -67,12 +59,12 @@ def _configure_server(self) -> 'Process': return server def _configure_trafficserver(self) -> 'Process': - '''Configure Traffic Server with a positive max_record_size clamp. + '''Configure Traffic Server with the requested record-size strategy. :return: The Traffic Server Process. ''' - ts = Test.MakeATSProcess(f'ts-{TestRecordSizeClamp._ts_counter}', enable_tls=True) - TestRecordSizeClamp._ts_counter += 1 + ts = Test.MakeATSProcess(f'ts-{TestRecordSize._ts_counter}', enable_tls=True) + TestRecordSize._ts_counter += 1 ts.addDefaultSSLFiles() ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') @@ -81,31 +73,45 @@ def _configure_trafficserver(self) -> 'Process': { 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', - # Positive cap -> the write path clamps each TLS record to this many bytes. - 'proxy.config.ssl.max_record_size': TestRecordSizeClamp._max_record, + 'proxy.config.ssl.max_record_size': self._max_record, }) + if self._max_record == -1: + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + r'proxy\.config\.ssl\.max_record_size.*Validity Check error', + 'The dynamic record-size sentinel should pass records validation') return ts def run(self) -> None: '''Configure and run the TestRun. - The client downloads the object and measures the TLS records on the wire, - asserting both that the body is intact and that no application-data record - exceeds the configured clamp. + The client downloads the object and measures the TLS records on the wire. ''' - tr = Test.AddTestRun("max_record_size>0 clamps records on a large TLS download") + if self._max_record == -1: + description = 'max_record_size=-1 dynamically sizes records on a large TLS download' + client_option = '--dynamic' + expected_output = 'PASS: TLS records ramp from small to large after the dynamic threshold' + else: + description = 'max_record_size>0 clamps records on a large TLS download' + client_option = f'--max-record {self._max_record}' + expected_output = 'PASS: every application-data record is within the configured clamp' + + tr = Test.AddTestRun(description) tr.Processes.Default.StartBefore(self._server) tr.Processes.Default.StartBefore(self._ts) tr.Processes.Default.Command = ( f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_record_size_client.py")} ' f'-p {self._ts.Variables.ssl_port} --host ex.test --path /obj ' - f'--max-record {TestRecordSizeClamp._max_record} --expect-bytes {TestRecordSizeClamp._body_len}') + f'{client_option} --expect-bytes {self._body_len}') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All += Testers.ContainsExpression( - "PASS: every application-data record is within the configured clamp", - "every TLS record must be clamped to the configured size") + expected_output, 'TLS records must follow the configured sizing strategy') tr.StillRunningAfter = self._ts tr.StillRunningAfter = self._server -TestRecordSizeClamp().run() +# The fixed-size test response is comfortably larger than its 4,096-byte clamp, +# ensuring that many records exercise the clamp. The dynamic-sizing test +# response must exceed its 1,000,000-byte threshold by enough data to demonstrate +# both phases. +TestRecordSize(4096, 1024 * 1024).run() +TestRecordSize(-1, 2 * 1024 * 1024).run() diff --git a/tests/gold_tests/tls/tls_record_size_client.py b/tests/gold_tests/tls/tls_record_size_client.py index a169944e5c4..b5a5ddce2c1 100644 --- a/tests/gold_tests/tls/tls_record_size_client.py +++ b/tests/gold_tests/tls/tls_record_size_client.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 ''' Download an object from ATS over TLS and inspect the TLS records on the wire: -confirm the body arrives intact AND that every application-data record is no -larger than the configured proxy.config.ssl.max_record_size (plus AEAD overhead). +confirm the body arrives intact AND that application-data records follow the +configured fixed or dynamic record-size strategy. A MemoryBIO drives the handshake so the raw ciphertext stream is visible; the 5-byte TLS record headers (type, version, length) are in cleartext, so record @@ -34,12 +34,16 @@ from collections.abc import Iterator TLS_APPLICATION_DATA = 23 +TLS12_GCM_OVERHEAD = 24 # A clamped plaintext record becomes ciphertext of plaintext + AEAD overhead # (TLS1.2 GCM: 8-byte explicit nonce + 16-byte tag = 24 bytes; the cipher is pinned # to AEAD below). 256 is a generous ceiling over that, far below an unclamped ~16 KB # record, so the clamp check stays decisive and cannot be tripped by the larger, # variable expansion of a CBC suite. RECORD_OVERHEAD = 256 +DYNAMIC_SMALL_RECORD = 1300 +DYNAMIC_MAX_RECORD = 16383 +DYNAMIC_BYTE_THRESHOLD = 1_000_000 def iter_record_lengths(buf: bytes | bytearray) -> Iterator[tuple[int, int]]: @@ -54,12 +58,42 @@ def iter_record_lengths(buf: bytes | bytearray) -> Iterator[tuple[int, int]]: i += 5 + length +def verify_dynamic_records(app_lengths: list[int]) -> bool: + '''Verify records ramp from single-segment to maximum-sized records.''' + small_limit = DYNAMIC_SMALL_RECORD + TLS12_GCM_OVERHEAD + max_limit = DYNAMIC_MAX_RECORD + TLS12_GCM_OVERHEAD + first_large = next((i for i, length in enumerate(app_lengths) if length > small_limit), None) + + if first_large is None: + print('FAIL: dynamic sizing never ramped up to large TLS records') + return False + + plaintext_before_ramp = sum(length - TLS12_GCM_OVERHEAD for length in app_lengths[:first_large]) + if plaintext_before_ramp < DYNAMIC_BYTE_THRESHOLD: + print( + f'FAIL: dynamic sizing ramped after only {plaintext_before_ramp} plaintext bytes; ' + f'expected at least {DYNAMIC_BYTE_THRESHOLD}') + return False + + max_record = max(app_lengths) + if max_record > max_limit: + print(f'FAIL: a dynamic application-data record ({max_record}) exceeds the maximum ({max_limit})') + return False + + print( + f'PASS: TLS records ramp from small to large after the dynamic threshold ' + f'(first_large={first_large}, plaintext_before_ramp={plaintext_before_ramp}, max_record_len={max_record})') + return True + + def main() -> int: parser = argparse.ArgumentParser(description='Measure ATS TLS record sizes on a download.') parser.add_argument('-p', '--port', type=int, required=True, help='ATS TLS port') parser.add_argument('--host', default='ex.test', help='Host header / SNI') parser.add_argument('--path', default='/obj', help='request path') - parser.add_argument('--max-record', type=int, required=True, help='configured proxy.config.ssl.max_record_size') + sizing = parser.add_mutually_exclusive_group(required=True) + sizing.add_argument('--max-record', type=int, help='positive proxy.config.ssl.max_record_size clamp') + sizing.add_argument('--dynamic', action='store_true', help='expect dynamic TLS record sizing') parser.add_argument('--expect-bytes', type=int, required=True, help='expected response body length') args = parser.parse_args() @@ -132,11 +166,8 @@ def feed() -> bytes: app_lengths = [length for content_type, length in iter_record_lengths(raw) if content_type == TLS_APPLICATION_DATA] max_record = max(app_lengths) if app_lengths else 0 - limit = args.max_record + RECORD_OVERHEAD - print( - f'app_data_records={len(app_lengths)} max_record_len={max_record} ' - f'limit={limit} body_len={body_len} expect={args.expect_bytes}') + print(f'app_data_records={len(app_lengths)} max_record_len={max_record} body_len={body_len} expect={args.expect_bytes}') if body_len != args.expect_bytes: print(f'FAIL: body length {body_len} != expected {args.expect_bytes}') @@ -144,6 +175,11 @@ def feed() -> bytes: if len(app_lengths) < 2: print('FAIL: too few application-data records to judge clamping') return 1 + if args.dynamic: + return 0 if verify_dynamic_records(app_lengths) else 1 + + assert args.max_record is not None + limit = args.max_record + RECORD_OVERHEAD if max_record > limit: print(f'FAIL: an application-data record ({max_record}) exceeds the clamp + overhead ({limit})') return 1 From 1954d5ae9c35e089f55160af95ed4e7c1faf390a Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Wed, 29 Jul 2026 11:53:32 -0500 Subject: [PATCH 04/11] Fix Fedora OTEL and WAMR builds (#13423) Fedora CI could silently omit the OTEL and WAMR plugins, while BoringSSL builds mixed system OpenSSL headers and libraries. This made the updated dependency image appear usable without proving either plugin could load. This makes the Fedora presets require both plugins and selects the matching curl and TLS roots for system OpenSSL and BoringSSL builds. This also distinguishes BoringSSL from OpenSSL 3 during configuration and gives the WASM targets explicit OpenSSL dependencies so both TLS variants build and load consistently. (cherry picked from commit fa297c39988da44f68ee627d5ca3d325645247db) --- CMakeLists.txt | 5 ++++- CMakePresets.json | 12 ++++++++---- plugins/experimental/wasm/CMakeLists.txt | 2 +- plugins/experimental/wasm/lib/CMakeLists.txt | 1 + 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16d262aff24..c4ac7478fb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -306,7 +306,10 @@ else() endif() check_openssl_is_quictls(SSLLIB_IS_QUICTLS "${OPENSSL_INCLUDE_DIR}") -if(OPENSSL_VERSION VERSION_GREATER_EQUAL "3.0.0") +if(NOT SSLLIB_IS_BORINGSSL + AND NOT SSLLIB_IS_AWSLC + AND OPENSSL_VERSION VERSION_GREATER_EQUAL "3.0.0" +) set(SSLLIB_IS_OPENSSL3 TRUE) add_compile_definitions(OPENSSL_API_COMPAT=10002 OPENSSL_IS_OPENSSL3) endif() diff --git a/CMakePresets.json b/CMakePresets.json index cec863644f3..bfa5cbb8de4 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -185,10 +185,12 @@ "inherits": ["ci"], "cacheVariables": { "ENABLE_PROBES": "ON", - "OPENSSL_ROOT_DIR": "/opt/openssl-quic", + "OPENSSL_ROOT_DIR": "/usr", "opentelemetry_ROOT": "/opt", - "CURL_ROOT": "/opt", + "CURL_ROOT": "/usr", "wamr_ROOT": "/opt", + "ENABLE_OTEL_TRACER": "ON", + "ENABLE_WASM_WAMR": "ON", "ENABLE_CRIPTS": "ON" } }, @@ -210,15 +212,17 @@ "OPENSSL_ROOT_DIR": "/opt/h3-tools-boringssl/boringssl", "quiche_ROOT": "/opt/h3-tools-boringssl/quiche", "opentelemetry_ROOT": "/opt", - "CURL_ROOT": "/opt", + "CURL_ROOT": "/opt/h3-tools-boringssl", "wamr_ROOT": "/opt", + "ENABLE_OTEL_TRACER": "ON", + "ENABLE_WASM_WAMR": "ON", "CMAKE_INSTALL_PREFIX": "/tmp/ats-quiche", "ENABLE_QUICHE": "ON" } }, { "name": "ci-fedora-autest", - "displayName": "CI Fedora Quiche Autest", + "displayName": "CI Fedora Autest", "description": "CI Pipeline config for Fedora Linux (autest build)", "inherits": ["ci-fedora", "autest"] }, diff --git a/plugins/experimental/wasm/CMakeLists.txt b/plugins/experimental/wasm/CMakeLists.txt index 0cccbc2082c..c10011e5fb4 100644 --- a/plugins/experimental/wasm/CMakeLists.txt +++ b/plugins/experimental/wasm/CMakeLists.txt @@ -29,7 +29,7 @@ if(wasmedge_FOUND) list(APPEND WASM_RUNTIME wasmedge::wasmedge) endif() -target_link_libraries(wasm PRIVATE ${WASM_RUNTIME}) +target_link_libraries(wasm PRIVATE OpenSSL::SSL ${WASM_RUNTIME}) if(wamr_FOUND) target_compile_options(wasm PRIVATE -DWAMR) diff --git a/plugins/experimental/wasm/lib/CMakeLists.txt b/plugins/experimental/wasm/lib/CMakeLists.txt index f838461d7bf..1a07cd3a882 100644 --- a/plugins/experimental/wasm/lib/CMakeLists.txt +++ b/plugins/experimental/wasm/lib/CMakeLists.txt @@ -38,6 +38,7 @@ endif() add_library(wasmlib STATIC ${CC_FILES}) target_compile_options(wasmlib PUBLIC -Wno-unused-parameter) +target_link_libraries(wasmlib PRIVATE OpenSSL::Crypto) if(wamr_FOUND) target_compile_options(wasmlib PRIVATE -Wno-missing-field-initializers) target_link_libraries(wasmlib PUBLIC wamr::wamr) From a823957a6f675fc5b4f5d8505721a85f32f4f9ed Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Tue, 28 Jul 2026 19:48:04 -0600 Subject: [PATCH 05/11] doc: clarify run-plugin argument is fixed at load time (#13429) The run-plugin action's plugin-argument is parsed once when the rule loads, like a remap.config argument, and is never re-evaluated per request, so variable interpolation (e.g. %{HEADER:bar}) does not work there. (cherry picked from commit 9a847a59e64b1af7990217f2316658c3c05b0ea4) --- doc/admin-guide/plugins/header_rewrite.en.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index 2d57f64f973..da446873201 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -1171,6 +1171,12 @@ run-plugin This allows to run an existing remap plugin, conditionally, from within a header rewrite rule. +.. note:: + ```` is fixed when the rule is loaded (or reloaded); + it behaves the same as a plugin argument in ``remap.config``. It is + not re-evaluated per request, so variable interpolation (e.g. + ``%{HEADER:bar}``) does not work here. + set-body ~~~~~~~~ :: From b2bc2c22719f8e38608d1c489ce254469e7525d5 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 30 Jul 2026 09:03:24 -0500 Subject: [PATCH 06/11] Add Prometheus v2 labeled stats (#13156) Prometheus consumers currently only get flat ATS stat names, which makes related counters hard to aggregate and can hide broken output behind parser leniency. The draft v2 output also exposed that risk by letting `completed` look like a request method and by interleaving samples from the same family. This adds a Prometheus v2 response format that groups samples by metric family and derives labels for methods, directions, status codes, cache results, time buckets, and cache volumes. This keeps lifecycle counters such as completed requests as their own metrics while preserving the existing v1 output. This extends the stats_over_http AuTest and Prometheus ingester to validate both the raw v2 exposition and the parser's view of it. This catches split families, missing TYPE metadata, malformed labels, and regressions in the expected labeled samples. (cherry picked from commit 8d35fd22efe9f86ab626368bea829417ad6a4ca3) --- .../plugins/stats_over_http.en.rst | 24 +- plugins/stats_over_http/stats_over_http.cc | 441 ++++++++++++++++-- ...over_http_prometheus_v2_accept_stderr.gold | 11 + .../stats_over_http_prometheus_v2_stderr.gold | 11 + .../prometheus_stats_ingester.py | 339 +++++++++++++- .../stats_over_http/stats_over_http.test.py | 201 ++++++++ 6 files changed, 979 insertions(+), 48 deletions(-) create mode 100644 tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_accept_stderr.gold create mode 100644 tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_stderr.gold diff --git a/doc/admin-guide/plugins/stats_over_http.en.rst b/doc/admin-guide/plugins/stats_over_http.en.rst index 93d82422341..264109cdfbd 100644 --- a/doc/admin-guide/plugins/stats_over_http.en.rst +++ b/doc/admin-guide/plugins/stats_over_http.en.rst @@ -105,21 +105,29 @@ if you wish to have it in CSV format you can do so by passing an ``Accept`` head .. option:: Accept: text/csv -Prometheus formatted output is also supported via the ``Accept`` header: +Prometheus formatted output is also supported via the ``Accept`` header. Version 0.0.4 +(flat metric names) and version 2.0.0 (labeled metrics for better aggregation) +are supported: .. option:: Accept: text/plain; version=0.0.4 +.. option:: Accept: text/plain; version=2.0.0 Alternatively, the output format can be specified as a suffix to the configured path in the HTTP request target. The supported suffixes are ``/json``, -``/csv``, and ``/prometheus``. For example, if the path is set to ``/_stats`` -(the default), you can access the stats in CSV format by using the URL:: +``/csv``, ``/prometheus``, and ``/prometheus_v2``. For example, if the path +is set to ``/_stats`` (the default), you can access the stats in CSV format by +using the URL:: http://host:port/_stats/csv -The Prometheus format can be requested by using the URL:: +The Prometheus version 0.0.4 format (flat) can be requested by using the URL:: http://host:port/_stats/prometheus +The Prometheus v2 labeled format can be requested by using the URL:: + + http://host:port/_stats/prometheus_v2 + The JSON format is the default, but you can also access it explicitly by using the URL:: http://host:port/_stats/json @@ -129,9 +137,11 @@ specify a path suffix, the plugin will return the data in that format regardless the ``Accept`` header. In either case the ``Content-Type`` header returned by ``stats_over_http.so`` will -reflect the content that has been returned: ``text/json``, ``text/csv``, or -``text/plain; version=0.0.4; charset=utf-8`` for JSON, CSV, and Prometheus -formats respectively. +reflect the content that has been returned: ``text/json``, ``text/csv``, +``text/plain; version=0.0.4; charset=utf-8``, or +``text/plain; version=2.0.0; charset=utf-8`` for JSON, CSV, Prometheus v1, and +Prometheus v2 formats respectively. + Stats over http also accepts returning data in gzip or br compressed format per the ``Accept-encoding`` header. If the header is present, the plugin will return the diff --git a/plugins/stats_over_http/stats_over_http.cc b/plugins/stats_over_http/stats_over_http.cc index b02b17c44e2..2eb4b8b5d91 100644 --- a/plugins/stats_over_http/stats_over_http.cc +++ b/plugins/stats_over_http/stats_over_http.cc @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include @@ -92,6 +94,25 @@ const int BROTLI_LGW = 16; static bool integer_counters = false; static bool wrap_counters = false; +#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L && (!defined(__clang__) || __clang_major__ > 16) +#define STATS_OVER_HTTP_HAS_CONSTEXPR_STRING 1 +#else +#define STATS_OVER_HTTP_HAS_CONSTEXPR_STRING 0 +#endif + +struct prometheus_v2_metric { + std::string name; + std::string labels; +}; + +struct prometheus_v2_metric_family { + TSRecordDataType data_type = TS_RECORDDATATYPE_NULL; + std::string help; + std::vector samples; +}; + +using prometheus_v2_metric_family_map = std::unordered_map; + struct config_t { unsigned int recordTypes; std::string stats_path; @@ -103,7 +124,7 @@ struct config_holder_t { config_t *config; }; -enum class output_format_t { JSON_OUTPUT, CSV_OUTPUT, PROMETHEUS_OUTPUT }; +enum class output_format_t { JSON_OUTPUT, CSV_OUTPUT, PROMETHEUS_OUTPUT, PROMETHEUS_V2_OUTPUT }; enum class encoding_format_t { NONE, DEFLATE, GZIP, BR }; int configReloadRequests = 0; @@ -147,11 +168,13 @@ struct stats_state { TSIOBuffer resp_buffer = nullptr; TSIOBufferReader resp_reader = nullptr; - int output_bytes = 0; - int body_written = 0; - output_format_t output_format = output_format_t::JSON_OUTPUT; - encoding_format_t encoding = encoding_format_t::NONE; - z_stream zstrm; + int64_t output_bytes = 0; + int body_written = 0; + output_format_t output_format = output_format_t::JSON_OUTPUT; + encoding_format_t encoding = encoding_format_t::NONE; + z_stream zstrm; + prometheus_v2_metric_family_map prometheus_v2_families; + std::vector prometheus_v2_family_order; #if HAVE_BROTLI_ENCODE_H b_stream bstrm; #endif @@ -168,6 +191,9 @@ struct stats_state { static char * nstr(const char *s) { + if (s == nullptr) { + return nullptr; + } char *mys = (char *)TSmalloc(strlen(s) + 1); strcpy(mys, s); return mys; @@ -246,7 +272,9 @@ stats_cleanup(TSCont contp, stats_state *my_state) my_state->resp_buffer = nullptr; } - TSVConnClose(my_state->net_vc); + if (my_state->net_vc != nullptr) { + TSVConnClose(my_state->net_vc); + } delete my_state; TSContDestroy(contp); } @@ -260,14 +288,20 @@ stats_process_accept(TSCont contp, stats_state *my_state) my_state->read_vio = TSVConnRead(my_state->net_vc, contp, my_state->req_buffer, INT64_MAX); } -static int +static int64_t stats_add_data_to_resp_buffer(const char *s, stats_state *my_state) { - int s_len = strlen(s); + if (s == nullptr) { + return 0; + } + int64_t s_len = strlen(s); - TSIOBufferWrite(my_state->resp_buffer, s, s_len); + int64_t bytes_written = TSIOBufferWrite(my_state->resp_buffer, s, s_len); + if (bytes_written == TS_ERROR) { + return 0; + } - return s_len; + return bytes_written; } static const char RESP_HEADER_JSON[] = "HTTP/1.0 200 OK\r\nContent-Type: text/json\r\nCache-Control: no-cache\r\n\r\n"; @@ -293,8 +327,17 @@ static const char RESP_HEADER_PROMETHEUS_DEFLATE[] = "no-cache\r\n\r\n"; static const char RESP_HEADER_PROMETHEUS_BR[] = "HTTP/1.0 200 OK\r\nContent-Type: text/plain; version=0.0.4; " "charset=utf-8\r\nContent-Encoding: br\r\nCache-Control: no-cache\r\n\r\n"; +static const char RESP_HEADER_PROMETHEUS_V2[] = + "HTTP/1.0 200 OK\r\nContent-Type: text/plain; version=2.0.0; charset=utf-8\r\nCache-Control: no-cache\r\n\r\n"; +static const char RESP_HEADER_PROMETHEUS_V2_GZIP[] = "HTTP/1.0 200 OK\r\nContent-Type: text/plain; version=2.0.0; " + "charset=utf-8\r\nContent-Encoding: gzip\r\nCache-Control: no-cache\r\n\r\n"; +static const char RESP_HEADER_PROMETHEUS_V2_DEFLATE[] = + "HTTP/1.0 200 OK\r\nContent-Type: text/plain; version=2.0.0; charset=utf-8\r\nContent-Encoding: deflate\r\nCache-Control: " + "no-cache\r\n\r\n"; +static const char RESP_HEADER_PROMETHEUS_V2_BR[] = "HTTP/1.0 200 OK\r\nContent-Type: text/plain; version=2.0.0; " + "charset=utf-8\r\nContent-Encoding: br\r\nCache-Control: no-cache\r\n\r\n"; -static int +static int64_t stats_add_resp_header(stats_state *my_state) { switch (my_state->output_format) { @@ -331,6 +374,17 @@ stats_add_resp_header(stats_state *my_state) return stats_add_data_to_resp_buffer(RESP_HEADER_PROMETHEUS, my_state); } break; + case output_format_t::PROMETHEUS_V2_OUTPUT: + if (my_state->encoding == encoding_format_t::GZIP) { + return stats_add_data_to_resp_buffer(RESP_HEADER_PROMETHEUS_V2_GZIP, my_state); + } else if (my_state->encoding == encoding_format_t::DEFLATE) { + return stats_add_data_to_resp_buffer(RESP_HEADER_PROMETHEUS_V2_DEFLATE, my_state); + } else if (my_state->encoding == encoding_format_t::BR) { + return stats_add_data_to_resp_buffer(RESP_HEADER_PROMETHEUS_V2_BR, my_state); + } else { + return stats_add_data_to_resp_buffer(RESP_HEADER_PROMETHEUS_V2, my_state); + } + break; } // Not reached. return stats_add_data_to_resp_buffer(RESP_HEADER_JSON, my_state); @@ -482,16 +536,12 @@ csv_out_stat(TSRecordType /* rec_type ATS_UNUSED */, void *edata, int /* registe * @param[in] name The metric name to sanitize. * @return A sanitized metric name. */ -static -// Remove this check when we drop support for pre-13 GCC versions. -#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -// Clang <= 16 doesn't fully support constexpr std::string. -#if !defined(__clang__) || __clang_major__ > 16 - constexpr +#if STATS_OVER_HTTP_HAS_CONSTEXPR_STRING +static constexpr std::string +#else +static std::string #endif -#endif - std::string - sanitize_metric_name_for_prometheus(std::string_view name) +sanitize_metric_name_for_prometheus(std::string_view name) { std::string sanitized_name(name); // If the first character is a digit, prepend an underscore since Prometheus @@ -509,32 +559,310 @@ static return sanitized_name; } +static std::string +escape_prometheus_v2_label_value(std::string_view val) +{ + size_t escaped_len = 0; + for (char c : val) { + if (c == '"' || c == '\\' || c == '\n') { + escaped_len += 2; + } else { + escaped_len += 1; + } + } + + std::string escaped; + if (escaped_len > 0) { + escaped.reserve(escaped_len); + for (char c : val) { + if (c == '"' || c == '\\') { + escaped += '\\'; + escaped += c; + } else if (c == '\n') { + escaped += "\\n"; + } else { + escaped += c; + } + } + } + return escaped; +} + +static void +append_prometheus_v2_label(std::string &labels, std::string_view key, std::string_view val) +{ + if (!labels.empty()) { + labels += ", "; + } + labels += key; + labels += "=\""; + labels += escape_prometheus_v2_label_value(val); + labels += "\""; +} + +static bool +contains_prometheus_v2_token(const std::string_view *tokens, size_t size, std::string_view token) +{ + for (size_t i = 0; i < size; ++i) { + if (tokens[i] == token) { + return true; + } + } + return false; +} + +static swoc::TextView +take_prometheus_v2_token(swoc::TextView &view) +{ + size_t sep = view.find_first_of("._[]"); + swoc::TextView token; + + if (sep == swoc::TextView::npos) { + token = view; + view.clear(); + } else { + token = view.prefix(sep); + view.remove_prefix(sep + 1); + } + return token; +} + +/** Parse a Prometheus v2 metric name and return the base name and labels. + * + * @param[in] name The metric name to parse. + * @return A prometheus_v2_metric struct containing the base name and labels. + */ +static prometheus_v2_metric +parse_metric_v2(std::string_view name) +{ + swoc::TextView name_view{name}; + std::string labels; + std::string base_name; + + constexpr std::string_view methods[] = {"get", "post", "head", "put", "delete", "options", "trace", "connect", "push", "purge"}; + constexpr std::string_view directions[] = {"incoming", "outgoing"}; + constexpr std::string_view results[] = {"hit", "miss", "error", "errors", "success", "failure"}; + constexpr std::string_view categories[] = {"volume", "thread", "interface", "net", "host", "port"}; + + while (!name_view.empty()) { + swoc::TextView token = take_prometheus_v2_token(name_view); + + if (token.empty()) { + continue; + } + + bool token_handled = false; + + // Status codes (200, 4xx, etc.) + if (token.length() == 3 && (token[0] >= '0' && token[0] <= '9') && ((token[1] >= '0' && token[1] <= '9') || token[1] == 'x') && + ((token[2] >= '0' && token[2] <= '9') || token[2] == 'x')) { + append_prometheus_v2_label(labels, "status", token); + token_handled = true; + } + // Direction (incoming / outgoing) + else if (contains_prometheus_v2_token(directions, sizeof(directions) / sizeof(directions[0]), token)) { + append_prometheus_v2_label(labels, "direction", token); + token_handled = true; + } + // Multi-token method categories. + else if (token == "extension" || token == "invalid") { + swoc::TextView next = name_view; + swoc::TextView next_token = take_prometheus_v2_token(next); + + if (token == "extension" && next_token == "method") { + append_prometheus_v2_label(labels, "method", "extension_method"); + name_view = next; + token_handled = true; + } else if (token == "invalid" && next_token == "client") { + append_prometheus_v2_label(labels, "method", "invalid_client"); + name_view = next; + token_handled = true; + } + } + // Methods + else if (contains_prometheus_v2_token(methods, sizeof(methods) / sizeof(methods[0]), token)) { + append_prometheus_v2_label(labels, "method", token); + token_handled = true; + } + // Generic Categories + Index (volume, 0, etc.) + else if (contains_prometheus_v2_token(categories, sizeof(categories) / sizeof(categories[0]), token)) { + swoc::TextView next = name_view; + swoc::TextView id = take_prometheus_v2_token(next); + + bool is_id = !id.empty(); + for (char c : id) { + if (!(c >= '0' && c <= '9') && c != 'x') { + is_id = false; + break; + } + } + if (is_id) { + append_prometheus_v2_label(labels, token, id); + if (!base_name.empty()) { + base_name += "."; + } + base_name += token; + name_view = next; + token_handled = true; + } + } + // Results (hit, miss) + else if (contains_prometheus_v2_token(results, sizeof(results) / sizeof(results[0]), token)) { + // 'hit' and 'miss' are almost always labels. + if (token == "hit" || token == "miss" || !name_view.empty()) { + append_prometheus_v2_label(labels, "result", token); + token_handled = true; + } + } + // Buckets (e.g., 10ms) + else { + constexpr std::string_view units[] = {"ms", "us", "s"}; + for (const auto &unit : units) { + size_t unit_len = unit.length(); + if (token.length() > unit_len && token.substr(token.length() - unit_len) == unit) { + bool all_digits = true; + for (size_t j = 0; j < token.length() - unit_len; ++j) { + if (!(token[j] >= '0' && token[j] <= '9')) { + all_digits = false; + break; + } + } + if (all_digits && token.length() > unit_len) { + append_prometheus_v2_label(labels, "le", token); + token_handled = true; + break; + } + } + } + } + + if (!token_handled) { + if (!base_name.empty()) { + base_name += "."; + } + base_name += token; + } + } + + return {base_name, labels}; +} + +static bool +format_prometheus_v2_sample(std::string &sample, const std::string &name, const std::string &labels, TSRecordDataType data_type, + TSRecordData *datum) +{ + char val_buffer[128]; + int len = 0; + + if (data_type == TS_RECORDDATATYPE_COUNTER) { + len = snprintf(val_buffer, sizeof(val_buffer), "%" PRIu64 "\n", wrap_unsigned_counter(datum->rec_counter)); + } else if (data_type == TS_RECORDDATATYPE_INT) { + len = snprintf(val_buffer, sizeof(val_buffer), "%" PRIu64 "\n", wrap_unsigned_counter(datum->rec_int)); + } else if (data_type == TS_RECORDDATATYPE_FLOAT) { + len = snprintf(val_buffer, sizeof(val_buffer), "%g\n", datum->rec_float); + } + + if (len <= 0 || len >= static_cast(sizeof(val_buffer))) { + return false; + } + + sample.reserve(name.size() + labels.size() + static_cast(len) + 3); + sample += name; + if (!labels.empty()) { + sample += "{"; + sample += labels; + sample += "}"; + } + sample += " "; + sample += val_buffer; + + return true; +} + +static void +prometheus_v2_out_stat(TSRecordType /* rec_type ATS_UNUSED */, void *edata, int /* registered ATS_UNUSED */, const char *name, + TSRecordDataType data_type, TSRecordData *datum) +{ + stats_state *my_state = static_cast(edata); + + if (data_type == TS_RECORDDATATYPE_STRING) { + return; // Prometheus does not support string values. + } + + auto v2 = parse_metric_v2(name); + std::string sanitized_name = sanitize_metric_name_for_prometheus(v2.name); + + if (sanitized_name.empty()) { + return; + } + + std::string sample; + if (!format_prometheus_v2_sample(sample, sanitized_name, v2.labels, data_type, datum)) { + return; + } + + // Note: Prometheus requires all metrics with the same name to have the same type. + // If Traffic Server metrics with different types (e.g., COUNTER and INT) are collapsed + // into the same base name, the first one encountered will determine the reported TYPE. + auto [it, inserted] = my_state->prometheus_v2_families.try_emplace(sanitized_name); + if (inserted) { + it->second.data_type = data_type; + it->second.help = name; + my_state->prometheus_v2_family_order.emplace_back(sanitized_name); + } else { + // Validate type consistency (at least between counter and gauge). + bool prev_is_counter = (it->second.data_type == TS_RECORDDATATYPE_COUNTER); + bool curr_is_counter = (data_type == TS_RECORDDATATYPE_COUNTER); + if (prev_is_counter != curr_is_counter) { + Dbg(dbg_ctl, "Inconsistent types for base metric %s: previously %s, now %s. Labels: %s", sanitized_name.c_str(), + prev_is_counter ? "counter" : "gauge", curr_is_counter ? "counter" : "gauge", v2.labels.c_str()); + } + } + + it->second.samples.emplace_back(std::move(sample)); +} + static void prometheus_out_stat(TSRecordType /* rec_type ATS_UNUSED */, void *edata, int /* registered ATS_UNUSED */, const char *name, TSRecordDataType data_type, TSRecordData *datum) { stats_state *my_state = static_cast(edata); std::string sanitized_name = sanitize_metric_name_for_prometheus(name); - char type_buffer[256]; - char help_buffer[256]; - snprintf(help_buffer, sizeof(help_buffer), "# HELP %s %s\n", sanitized_name.c_str(), name); + if (sanitized_name.empty()) { + return; + } + switch (data_type) { case TS_RECORDDATATYPE_COUNTER: - APPEND(help_buffer); - snprintf(type_buffer, sizeof(type_buffer), "# TYPE %s counter\n", sanitized_name.c_str()); - APPEND(type_buffer); + APPEND("# HELP "); + APPEND(sanitized_name.c_str()); + APPEND(" "); + APPEND(name); + APPEND("\n"); + APPEND("# TYPE "); + APPEND(sanitized_name.c_str()); + APPEND(" counter\n"); APPEND_STAT_PROMETHEUS_NUMERIC(sanitized_name.c_str(), "%" PRIu64, wrap_unsigned_counter(datum->rec_counter)); break; case TS_RECORDDATATYPE_INT: - APPEND(help_buffer); - snprintf(type_buffer, sizeof(type_buffer), "# TYPE %s gauge\n", sanitized_name.c_str()); - APPEND(type_buffer); + APPEND("# HELP "); + APPEND(sanitized_name.c_str()); + APPEND(" "); + APPEND(name); + APPEND("\n"); + APPEND("# TYPE "); + APPEND(sanitized_name.c_str()); + APPEND(" gauge\n"); APPEND_STAT_PROMETHEUS_NUMERIC(sanitized_name.c_str(), "%" PRIu64, wrap_unsigned_counter(datum->rec_int)); break; case TS_RECORDDATATYPE_FLOAT: - APPEND(help_buffer); - APPEND_STAT_PROMETHEUS_NUMERIC(sanitized_name.c_str(), "%f", datum->rec_float); + APPEND("# HELP "); + APPEND(sanitized_name.c_str()); + APPEND(" "); + APPEND(name); + APPEND("\n"); + APPEND_STAT_PROMETHEUS_NUMERIC(sanitized_name.c_str(), "%g", datum->rec_float); break; case TS_RECORDDATATYPE_STRING: Dbg(dbg_ctl, "Prometheus does not support string values, skipping: %s", sanitized_name.c_str()); @@ -644,6 +972,37 @@ prometheus_out_stats(stats_state *my_state) // No version printed, since string stats are not supported by Prometheus. } +static void +prometheus_v2_out_stats(stats_state *my_state) +{ + TSRecordDump((TSRecordType)(TS_RECORDTYPE_PLUGIN | TS_RECORDTYPE_NODE | TS_RECORDTYPE_PROCESS), prometheus_v2_out_stat, my_state); + + for (const auto &sanitized_name : my_state->prometheus_v2_family_order) { + const auto &family = my_state->prometheus_v2_families.at(sanitized_name); + + APPEND("# HELP "); + APPEND(sanitized_name.c_str()); + APPEND(" "); + APPEND(family.help.c_str()); + APPEND("\n"); + + const char *type_str = (family.data_type == TS_RECORDDATATYPE_COUNTER) ? "counter" : "gauge"; + APPEND("# TYPE "); + APPEND(sanitized_name.c_str()); + APPEND(" "); + APPEND(type_str); + APPEND("\n"); + + for (const auto &sample : family.samples) { + APPEND(sample.c_str()); + } + } + + APPEND("# HELP current_time_epoch_ms Current time in milliseconds since epoch.\n"); + APPEND("# TYPE current_time_epoch_ms gauge\n"); + APPEND_STAT_PROMETHEUS_NUMERIC("current_time_epoch_ms", "%" PRIu64, ms_since_epoch()); +} + static void stats_process_write(TSCont contp, TSEvent event, stats_state *my_state) { @@ -660,6 +1019,9 @@ stats_process_write(TSCont contp, TSEvent event, stats_state *my_state) case output_format_t::PROMETHEUS_OUTPUT: prometheus_out_stats(my_state); break; + case output_format_t::PROMETHEUS_V2_OUTPUT: + prometheus_v2_out_stats(my_state); + break; } if ((my_state->encoding == encoding_format_t::GZIP) || (my_state->encoding == encoding_format_t::DEFLATE)) { @@ -753,6 +1115,8 @@ stats_origin(TSCont contp, TSEvent /* event ATS_UNUSED */, void *edata) format_per_path = output_format_t::CSV_OUTPUT; } else if (request_path_suffix == "/prometheus") { format_per_path = output_format_t::PROMETHEUS_OUTPUT; + } else if (request_path_suffix == "/prometheus_v2") { + format_per_path = output_format_t::PROMETHEUS_V2_OUTPUT; } else { Dbg(dbg_ctl, "Unknown suffix for stats path: %.*s", static_cast(request_path_suffix.length()), request_path_suffix.data()); @@ -795,6 +1159,9 @@ stats_origin(TSCont contp, TSEvent /* event ATS_UNUSED */, void *edata) } else if (!strncasecmp(str, "text/plain; version=0.0.4", len)) { Dbg(dbg_ctl, "Saw text/plain; version=0.0.4 in accept header, sending Prometheus output."); my_state->output_format = output_format_t::PROMETHEUS_OUTPUT; + } else if (!strncasecmp(str, "text/plain; version=2.0.0", len)) { + Dbg(dbg_ctl, "Saw text/plain; version=2.0.0 in accept header, sending Prometheus v2 output."); + my_state->output_format = output_format_t::PROMETHEUS_V2_OUTPUT; } else { Dbg(dbg_ctl, "Saw %.*s in accept header, defaulting to JSON output.", len, str); my_state->output_format = output_format_t::JSON_OUTPUT; @@ -1135,11 +1502,7 @@ config_handler(TSCont cont, TSEvent /* event ATS_UNUSED */, void * /* edata ATS_ // // Compilation time unit tests. // -#ifdef DEBUG -// Remove this check when we drop support for pre-13 GCC versions. -#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -// Clang <= 16 doesn't fully support constexpr std::string. -#if !defined(__clang__) || __clang_major__ > 16 +#if defined(DEBUG) && STATS_OVER_HTTP_HAS_CONSTEXPR_STRING constexpr void test_sanitize_metric_name_for_prometheus() { @@ -1211,6 +1574,4 @@ test_sanitize_metric_name_for_prometheus() static_assert(sanitize_metric_name_for_prometheus("foo [[[bar]]]") == "foo____bar___"); static_assert(sanitize_metric_name_for_prometheus("foo@#$%bar") == "foo____bar"); } -#endif // !defined(__clang__) || __clang_major__ > 16 -#endif // defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#endif // DEBUG +#endif // defined(DEBUG) && STATS_OVER_HTTP_HAS_CONSTEXPR_STRING diff --git a/tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_accept_stderr.gold b/tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_accept_stderr.gold new file mode 100644 index 00000000000..bc0e58685e6 --- /dev/null +++ b/tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_accept_stderr.gold @@ -0,0 +1,11 @@ +`` +> GET /_stats``HTTP/1.1 +`` +< HTTP/1.1 200 OK +< Content-Type: text/plain; version=2.0.0; charset=utf-8 +< Cache-Control: no-cache +< Date:`` +< Age:`` +< Transfer-Encoding: chunked +< Connection:`` +`` diff --git a/tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_stderr.gold b/tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_stderr.gold new file mode 100644 index 00000000000..9da19b0ad7f --- /dev/null +++ b/tests/gold_tests/pluginTest/stats_over_http/gold/stats_over_http_prometheus_v2_stderr.gold @@ -0,0 +1,11 @@ +`` +> GET /_stats/prometheus_v2``HTTP/1.1 +`` +< HTTP/1.1 200 OK +< Content-Type: text/plain; version=2.0.0; charset=utf-8 +< Cache-Control: no-cache +< Date:`` +< Age:`` +< Transfer-Encoding: chunked +< Connection:`` +`` diff --git a/tests/gold_tests/pluginTest/stats_over_http/prometheus_stats_ingester.py b/tests/gold_tests/pluginTest/stats_over_http/prometheus_stats_ingester.py index 16b3701eccb..89b3ea20361 100644 --- a/tests/gold_tests/pluginTest/stats_over_http/prometheus_stats_ingester.py +++ b/tests/gold_tests/pluginTest/stats_over_http/prometheus_stats_ingester.py @@ -16,10 +16,19 @@ # limitations under the License. import argparse +from collections import Counter +import re import sys from urllib.request import urlopen from prometheus_client.parser import text_string_to_metric_families +HELP_RE = re.compile(r"^# HELP (?P[a-zA-Z_:][a-zA-Z0-9_:]*) (?P.*)$") +TYPE_RE = re.compile(r"^# TYPE (?P[a-zA-Z_:][a-zA-Z0-9_:]*) (?P[a-zA-Z]+)$") +SAMPLE_RE = re.compile( + r"^(?P[a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{(?P.*)\})?\s+" + r"(?P(?:[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)|(?:[+-]?(?:Inf|inf))|(?:NaN|nan))$") +LABEL_RE = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*)="(?P(?:\\.|[^"\\])*)"') + def parse_args() -> argparse.Namespace: """ @@ -28,6 +37,16 @@ def parse_args() -> argparse.Namespace: :return: Parsed arguments with the 'url' attribute. """ parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--strict-family-metadata", + action="store_true", + help="Fail if parsed metric families are split or emitted without TYPE metadata.", + ) + parser.add_argument( + "--validate-v2-format", + action="store_true", + help="Fail if the raw v2 exposition is not grouped into complete, labeled metric families.", + ) parser.add_argument("url", help="URL to fetch metrics from") return parser.parse_args() @@ -54,7 +73,7 @@ def parse_ats_metrics(text: str) -> list: :return: List of parsed metric families. """ try: - families = text_string_to_metric_families(text) + families = list(text_string_to_metric_families(text)) except Exception as e: raise RuntimeError(f"Failed to parse metrics: {e}") @@ -63,6 +82,310 @@ def parse_ats_metrics(text: str) -> list: return families +def validate_metric_families(families: list) -> None: + """ + Verify each metric family is complete and emitted once. + + Prometheus' parser accepts samples without adjacent HELP/TYPE metadata by + parsing them as unknown families. That is useful leniency, but it can hide + broken exposition output where samples for one metric family are interleaved + with unrelated metrics. + + :param families: List of parsed metric families. + """ + family_counts = Counter(family.name for family in families) + duplicate_families = sorted(name for name, count in family_counts.items() if count > 1) + if duplicate_families: + raise RuntimeError(f"Duplicate metric families found: {', '.join(duplicate_families)}") + + unknown_families = sorted(family.name for family in families if family.type == "unknown") + if unknown_families: + raise RuntimeError(f"Metric families without TYPE metadata found: {', '.join(unknown_families)}") + + +def decode_label_value(value: str, line_no: int) -> str: + """ + Decode a Prometheus label value and reject unsupported escape sequences. + + :param value: Escaped label value without the surrounding quotes. + :param line_no: Line number used for diagnostics. + :return: Decoded label value. + """ + decoded = [] + i = 0 + while i < len(value): + if value[i] != "\\": + decoded.append(value[i]) + i += 1 + continue + + if i + 1 >= len(value): + raise RuntimeError(f"Line {line_no}: label value ends with an incomplete escape") + + escaped = value[i + 1] + if escaped == "n": + decoded.append("\n") + elif escaped in {'"', "\\"}: + decoded.append(escaped) + else: + raise RuntimeError(f"Line {line_no}: unsupported label escape \\{escaped}") + i += 2 + + return "".join(decoded) + + +def parse_labels(labels_text: str | None, line_no: int) -> dict[str, str]: + """ + Parse a Prometheus label block. + + :param labels_text: Label block without braces, or None if absent. + :param line_no: Line number used for diagnostics. + :return: Mapping of label names to decoded values. + """ + if labels_text is None: + return {} + if not labels_text: + raise RuntimeError(f"Line {line_no}: empty label block") + + labels = {} + pos = 0 + while pos < len(labels_text): + match = LABEL_RE.match(labels_text, pos) + if match is None: + raise RuntimeError(f"Line {line_no}: invalid label syntax near: {labels_text[pos:]}") + + name = match.group("name") + if name in labels: + raise RuntimeError(f"Line {line_no}: duplicate label {name}") + labels[name] = decode_label_value(match.group("value"), line_no) + + pos = match.end() + if pos == len(labels_text): + break + if labels_text[pos] != ",": + raise RuntimeError(f"Line {line_no}: expected comma after label {name}") + pos += 1 + if pos < len(labels_text) and labels_text[pos] == " ": + pos += 1 + if pos == len(labels_text): + raise RuntimeError(f"Line {line_no}: trailing comma in label block") + + return labels + + +def require_family(samples_by_family: dict[str, list[dict[str, str]]], family: str) -> list[dict[str, str]]: + """ + Retrieve samples for a required metric family. + + :param samples_by_family: Mapping of family names to their parsed labels. + :param family: Required family name. + :return: Samples for the family. + """ + try: + return samples_by_family[family] + except KeyError: + raise RuntimeError(f"Required metric family missing: {family}") + + +def require_label_values(samples: list[dict[str, str]], family: str, label: str, expected_values: set[str]) -> None: + """ + Verify a family has samples for each expected value of a label. + + :param samples: Parsed labels for all samples in the family. + :param family: Family name used for diagnostics. + :param label: Label name to inspect. + :param expected_values: Required label values. + """ + actual_values = {sample[label] for sample in samples if label in sample} + missing_values = sorted(expected_values - actual_values) + if missing_values: + raise RuntimeError(f"{family} is missing {label} values: {', '.join(missing_values)}") + + +def require_sample(samples: list[dict[str, str]], family: str, required_labels: dict[str, str]) -> None: + """ + Verify a family has a sample containing a set of labels. + + :param samples: Parsed labels for all samples in the family. + :param family: Family name used for diagnostics. + :param required_labels: Required labels and values. + """ + for sample in samples: + if all(sample.get(label) == value for label, value in required_labels.items()): + return + + labels = ", ".join(f'{label}="{value}"' for label, value in required_labels.items()) + raise RuntimeError(f"{family} is missing a sample with labels: {labels}") + + +def validate_prometheus_v2_label_coverage(samples_by_family: dict[str, list[dict[str, str]]]) -> None: + """ + Verify the v2 output exercises the expected label transformations. + + :param samples_by_family: Mapping of family names to their parsed labels. + """ + http_request_samples = require_family(samples_by_family, "proxy_process_http_requests") + require_label_values( + http_request_samples, + "proxy_process_http_requests", + "method", + { + "connect", + "delete", + "extension_method", + "get", + "head", + "invalid_client", + "options", + "post", + "purge", + "push", + "put", + "trace", + }, + ) + require_label_values(http_request_samples, "proxy_process_http_requests", "direction", {"incoming", "outgoing"}) + for sample in http_request_samples: + if set(sample) not in ({"method"}, {"direction"}): + raise RuntimeError(f"proxy_process_http_requests has unexpected labels: {sample}") + + completed_samples = require_family(samples_by_family, "proxy_process_http_completed_requests") + for sample in completed_samples: + if sample: + raise RuntimeError("proxy_process_http_completed_requests should not have labels") + + response_samples = require_family(samples_by_family, "proxy_process_http_responses") + require_label_values(response_samples, "proxy_process_http_responses", "direction", {"incoming"}) + require_label_values( + response_samples, + "proxy_process_http_responses", + "status", + {"000", "100", "1xx", "200", "2xx", "404", "4xx", "500", "5xx"}, + ) + + require_sample( + require_family(samples_by_family, "proxy_process_http_disallowed_continue"), + "proxy_process_http_disallowed_continue", + { + "method": "post", + "status": "100" + }, + ) + require_label_values( + require_family(samples_by_family, "proxy_process_http_cache_ims"), "proxy_process_http_cache_ims", "result", + {"hit", "miss"}) + require_label_values( + require_family(samples_by_family, "proxy_process_http_cache_fresh"), "proxy_process_http_cache_fresh", "result", {"hit"}) + require_sample( + require_family(samples_by_family, "proxy_process_http_transaction_counts_failed"), + "proxy_process_http_transaction_counts_failed", + { + "result": "errors", + "method": "connect" + }, + ) + require_label_values( + require_family(samples_by_family, "proxy_process_eventloop_count"), "proxy_process_eventloop_count", "le", + {"10s", "100s", "1000s"}) + require_label_values( + require_family(samples_by_family, "proxy_process_eventloop_time"), "proxy_process_eventloop_time", "le", + {"0ms", "100ms", "2560ms"}) + require_label_values( + require_family(samples_by_family, "proxy_process_cache_volume_lookup_active"), "proxy_process_cache_volume_lookup_active", + "volume", {"0"}) + require_label_values( + require_family(samples_by_family, "proxy_process_cache_volume_lookup_success"), "proxy_process_cache_volume_lookup_success", + "volume", {"0"}) + + for family, samples in samples_by_family.items(): + for sample in samples: + if sample.get("method") == "completed": + raise RuntimeError(f"{family} incorrectly labels completed as an HTTP method") + + +def validate_prometheus_v2_text(text: str) -> None: + """ + Verify the raw v2 exposition has complete grouped metric families. + + :param text: Raw ATS Prometheus v2 output. + """ + current_name = None + current_type = None + current_has_sample = False + seen_families = set() + samples_by_family = {} + help_count = 0 + type_count = 0 + sample_count = 0 + + for line_no, line in enumerate(text.splitlines(), 1): + if not line: + continue + + help_match = HELP_RE.match(line) + if help_match is not None: + if current_name is not None and not current_has_sample: + raise RuntimeError(f"Line {line_no}: family {current_name} has no samples") + + current_name = help_match.group("name") + if current_name in seen_families: + raise RuntimeError(f"Line {line_no}: duplicate HELP for metric family {current_name}") + seen_families.add(current_name) + samples_by_family[current_name] = [] + current_type = None + current_has_sample = False + help_count += 1 + continue + + type_match = TYPE_RE.match(line) + if type_match is not None: + type_name = type_match.group("name") + if current_name is None: + raise RuntimeError(f"Line {line_no}: TYPE appears before HELP") + if type_name != current_name: + raise RuntimeError(f"Line {line_no}: TYPE name {type_name} does not match HELP name {current_name}") + if current_type is not None: + raise RuntimeError(f"Line {line_no}: duplicate TYPE for metric family {current_name}") + + current_type = type_match.group("type") + if current_type not in ("counter", "gauge"): + raise RuntimeError(f"Line {line_no}: unsupported TYPE for {current_name}: {current_type}") + type_count += 1 + continue + + if line.startswith("#"): + raise RuntimeError(f"Line {line_no}: unsupported metadata line: {line}") + + sample_match = SAMPLE_RE.match(line) + if sample_match is None: + raise RuntimeError(f"Line {line_no}: invalid sample line: {line}") + if current_name is None or current_type is None: + raise RuntimeError(f"Line {line_no}: sample appears before HELP/TYPE") + + sample_name = sample_match.group("name") + expected_names = {current_name} + if current_type == "counter": + expected_names.add(f"{current_name}_total") + if sample_name not in expected_names: + raise RuntimeError(f"Line {line_no}: sample {sample_name} does not belong to family {current_name}") + + labels = parse_labels(sample_match.group("labels"), line_no) + samples_by_family[current_name].append(labels) + current_has_sample = True + sample_count += 1 + + if help_count == 0: + raise RuntimeError("No metric families found") + if current_name is not None and not current_has_sample: + raise RuntimeError(f"Metric family {current_name} has no samples") + if help_count != type_count: + raise RuntimeError(f"HELP/TYPE count mismatch: {help_count} HELP lines, {type_count} TYPE lines") + if sample_count < help_count: + raise RuntimeError(f"Expected at least one sample per family, saw {sample_count} samples for {help_count} families") + + validate_prometheus_v2_label_coverage(samples_by_family) + + def print_metrics(families: list) -> None: """ Print parsed metric families in Prometheus format. @@ -98,12 +421,26 @@ def main() -> int: print(f"Error fetching URL {args.url}: {e}", file=sys.stderr) return 1 + if args.validate_v2_format: + try: + validate_prometheus_v2_text(ats_output) + except RuntimeError as e: + print(f"Error validating Prometheus v2 metrics: {e}", file=sys.stderr) + return 1 + try: families = parse_ats_metrics(ats_output) except RuntimeError as e: print(f"Error parsing ATS metrics: {e}", file=sys.stderr) return 1 + if args.strict_family_metadata: + try: + validate_metric_families(families) + except RuntimeError as e: + print(f"Error validating metric families: {e}", file=sys.stderr) + return 1 + # Parsing issues may not arise until we try to print the metrics. try: print_metrics(families) diff --git a/tests/gold_tests/pluginTest/stats_over_http/stats_over_http.test.py b/tests/gold_tests/pluginTest/stats_over_http/stats_over_http.test.py index ab4b450373c..a93ef0491c5 100644 --- a/tests/gold_tests/pluginTest/stats_over_http/stats_over_http.test.py +++ b/tests/gold_tests/pluginTest/stats_over_http/stats_over_http.test.py @@ -17,6 +17,7 @@ # limitations under the License. from enum import Enum +import re import sys Test.Summary = 'Exercise stats-over-http plugin' @@ -66,6 +67,12 @@ def __checkProcessAfter(self, tr): assert (self.state == self.State.RUNNING) tr.StillRunningAfter = self.ts + def __containsLiteral(self, p: "Test.Process", expression: str, description: str): + p.Streams.stdout += Testers.ContainsExpression(re.escape(expression), description) + + def __excludesLiteral(self, p: "Test.Process", expression: str, description: str): + p.Streams.stdout += Testers.ExcludesExpression(re.escape(expression), description) + def __checkPrometheusMetrics(self, p: 'Test.Process', from_prometheus: bool): '''Check the Prometheus metrics output. :param p: The process whose output to check. @@ -93,6 +100,154 @@ def __checkPrometheusMetrics(self, p: 'Test.Process', from_prometheus: bool): p.Streams.stdout += Testers.ContainsExpression( 'proxy_process_http_delete_requests 0', 'Verify the successful parsing of Prometheus metrics for a counter.') + def __checkPrometheusV2Metrics(self, p: "Test.Process"): + """Check the Prometheus v2 metrics output. + :param p: The process whose output to check. + """ + p.Streams.stdout += Testers.ContainsExpression( + "# HELP proxy_process_http_requests", + "Output should have a help line for the base metric name.", + ) + p.Streams.stdout += Testers.ContainsExpression( + "# TYPE proxy_process_http_requests counter", + "Output should have a type line for the base metric name.", + ) + + self.__containsLiteral( + p, + 'proxy_process_http_requests{method="delete"}', + "Verify that HTTP method labels (GET, POST, DELETE, etc.) are extracted correctly.", + ) + + self.__containsLiteral( + p, + 'proxy_process_http_requests{method="extension_method"}', + "Verify that multi-token HTTP method labels are extracted correctly.", + ) + + self.__containsLiteral( + p, + 'proxy_process_http_requests{method="invalid_client"}', + "Verify that invalid client request labels are extracted correctly.", + ) + + self.__containsLiteral( + p, + 'proxy_process_http_requests{direction="incoming"}', + "Verify that direction labels (incoming / outgoing) are extracted correctly.", + ) + + p.Streams.stdout += Testers.ContainsExpression( + "proxy_process_http_completed_requests", + "Verify that completed_requests remains its own lifecycle counter.", + ) + self.__excludesLiteral( + p, + 'method="completed"', + "completed is not an HTTP method label.", + ) + + self.__containsLiteral( + p, + 'proxy_process_http_cache_fresh{result="hit"}', + "Verify that result labels are extracted correctly.", + ) + + self.__containsLiteral( + p, + 'proxy_process_http_disallowed_continue{method="post", status="100"}', + "Verify that status code labels are extracted correctly.", + ) + + self.__containsLiteral( + p, + 'proxy_process_cache_volume_lookup_active{volume="0"}', + "Verify that volume labels are extracted from volume_N patterns.", + ) + + self.__containsLiteral( + p, + 'proxy_process_eventloop_count{le="', + "Verify that time buckets are correctly transformed into le labels.", + ) + + def __checkParsedPrometheusV2Metrics(self, p: "Test.Process"): + """Check the Prometheus parser's view of the v2 metrics output. + :param p: The process whose output to check. + """ + p.Streams.stdout += Testers.ContainsExpression( + "# TYPE proxy_process_http_requests counter", + "Prometheus parser should recognize HTTP request metrics as one counter family.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_requests_total{method="delete"}', + "Parsed output should retain HTTP method labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_requests_total{method="extension_method"}', + "Parsed output should retain multi-token HTTP method labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_requests_total{direction="incoming"}', + "Parsed output should retain direction labels.", + ) + p.Streams.stdout += Testers.ContainsExpression( + "proxy_process_http_completed_requests_total", + "Parsed output should keep completed_requests as its own counter family.", + ) + self.__excludesLiteral( + p, + 'method="completed"', + "completed should not be parsed as an HTTP method label.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_disallowed_continue_total{method="post",status="100"}', + "Parsed output should preserve multiple labels.", + ) + p.Streams.stdout += Testers.ContainsExpression( + "# TYPE proxy_process_http_responses counter", + "Prometheus parser should recognize HTTP response metrics as one counter family.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_responses_total{direction="incoming"}', + "Parsed output should retain direction labels on response metrics.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_responses_total{status="2xx"}', + "Parsed output should retain status code class labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_cache_ims_total{result="miss"}', + "Parsed output should retain cache result labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_http_transaction_counts_failed_total{result="errors",method="connect"}', + "Parsed output should preserve combined result and method labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_eventloop_count{le="100s"}', + "Parsed output should retain bucket labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_cache_volume_lookup_active{volume="0"}', + "Parsed output should preserve gauge labels.", + ) + self.__containsLiteral( + p, + 'proxy_process_cache_volume_lookup_success_total{volume="0"}', + "Parsed output should preserve counter labels for volume metrics.", + ) + def __testCaseNoAccept(self): tr = Test.AddTestRun('Fetch stats over HTTP in JSON format: no Accept and default path') self.__checkProcessBefore(tr) @@ -127,6 +282,19 @@ def __testCaseAcceptPrometheus(self): tr.Processes.Default.TimeOut = 3 self.__checkProcessAfter(tr) + def __testCaseAcceptPrometheusV2(self): + tr = Test.AddTestRun("Fetch stats over HTTP in Prometheus v2 format via Accept header") + self.__checkProcessBefore(tr) + tr.MakeCurlCommand( + f"-vs -H'Accept: text/plain; version=2.0.0' --http1.1 http://127.0.0.1:{self.ts.Variables.port}/_stats", + ts=self.ts, + ) + tr.Processes.Default.ReturnCode = 0 + self.__checkPrometheusV2Metrics(tr.Processes.Default) + tr.Processes.Default.Streams.stderr = ("gold/stats_over_http_prometheus_v2_accept_stderr.gold") + tr.Processes.Default.TimeOut = 3 + self.__checkProcessAfter(tr) + def __testCasePathJSON(self): tr = Test.AddTestRun('Fetch stats over HTTP in JSON format via /_stats/json') self.__checkProcessBefore(tr) @@ -160,6 +328,19 @@ def __testCasePathPrometheus(self): tr.Processes.Default.TimeOut = 3 self.__checkProcessAfter(tr) + def __testCasePathPrometheusV2(self): + tr = Test.AddTestRun("Fetch stats over HTTP in Prometheus v2 format via /_stats/prometheus_v2") + self.__checkProcessBefore(tr) + tr.MakeCurlCommand( + f"-vs --http1.1 http://127.0.0.1:{self.ts.Variables.port}/_stats/prometheus_v2", + ts=self.ts, + ) + tr.Processes.Default.ReturnCode = 0 + self.__checkPrometheusV2Metrics(tr.Processes.Default) + tr.Processes.Default.Streams.stderr = ("gold/stats_over_http_prometheus_v2_stderr.gold") + tr.Processes.Default.TimeOut = 3 + self.__checkProcessAfter(tr) + def __testCaseAcceptIgnoredIfPathExplicit(self): tr = Test.AddTestRun('Fetch stats over HTTP in Prometheus format with Accept csv header') self.__checkProcessBefore(tr) @@ -184,16 +365,36 @@ def __queryAndParsePrometheusMetrics(self): p.Command = f'{sys.executable} {ingester} http://127.0.0.1:{self.ts.Variables.port}/_stats/prometheus' p.ReturnCode = 0 self.__checkPrometheusMetrics(p, from_prometheus=True) + self.__checkProcessAfter(tr) + + def __queryAndParsePrometheusV2Metrics(self): + """ + Query the ATS stats over HTTP in Prometheus v2 format and parse the output. + """ + tr = Test.AddTestRun('Query and parse Prometheus v2 metrics') + ingester = 'prometheus_stats_ingester.py' + tr.Setup.CopyAs(ingester) + self.__checkProcessBefore(tr) + p = tr.Processes.Default + p.Command = ( + f'{sys.executable} {ingester} --validate-v2-format --strict-family-metadata ' + f'http://127.0.0.1:{self.ts.Variables.port}/_stats/prometheus_v2') + p.ReturnCode = 0 + self.__checkParsedPrometheusV2Metrics(p) + self.__checkProcessAfter(tr) def run(self): self.__testCaseNoAccept() self.__testCaseAcceptCSV() self.__testCaseAcceptPrometheus() + self.__testCaseAcceptPrometheusV2() self.__testCasePathJSON() self.__testCasePathCSV() self.__testCasePathPrometheus() + self.__testCasePathPrometheusV2() self.__testCaseAcceptIgnoredIfPathExplicit() self.__queryAndParsePrometheusMetrics() + self.__queryAndParsePrometheusV2Metrics() StatsOverHttpPluginTest().run() From 5e248c31221dcbf5468e2dca60f225e247c797bc Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 30 Jul 2026 08:45:02 -0500 Subject: [PATCH 07/11] Add H3 quiche traffic handling tests and provide fixes (#13213) This patch extends the HTTP/3 autest coverage, using curl, Go, Python/aioquic, and Proxy Verifier HTTP/3 clients to generate their implementations of H3 traffic. It also adds request and response bodies of various sizes, including "large" 300k bodies to exercise multiple packet, buffer, and flow control ATS HTTP/3 implementations. It also exercises interesting requests and responses, such as HEAD, 204, PUT, DELETE, OPTIONS, range responses over cached objects, and malformed HTTP/3 frame behavior. This patch also includes the various production fixes needed for these tests. Large request and response bodies exposed a UDP receive starvation bug in the UDP read path. On systems using `recvmmsg()` with edge-triggered readiness, ATS could read one full batch of datagrams and then leave the rest queued in the kernel without another readable event to wake the QUIC stack. This changes `UDPNetProcessorInternal::read_multiple_messages_from_net()` in `src/iocore/net/UnixUDPNet.cc` to return whether the kernel supplied a full batch. `udp_read_from_net()` now processes a bounded number of full batches per event, preserving UDP batching for H3 while avoiding both unread UDP bursts and unbounded net-thread monopolization under sustained QUIC load. The stream write path consumed the `QUICStreamVCAdapter` write reader inside `_read()`, before `QUICStream::send_data()` knew whether `quiche_conn_stream_send()` had accepted the bytes. When quiche accepted only a partial write or returned a flow-control error, ATS could lose stream data and report write progress too early. This makes `QUICStream::send_data()` keep a pending `IOBufferBlock`/FIN pair until quiche reports successful consumption, and only then calls the new `QUICStreamAdapter::consume()` hook. The concrete reader accounting lives in `QUICStreamVCAdapter::_consume()`, while `QUICStream::has_data_to_send()`, `QUICStream::on_write()`, and `QUICNetVConnection::on_stream_updated()` make newly writable stream data schedule packet writes again. This also treats completed finite writes with only FIN left as writable stream state, so empty bodies and fully consumed bodies still close the H3 stream cleanly. The large-body tests exposed that `QUICStreamVCAdapter::_read()` could hand more data to the transaction than the read VIO requested. That was usually hidden by small bodies, but larger reads made finite request-body accounting fragile. This clamps cloned input blocks in `QUICStreamVCAdapter::_read()` to the requested and available byte count before filling the read VIO. The adapter now also checks for a missing reader before touching the read buffer, which makes late stream cleanup paths more defensive. The timeout and stream lifetime tests exposed cases where an `HQTransaction` could be deleted while an event handler was still active, or while the QUIC stream adapter still had read/write cleanup to finish. That left later stream-close and timeout paths touching state that had already been torn down. This makes the transaction and stream closed state derive from the active event handlers instead of separate booleans that could drift from the adapter state. `Http3App::on_stream_close()` now calls `HQTransaction::stream_closed()` while holding the transaction mutex, and `HQTransaction::_delete_if_possible()` waits until the transaction is done, the stream is closed or no longer readable, and pending writes have flushed before deleting the transaction. The aioquic edge-case probes found malformed request streams that were correctly rejected at the H3 layer but still left partially constructed transactions attached to the session. Session teardown then either asserted because the transaction list was not empty or touched the H3 session after `Http3Session` had already nulled its network connection. This adds `HQSession::_close_transactions()` and drains any remaining transactions before destroying the H3 session-specific state. It also lets `Http3App::on_stream_close()` attach a cleanup callback to the transaction so the application stream map is erased when the transaction is actually destroyed, rather than when quiche first reports stream closure. The H3 request read path could signal completion before asynchronous QPACK header decode and buffered DATA delivery had finished updating the sink VIO. That showed up around HEAD, 204, and stream-close timing because the HTTP state machine needed a stable view of whether headers were decoded and whether a request body existed. This updates `Http3HeaderVIOAdaptor::_on_qpack_decode_complete()` to add the printed header length to the sink VIO and notify `Http3Transaction::on_header_decode_complete()`, which schedules the appropriate read event. `Http3StreamDataVIOAdaptor::finalize()` now uses a persistent reader, writes buffered DATA into the sink VIO exactly once, and updates `ndone`/`nbytes` consistently before the transaction is signaled. The aioquic client can write raw QUIC stream data, which exposed gaps in ATS's HTTP/3 frame validation. Reserved frames on request streams, DATA-before-HEADERS, client-created push streams, and duplicate control streams did not all reliably close the QUIC connection with an H3 application error. This adds request-stream enforcement through `Http3ProtocolEnforcer` in `Http3Transaction`, recognizes reserved HTTP/3 frame types in `Http3Frame`, and routes connection-level errors through `Http3App::_handle_error()` and `Http3Transaction::_handle_error()` to close the QUIC connection. The transaction signal path now also avoids calling the HTTP state machine through closed transactions or the initial zero-byte write VIO created before the HTTP response handler is installed. The HEAD, 204, and quic-go coverage exposed that ATS's static QPACK table was not the table used by external HTTP/3 implementations. The extra zstd entry and modified `accept-encoding` value in `src/proxy/http3/QPACK.cc` shifted later static indexes, so an externally encoded `:status 204` could decode as a different status. This restores the standard static table entries by using `accept-encoding: gzip, deflate, br` and removing the non-standard `content-encoding: zstd` entry. The new 204 cases in `tests/gold_tests/h3/replays/h3_proxy_verifier.replay.yaml`, `tests/gold_tests/h3/replays/h3_server_for_go_client.replay.yaml`, and `tests/gold_tests/h3/replays/h3_server_for_python_client.replay.yaml` cover this interoperability point with Proxy Verifier, quic-go, and aioquic. (cherry picked from commit 3076c17172578d21a631ddb67171342a856caed9) --- ci/rat-exclude.txt | 2 + include/iocore/net/quic/Mock.h | 22 +- include/iocore/net/quic/QUICConnection.h | 1 + include/iocore/net/quic/QUICStream.h | 6 + include/iocore/net/quic/QUICStreamAdapter.h | 4 +- include/iocore/net/quic/QUICStreamVCAdapter.h | 3 + include/iocore/net/quic/QUICTypes.h | 6 +- include/proxy/http3/Http3App.h | 1 + include/proxy/http3/Http3ProtocolEnforcer.h | 1 + include/proxy/http3/Http3Session.h | 3 + .../proxy/http3/Http3StreamDataVIOAdaptor.h | 8 +- include/proxy/http3/Http3Transaction.h | 28 +- include/proxy/http3/Http3Types.h | 1 + include/proxy/logging/TransactionLogData.h | 2 + src/iocore/net/P_QUICNetVConnection.h | 1 + src/iocore/net/P_UDPNet.h | 2 +- src/iocore/net/QUICNetVConnection.cc | 30 +- src/iocore/net/UnixUDPNet.cc | 13 +- src/iocore/net/quic/QUICStream.cc | 97 +++- src/iocore/net/quic/QUICStreamAdapter.cc | 9 +- src/iocore/net/quic/QUICStreamVCAdapter.cc | 74 ++- src/proxy/http3/Http3App.cc | 23 +- src/proxy/http3/Http3DebugNames.cc | 2 + src/proxy/http3/Http3Frame.cc | 15 +- src/proxy/http3/Http3HeaderVIOAdaptor.cc | 14 +- src/proxy/http3/Http3ProtocolEnforcer.cc | 17 +- src/proxy/http3/Http3Session.cc | 20 +- src/proxy/http3/Http3StreamDataVIOAdaptor.cc | 24 +- src/proxy/http3/Http3Transaction.cc | 212 ++++++-- src/proxy/http3/QPACK.cc | 3 +- .../http3/test/test_Http3FrameDispatcher.cc | 30 +- src/proxy/http3/test/test_QPACK.cc | 4 +- .../autest-site/ats_replay.test.ext | 2 + .../autest-site/conditions.test.ext | 51 +- .../early_hints/early_hints.test.py | 11 +- tests/gold_tests/h3/go_h3_client/go.mod | 13 + tests/gold_tests/h3/go_h3_client/go.sum | 38 ++ tests/gold_tests/h3/go_h3_client/main.go | 300 +++++++++++ tests/gold_tests/h3/h3_active_timeout.test.py | 26 + tests/gold_tests/h3/h3_curl.test.py | 139 +++++ tests/gold_tests/h3/h3_flow_control.test.py | 27 + tests/gold_tests/h3/h3_go_client.test.py | 124 +++++ tests/gold_tests/h3/h3_proxy_verifier.test.py | 27 + tests/gold_tests/h3/h3_python_client.test.py | 132 +++++ tests/gold_tests/h3/h3_range_cache.test.py | 140 +++++ tests/gold_tests/h3/h3_session_ticket.sh | 41 ++ tests/gold_tests/h3/h3_session_ticket.test.py | 110 ++++ tests/gold_tests/h3/h3_sni_check.test.py | 15 +- .../gold_tests/h3/h3_stream_lifetime.test.py | 27 + tests/gold_tests/h3/py_h3_client/h3_client.py | 332 ++++++++++++ .../h3/replays/h3_active_timeout.replay.yaml | 85 ++++ .../h3/replays/h3_flow_control.replay.yaml | 132 +++++ .../h3/replays/h3_proxy_verifier.replay.yaml | 481 ++++++++++++++++++ .../h3_server_for_go_client.replay.yaml | 268 ++++++++++ .../h3_server_for_python_client.replay.yaml | 297 +++++++++++ .../gold_tests/h3/replays/h3_sni.replay.yaml | 6 +- .../h3/replays/h3_stream_lifetime.replay.yaml | 198 +++++++ .../gold_tests/timeout/active_timeout.test.py | 2 +- .../timeout/quic_no_activity_timeout.test.py | 33 +- tests/pyproject.toml | 2 +- 60 files changed, 3599 insertions(+), 138 deletions(-) create mode 100644 tests/gold_tests/h3/go_h3_client/go.mod create mode 100644 tests/gold_tests/h3/go_h3_client/go.sum create mode 100644 tests/gold_tests/h3/go_h3_client/main.go create mode 100644 tests/gold_tests/h3/h3_active_timeout.test.py create mode 100644 tests/gold_tests/h3/h3_curl.test.py create mode 100644 tests/gold_tests/h3/h3_flow_control.test.py create mode 100644 tests/gold_tests/h3/h3_go_client.test.py create mode 100644 tests/gold_tests/h3/h3_proxy_verifier.test.py create mode 100644 tests/gold_tests/h3/h3_python_client.test.py create mode 100644 tests/gold_tests/h3/h3_range_cache.test.py create mode 100755 tests/gold_tests/h3/h3_session_ticket.sh create mode 100644 tests/gold_tests/h3/h3_session_ticket.test.py create mode 100644 tests/gold_tests/h3/h3_stream_lifetime.test.py create mode 100644 tests/gold_tests/h3/py_h3_client/h3_client.py create mode 100644 tests/gold_tests/h3/replays/h3_active_timeout.replay.yaml create mode 100644 tests/gold_tests/h3/replays/h3_flow_control.replay.yaml create mode 100644 tests/gold_tests/h3/replays/h3_proxy_verifier.replay.yaml create mode 100644 tests/gold_tests/h3/replays/h3_server_for_go_client.replay.yaml create mode 100644 tests/gold_tests/h3/replays/h3_server_for_python_client.replay.yaml create mode 100644 tests/gold_tests/h3/replays/h3_stream_lifetime.replay.yaml diff --git a/ci/rat-exclude.txt b/ci/rat-exclude.txt index 39ad1b11ca8..bb8926ef25c 100644 --- a/ci/rat-exclude.txt +++ b/ci/rat-exclude.txt @@ -24,6 +24,8 @@ blib/** **/*.default.in **/*.config **/*.gold +**/go.mod +**/go.sum **/*.hrw4u **/.gitignore **/.gitmodules diff --git a/include/iocore/net/quic/Mock.h b/include/iocore/net/quic/Mock.h index 95333ee2001..3332f82926f 100644 --- a/include/iocore/net/quic/Mock.h +++ b/include/iocore/net/quic/Mock.h @@ -28,6 +28,8 @@ #include "iocore/net/quic/QUICStreamAdapter.h" #include "iocore/net/quic/QUICStream.h" +#include + class MockQUICContext; using namespace std::literals; @@ -191,6 +193,11 @@ class MockQUICConnectionInfoProvider : public QUICConnectionInfoProvider { return negotiated_application_name_sv; } + + void + on_stream_updated() override + { + } }; class MockQUICStreamManager : public QUICStreamManager @@ -431,6 +438,11 @@ class MockQUICConnection : public QUICConnection return negotiated_application_name_sv; } + void + on_stream_updated() override + { + } + int _transmit_count = 0; int _retransmit_count = 0; Ptr _mutex; @@ -519,13 +531,19 @@ class MockQUICStreamAdapter : public QUICStreamAdapter Ptr _read(size_t len) override { - this->_sending_data_len -= len; - Ptr block = make_ptr(new_IOBufferBlock()); + len = std::min(len, this->_sending_data_len); + Ptr block = make_ptr(new_IOBufferBlock()); block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); block->fill(len); return block; } + void + _consume(size_t len) override + { + this->_sending_data_len -= std::min(len, this->_sending_data_len); + } + private: size_t _sending_data_len = 0; size_t _total_sending_data_len = 0; diff --git a/include/iocore/net/quic/QUICConnection.h b/include/iocore/net/quic/QUICConnection.h index 96f8f25b895..5a91c10380d 100644 --- a/include/iocore/net/quic/QUICConnection.h +++ b/include/iocore/net/quic/QUICConnection.h @@ -60,6 +60,7 @@ class QUICConnectionInfoProvider virtual bool is_handshake_completed() const = 0; virtual QUICVersion negotiated_version() const = 0; virtual std::string_view negotiated_application_name() const = 0; + virtual void on_stream_updated() = 0; }; class QUICConnection : public QUICConnectionInfoProvider diff --git a/include/iocore/net/quic/QUICStream.h b/include/iocore/net/quic/QUICStream.h index e0cb94c8da0..ef84c9cb6aa 100644 --- a/include/iocore/net/quic/QUICStream.h +++ b/include/iocore/net/quic/QUICStream.h @@ -26,6 +26,7 @@ #include "tscore/List.h" #include "iocore/eventsystem/Event.h" +#include "iocore/eventsystem/IOBuffer.h" #include "iocore/net/quic/QUICConnection.h" #include "iocore/net/quic/QUICDebugNames.h" @@ -53,6 +54,7 @@ class QUICStream QUICStreamDirection direction() const; bool is_bidirectional() const; bool has_no_more_data() const; + bool has_data_to_send(); QUICOffset final_offset() const; @@ -66,6 +68,7 @@ class QUICStream * QUICApplication need to call one of these functions when it process VC_EVENT_* */ void on_read(); + void on_write(); void on_eos(); /** @@ -85,6 +88,9 @@ class QUICStream uint64_t _received_bytes = 0; uint64_t _sent_bytes = 0; bool _has_no_more_data = false; + Ptr _pending_send_block; + bool _pending_send_fin = false; + bool _sent_fin = false; }; class QUICStreamStateListener diff --git a/include/iocore/net/quic/QUICStreamAdapter.h b/include/iocore/net/quic/QUICStreamAdapter.h index 1bf36cbcc1a..87afe51a5b8 100644 --- a/include/iocore/net/quic/QUICStreamAdapter.h +++ b/include/iocore/net/quic/QUICStreamAdapter.h @@ -39,6 +39,7 @@ class QUICStreamAdapter virtual int64_t write(QUICOffset offset, const uint8_t *data, uint64_t data_length, bool fin) = 0; Ptr read(size_t len); + void consume(size_t len); virtual bool is_eos() = 0; virtual uint64_t unread_len() = 0; virtual uint64_t read_len() = 0; @@ -60,6 +61,7 @@ class QUICStreamAdapter virtual void notify_eos() = 0; protected: - virtual Ptr _read(size_t len) = 0; + virtual Ptr _read(size_t len) = 0; + virtual void _consume(size_t len) = 0; QUICStream &_stream; }; diff --git a/include/iocore/net/quic/QUICStreamVCAdapter.h b/include/iocore/net/quic/QUICStreamVCAdapter.h index ce27422560c..e245f0b4c6b 100644 --- a/include/iocore/net/quic/QUICStreamVCAdapter.h +++ b/include/iocore/net/quic/QUICStreamVCAdapter.h @@ -53,6 +53,8 @@ class QUICStreamVCAdapter : public VConnection, public QUICStreamAdapter // Helpers to check VIO states bool is_readable(); bool is_writable(); + void mark_stream_closed(); + bool is_stream_closed() const; void clear_read_ready_event(Event *e); void clear_read_complete_event(Event *e); @@ -65,6 +67,7 @@ class QUICStreamVCAdapter : public VConnection, public QUICStreamAdapter protected: Ptr _read(size_t len) override; + void _consume(size_t len) override; VIO _read_vio; VIO _write_vio; diff --git a/include/iocore/net/quic/QUICTypes.h b/include/iocore/net/quic/QUICTypes.h index 706e807cadb..951226b054e 100644 --- a/include/iocore/net/quic/QUICTypes.h +++ b/include/iocore/net/quic/QUICTypes.h @@ -455,9 +455,9 @@ class QUICFiveTuple int protocol() const; private: - IpEndpoint _source; - IpEndpoint _destination; - int _protocol; + IpEndpoint _source{}; + IpEndpoint _destination{}; + int _protocol = 0; uint64_t _hash_code = 0; }; diff --git a/include/proxy/http3/Http3App.h b/include/proxy/http3/Http3App.h index 80c05cf12c9..70e7377beff 100644 --- a/include/proxy/http3/Http3App.h +++ b/include/proxy/http3/Http3App.h @@ -79,6 +79,7 @@ class Http3App : public QUICApplication void _handle_bidi_stream_on_write_complete(int event, VIO *vio); void _handle_bidi_stream_on_eos(int event, VIO *vio); + void _handle_error(const Http3Error &error); void _set_qpack_stream(Http3StreamType type, QUICStreamVCAdapter *adapter); QUICStreamVCAdapter::IOInfo &_get_stream_info(QUICStreamId stream_id); diff --git a/include/proxy/http3/Http3ProtocolEnforcer.h b/include/proxy/http3/Http3ProtocolEnforcer.h index ee291bdeca1..e9801f6802b 100644 --- a/include/proxy/http3/Http3ProtocolEnforcer.h +++ b/include/proxy/http3/Http3ProtocolEnforcer.h @@ -37,4 +37,5 @@ class Http3ProtocolEnforcer : public Http3FrameHandler private: bool _is_first_frame_received_on_control = false; + bool _is_headers_frame_received = false; }; diff --git a/include/proxy/http3/Http3Session.h b/include/proxy/http3/Http3Session.h index 6528b1f0e6f..4785390eba7 100644 --- a/include/proxy/http3/Http3Session.h +++ b/include/proxy/http3/Http3Session.h @@ -56,6 +56,9 @@ class HQSession : public ProxySession void remove_transaction(HQTransaction *trans); HQTransaction *get_transaction(QUICStreamId); +protected: + void _close_transactions(); + private: // this should be unordered map? Queue _transaction_list; diff --git a/include/proxy/http3/Http3StreamDataVIOAdaptor.h b/include/proxy/http3/Http3StreamDataVIOAdaptor.h index 4eff081d9d7..8968ab65cc2 100644 --- a/include/proxy/http3/Http3StreamDataVIOAdaptor.h +++ b/include/proxy/http3/Http3StreamDataVIOAdaptor.h @@ -42,7 +42,9 @@ class Http3StreamDataVIOAdaptor : public Http3FrameHandler bool has_data(); private: - VIO *_sink_vio = nullptr; - int64_t _total_data_length = 0; - MIOBuffer *_buffer; + VIO *_sink_vio = nullptr; + int64_t _total_data_length = 0; + MIOBuffer *_buffer = nullptr; + IOBufferReader *_reader = nullptr; + bool _finalized = false; }; diff --git a/include/proxy/http3/Http3Transaction.h b/include/proxy/http3/Http3Transaction.h index 1b0eb48806c..b253ecbb8be 100644 --- a/include/proxy/http3/Http3Transaction.h +++ b/include/proxy/http3/Http3Transaction.h @@ -29,6 +29,8 @@ #include "proxy/http3/Http3FrameDispatcher.h" #include "proxy/http3/Http3FrameCollector.h" +#include + class QUICStreamIO; class HQSession; class Http09Session; @@ -36,6 +38,7 @@ class Http3Session; class Http3HeaderFramer; class Http3DataFramer; class Http3HeaderVIOAdaptor; +class Http3ProtocolEnforcer; class Http3StreamDataVIOAdaptor; class HQTransaction : public ProxyTransaction @@ -53,6 +56,8 @@ class HQTransaction : public ProxyTransaction void transaction_done() override; void release() override; int get_transaction_id() const override; + void stream_closed(); + void set_stream_cleanup(std::function cleanup); void increment_transactions_stat() override; void decrement_transactions_stat() override; @@ -81,6 +86,7 @@ class HQTransaction : public ProxyTransaction void _schedule_read_complete_event(); void _unschedule_read_complete_event(); void _close_read_complete_event(Event *e); + void _schedule_read_event(); void _schedule_write_ready_event(); void _unschedule_write_ready_event(); void _close_write_ready_event(Event *e); @@ -90,12 +96,16 @@ class HQTransaction : public ProxyTransaction void _signal_event(int event, Event *e); void _signal_read_event(); void _signal_write_event(); + bool _is_write_buffer_flushed(); + virtual bool _is_closed() const = 0; + bool _is_stream_closed() const; void _delete_if_possible(); EThread *_thread = nullptr; MIOBuffer _read_vio_buf{BUFFER_SIZE_INDEX_4K}; QUICStreamVCAdapter::IOInfo &_info; + QUICStreamId _stream_id = 0; size_t _sent_bytes = 0; @@ -106,7 +116,10 @@ class HQTransaction : public ProxyTransaction Event *_write_ready_event = nullptr; Event *_write_complete_event = nullptr; - bool _transaction_done = false; + bool _transaction_done = false; + bool _event_handler_active = false; + + std::function _stream_cleanup; }; class Http3Transaction : public HQTransaction @@ -121,6 +134,7 @@ class Http3Transaction : public HQTransaction int state_stream_closed(int event, Event *data) override; void do_io_close(int lerrno = -1) override; + void on_header_decode_complete(); bool is_response_header_sent() const; bool is_response_body_sent() const; @@ -131,14 +145,17 @@ class Http3Transaction : public HQTransaction private: int64_t _process_read_vio() override; int64_t _process_write_vio() override; + bool _is_closed() const override; + void _handle_error(const Http3Error &error); // These are for HTTP/3 Http3FrameDispatcher _frame_dispatcher; Http3FrameCollector _frame_collector; - Http3FrameGenerator *_header_framer = nullptr; - Http3FrameGenerator *_data_framer = nullptr; - Http3HeaderVIOAdaptor *_header_handler = nullptr; - Http3StreamDataVIOAdaptor *_data_handler = nullptr; + Http3ProtocolEnforcer *_protocol_enforcer = nullptr; + Http3FrameGenerator *_header_framer = nullptr; + Http3FrameGenerator *_data_framer = nullptr; + Http3HeaderVIOAdaptor *_header_handler = nullptr; + Http3StreamDataVIOAdaptor *_data_handler = nullptr; }; /** @@ -160,6 +177,7 @@ class Http09Transaction : public HQTransaction private: int64_t _process_read_vio() override; int64_t _process_write_vio() override; + bool _is_closed() const override; // These are for HTTP/0.9 bool _protocol_detected = false; diff --git a/include/proxy/http3/Http3Types.h b/include/proxy/http3/Http3Types.h index 9d98578827c..5eb6eac2722 100644 --- a/include/proxy/http3/Http3Types.h +++ b/include/proxy/http3/Http3Types.h @@ -62,6 +62,7 @@ enum class Http3FrameType : uint64_t { X_RESERVED_4 = 0x09, MAX_PUSH_ID = 0x0D, X_MAX_DEFINED = 0x0D, + RESERVED = 0x21, UNKNOWN = 0x0E, }; diff --git a/include/proxy/logging/TransactionLogData.h b/include/proxy/logging/TransactionLogData.h index 908e036e389..09b1c6fefb0 100644 --- a/include/proxy/logging/TransactionLogData.h +++ b/include/proxy/logging/TransactionLogData.h @@ -27,6 +27,8 @@ #include "proxy/hdrs/HTTP.h" #include "tscore/ink_inet.h" +#include +#include #include #include diff --git a/src/iocore/net/P_QUICNetVConnection.h b/src/iocore/net/P_QUICNetVConnection.h index e6fec2b8ea5..89af3685e4e 100644 --- a/src/iocore/net/P_QUICNetVConnection.h +++ b/src/iocore/net/P_QUICNetVConnection.h @@ -129,6 +129,7 @@ class QUICNetVConnection : public UnixNetVConnection, bool is_at_anti_amplification_limit() const override; bool is_address_validation_completed() const override; bool is_handshake_completed() const override; + void on_stream_updated() override; // QUICSupport QUICConnection *get_quic_connection() override; diff --git a/src/iocore/net/P_UDPNet.h b/src/iocore/net/P_UDPNet.h index 04af410c371..42438b2c3ba 100644 --- a/src/iocore/net/P_UDPNet.h +++ b/src/iocore/net/P_UDPNet.h @@ -56,7 +56,7 @@ class UDPNetProcessorInternal : public UDPNetProcessor private: void read_single_message_from_net(UDPNetHandler *nh, UDPConnection *uc); - void read_multiple_messages_from_net(UDPNetHandler *nh, UDPConnection *xuc); + bool read_multiple_messages_from_net(UDPNetHandler *nh, UDPConnection *xuc); }; extern UDPNetProcessorInternal udpNetInternal; diff --git a/src/iocore/net/QUICNetVConnection.cc b/src/iocore/net/QUICNetVConnection.cc index d113ae593e1..18a85913739 100644 --- a/src/iocore/net/QUICNetVConnection.cc +++ b/src/iocore/net/QUICNetVConnection.cc @@ -37,6 +37,7 @@ #include #include +#include namespace { @@ -390,8 +391,28 @@ QUICNetVConnection::stream_manager() } void -QUICNetVConnection::close_quic_connection(QUICConnectionErrorUPtr /* error ATS_UNUSED */) +QUICNetVConnection::close_quic_connection(QUICConnectionErrorUPtr error) { + if (this->_quiche_con == nullptr || quiche_conn_is_closed(this->_quiche_con) || quiche_conn_is_draining(this->_quiche_con)) { + return; + } + + const bool is_app_error = error != nullptr && error->cls == QUICErrorClass::APPLICATION; + const uint64_t code = error == nullptr ? static_cast(QUICTransErrorCode::NO_ERROR) : error->code; + const uint8_t *reason = nullptr; + size_t reason_len = 0; + + if (error != nullptr && error->msg != nullptr) { + reason = reinterpret_cast(error->msg); + reason_len = strlen(error->msg); + } + + if (quiche_conn_close(this->_quiche_con, is_app_error, code, reason, reason_len) != 0) { + QUICConDebug("failed to close QUIC connection with code %" PRIu64, code); + return; + } + + this->_schedule_packet_write_ready(false); } void @@ -500,6 +521,12 @@ QUICNetVConnection::negotiated_application_name() const return std::string_view(reinterpret_cast(name), name_len); } +void +QUICNetVConnection::on_stream_updated() +{ + this->_schedule_packet_write_ready(false); +} + bool QUICNetVConnection::is_closed() const { @@ -690,7 +717,6 @@ QUICNetVConnection::_handle_write_ready() while (written + max_udp_payload_size <= quantum) { res = quiche_conn_send(this->_quiche_con, reinterpret_cast(udp_payload->end()) + written, max_udp_payload_size, &send_info); - #ifdef HAVE_SO_TXTIME if (written == 0) { memcpy(&send_at_hint, &send_info.at, sizeof(struct timespec)); diff --git a/src/iocore/net/UnixUDPNet.cc b/src/iocore/net/UnixUDPNet.cc index b0a012f9827..45973a7c512 100644 --- a/src/iocore/net/UnixUDPNet.cc +++ b/src/iocore/net/UnixUDPNet.cc @@ -70,7 +70,8 @@ EventType ET_UDP; namespace { #ifdef HAVE_RECVMMSG -const uint32_t MAX_RECEIVE_MSG_PER_CALL{16}; //< VLEN parameter for the recvmmsg call. +const uint32_t MAX_RECEIVE_MSG_PER_CALL{16}; //< VLEN parameter for the recvmmsg call. +const uint32_t MAX_RECEIVE_MSG_BATCHES_PER_EVENT{8}; //< Maximum number of full recvmmsg batches per event. #endif DbgCtl dbg_ctl_udpnet{"udpnet"}; @@ -516,7 +517,7 @@ UDPNetProcessorInternal::read_single_message_from_net(UDPNetHandler *nh, UDPConn } #ifdef HAVE_RECVMMSG -void +bool UDPNetProcessorInternal::read_multiple_messages_from_net(UDPNetHandler *nh, UDPConnection *xuc) { UnixUDPConnection *uc = static_cast(xuc); @@ -575,7 +576,7 @@ UDPNetProcessorInternal::read_multiple_messages_from_net(UDPNetHandler *nh, UDPC if (return_val <= 0) { Dbg(dbg_ctl_udp_read, "Done. recvmmsg() ret is %d, errno %s", return_val, strerror(errno)); - return; + return false; } Dbg(dbg_ctl_udp_read, "recvmmsg() read %d packets", return_val); @@ -593,7 +594,7 @@ UDPNetProcessorInternal::read_multiple_messages_from_net(UDPNetHandler *nh, UDPC if (mhdr.msg_namelen <= 0) { Dbg(dbg_ctl_udp_read, "Unable to get remote address from recvmmsg() for fd: %d", uc->getFd()); - return; + return false; } toaddr[packet_num].ss_family = AF_UNSPEC; @@ -666,6 +667,8 @@ UDPNetProcessorInternal::read_multiple_messages_from_net(UDPNetHandler *nh, UDPC nh->udp_callbacks.enqueue(uc); uc->onCallbackQueue = 1; } + + return return_val == static_cast(MAX_RECEIVE_MSG_PER_CALL); } #endif @@ -673,7 +676,7 @@ void UDPNetProcessorInternal::udp_read_from_net(UDPNetHandler *nh, UDPConnection *xuc) { #if HAVE_RECVMMSG - read_multiple_messages_from_net(nh, xuc); + for (uint32_t batch = 0; batch < MAX_RECEIVE_MSG_BATCHES_PER_EVENT && read_multiple_messages_from_net(nh, xuc); ++batch) {} #else read_single_message_from_net(nh, xuc); #endif diff --git a/src/iocore/net/quic/QUICStream.cc b/src/iocore/net/quic/QUICStream.cc index e221df25163..cdc427c9af8 100644 --- a/src/iocore/net/quic/QUICStream.cc +++ b/src/iocore/net/quic/QUICStream.cc @@ -24,7 +24,8 @@ #include "iocore/net/quic/QUICStream.h" #include "iocore/net/quic/QUICStreamAdapter.h" -constexpr uint32_t MAX_STREAM_FRAME_OVERHEAD = 24; +constexpr uint32_t MAX_STREAM_FRAME_OVERHEAD = 24; +constexpr size_t MAX_STREAM_SEND_BYTES_PER_EVENT = 16 * 1024; QUICStream::QUICStream(QUICConnectionInfoProvider *cinfo, QUICStreamId sid) : _connection_info(cinfo), _id(sid) {} @@ -60,6 +61,22 @@ QUICStream::has_no_more_data() const return this->_has_no_more_data; } +bool +QUICStream::has_data_to_send() +{ + if (this->_pending_send_block) { + return true; + } + if (this->_adapter == nullptr) { + return false; + } + + const bool has_buffered_data = this->_adapter->unread_len() > 0; + const bool needs_fin = !this->_sent_fin && this->_adapter->is_eos() && this->_adapter->total_len() == this->_sent_bytes; + + return has_buffered_data || needs_fin; +} + void QUICStream::set_io_adapter(QUICStreamAdapter *adapter) { @@ -87,6 +104,14 @@ QUICStream::on_read() { } +void +QUICStream::on_write() +{ + if (this->_connection_info != nullptr) { + this->_connection_info->on_stream_updated(); + } +} + void QUICStream::on_eos() { @@ -117,21 +142,63 @@ QUICStream::send_data(quiche_conn *quiche_con) ssize_t len = 0; [[maybe_unused]] ErrorCode error_code{0}; // Only set if QUICHE_ERR_STREAM_STOPPED(-15) or QUICHE_ERR_STREAM_RESET(-16) are // returned by quiche_conn_stream_send. + size_t written_this_event = 0; - len = quiche_conn_stream_capacity(quiche_con, this->_id); - if (len <= 0) { - return; - } - Ptr block = this->_adapter->read(len); - if (this->_adapter->total_len() == this->_sent_bytes + block->size()) { - fin = true; - } - if (block->size() > 0 || fin) { - ssize_t written_len = - quiche_conn_stream_send(quiche_con, this->_id, reinterpret_cast(block->start()), block->size(), fin, &error_code); - if (written_len >= 0) { - this->_sent_bytes += written_len; + while (written_this_event < MAX_STREAM_SEND_BYTES_PER_EVENT) { + len = quiche_conn_stream_capacity(quiche_con, this->_id); + if (len <= 0) { + return; + } + + if (!this->_pending_send_block) { + size_t read_len = std::min(static_cast(len), MAX_STREAM_SEND_BYTES_PER_EVENT - written_this_event); + this->_pending_send_block = this->_adapter->read(read_len); + if (!this->_pending_send_block) { + if (!this->_sent_fin && this->_adapter->is_eos() && this->_adapter->total_len() == this->_sent_bytes) { + static constexpr uint8_t empty_data = 0; + ssize_t written_len = quiche_conn_stream_send(quiche_con, this->_id, &empty_data, 0, true, &error_code); + if (written_len >= 0) { + this->_sent_fin = true; + } + } + this->_adapter->encourge_write(); + return; + } + this->_pending_send_fin = this->_adapter->total_len() == this->_sent_bytes + this->_pending_send_block->size(); } + + Ptr block = this->_pending_send_block; + fin = this->_pending_send_fin; + if (block->size() == 0 && !fin) { + this->_pending_send_block = nullptr; + this->_pending_send_fin = false; + this->_adapter->encourge_write(); + continue; + } + + if (block->size() > 0 || fin) { + ssize_t written_len = quiche_conn_stream_send(quiche_con, this->_id, reinterpret_cast(block->start()), + block->size(), fin, &error_code); + if (written_len >= 0) { + this->_adapter->consume(written_len); + this->_sent_bytes += written_len; + written_this_event += written_len; + if (written_len >= block->size()) { + this->_pending_send_block = nullptr; + this->_pending_send_fin = false; + this->_sent_fin = fin; + } else { + block->consume(written_len); + return; + } + if (!this->has_data_to_send()) { + this->_adapter->encourge_write(); + return; + } + continue; + } + } + this->_adapter->encourge_write(); + return; } - this->_adapter->encourge_write(); } diff --git a/src/iocore/net/quic/QUICStreamAdapter.cc b/src/iocore/net/quic/QUICStreamAdapter.cc index 9992b8a661e..2e49214ad50 100644 --- a/src/iocore/net/quic/QUICStreamAdapter.cc +++ b/src/iocore/net/quic/QUICStreamAdapter.cc @@ -26,7 +26,12 @@ Ptr QUICStreamAdapter::read(size_t len) { - auto ret = this->_read(len); + return this->_read(len); +} + +void +QUICStreamAdapter::consume(size_t len) +{ + this->_consume(len); this->_stream.on_read(); - return ret; } diff --git a/src/iocore/net/quic/QUICStreamVCAdapter.cc b/src/iocore/net/quic/QUICStreamVCAdapter.cc index d79abe51db9..a03cc59f59d 100644 --- a/src/iocore/net/quic/QUICStreamVCAdapter.cc +++ b/src/iocore/net/quic/QUICStreamVCAdapter.cc @@ -24,6 +24,8 @@ #include "iocore/eventsystem/VConnection.h" #include "iocore/net/quic/QUICStreamVCAdapter.h" +#include + QUICStreamVCAdapter::QUICStreamVCAdapter(QUICStream &stream) : VConnection(new_ProxyMutex()), QUICStreamAdapter(stream) { SET_HANDLER(&QUICStreamVCAdapter::state_stream_open); @@ -84,18 +86,45 @@ QUICStreamVCAdapter::_read(size_t len) SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); IOBufferReader *reader = this->_write_vio.get_reader(); - block = make_ptr(reader->get_current_block()->clone()); + if (reader == nullptr || reader->get_current_block() == nullptr || reader->block_read_avail() <= 0) { + return block; + } + + const size_t read_len = std::min(len, static_cast(reader->block_read_avail())); + block = make_ptr(reader->get_current_block()->clone()); if (block->size()) { block->consume(reader->start_offset); - block->_end = std::min(block->start() + len, block->_buf_end); - this->_write_vio.ndone += block->size(); + block->_end = block->start() + read_len; + } + if (block->size() == 0) { + block = nullptr; } - reader->consume(block->size()); } return block; } +void +QUICStreamVCAdapter::_consume(size_t len) +{ + if (len == 0 || this->_write_vio.op != VIO::WRITE) { + return; + } + + SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); + + IOBufferReader *reader = this->_write_vio.get_reader(); + if (reader == nullptr) { + return; + } + + const size_t consume_len = std::min(len, static_cast(std::max(reader->read_avail(), 0))); + if (consume_len > 0) { + reader->consume(consume_len); + this->_write_vio.ndone += consume_len; + } +} + bool QUICStreamVCAdapter::is_eos() { @@ -119,7 +148,8 @@ QUICStreamVCAdapter::unread_len() { if (this->_write_vio.op == VIO::WRITE) { SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); - return this->_write_vio.get_reader()->block_read_avail(); + IOBufferReader *reader = this->_write_vio.get_reader(); + return reader == nullptr ? 0 : reader->block_read_avail(); } else { return 0; } @@ -321,23 +351,45 @@ QUICStreamVCAdapter::do_io_shutdown(ShutdownHowTo_t /* howto ATS_UNUSED */) } void -QUICStreamVCAdapter::reenable(VIO * /* vio ATS_UNUSED */) +QUICStreamVCAdapter::reenable(VIO *vio) { - // TODO We probably need to tell QUICStream that the application consumed received data - // to update receive window here. In other words, we should not update receive window - // until the application consume data. + if (vio == nullptr || vio->op != VIO::WRITE) { + // TODO We probably need to tell QUICStream that the application consumed received data + // to update receive window here. In other words, we should not update receive window + // until the application consume data. + return; + } + + const bool has_buffered_data = vio->get_reader() != nullptr && vio->get_reader()->read_avail() > 0; + const bool needs_fin = vio->nbytes != INT64_MAX && vio->ntodo() == 0; + if (has_buffered_data || needs_fin) { + this->stream().on_write(); + } } bool QUICStreamVCAdapter::is_readable() { - return this->stream().direction() != QUICStreamDirection::SEND && _read_vio.nbytes != _read_vio.ndone; + return this->stream().direction() != QUICStreamDirection::SEND && this->_read_vio.op == VIO::READ && + this->_read_vio.nbytes != this->_read_vio.ndone; } bool QUICStreamVCAdapter::is_writable() { - return this->stream().direction() != QUICStreamDirection::RECEIVE && _write_vio.nbytes != _read_vio.ndone; + return this->stream().direction() != QUICStreamDirection::RECEIVE; +} + +void +QUICStreamVCAdapter::mark_stream_closed() +{ + SET_HANDLER(&QUICStreamVCAdapter::state_stream_closed); +} + +bool +QUICStreamVCAdapter::is_stream_closed() const +{ + return this->handler == continuation_handler_void_ptr(&QUICStreamVCAdapter::state_stream_closed); } int diff --git a/src/proxy/http3/Http3App.cc b/src/proxy/http3/Http3App.cc index 09d349a6a69..1f3d1a3172a 100644 --- a/src/proxy/http3/Http3App.cc +++ b/src/proxy/http3/Http3App.cc @@ -103,6 +103,15 @@ Http3App::start() // } } +void +Http3App::_handle_error(const Http3Error &error) +{ + if (error.cls == Http3ErrorClass::CONNECTION) { + this->_qc->close_quic_connection( + std::make_unique(QUICErrorClass::APPLICATION, static_cast(error.code))); + } +} + void Http3App::on_stream_open(QUICStream &stream) { @@ -131,7 +140,15 @@ Http3App::on_stream_open(QUICStream &stream) void Http3App::on_stream_close(QUICStream &stream) { - this->_streams.erase(stream.id()); + QUICStreamId const stream_id = stream.id(); + + if (auto *txn = this->_ssn->get_transaction(stream_id); txn != nullptr) { + SCOPED_MUTEX_LOCK(lock, txn->mutex, this_ethread()); + txn->set_stream_cleanup([this, stream_id]() { this->_streams.erase(stream_id); }); + txn->stream_closed(); + } else { + this->_streams.erase(stream_id); + } } int @@ -286,6 +303,10 @@ Http3App::_handle_uni_stream_on_read_ready(int /* event */, VIO *vio) default: break; } + + if (error && error->cls != Http3ErrorClass::UNDEFINED) { + this->_handle_error(*error); + } } void diff --git a/src/proxy/http3/Http3DebugNames.cc b/src/proxy/http3/Http3DebugNames.cc index 4208f6c4a43..1ea186fa19b 100644 --- a/src/proxy/http3/Http3DebugNames.cc +++ b/src/proxy/http3/Http3DebugNames.cc @@ -48,6 +48,8 @@ Http3DebugNames::frame_type(Http3FrameType type) return "X_RESERVED_3"; case Http3FrameType::X_RESERVED_4: return "X_RESERVED_4"; + case Http3FrameType::RESERVED: + return "RESERVED"; case Http3FrameType::UNKNOWN: default: return "UNKNOWN"; diff --git a/src/proxy/http3/Http3Frame.cc b/src/proxy/http3/Http3Frame.cc index 64b0a0a4651..6fc7d5b5441 100644 --- a/src/proxy/http3/Http3Frame.cc +++ b/src/proxy/http3/Http3Frame.cc @@ -42,6 +42,13 @@ constexpr int HEADER_OVERHEAD = 10; // This should work as long as a payloa DbgCtl dbg_ctl_http3_frame_factory{"http3_frame_factory"}; +bool +is_reserved_frame_type(uint64_t type) +{ + return type >= static_cast(Http3FrameType::RESERVED) && + (type - static_cast(Http3FrameType::RESERVED)) % 0x1f == 0; +} + } // end anonymous namespace // @@ -64,6 +71,8 @@ Http3Frame::type(const uint8_t *buf, size_t buf_len) ink_assert(ret != 1); if (type <= static_cast(Http3FrameType::X_MAX_DEFINED)) { return static_cast(type); + } else if (is_reserved_frame_type(type)) { + return Http3FrameType::RESERVED; } else { return Http3FrameType::UNKNOWN; } @@ -143,8 +152,12 @@ Http3Frame::length() const Http3FrameType Http3Frame::type() const { - if (static_cast(this->_type) <= static_cast(Http3FrameType::X_MAX_DEFINED)) { + const auto type = static_cast(this->_type); + + if (type <= static_cast(Http3FrameType::X_MAX_DEFINED)) { return this->_type; + } else if (is_reserved_frame_type(type)) { + return Http3FrameType::RESERVED; } else { return Http3FrameType::UNKNOWN; } diff --git a/src/proxy/http3/Http3HeaderVIOAdaptor.cc b/src/proxy/http3/Http3HeaderVIOAdaptor.cc index 9d3e256b7f6..7f489d4fa28 100644 --- a/src/proxy/http3/Http3HeaderVIOAdaptor.cc +++ b/src/proxy/http3/Http3HeaderVIOAdaptor.cc @@ -139,7 +139,8 @@ Http3HeaderVIOAdaptor::_on_qpack_decode_complete() // or // c). Add interface to HttpSM to handle HTTPHdr directly int bufindex; - int dumpoffset = 0; + int dumpoffset = 0; + int64_t header_length = 0; int done, tmp; IOBufferBlock *block; do { @@ -150,14 +151,19 @@ Http3HeaderVIOAdaptor::_on_qpack_decode_complete() writer->add_block(); block = writer->get_current_block(); } - done = this->_header.print(block->end(), block->write_avail(), &bufindex, &tmp); - dumpoffset += bufindex; + done = this->_header.print(block->end(), block->write_avail(), &bufindex, &tmp); + dumpoffset += bufindex; + header_length += bufindex; writer->fill(bufindex); if (!done) { writer->add_block(); } } while (!done); - this->_is_complete = true; + this->_sink_vio->ndone += header_length; + this->_is_complete = true; + if (auto *transaction = dynamic_cast(this->_txn); transaction != nullptr) { + transaction->on_header_decode_complete(); + } return 1; } diff --git a/src/proxy/http3/Http3ProtocolEnforcer.cc b/src/proxy/http3/Http3ProtocolEnforcer.cc index 9f7e8a8963a..a37cb543ae4 100644 --- a/src/proxy/http3/Http3ProtocolEnforcer.cc +++ b/src/proxy/http3/Http3ProtocolEnforcer.cc @@ -30,7 +30,7 @@ Http3ProtocolEnforcer::interests() return {Http3FrameType::DATA, Http3FrameType::HEADERS, Http3FrameType::X_RESERVED_1, Http3FrameType::CANCEL_PUSH, Http3FrameType::SETTINGS, Http3FrameType::PUSH_PROMISE, Http3FrameType::X_RESERVED_2, Http3FrameType::GOAWAY, Http3FrameType::X_RESERVED_3, Http3FrameType::X_RESERVED_4, Http3FrameType::MAX_PUSH_ID, Http3FrameType::X_MAX_DEFINED, - Http3FrameType::UNKNOWN}; + Http3FrameType::RESERVED, Http3FrameType::UNKNOWN}; } Http3ErrorUPtr @@ -47,9 +47,8 @@ Http3ProtocolEnforcer::handle_frame(std::shared_ptr frame, Htt "only one SETTINGS frame is allowed per the control stream"); } else if (f_type == Http3FrameType::DATA || f_type == Http3FrameType::HEADERS || f_type == Http3FrameType::X_RESERVED_1 || f_type == Http3FrameType::X_RESERVED_2 || f_type == Http3FrameType::X_RESERVED_3) { - std::string error_msg = Http3DebugNames::frame_type(f_type); - error_msg.append(" frame is not allowed on control stream"); - error = std::make_unique(Http3ErrorClass::CONNECTION, Http3ErrorCode::H3_FRAME_UNEXPECTED, error_msg.c_str()); + error = std::make_unique(Http3ErrorClass::CONNECTION, Http3ErrorCode::H3_FRAME_UNEXPECTED, + "frame is not allowed on control stream"); } if (!this->_is_first_frame_received_on_control) { this->_is_first_frame_received_on_control = true; @@ -57,9 +56,13 @@ Http3ProtocolEnforcer::handle_frame(std::shared_ptr frame, Htt } else { if (f_type == Http3FrameType::X_RESERVED_1 || f_type == Http3FrameType::X_RESERVED_2 || f_type == Http3FrameType::X_RESERVED_3) { - std::string error_msg = Http3DebugNames::frame_type(f_type); - error_msg.append(" frame is not allowed on any stream"); - error = std::make_unique(Http3ErrorClass::CONNECTION, Http3ErrorCode::H3_FRAME_UNEXPECTED, error_msg.c_str()); + error = std::make_unique(Http3ErrorClass::CONNECTION, Http3ErrorCode::H3_FRAME_UNEXPECTED, + "frame is not allowed on any stream"); + } else if (!this->_is_headers_frame_received && f_type == Http3FrameType::DATA) { + error = std::make_unique(Http3ErrorClass::CONNECTION, Http3ErrorCode::H3_FRAME_UNEXPECTED, + "DATA frame is not allowed before HEADERS"); + } else if (f_type == Http3FrameType::HEADERS) { + this->_is_headers_frame_received = true; } } diff --git a/src/proxy/http3/Http3Session.cc b/src/proxy/http3/Http3Session.cc index 9a001306bf7..1c92f031b21 100644 --- a/src/proxy/http3/Http3Session.cc +++ b/src/proxy/http3/Http3Session.cc @@ -40,7 +40,9 @@ HQSession::HQSession(NetVConnection *vc) : ProxySession(vc) HQSession::~HQSession() { - // Transactions should be deleted first before HQSesson gets deleted. + this->_close_transactions(); + + // Transactions should be deleted first before HQSession gets deleted. ink_assert(this->_transaction_list.head == nullptr); } @@ -60,6 +62,17 @@ HQSession::remove_transaction(HQTransaction *trans) return; } +void +HQSession::_close_transactions() +{ + while (this->_transaction_list.head != nullptr) { + auto *transaction = this->_transaction_list.head; + + transaction->do_io_close(); + delete transaction; + } +} + const char * HQSession::get_protocol_string() const { @@ -163,9 +176,11 @@ HQSession::main_event_handler(int event, void *edata) case VC_EVENT_ERROR: case VC_EVENT_EOS: this->do_io_close(); - for (HQTransaction *t = this->_transaction_list.head; t; t = static_cast(t->link.next)) { + for (HQTransaction *t = this->_transaction_list.head; t != nullptr;) { + HQTransaction *next = static_cast(t->link.next); SCOPED_MUTEX_LOCK(lock, t->mutex, this_ethread()); t->handleEvent(event, edata); + t = next; } break; } @@ -195,6 +210,7 @@ Http3Session::Http3Session(NetVConnection *vc) : HQSession(vc) Http3Session::~Http3Session() { + this->_close_transactions(); this->_vc = nullptr; delete this->_local_qpack; delete this->_remote_qpack; diff --git a/src/proxy/http3/Http3StreamDataVIOAdaptor.cc b/src/proxy/http3/Http3StreamDataVIOAdaptor.cc index 7f19c277d9a..296763972b8 100644 --- a/src/proxy/http3/Http3StreamDataVIOAdaptor.cc +++ b/src/proxy/http3/Http3StreamDataVIOAdaptor.cc @@ -24,10 +24,14 @@ #include "proxy/http3/Http3StreamDataVIOAdaptor.h" #include "iocore/eventsystem/VIO.h" -Http3StreamDataVIOAdaptor::Http3StreamDataVIOAdaptor(VIO *sink) : _sink_vio(sink), _buffer(new_MIOBuffer(BUFFER_SIZE_INDEX_4K)) {} +Http3StreamDataVIOAdaptor::Http3StreamDataVIOAdaptor(VIO *sink) : _sink_vio(sink), _buffer(new_MIOBuffer(BUFFER_SIZE_INDEX_4K)) +{ + this->_reader = this->_buffer->alloc_reader(); +} Http3StreamDataVIOAdaptor::~Http3StreamDataVIOAdaptor() { + this->_buffer->dealloc_reader(this->_reader); free_MIOBuffer(this->_buffer); } @@ -53,17 +57,17 @@ Http3StreamDataVIOAdaptor::handle_frame(std::shared_ptr frame, void Http3StreamDataVIOAdaptor::finalize() { - SCOPED_MUTEX_LOCK(lock, this->_sink_vio->mutex, this_ethread()); - MIOBuffer *writer = this->_sink_vio->get_writer(); - IOBufferReader *reader = this->_buffer->alloc_reader(); - IOBufferBlock *block; - while (reader->read_avail() > 0 && (block = reader->get_current_block()) != nullptr) { - writer->append_block(block); - reader->consume(block->size()); + if (this->_finalized) { + return; } - this->_buffer->dealloc_reader(reader); - this->_sink_vio->nbytes = this->_total_data_length; + SCOPED_MUTEX_LOCK(lock, this->_sink_vio->mutex, this_ethread()); + MIOBuffer *writer = this->_sink_vio->get_writer(); + int64_t delivered = writer->write(this->_reader, this->_reader->read_avail()); + this->_reader->consume(delivered); + this->_sink_vio->ndone += delivered; + this->_sink_vio->nbytes = this->_sink_vio->ndone; + this->_finalized = true; } bool diff --git a/src/proxy/http3/Http3Transaction.cc b/src/proxy/http3/Http3Transaction.cc index d525720db5b..5927f3e47b3 100644 --- a/src/proxy/http3/Http3Transaction.cc +++ b/src/proxy/http3/Http3Transaction.cc @@ -31,6 +31,7 @@ #include "proxy/http3/Http3HeaderVIOAdaptor.h" #include "proxy/http3/Http3HeaderFramer.h" #include "proxy/http3/Http3DataFramer.h" +#include "proxy/http3/Http3ProtocolEnforcer.h" #include "proxy/http/HttpSM.h" #define NetVC2QUICCon(netvc) netvc->get_service()->get_quic_connection() @@ -63,7 +64,8 @@ DbgCtl dbg_ctl_v_http3_trans{"v_http3_trans"}; // // HQTransaction // -HQTransaction::HQTransaction(HQSession *session, QUICStreamVCAdapter::IOInfo &info) : super(session), _info(info) +HQTransaction::HQTransaction(HQSession *session, QUICStreamVCAdapter::IOInfo &info) + : super(session), _info(info), _stream_id(info.adapter.stream().id()) { this->mutex = new_ProxyMutex(); this->_thread = this_ethread(); @@ -81,12 +83,16 @@ HQTransaction::~HQTransaction() this->_unschedule_write_complete_event(); static_cast(this->_proxy_ssn)->remove_transaction(this); + + if (this->_stream_cleanup) { + this->_stream_cleanup(); + } } void HQTransaction::set_active_timeout(ink_hrtime timeout_in) { - if (this->_proxy_ssn) { + if (!this->_is_closed() && this->_proxy_ssn) { this->_proxy_ssn->set_active_timeout(timeout_in); } } @@ -94,7 +100,7 @@ HQTransaction::set_active_timeout(ink_hrtime timeout_in) void HQTransaction::set_inactivity_timeout(ink_hrtime timeout_in) { - if (this->_proxy_ssn) { + if (!this->_is_closed() && this->_proxy_ssn) { this->_proxy_ssn->set_inactivity_timeout(timeout_in); } } @@ -102,7 +108,7 @@ HQTransaction::set_inactivity_timeout(ink_hrtime timeout_in) void HQTransaction::cancel_inactivity_timeout() { - if (this->_proxy_ssn) { + if (!this->_is_closed() && this->_proxy_ssn) { this->_proxy_ssn->cancel_inactivity_timeout(); } } @@ -132,7 +138,7 @@ HQTransaction::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) if (buf) { this->_process_read_vio(); - this->_schedule_read_ready_event(); + this->_schedule_read_event(); } return &this->_read_vio; @@ -186,6 +192,10 @@ HQTransaction::do_io_shutdown(ShutdownHowTo_t /* howto ATS_UNUSED */) void HQTransaction::reenable(VIO *vio) { + if (this->_is_closed() || this->_is_stream_closed()) { + return; + } + if (vio->op == VIO::READ) { int64_t len = this->_process_read_vio(); this->_info.read_vio->reenable(); @@ -209,13 +219,28 @@ HQTransaction::transaction_done() // TODO: start closing transaction super::transaction_done(); this->_transaction_done = true; + this->_delete_if_possible(); return; } int HQTransaction::get_transaction_id() const { - return this->_info.adapter.stream().id(); + return this->_stream_id; +} + +void +HQTransaction::stream_closed() +{ + this->_info.adapter.mark_stream_closed(); + this->do_io_close(); + this->_delete_if_possible(); +} + +void +HQTransaction::set_stream_cleanup(std::function cleanup) +{ + this->_stream_cleanup = cleanup; } void @@ -276,6 +301,20 @@ HQTransaction::_schedule_read_complete_event() this->_read_complete_event = this->_thread->schedule_imm(this, VC_EVENT_READ_COMPLETE, &this->_read_vio); } +void +HQTransaction::_schedule_read_event() +{ + if (this->_read_vio.nbytes == 0) { + return; + } + + if (this->_info.read_vio->nbytes == INT64_MAX) { + this->_schedule_read_ready_event(); + } else { + this->_schedule_read_complete_event(); + } +} + void HQTransaction::_unschedule_read_complete_event() { @@ -353,17 +392,23 @@ HQTransaction::_close_write_complete_event(Event *e) } void -HQTransaction::_signal_event(int event, Event * /* edata ATS_UNUSED */) +HQTransaction::_signal_event(int event, Event *) { - // HttpSM::main_handler expects a VIO* as the event data for VC events so it - // can locate the vc_table entry. - if (this->_write_vio.cont) { - SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); - this->_write_vio.cont->handleEvent(event, &this->_write_vio); + if (this->_is_closed() || this->_is_stream_closed()) { + return; } - if (this->_read_vio.cont && this->_read_vio.cont != this->_write_vio.cont) { + + // HttpSM::main_handler expects a VIO* as the event data for VC events so it + // can locate the vc_table entry. Prefer the read side because H3 creates a + // zero-byte write VIO before HttpSM installs a client write handler. + if (this->_read_vio.cont && this->_read_vio.op != VIO::NONE) { SCOPED_MUTEX_LOCK(lock, this->_read_vio.mutex, this_ethread()); this->_read_vio.cont->handleEvent(event, &this->_read_vio); + return; + } + if (this->_write_vio.cont && this->_write_vio.op != VIO::NONE && this->_write_vio.nbytes > 0) { + SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); + this->_write_vio.cont->handleEvent(event, &this->_write_vio); } } @@ -373,10 +418,10 @@ HQTransaction::_signal_event(int event, Event * /* edata ATS_UNUSED */) void HQTransaction::_signal_read_event() { - if (this->_read_vio.cont == nullptr || this->_read_vio.op == VIO::NONE) { + if (this->_is_closed() || this->_is_stream_closed() || this->_read_vio.cont == nullptr || this->_read_vio.op == VIO::NONE) { return; } - int event = this->_read_vio.nbytes == INT64_MAX ? VC_EVENT_READ_READY : VC_EVENT_READ_COMPLETE; + int event = this->_read_vio.nbytes == INT64_MAX || this->_read_vio.ntodo() > 0 ? VC_EVENT_READ_READY : VC_EVENT_READ_COMPLETE; SCOPED_MUTEX_LOCK(lock, this->_read_vio.mutex, this_ethread()); this->_read_vio.cont->handleEvent(event, &this->_read_vio); @@ -390,9 +435,13 @@ HQTransaction::_signal_read_event() void HQTransaction::_signal_write_event() { - if (this->_write_vio.cont == nullptr || this->_write_vio.op == VIO::NONE) { + if (this->_is_closed() || this->_is_stream_closed() || this->_write_vio.cont == nullptr || this->_write_vio.op == VIO::NONE) { + return; + } + if (this->_write_vio.ntodo() == 0 && !this->_is_write_buffer_flushed()) { return; } + int event = this->_write_vio.ntodo() ? VC_EVENT_WRITE_READY : VC_EVENT_WRITE_COMPLETE; SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); @@ -401,6 +450,28 @@ HQTransaction::_signal_write_event() Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); } +bool +HQTransaction::_is_write_buffer_flushed() +{ + if (this->_is_closed() || this->_is_stream_closed()) { + return true; + } + + if (this->_info.write_vio->op == VIO::NONE) { + return true; + } + + SCOPED_MUTEX_LOCK(lock, this->_info.write_vio->mutex, this_ethread()); + + return this->_info.write_vio->ntodo() == 0; +} + +bool +HQTransaction::_is_stream_closed() const +{ + return this->_info.adapter.is_stream_closed(); +} + /** * Deletes this transaction itself. * This must be called only at the end of event handlers to avoid touching itself after deletion. @@ -408,7 +479,12 @@ HQTransaction::_signal_write_event() void HQTransaction::_delete_if_possible() { - if (this->_transaction_done) { + if (this->_event_handler_active) { + return; + } + + if (this->_transaction_done && this->_is_write_buffer_flushed() && + (this->_is_stream_closed() || !this->_info.adapter.is_readable())) { delete this; } } @@ -432,10 +508,12 @@ Http3Transaction::Http3Transaction(Http3Session *session, QUICStreamVCAdapter::I } else { http_type = HTTPType::REQUEST; } - this->_header_handler = new Http3HeaderVIOAdaptor(&this->_read_vio, http_type, session->remote_qpack(), stream_id, this); - this->_data_handler = new Http3StreamDataVIOAdaptor(&this->_read_vio); + this->_protocol_enforcer = new Http3ProtocolEnforcer(); + this->_header_handler = new Http3HeaderVIOAdaptor(&this->_read_vio, http_type, session->remote_qpack(), stream_id, this); + this->_data_handler = new Http3StreamDataVIOAdaptor(&this->_read_vio); this->_frame_dispatcher.add_handler(session->get_received_frame_counter()); + this->_frame_dispatcher.add_handler(this->_protocol_enforcer); this->_frame_dispatcher.add_handler(this->_header_handler); this->_frame_dispatcher.add_handler(this->_data_handler); @@ -453,6 +531,8 @@ Http3Transaction::~Http3Transaction() this->_header_framer = nullptr; delete this->_data_framer; this->_data_framer = nullptr; + delete this->_protocol_enforcer; + this->_protocol_enforcer = nullptr; delete this->_header_handler; this->_header_handler = nullptr; delete this->_data_handler; @@ -465,6 +545,7 @@ Http3Transaction::state_stream_open(int event, Event *edata) // TODO: should check recursive call? ink_release_assert(this->_thread == this_ethread()); SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + this->_event_handler_active = true; switch (event) { case VC_EVENT_READ_READY: @@ -474,22 +555,29 @@ Http3Transaction::state_stream_open(int event, Event *edata) if (this->_process_read_vio() > 0) { this->_signal_read_event(); } - this->_info.read_vio->reenable(); + if (!this->_is_closed()) { + this->_info.read_vio->reenable(); + } break; - case VC_EVENT_READ_COMPLETE: + case VC_EVENT_READ_COMPLETE: { Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); this->_close_read_complete_event(edata); - this->_process_read_vio(); + int64_t nread = this->_process_read_vio(); if (!this->_header_handler->is_complete()) { - // Delay processing READ_COMPLETE - this->_schedule_read_complete_event(); + if (nread > 0) { + // Delay processing READ_COMPLETE until the header block can be fully decoded. + this->_schedule_read_complete_event(); + } break; } this->_data_handler->finalize(); // always signal regardless of progress this->_signal_read_event(); - this->_info.read_vio->reenable(); + if (!this->_is_closed()) { + this->_info.read_vio->reenable(); + } break; + } case VC_EVENT_WRITE_READY: this->_close_write_ready_event(edata); Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); @@ -497,7 +585,9 @@ Http3Transaction::state_stream_open(int event, Event *edata) if (this->_process_write_vio() > 0) { this->_signal_write_event(); } - this->_info.write_vio->reenable(); + if (!this->_is_closed()) { + this->_info.write_vio->reenable(); + } break; case VC_EVENT_WRITE_COMPLETE: this->_close_write_complete_event(edata); @@ -505,7 +595,9 @@ Http3Transaction::state_stream_open(int event, Event *edata) this->_process_write_vio(); // always signal regardless of progress this->_signal_write_event(); - this->_info.write_vio->reenable(); + if (!this->_is_closed()) { + this->_info.write_vio->reenable(); + } break; case VC_EVENT_EOS: case VC_EVENT_ERROR: @@ -513,12 +605,14 @@ Http3Transaction::state_stream_open(int event, Event *edata) case VC_EVENT_ACTIVE_TIMEOUT: { Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); this->_signal_event(event, edata); + this->do_io_close(); break; } default: Http3TransDebug("Unknown event %d", event); } + this->_event_handler_active = false; this->_delete_if_possible(); return EVENT_DONE; } @@ -527,6 +621,7 @@ int Http3Transaction::state_stream_closed(int event, Event *data) { Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); + this->_event_handler_active = true; switch (event) { case VC_EVENT_READ_READY: @@ -553,6 +648,7 @@ Http3Transaction::state_stream_closed(int event, Event *data) Http3TransDebug("Unknown event %d", event); } + this->_event_handler_active = false; this->_delete_if_possible(); return EVENT_DONE; } @@ -564,6 +660,18 @@ Http3Transaction::do_io_close(int lerrno) super::do_io_close(lerrno); } +bool +Http3Transaction::_is_closed() const +{ + return this->handler == continuation_handler_void_ptr(&Http3Transaction::state_stream_closed); +} + +void +Http3Transaction::on_header_decode_complete() +{ + this->_schedule_read_event(); +} + bool Http3Transaction::is_response_header_sent() const { @@ -576,9 +684,26 @@ Http3Transaction::is_response_body_sent() const return this->_data_framer->is_done(); } +void +Http3Transaction::_handle_error(const Http3Error &error) +{ + if (error.cls == Http3ErrorClass::CONNECTION) { + this->_info.adapter.mark_stream_closed(); + this->do_io_close(); + this->_transaction_done = true; + NetVC2QUICCon(this->_proxy_ssn->get_netvc()) + ->close_quic_connection( + std::make_unique(QUICErrorClass::APPLICATION, static_cast(error.code))); + } +} + int64_t Http3Transaction::_process_read_vio() { + if (this->_is_stream_closed()) { + return 0; + } + if (this->_info.read_vio->cont == nullptr || this->_info.read_vio->op == VIO::NONE) { return 0; } @@ -590,7 +715,8 @@ Http3Transaction::_process_read_vio() auto error = this->_frame_dispatcher.on_read_ready(this->_info.adapter.stream().id(), Http3StreamType::UNKNOWN, *this->_info.read_vio->get_reader(), nread); if (error && error->cls != Http3ErrorClass::UNDEFINED) { - Http3TransDebug("Error occured while processing read vio: %hu, %s", error->get_code(), error->msg); + Http3TransDebug("Error occurred while processing read vio: %hu", error->get_code()); + this->_handle_error(*error); return 0; } this->_info.read_vio->ndone += nread; @@ -600,6 +726,10 @@ Http3Transaction::_process_read_vio() int64_t Http3Transaction::_process_write_vio() { + if (this->_is_stream_closed()) { + return 0; + } + if (this->_info.write_vio->cont == nullptr || this->_info.write_vio->op == VIO::NONE) { return 0; } @@ -612,7 +742,7 @@ Http3Transaction::_process_write_vio() auto error = this->_frame_collector.on_write_ready(this->_info.adapter.stream().id(), *this->_info.write_vio->get_writer(), nwritten, all_done); if (error && error->cls != Http3ErrorClass::UNDEFINED) { - Http3TransDebug("Error occured while processing write vio: %hu, %s", error->get_code(), error->msg); + Http3TransDebug("Error occurred while processing write vio: %hu", error->get_code()); return 0; } this->_sent_bytes += nwritten; @@ -636,6 +766,10 @@ Http3Transaction::has_request_body(int64_t content_length, bool /* is_chunked_se return true; } + if (this->_is_stream_closed()) { + return false; + } + // No body if stream is already closed and DATA frame is not received yet if (this->_info.adapter.stream().has_no_more_data()) { return false; @@ -669,6 +803,7 @@ Http09Transaction::state_stream_open(int event, Event *edata) ink_release_assert(this->_thread == this_ethread()); SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + this->_event_handler_active = true; switch (event) { case VC_EVENT_READ_READY: @@ -706,12 +841,15 @@ Http09Transaction::state_stream_open(int event, Event *edata) case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ACTIVE_TIMEOUT: { Http3TransDebug("%d", event); + this->_signal_event(event, edata); + this->do_io_close(); break; } default: Http3TransDebug("Unknown event %d", event); } + this->_event_handler_active = false; this->_delete_if_possible(); return EVENT_DONE; } @@ -723,10 +861,17 @@ Http09Transaction::do_io_close(int lerrno) super::do_io_close(lerrno); } +bool +Http09Transaction::_is_closed() const +{ + return this->handler == continuation_handler_void_ptr(&Http09Transaction::state_stream_closed); +} + int Http09Transaction::state_stream_closed(int event, Event *data) { Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); + this->_event_handler_active = true; switch (event) { case VC_EVENT_READ_READY: @@ -752,6 +897,7 @@ Http09Transaction::state_stream_closed(int event, Event *data) Http3TransDebug("Unknown event %d", event); } + this->_event_handler_active = false; this->_delete_if_possible(); return EVENT_DONE; } @@ -760,6 +906,10 @@ Http09Transaction::state_stream_closed(int event, Event *data) int64_t Http09Transaction::_process_read_vio() { + if (this->_is_stream_closed()) { + return 0; + } + if (this->_read_vio.cont == nullptr || this->_read_vio.op == VIO::NONE) { return 0; } @@ -840,6 +990,10 @@ static constexpr char http_1_1_version[] = "HTTP/1.1"; int64_t Http09Transaction::_process_write_vio() { + if (this->_is_stream_closed()) { + return 0; + } + if (this->_write_vio.cont == nullptr || this->_write_vio.op == VIO::NONE) { return 0; } diff --git a/src/proxy/http3/QPACK.cc b/src/proxy/http3/QPACK.cc index 6e915334dfc..45da0384942 100644 --- a/src/proxy/http3/QPACK.cc +++ b/src/proxy/http3/QPACK.cc @@ -69,7 +69,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {":status", "503" }, {"accept", "*/*" }, {"accept", "application/dns-message" }, - {"accept-encoding", "gzip, deflate, br, zstd" }, + {"accept-encoding", "gzip, deflate, br" }, {"accept-ranges", "bytes" }, {"access-control-allow-headers", "cache-control" }, {"access-control-allow-headers", "content-type" }, @@ -80,7 +80,6 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {"cache-control", "no-cache" }, {"cache-control", "no-store" }, {"cache-control", "public, max-age=31536000" }, - {"content-encoding", "zstd" }, {"content-encoding", "br" }, {"content-encoding", "gzip" }, {"content-type", "application/dns-message" }, diff --git a/src/proxy/http3/test/test_Http3FrameDispatcher.cc b/src/proxy/http3/test/test_Http3FrameDispatcher.cc index 0c7a65b749c..37560e9a157 100644 --- a/src/proxy/http3/test/test_Http3FrameDispatcher.cc +++ b/src/proxy/http3/test/test_Http3FrameDispatcher.cc @@ -231,7 +231,7 @@ TEST_CASE("control stream tests", "[http3]") CHECK(nread == sizeof(input)); } - SECTION("RESERVED frame is not allowed on control stream") + SECTION("HTTP/2 reserved frame is not allowed on control stream") { uint8_t input[] = {0x04, // Type 0x08, // Length @@ -258,6 +258,32 @@ TEST_CASE("control stream tests", "[http3]") CHECK(nread == sizeof(input)); } + SECTION("GREASE reserved frame is ignored on control stream") + { + uint8_t input[] = {0x04, // Type + 0x08, // Length + 0x06, // Identifier + 0x44, 0x00, // Value + 0x09, // Identifier + 0x0f, // Value + 0x4a, 0x0a, // Identifier + 0x00, // Value + 0x21, // Type: reserved by the 0x21 + 0x1f * N pattern + 0x04, // Length + 0x11, 0x22, 0x33, 0x44}; + + buf->write(input, sizeof(input)); + + // Initial state + CHECK(handler.total_frame_received == 0); + CHECK(nread == 0); + + error = http3FrameDispatcher.on_read_ready(0, Http3StreamType::CONTROL, *reader, nread); + CHECK(!error); + CHECK(handler.total_frame_received == 1); + CHECK(nread == sizeof(input)); + } + SECTION("padding should not be interpreted as a DATA frame", "[http3]") { uint8_t input[] = { @@ -311,7 +337,7 @@ TEST_CASE("ignore unknown frames", "[http3]") } } -TEST_CASE("Reserved frame type not allowed", "[http3]") +TEST_CASE("HTTP/2 reserved frame type not allowed", "[http3]") { SECTION("Reject reserved frame type in non control stream") { diff --git a/src/proxy/http3/test/test_QPACK.cc b/src/proxy/http3/test/test_QPACK.cc index df4d5495f1d..7059c773f94 100644 --- a/src/proxy/http3/test/test_QPACK.cc +++ b/src/proxy/http3/test/test_QPACK.cc @@ -86,7 +86,9 @@ class TestQUICStream : public QUICStream auto ibb = this->_adapter->read(buf_len); IOBufferReader reader; reader.block = ibb; - return reader.read(buf, buf_len); + auto nread = reader.read(buf, buf_len); + this->_adapter->consume(nread); + return nread; } }; diff --git a/tests/gold_tests/autest-site/ats_replay.test.ext b/tests/gold_tests/autest-site/ats_replay.test.ext index 04196e8711f..85516e92abe 100644 --- a/tests/gold_tests/autest-site/ats_replay.test.ext +++ b/tests/gold_tests/autest-site/ats_replay.test.ext @@ -91,6 +91,8 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti name = ats_config.get('name', 'ts') process_config = ats_config.get('process_config', {}) ts = obj.MakeATSProcess(name, **process_config) + if 'startup_timeout' in ats_config: + ts.StartupTimeout = ats_config['startup_timeout'] # Cripts are compiled with clang at TS startup (during remap load), which is # slow and scales with the number of cripts. diff --git a/tests/gold_tests/autest-site/conditions.test.ext b/tests/gold_tests/autest-site/conditions.test.ext index 41c5a6cacdf..090075c4ff8 100644 --- a/tests/gold_tests/autest-site/conditions.test.ext +++ b/tests/gold_tests/autest-site/conditions.test.ext @@ -33,6 +33,12 @@ OPENSSL_TLS_FLAGS = { } +def _version_tuple(value, width=3): + parts = [int(part) for part in re.findall(r'\d+', value)[:width]] + parts.extend([0] * (width - len(parts))) + return tuple(parts) + + def _terminate_process(process): if process.poll() is not None: return @@ -130,10 +136,27 @@ def HasOpenSSLVersion(self, version): output = subprocess.check_output(os.path.join(self.Variables.BINDIR, "traffic_layout") + " info --versions --json", shell=True) json_data = output.decode('utf-8') openssl_str = json.loads(json_data)['openssl_str'] - exe_ver = re.search(r'\d\.\d\.\d', openssl_str).group(0) - if exe_ver == '': + match = re.search(r'\d+(?:\.\d+)+', openssl_str) + if match is None: raise ValueError("Error determining version of OpenSSL library needed by traffic_server executable") - return self.Condition(lambda: exe_ver >= version, "OpenSSL library version is " + exe_ver + ", must be at least " + version) + exe_ver = match.group(0) + return self.Condition( + lambda: _version_tuple(exe_ver) >= _version_tuple(version), + "OpenSSL library version is " + exe_ver + ", must be at least " + version) + + +def HasOpenSSLQuicClient(self): + """Check whether the openssl CLI supports s_client -quic.""" + + def check_openssl_quic_client(): + try: + result = subprocess.run(["openssl", "s_client", "-help"], capture_output=True, text=True, timeout=5) + except (OSError, subprocess.SubprocessError): + return False + + return "-quic" in result.stdout or "-quic" in result.stderr + + return self.Condition(check_openssl_quic_client, "OpenSSL CLI must support s_client -quic") def IsBoringSSL(self): @@ -217,6 +240,26 @@ def HasProxyVerifierVersion(self, version): return self.EnsureVersion([verifier_path, "--version"], min_version=version) +def HasGoVersion(self, version): + """Check whether the go command is available at the requested version.""" + + def check_go_version(): + try: + output = subprocess.check_output(["go", "version"], stderr=subprocess.STDOUT, text=True) + except (OSError, subprocess.SubprocessError): + return False + + match = re.search(r'\bgo(\d+\.\d+(?:\.\d+)?)\b', output) + if match is None: + return False + + found = _version_tuple(match.group(1)) + required = _version_tuple(version) + return found >= required + + return self.Condition(check_go_version, "Go must be installed and at least version " + version) + + def HasCurlFeature(self, feature): def default(output): @@ -325,7 +368,9 @@ def CurlUsingUnixDomainSocket(self): ExtendCondition(HasOpenSSLVersion) +ExtendCondition(HasOpenSSLQuicClient) ExtendCondition(HasProxyVerifierVersion) +ExtendCondition(HasGoVersion) ExtendCondition(IsBoringSSL) ExtendCondition(IsOpenSSL) ExtendCondition(HasLegacyTLSSupport) diff --git a/tests/gold_tests/early_hints/early_hints.test.py b/tests/gold_tests/early_hints/early_hints.test.py index 84c7a760edf..79e659fc9bd 100644 --- a/tests/gold_tests/early_hints/early_hints.test.py +++ b/tests/gold_tests/early_hints/early_hints.test.py @@ -28,6 +28,7 @@ class Protocol(Enum): HTTP = auto() HTTPS = auto() HTTP2 = auto() + HTTP3 = auto() @classmethod def to_string(cls, protocol): @@ -37,6 +38,8 @@ def to_string(cls, protocol): return 'HTTPS' elif protocol == cls.HTTP2: return 'HTTP2' + elif protocol == cls.HTTP3: + return 'HTTP3' else: return None @@ -87,7 +90,7 @@ def _configure_ts(self, tr: 'TestRun'): :param tr: The TestRun for the traffic server. ''' - ts = Test.MakeATSProcess(f'ts_{self._protocol_str}', enable_tls=True) + ts = Test.MakeATSProcess(f'ts_{self._protocol_str}', enable_tls=True, enable_quic=self._protocol == Protocol.HTTP3) self._ts = ts ts.Disk.remap_config.AddLine(f'map / http://backend.server.com:{self._server.Variables.http_port}') ts.addDefaultSSLFiles() @@ -131,6 +134,10 @@ def _configure_client(self, tr: 'TestRun'): protocol_arg = '-k --http2' scheme = 'https' ts_port = self._ts.Variables.ssl_port + elif self._protocol == Protocol.HTTP3: + protocol_arg = '-k --http3-only' + scheme = 'https' + ts_port = self._ts.Variables.ssl_port tr.MakeCurlCommand( f'-v {protocol_arg} ' f'--resolve "server.com:{ts_port}:127.0.0.1" ' @@ -158,3 +165,5 @@ def _configure_client(self, tr: 'TestRun'): if not Condition.CurlUsingUnixDomainSocket(): TestEarlyHints(Protocol.HTTPS) TestEarlyHints(Protocol.HTTP2) + if Condition.HasATSFeature('TS_USE_QUIC') and Condition.HasCurlFeature('http3') and Condition.HasCurlOption('--http3-only'): + TestEarlyHints(Protocol.HTTP3) diff --git a/tests/gold_tests/h3/go_h3_client/go.mod b/tests/gold_tests/h3/go_h3_client/go.mod new file mode 100644 index 00000000000..9f8dd05a0be --- /dev/null +++ b/tests/gold_tests/h3/go_h3_client/go.mod @@ -0,0 +1,13 @@ +module trafficserver.apache.org/h3-go-client + +go 1.24 + +require github.com/quic-go/quic-go v0.59.1 + +require ( + github.com/quic-go/qpack v0.6.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect +) diff --git a/tests/gold_tests/h3/go_h3_client/go.sum b/tests/gold_tests/h3/go_h3_client/go.sum new file mode 100644 index 00000000000..314cb107874 --- /dev/null +++ b/tests/gold_tests/h3/go_h3_client/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jordanlewis/gcassert v0.0.0-20250430164644-389ef753e22e h1:a+PGEeXb+exwBS3NboqXHyxarD9kaboBbrSp+7GuBuc= +github.com/jordanlewis/gcassert v0.0.0-20250430164644-389ef753e22e/go.mod h1:ZybsQk6DWyN5t7An1MuPm1gtSZ1xDaTXS9ZjIOxvQrk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= +github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/gold_tests/h3/go_h3_client/main.go b/tests/gold_tests/h3/go_h3_client/main.go new file mode 100644 index 00000000000..8e73127a333 --- /dev/null +++ b/tests/gold_tests/h3/go_h3_client/main.go @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information regarding +// copyright ownership. The ASF licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may +// obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/tls" + "flag" + "fmt" + "io" + "net/http" + "os" + "sync" + "time" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +const ( + largeBodySize = 300000 + largeBodySuffix = "000927b " + reusedHeaderValue = "stable-qpack-value" +) + +type requestCase struct { + name string + method string + path string + requestSize int + responseSize int + status int +} + +func generatedBody(size int) []byte { + var body bytes.Buffer + for i := 0; body.Len() < size; i++ { + fmt.Fprintf(&body, "%07x ", i) + } + return body.Bytes()[:size] +} + +func newTLSConfig(serverName string) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{http3.NextProtoH3}, + ServerName: serverName, + } +} + +func newQUICConfig() *quic.Config { + return &quic.Config{ + MaxIdleTimeout: 10 * time.Second, + } +} + +func newClient(serverName string) (*http.Client, *http3.Transport) { + transport := &http3.Transport{ + TLSClientConfig: newTLSConfig(serverName), + QUICConfig: newQUICConfig(), + DisableCompression: true, + } + client := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + } + return client, transport +} + +func newRequest(ctx context.Context, baseURL string, authority string, tc requestCase) (*http.Request, error) { + var body io.Reader + if tc.requestSize > 0 { + body = bytes.NewReader(generatedBody(tc.requestSize)) + } + + req, err := http.NewRequestWithContext(ctx, tc.method, baseURL+tc.path, body) + if err != nil { + return nil, err + } + + req.Host = authority + req.Header.Set("User-Agent", "ats-h3-quic-go-autest") + req.Header.Set("X-H3-Go-Client", "quic-go") + req.Header.Set("X-H3-Reused-Header", reusedHeaderValue) + req.Header.Set("X-H3-Test-Case", tc.name) + req.Header.Set("uuid", tc.name) + if tc.requestSize > 0 { + req.Header.Set("Content-Type", "application/octet-stream") + } + + return req, nil +} + +func verifyResponse(tc requestCase, resp *http.Response) error { + defer resp.Body.Close() + + if resp.ProtoMajor != 3 { + return fmt.Errorf("%s: expected HTTP/3, got %s", tc.name, resp.Proto) + } + if resp.StatusCode != tc.status { + return fmt.Errorf("%s: expected status %d, got %d", tc.name, tc.status, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%s: read response body: %w", tc.name, err) + } + + if tc.method == http.MethodHead || tc.status == http.StatusNoContent { + if len(body) != 0 { + return fmt.Errorf("%s: expected no response body, got %d bytes", tc.name, len(body)) + } + return nil + } + + expected := generatedBody(tc.responseSize) + if !bytes.Equal(body, expected) { + return fmt.Errorf("%s: response body mismatch: got %d bytes, expected %d", tc.name, len(body), len(expected)) + } + if tc.responseSize == largeBodySize && !bytes.HasSuffix(body, []byte(largeBodySuffix)) { + return fmt.Errorf("%s: large response body does not end with %q", tc.name, largeBodySuffix) + } + + return nil +} + +func doRequest( + ctx context.Context, + roundTrip func(*http.Request) (*http.Response, error), + baseURL string, + authority string, + tc requestCase, +) error { + req, err := newRequest(ctx, baseURL, authority, tc) + if err != nil { + return err + } + + resp, err := roundTrip(req) + if err != nil { + return fmt.Errorf("%s: request failed: %w", tc.name, err) + } + + if err := verifyResponse(tc, resp); err != nil { + return err + } + + fmt.Printf("ok %s\n", tc.name) + return nil +} + +func runSequential(ctx context.Context, baseURL string, authority string, serverName string, cases []requestCase) error { + client, transport := newClient(serverName) + defer transport.Close() + + for _, tc := range cases { + if err := doRequest(ctx, client.Do, baseURL, authority, tc); err != nil { + return err + } + } + + return nil +} + +func runConcurrent(ctx context.Context, addr string, baseURL string, authority string, serverName string, cases []requestCase) error { + transport := &http3.Transport{} + defer transport.Close() + + conn, err := quic.DialAddr(ctx, addr, newTLSConfig(serverName), newQUICConfig()) + if err != nil { + return fmt.Errorf("dial concurrent HTTP/3 connection: %w", err) + } + clientConn := transport.NewClientConn(conn) + defer clientConn.CloseWithError(0, "done") + + var wg sync.WaitGroup + errs := make(chan error, len(cases)) + for _, tc := range cases { + tc := tc + wg.Add(1) + go func() { + defer wg.Done() + errs <- doRequest(ctx, clientConn.RoundTrip, baseURL, authority, tc) + }() + } + + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + return err + } + } + + return nil +} + +func main() { + addr := flag.String("addr", "", "ATS HTTP/3 address in host:port form") + authority := flag.String("authority", "", "HTTP/3 request authority") + serverName := flag.String("server-name", "", "TLS SNI server name") + flag.Parse() + + if *addr == "" || *authority == "" || *serverName == "" { + flag.Usage() + os.Exit(2) + } + + baseURL := "https://" + *addr + ctx := context.Background() + sequentialCases := []requestCase{ + {name: "go-get-empty", method: http.MethodGet, path: "/go-get-empty", status: http.StatusOK}, + {name: "go-get-small", method: http.MethodGet, path: "/go-get-small", responseSize: 100, status: http.StatusOK}, + {name: "go-head-no-body", method: http.MethodHead, path: "/go-head-no-body", responseSize: 100, status: http.StatusOK}, + {name: "go-204-no-body", method: http.MethodGet, path: "/go-204-no-body", status: http.StatusNoContent}, + { + name: "go-post-small", + method: http.MethodPost, + path: "/go-post-small", + requestSize: 100, + responseSize: 100, + status: http.StatusOK, + }, + { + name: "go-put-small", + method: http.MethodPut, + path: "/go-put-small", + requestSize: 100, + responseSize: 100, + status: http.StatusOK, + }, + {name: "go-delete-empty", method: http.MethodDelete, path: "/go-delete-empty", status: http.StatusNoContent}, + {name: "go-options-small", method: http.MethodOptions, path: "/go-options-small", responseSize: 100, status: http.StatusOK}, + } + concurrentCases := []requestCase{ + { + name: "go-get-concurrent-large", + method: http.MethodGet, + path: "/go-get-concurrent-large", + responseSize: largeBodySize, + status: http.StatusOK, + }, + { + name: "go-get-concurrent-small", + method: http.MethodGet, + path: "/go-get-concurrent-small", + responseSize: 100, + status: http.StatusOK, + }, + } + largeCases := []requestCase{ + {name: "go-get-large", method: http.MethodGet, path: "/go-get-large", responseSize: largeBodySize, status: http.StatusOK}, + { + name: "go-post-large", + method: http.MethodPost, + path: "/go-post-large", + requestSize: largeBodySize, + responseSize: largeBodySize, + status: http.StatusOK, + }, + { + name: "go-put-large", + method: http.MethodPut, + path: "/go-put-large", + requestSize: largeBodySize, + responseSize: largeBodySize, + status: http.StatusOK, + }, + } + + if err := runSequential(ctx, baseURL, *authority, *serverName, sequentialCases); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := runConcurrent(ctx, *addr, baseURL, *authority, *serverName, concurrentCases); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := runSequential(ctx, baseURL, *authority, *serverName, largeCases); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + fmt.Println("completed 13 HTTP/3 requests") +} diff --git a/tests/gold_tests/h3/h3_active_timeout.test.py b/tests/gold_tests/h3/h3_active_timeout.test.py new file mode 100644 index 00000000000..9a159fb49c6 --- /dev/null +++ b/tests/gold_tests/h3/h3_active_timeout.test.py @@ -0,0 +1,26 @@ +''' +Verify HTTP/3 transaction cleanup on active timeout. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify HTTP/3 transactions are removed cleanly after transaction active timeout. +''' + +Test.SkipUnless(Condition.HasATSFeature('TS_USE_QUIC')) + +Test.ATSReplayTest(replay_file="replays/h3_active_timeout.replay.yaml") diff --git a/tests/gold_tests/h3/h3_curl.test.py b/tests/gold_tests/h3/h3_curl.test.py new file mode 100644 index 00000000000..6e176fc382a --- /dev/null +++ b/tests/gold_tests/h3/h3_curl.test.py @@ -0,0 +1,139 @@ +''' +Verify HTTP/3 client interop with curl. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +This test is written specifically to verify that an HTTP/3 curl client can +complete a request through ATS. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_QUIC'), + Condition.HasCurlFeature('http3'), + Condition.HasCurlOption('--http3-only'), +) +Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) + + +class TestHttp3Curl: + """Configure a test to verify HTTP/3 curl client interoperability.""" + + response_body = "0123456789" * 30000 + + def __init__(self, name: str): + """Initialize the test. + + :param name: The name of the test. + """ + self.name = name + self._body_path = os.path.join(Test.RunDirectory, "h3_curl_body.txt") + self._configure_server() + self._configure_traffic_server() + self._configure_client() + + def _configure_server(self): + """Configure the origin server.""" + server = Test.MakeOriginServer("server") + server.addResponse( + "sessionlog.json", { + "headers": "GET /h3-curl HTTP/1.1\r\nHost: localhost\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": f"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: {len(self.response_body)}\r\n\r\n", + "timestamp": "1469733493.993", + "body": self.response_body + }) + + self._server = server + + def _configure_traffic_server(self): + """Configure Traffic Server.""" + ts = Test.MakeATSProcess("ts", enable_tls=True, enable_quic=True, enable_cache=False) + ts.StartupTimeout = 60 + ts.addDefaultSSLFiles() + ts.addSSLfile("../tls/ssl/signed-foo.pem") + ts.addSSLfile("../tls/ssl/signed-foo.key") + ts.Disk.ssl_multicert_config.AddLines( + [ + 'ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key', + ]) + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'quic|http3', + 'proxy.config.quic.server.stateless_retry_enabled': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.Port}') + ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: h3_access + format: 'c_alpn=% client_version=% c_ssl_version=% c_method=% c_url=%' + + logs: + - filename: h3_access + format: h3_access +'''.split("\n")) + + self._access_log = Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'h3_access.log'), exists=True) + self._access_log.Content = Testers.ContainsExpression( + r'c_alpn=h3 client_version=http/3 c_ssl_version=[^ ]+ c_method=GET c_url=https://foo.com:[0-9]+/h3-curl', + "ATS should log the curl request as HTTP/3") + + self._ts = ts + + def _check_curl_response(self, tr): + """Verify that curl received the response over HTTP/3.""" + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + f"size_download={len(self.response_body)}", "curl should receive the complete HTTP/3 response body") + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression("http_version=3", "curl should report HTTP/3") + + def _configure_client(self): + """Configure the curl client test runs.""" + tr = Test.AddTestRun(self.name) + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.MakeCurlCommand( + '--silent --show-error --fail --ipv4 --http3-only --insecure ' + f'--resolve "foo.com:{self._ts.Variables.ssl_port}:127.0.0.1" ' + f'--output "{self._body_path}" ' + '--write-out "\\nhttp_version=%{http_version}\\nsize_download=%{size_download}\\n" ' + f'https://foo.com:{self._ts.Variables.ssl_port}/h3-curl', + ts=self._ts) + self._check_curl_response(tr) + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + tr = Test.AddTestRun("Wait for HTTP/3 access log") + tr.Processes.Default.Command = ( + os.path.join(Test.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' + + os.path.join(self._ts.Variables.LOGDIR, 'h3_access.log')) + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + +TestHttp3Curl("curl forced HTTP/3 request") diff --git a/tests/gold_tests/h3/h3_flow_control.test.py b/tests/gold_tests/h3/h3_flow_control.test.py new file mode 100644 index 00000000000..85c4c5b5573 --- /dev/null +++ b/tests/gold_tests/h3/h3_flow_control.test.py @@ -0,0 +1,27 @@ +''' +Verify HTTP/3 traffic progresses with small QUIC flow-control windows. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify that large HTTP/3 request and response bodies complete when ATS +advertises small initial QUIC flow-control windows. +''' + +Test.SkipUnless(Condition.HasATSFeature('TS_USE_QUIC')) + +Test.ATSReplayTest(replay_file="replays/h3_flow_control.replay.yaml") diff --git a/tests/gold_tests/h3/h3_go_client.test.py b/tests/gold_tests/h3/h3_go_client.test.py new file mode 100644 index 00000000000..8857f58d93a --- /dev/null +++ b/tests/gold_tests/h3/h3_go_client.test.py @@ -0,0 +1,124 @@ +''' +Verify HTTP/3 client interop with a quic-go client. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify that a quic-go HTTP/3 client can complete sequential and concurrent +transactions through ATS. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_QUIC'), + Condition.HasGoVersion('1.24'), +) + + +def add_default_ssl_multicert(ts): + """Configure the default server certificate.""" + ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + + +class TestHttp3GoClient: + """Configure a test to verify HTTP/3 quic-go client interoperability.""" + + replay_file = "replays/h3_server_for_go_client.replay.yaml" + + def __init__(self, name: str): + """Initialize the test.""" + self.name = name + self._configure_server() + self._configure_traffic_server() + self._configure_client() + + def _configure_server(self): + """Configure the Proxy Verifier origin server.""" + self._server = Test.MakeVerifierServerProcess( + "server-go-h3-client", self.replay_file, verbose=False, other_args="--poll-timeout 30000") + + def _configure_traffic_server(self): + """Configure Traffic Server.""" + ts = Test.MakeATSProcess("ts-go-h3-client", enable_tls=True, enable_quic=True, enable_cache=False) + ts.StartupTimeout = 60 + ts.addDefaultSSLFiles() + add_default_ssl_multicert(ts) + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'quic|http3', + 'proxy.config.quic.initial_max_data_in': 1000000, + 'proxy.config.quic.initial_max_stream_data_bidi_remote_in': 1000000, + 'proxy.config.quic.server.stateless_retry_enabled': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') + ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: h3_go_access + format: 'c_alpn=% client_version=% c_method=% c_url=%' + + logs: + - filename: h3_go_access + format: h3_go_access +'''.split("\n")) + + self._access_log = Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'h3_go_access.log'), exists=True) + self._access_log.Content = Testers.ContainsExpression( + r'c_alpn=h3 client_version=http/3 c_method=GET c_url=https://go\.example\.com:[0-9]+/go-get-empty', + "ATS should log the quic-go request as HTTP/3") + self._access_log.Content += Testers.ContainsExpression( + r'c_alpn=h3 client_version=http/3 c_method=POST c_url=https://go\.example\.com:[0-9]+/go-post-large', + "ATS should log the quic-go large POST as HTTP/3") + + self._ts = ts + + def _configure_client(self): + """Configure the quic-go client test runs.""" + tr = Test.AddTestRun(self.name) + tr.Setup.Copy("go_h3_client") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Env['GOFLAGS'] = '-mod=readonly' + tr.Processes.Default.Env['GOCACHE'] = os.path.join(tr.RunDirectory, 'gocache') + tr.Processes.Default.Env['GOMODCACHE'] = os.path.join(tr.RunDirectory, 'gomodcache') + tr.Processes.Default.Env['GOTOOLCHAIN'] = 'local' + tr.Processes.Default.Command = ( + f'cd "{os.path.join(tr.RunDirectory, "go_h3_client")}" && ' + f'go run . --addr 127.0.0.1:{self._ts.Variables.ssl_port} ' + f'--authority go.example.com:{self._ts.Variables.ssl_port} ' + '--server-name go.example.com') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "completed 13 HTTP/3 requests", "The quic-go client should complete all HTTP/3 requests.") + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + tr = Test.AddTestRun("Wait for quic-go HTTP/3 access log") + tr.Processes.Default.Command = ( + os.path.join(Test.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' + + os.path.join(self._ts.Variables.LOGDIR, 'h3_go_access.log')) + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + +TestHttp3GoClient("quic-go HTTP/3 client requests") diff --git a/tests/gold_tests/h3/h3_proxy_verifier.test.py b/tests/gold_tests/h3/h3_proxy_verifier.test.py new file mode 100644 index 00000000000..a32b21dfd59 --- /dev/null +++ b/tests/gold_tests/h3/h3_proxy_verifier.test.py @@ -0,0 +1,27 @@ +''' +Verify HTTP/3 client interop with Proxy Verifier. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify a real HTTP/3 Proxy Verifier client can complete multiple transactions +across multiple QUIC connections through ATS. +''' + +Test.SkipUnless(Condition.HasATSFeature('TS_USE_QUIC')) + +Test.ATSReplayTest(replay_file="replays/h3_proxy_verifier.replay.yaml") diff --git a/tests/gold_tests/h3/h3_python_client.test.py b/tests/gold_tests/h3/h3_python_client.test.py new file mode 100644 index 00000000000..2d809977ef4 --- /dev/null +++ b/tests/gold_tests/h3/h3_python_client.test.py @@ -0,0 +1,132 @@ +''' +Verify HTTP/3 client interop with an aioquic Python client. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = ''' +Verify that an aioquic HTTP/3 client can complete normal requests and selected +HTTP/3 edge-case probes through ATS. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_QUIC'), + Condition.HasProgram("python3", "python3 is required for the aioquic HTTP/3 client"), +) + + +def add_default_ssl_multicert(ts): + """Configure the default server certificate.""" + if hasattr(ts.Disk, "ssl_multicert_yaml"): + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + else: + ts.Disk.ssl_multicert_config.AddLine("dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key") + + +class TestHttp3PythonClient: + """Configure a test to verify HTTP/3 aioquic client interoperability.""" + + replay_file = "replays/h3_server_for_python_client.replay.yaml" + + def __init__(self, name: str): + """Initialize the test.""" + self.name = name + self._configure_server() + self._configure_traffic_server() + self._configure_client() + + def _configure_server(self): + """Configure the Proxy Verifier origin server.""" + self._server = Test.MakeVerifierServerProcess( + "server-python-h3-client", self.replay_file, verbose=False, other_args="--poll-timeout 30000") + + def _configure_traffic_server(self): + """Configure Traffic Server.""" + ts = Test.MakeATSProcess("ts-python-h3-client", enable_tls=True, enable_quic=True, enable_cache=False) + ts.StartupTimeout = 60 + ts.addDefaultSSLFiles() + add_default_ssl_multicert(ts) + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'quic|http3', + 'proxy.config.quic.initial_max_data_in': 1000000, + 'proxy.config.quic.initial_max_stream_data_bidi_remote_in': 1000000, + 'proxy.config.quic.max_send_udp_payload_size_in': 1200, + 'proxy.config.quic.server.stateless_retry_enabled': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') + ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: h3_python_access + format: 'c_alpn=% client_version=% c_method=% c_url=%' + + logs: + - filename: h3_python_access + format: h3_python_access +'''.split("\n")) + + self._access_log = Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'h3_python_access.log'), exists=True) + self._access_log.Content = Testers.ContainsExpression( + r'c_alpn=h3 client_version=http/3 c_method=GET c_url=https://py\.example\.com:[0-9]+/py-get-empty', + "ATS should log the aioquic request as HTTP/3") + self._access_log.Content += Testers.ContainsExpression( + r'c_alpn=h3 client_version=http/3 c_method=PUT c_url=https://py\.example\.com:[0-9]+/py-put-large', + "ATS should log the aioquic large PUT as HTTP/3") + + self._ts = ts + + def _configure_client(self): + """Configure the aioquic client test runs.""" + tr = Test.AddTestRun(self.name) + tr.Setup.Copy("py_h3_client") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + client_dir = os.path.join(tr.RunDirectory, "py_h3_client") + tr.Processes.Default.Command = ( + f'"{sys.executable}" "{os.path.join(client_dir, "h3_client.py")}" ' + f'--addr 127.0.0.1:{self._ts.Variables.ssl_port} ' + f'--authority py.example.com:{self._ts.Variables.ssl_port} ' + '--server-name py.example.com') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "completed 18 Python HTTP/3 checks", "The aioquic client should complete all HTTP/3 checks.") + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + tr = Test.AddTestRun("Wait for aioquic HTTP/3 access log") + tr.Processes.Default.Command = ( + os.path.join(Test.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' + + os.path.join(self._ts.Variables.LOGDIR, 'h3_python_access.log')) + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + +TestHttp3PythonClient("aioquic HTTP/3 client requests") diff --git a/tests/gold_tests/h3/h3_range_cache.test.py b/tests/gold_tests/h3/h3_range_cache.test.py new file mode 100644 index 00000000000..2222e2e47dc --- /dev/null +++ b/tests/gold_tests/h3/h3_range_cache.test.py @@ -0,0 +1,140 @@ +''' +Verify HTTP/3 range requests over cached content. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify that HTTP/3 clients can populate cache and receive range responses from +cached objects. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_QUIC'), + Condition.HasCurlFeature('http3'), + Condition.HasCurlOption('--http3-only'), +) +Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) + + +def add_default_ssl_multicert(ts): + """Configure the default server certificate.""" + if hasattr(ts.Disk, "ssl_multicert_yaml"): + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + else: + ts.Disk.ssl_multicert_config.AddLine("dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key") + + +class TestHttp3RangeCache: + """Configure an HTTP/3 range-over-cache test.""" + + response_body = "0123456789" * 30000 + range_body = "6789012345678901" + + def __init__(self): + """Initialize the test.""" + self._configure_server() + self._configure_traffic_server() + self._configure_clients() + + def _configure_server(self): + """Configure the origin server.""" + server = Test.MakeOriginServer("server-h3-range-cache") + server.addResponse( + "sessionlog.json", { + "headers": "GET /h3-range-cache HTTP/1.1\r\nHost: localhost\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": + ( + "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Cache-Control: public, max-age=60\r\n" + f"Content-Length: {len(self.response_body)}\r\n\r\n"), + "timestamp": "1469733493.993", + "body": self.response_body + }) + self._server = server + + def _configure_traffic_server(self): + """Configure Traffic Server.""" + ts = Test.MakeATSProcess("ts-h3-range-cache", enable_tls=True, enable_quic=True, enable_cache=True) + ts.StartupTimeout = 60 + ts.addDefaultSSLFiles() + add_default_ssl_multicert(ts) + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'quic|http3|http', + 'proxy.config.quic.server.stateless_retry_enabled': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.Port}') + self._ts = ts + + def _curl_base(self): + """Build the shared curl arguments.""" + return ( + '--silent --show-error --fail --ipv4 --http3-only --insecure ' + f'--resolve "range.example.com:{self._ts.Variables.ssl_port}:127.0.0.1" ' + f'https://range.example.com:{self._ts.Variables.ssl_port}/h3-range-cache') + + def _configure_clients(self): + """Configure the cache fill and range request clients.""" + full_body_path = os.path.join(Test.RunDirectory, "h3-range-full.txt") + range_body_path = os.path.join(Test.RunDirectory, "h3-range-part.txt") + + tr = Test.AddTestRun("HTTP/3 cache fill") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.MakeCurlCommand( + f'{self._curl_base()} --output "{full_body_path}" ' + '--write-out "\\nhttp_code=%{http_code}\\nsize_download=%{size_download}\\n"', + ts=self._ts) + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("http_code=200", "The fill request should return 200.") + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + f"size_download={len(self.response_body)}", "The fill request should receive the full object.") + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + tr = Test.AddTestRun("HTTP/3 cached range request") + tr.MakeCurlCommand( + f'{self._curl_base()} --header "Range: bytes=16-31" --output "{range_body_path}" ' + '--write-out "\\nhttp_code=%{http_code}\\nsize_download=%{size_download}\\n"', + ts=self._ts) + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("http_code=206", "The range request should return 206.") + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "size_download=16", "The range request should receive 16 bytes.") + Test.Disk.File( + range_body_path, exists=True).Content = Testers.ContainsExpression( + self.range_body, "The cached range response body should match the requested byte range.") + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + +TestHttp3RangeCache() diff --git a/tests/gold_tests/h3/h3_session_ticket.sh b/tests/gold_tests/h3/h3_session_ticket.sh new file mode 100755 index 00000000000..ca7ade26397 --- /dev/null +++ b/tests/gold_tests/h3/h3_session_ticket.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +session_option=$1 +session_file=$2 +port=$3 + +set +e +sleep 1 | timeout 5 openssl s_client \ + -quic \ + -alpn h3 \ + -connect "127.0.0.1:${port}" \ + -servername foo.com \ + "${session_option}" "${session_file}" \ + -brief \ + -ign_eof +status=${PIPESTATUS[1]} +set -e + +if [[ ${status} -ne 0 && ${status} -ne 1 && ${status} -ne 124 ]]; then + exit "${status}" +fi + +test -s "${session_file}" diff --git a/tests/gold_tests/h3/h3_session_ticket.test.py b/tests/gold_tests/h3/h3_session_ticket.test.py new file mode 100644 index 00000000000..fd839aad1cb --- /dev/null +++ b/tests/gold_tests/h3/h3_session_ticket.test.py @@ -0,0 +1,110 @@ +''' +Verify HTTP/3 QUIC TLS session ticket handling. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shlex + +Test.Summary = ''' +Verify that HTTP/3 QUIC connections can receive and offer TLS session tickets. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_QUIC'), + Condition.HasOpenSSLVersion('3.5.0'), + Condition.HasOpenSSLQuicClient(), +) +Test.Setup.Copy('../tls/file.ticket') + + +def add_default_ssl_multicert(ts): + """Configure the default server certificate.""" + if hasattr(ts.Disk, "ssl_multicert_yaml"): + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + else: + ts.Disk.ssl_multicert_config.AddLine("dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key") + + +class TestHttp3SessionTicket: + """Configure an HTTP/3 QUIC TLS session ticket test.""" + + def __init__(self, name: str): + """Initialize the test.""" + self.name = name + self.session_file = os.path.join(Test.RunDirectory, "h3-quic-session.pem") + self.ticket_file = os.path.join(Test.RunDirectory, "file.ticket") + self._configure_traffic_server() + self._configure_ticket_save() + self._configure_ticket_reuse() + + def _configure_traffic_server(self): + """Configure Traffic Server.""" + ts = Test.MakeATSProcess("ts", enable_tls=True, enable_quic=True, enable_cache=False) + ts.StartupTimeout = 60 + ts.addDefaultSSLFiles() + add_default_ssl_multicert(ts) + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'quic|ssl', + 'proxy.config.quic.server.stateless_retry_enabled': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.session_ticket.enable': 1, + 'proxy.config.ssl.server.session_ticket.number': 2, + 'proxy.config.ssl.server.ticket_key.filename': self.ticket_file, + }) + + self._ts = ts + + def _s_client_command(self, session_option: str): + """Build an OpenSSL QUIC client command for ticket save or reuse.""" + script = os.path.join(Test.TestDirectory, "h3_session_ticket.sh") + return f"{shlex.quote(script)} {session_option} {shlex.quote(self.session_file)} {self._ts.Variables.ssl_port}" + + def _check_s_client_handshake(self, tr): + """Verify that OpenSSL completed the QUIC handshake.""" + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All = Testers.ContainsExpression( + "CONNECTION ESTABLISHED", "OpenSSL should complete the QUIC handshake.") + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + "Protocol version: QUICv1", "OpenSSL should negotiate QUICv1.") + + def _configure_ticket_save(self): + """Configure the ticket save test run.""" + tr = Test.AddTestRun(self.name) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = f"rm -f {shlex.quote(self.session_file)}; {self._s_client_command('-sess_out')}" + self._check_s_client_handshake(tr) + tr.StillRunningAfter = self._ts + + def _configure_ticket_reuse(self): + """Configure the ticket reuse test run.""" + tr = Test.AddTestRun("OpenSSL QUIC offers saved session ticket") + tr.Processes.Default.Command = self._s_client_command("-sess_in") + self._check_s_client_handshake(tr) + tr.StillRunningAfter = self._ts + + +TestHttp3SessionTicket("OpenSSL QUIC saves session ticket") diff --git a/tests/gold_tests/h3/h3_sni_check.test.py b/tests/gold_tests/h3/h3_sni_check.test.py index 4ac57c66206..08778162aa4 100644 --- a/tests/gold_tests/h3/h3_sni_check.test.py +++ b/tests/gold_tests/h3/h3_sni_check.test.py @@ -21,7 +21,7 @@ Verify h3 SNI checking behavior. ''' -Test.SkipUnless(Condition.HasATSFeature('TS_HAS_QUICHE'), Condition.HasCurlFeature('http3')) +Test.SkipUnless(Condition.HasATSFeature('TS_USE_QUIC')) Test.ContinueOnFail = True @@ -62,19 +62,26 @@ def _configure_traffic_server(self, tr: 'TestRun'): :param tr: The TestRun object to associate the ts process with. """ ts = tr.MakeATSProcess(f"ts-{Test_sni_check.ts_counter}", enable_quic=True, enable_tls=True) + ts.StartupTimeout = 60 Test_sni_check.ts_counter += 1 self._ts = ts # Configure TLS for Traffic Server. self._ts.addDefaultSSLFiles() - self._ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + self._ts.addSSLfile("../tls/ssl/signed-foo.pem") + self._ts.addSSLfile("../tls/ssl/signed-foo.key") + self._ts.Disk.ssl_multicert_config.AddLines( + [ + 'ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key', + ]) self._ts.Disk.records_config.update( { 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, 'proxy.config.quic.no_activity_timeout_in': 0, - 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', }) diff --git a/tests/gold_tests/h3/h3_stream_lifetime.test.py b/tests/gold_tests/h3/h3_stream_lifetime.test.py new file mode 100644 index 00000000000..9715392ea78 --- /dev/null +++ b/tests/gold_tests/h3/h3_stream_lifetime.test.py @@ -0,0 +1,27 @@ +''' +Verify HTTP/3 stream lifetime handling with concurrent streams. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify HTTP/3 transactions survive concurrent stream close and write-ready +events on the same QUIC connection. +''' + +Test.SkipUnless(Condition.HasATSFeature('TS_USE_QUIC')) + +Test.ATSReplayTest(replay_file="replays/h3_stream_lifetime.replay.yaml") diff --git a/tests/gold_tests/h3/py_h3_client/h3_client.py b/tests/gold_tests/h3/py_h3_client/h3_client.py new file mode 100644 index 00000000000..ca6e83fbb7f --- /dev/null +++ b/tests/gold_tests/h3/py_h3_client/h3_client.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Exercise ATS HTTP/3 client-side behavior with aioquic.""" + +from __future__ import annotations + +import argparse +import asyncio +import ssl +from dataclasses import dataclass, field +from typing import Callable + +from aioquic.asyncio.client import connect +from aioquic.asyncio.protocol import QuicConnectionProtocol +from aioquic.buffer import encode_uint_var +from aioquic.h3.connection import H3Connection, H3_ALPN, FrameType, StreamType, encode_frame +from aioquic.h3.events import DataReceived, HeadersReceived +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.events import ConnectionTerminated, QuicEvent, StreamDataReceived + +LARGE_BODY_SIZE = 300000 +LARGE_BODY_SUFFIX = b"000927b " +REUSED_HEADER_VALUE = b"stable-python-qpack-value" + + +@dataclass +class RequestCase: + """A single HTTP/3 request/response expectation.""" + + name: str + method: bytes + path: str + request_size: int = 0 + response_size: int = 0 + status: int = 200 + + +@dataclass +class ResponseState: + """Accumulate response headers and data for one HTTP/3 stream.""" + + header_blocks: list[list[tuple[bytes, bytes]]] = field(default_factory=list) + body: bytearray = field(default_factory=bytearray) + + @property + def status(self) -> int: + for header_block in reversed(self.header_blocks): + for name, value in header_block: + if name == b":status": + return int(value) + raise RuntimeError("response did not contain :status") + + +class H3ClientProtocol(QuicConnectionProtocol): + """Minimal HTTP/3 client protocol with raw QUIC stream helpers.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._http = H3Connection(self._quic) + self._responses: dict[int, ResponseState] = {} + self._waiters: dict[int, asyncio.Future[ResponseState]] = {} + self._raw_response_bytes: dict[int, int] = {} + self._event_counts: dict[str, int] = {} + self._terminated: asyncio.Future[ConnectionTerminated] = asyncio.get_running_loop().create_future() + + def quic_event_received(self, event: QuicEvent) -> None: + event_name = type(event).__name__ + self._event_counts[event_name] = self._event_counts.get(event_name, 0) + 1 + + if isinstance(event, ConnectionTerminated) and not self._terminated.done(): + self._terminated.set_result(event) + for waiter in self._waiters.values(): + if not waiter.done(): + waiter.set_exception(RuntimeError(f"connection terminated: {event.error_code} {event.reason_phrase}")) + + for http_event in self._http.handle_event(event): + if isinstance(http_event, HeadersReceived): + response = self._responses.setdefault(http_event.stream_id, ResponseState()) + response.header_blocks.append(http_event.headers) + if http_event.stream_ended: + self._complete_response(http_event.stream_id) + elif isinstance(http_event, DataReceived): + response = self._responses.setdefault(http_event.stream_id, ResponseState()) + response.body.extend(http_event.data) + if http_event.stream_ended: + self._complete_response(http_event.stream_id) + + if isinstance(event, StreamDataReceived): + self._raw_response_bytes[event.stream_id] = self._raw_response_bytes.get(event.stream_id, 0) + len(event.data) + if event.end_stream and event.stream_id in self._responses: + self._complete_response(event.stream_id) + + self.transmit() + + def _complete_response(self, stream_id: int) -> None: + waiter = self._waiters.get(stream_id) + if waiter is not None and not waiter.done(): + waiter.set_result(self._responses[stream_id]) + + async def request(self, authority: str, request_case: RequestCase) -> ResponseState: + stream_id = self._quic.get_next_available_stream_id() + waiter: asyncio.Future[ResponseState] = asyncio.get_running_loop().create_future() + self._waiters[stream_id] = waiter + + headers = [ + (b":method", request_case.method), + (b":scheme", b"https"), + (b":authority", authority.encode()), + (b":path", request_case.path.encode()), + (b"user-agent", b"ats-h3-aioquic-autest"), + (b"x-h3-python-client", b"aioquic"), + (b"x-h3-reused-header", REUSED_HEADER_VALUE), + (b"x-h3-test-case", request_case.name.encode()), + (b"uuid", request_case.name.encode()), + ] + if request_case.request_size > 0: + headers.extend( + [ + (b"content-type", b"application/octet-stream"), + (b"content-length", str(request_case.request_size).encode()), + ]) + + request_body = generated_body(request_case.request_size) + self._http.send_headers(stream_id, headers, end_stream=not request_body) + if request_body: + self._http.send_data(stream_id, request_body, end_stream=True) + self.transmit() + + try: + return await asyncio.wait_for(waiter, timeout=30) + except TimeoutError as e: + response = self._responses.get(stream_id) + raw_response_bytes = self._raw_response_bytes.get(stream_id, 0) + event_summary = ", ".join(f"{name}={count}" for name, count in sorted(self._event_counts.items())) + if response is None: + raise TimeoutError( + f"{request_case.name}: timed out before receiving response headers; raw QUIC stream bytes={raw_response_bytes}; " + f"events=[{event_summary}]") from e + raise TimeoutError( + f"{request_case.name}: timed out after receiving {len(response.header_blocks)} header block(s) and " + f"{len(response.body)} response byte(s); raw QUIC stream bytes={raw_response_bytes}; events=[{event_summary}]" + ) from e + + async def wait_for_termination(self) -> ConnectionTerminated: + return await asyncio.wait_for(self._terminated, timeout=5) + + def send_unknown_unidirectional_stream(self) -> None: + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=True) + self._quic.send_stream_data(stream_id, encode_uint_var(0x21) + b"ignored", end_stream=True) + self.transmit() + + def send_client_push_stream(self) -> None: + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=True) + self._quic.send_stream_data(stream_id, encode_uint_var(StreamType.PUSH) + encode_uint_var(0), end_stream=False) + self.transmit() + + def send_duplicate_control_stream(self) -> None: + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=True) + payload = encode_uint_var(StreamType.CONTROL) + encode_frame(FrameType.SETTINGS, b"") + self._quic.send_stream_data(stream_id, payload, end_stream=False) + self.transmit() + + def send_reserved_request_frame(self) -> None: + stream_id = self._quic.get_next_available_stream_id() + self._quic.send_stream_data(stream_id, encode_frame(0x21, b""), end_stream=True) + self.transmit() + + def send_data_before_headers(self) -> None: + stream_id = self._quic.get_next_available_stream_id() + self._quic.send_stream_data(stream_id, encode_frame(FrameType.DATA, b"bad"), end_stream=True) + self.transmit() + + +def generated_body(size: int) -> bytes: + """Generate deterministic content matching Proxy Verifier size bodies.""" + chunks: list[bytes] = [] + total = 0 + value = 0 + while total < size: + chunk = f"{value:07x} ".encode() + chunks.append(chunk) + total += len(chunk) + value += 1 + return b"".join(chunks)[:size] + + +def quic_configuration(server_name: str) -> QuicConfiguration: + """Create an insecure test-only HTTP/3 client configuration.""" + configuration = QuicConfiguration(is_client=True, alpn_protocols=H3_ALPN, server_name=server_name) + configuration.verify_mode = ssl.CERT_NONE + return configuration + + +async def connect_h3(host: str, port: int, server_name: str): + """Open an HTTP/3 connection using the test protocol.""" + return connect(host, port, configuration=quic_configuration(server_name), create_protocol=H3ClientProtocol) + + +def verify_response(request_case: RequestCase, response: ResponseState) -> None: + """Verify one response matches the expected status and body.""" + body = bytes(response.body) + if response.status != request_case.status: + raise AssertionError(f"{request_case.name}: expected status {request_case.status}, got {response.status}") + + if request_case.method == b"HEAD" or request_case.status == 204: + if body: + raise AssertionError(f"{request_case.name}: expected no response body, got {len(body)} bytes") + return + + expected = generated_body(request_case.response_size) + if body != expected: + raise AssertionError(f"{request_case.name}: response body mismatch: got {len(body)}, expected {len(expected)}") + if request_case.response_size == LARGE_BODY_SIZE and not body.endswith(LARGE_BODY_SUFFIX): + raise AssertionError(f"{request_case.name}: large body suffix mismatch") + + +async def run_requests(host: str, port: int, authority: str, server_name: str, request_cases: list[RequestCase]) -> None: + """Run a sequence of request cases on one HTTP/3 connection.""" + async with await connect_h3(host, port, server_name) as client: + for request_case in request_cases: + response = await client.request(authority, request_case) + verify_response(request_case, response) + print(f"ok {request_case.name}") + + +async def run_concurrent_requests(host: str, port: int, authority: str, server_name: str, request_cases: list[RequestCase]) -> None: + """Run request cases concurrently on one HTTP/3 connection.""" + async with await connect_h3(host, port, server_name) as client: + responses = await asyncio.gather(*(client.request(authority, request_case) for request_case in request_cases)) + for request_case, response in zip(request_cases, responses): + verify_response(request_case, response) + print(f"ok {request_case.name}") + + +async def expect_connection_error( + host: str, + port: int, + server_name: str, + name: str, + action: Callable[[H3ClientProtocol], None], +) -> None: + """Run a malformed action and require ATS to close the QUIC connection.""" + async with await connect_h3(host, port, server_name) as client: + action(client) + terminated = await client.wait_for_termination() + if terminated.error_code == 0: + raise AssertionError(f"{name}: expected non-zero H3/QUIC close error") + print(f"ok {name} error={terminated.error_code}") + + +async def run_edge_cases(host: str, port: int, authority: str, server_name: str) -> None: + """Exercise H3 control stream and frame behavior with raw stream writes.""" + async with await connect_h3(host, port, server_name) as client: + client.send_unknown_unidirectional_stream() + request_case = RequestCase("py-edge-after-unknown", b"GET", "/py-edge-after-unknown", response_size=100) + response = await client.request(authority, request_case) + verify_response(request_case, response) + print("ok py-unknown-unidirectional-stream") + + await expect_connection_error( + host, port, server_name, "py-client-push-stream-rejected", lambda client: client.send_client_push_stream()) + await expect_connection_error( + host, port, server_name, "py-duplicate-control-stream-rejected", lambda client: client.send_duplicate_control_stream()) + async with await connect_h3(host, port, server_name) as client: + client.send_reserved_request_frame() + request_case = RequestCase("py-edge-after-reserved", b"GET", "/py-edge-after-reserved", response_size=100) + response = await client.request(authority, request_case) + verify_response(request_case, response) + print("ok py-reserved-request-frame-ignored") + + await expect_connection_error( + host, port, server_name, "py-data-before-headers-rejected", lambda client: client.send_data_before_headers()) + + +async def async_main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--addr", required=True, help="ATS HTTP/3 address in host:port form") + parser.add_argument("--authority", required=True, help="HTTP/3 request authority") + parser.add_argument("--server-name", required=True, help="TLS SNI server name") + args = parser.parse_args() + + host, port_text = args.addr.rsplit(":", 1) + port = int(port_text) + + sequential_cases = [ + RequestCase("py-get-empty", b"GET", "/py-get-empty"), + RequestCase("py-get-small", b"GET", "/py-get-small", response_size=100), + RequestCase("py-head-no-body", b"HEAD", "/py-head-no-body", response_size=100), + RequestCase("py-204-no-body", b"GET", "/py-204-no-body", status=204), + RequestCase("py-post-small", b"POST", "/py-post-small", request_size=100, response_size=100), + RequestCase("py-put-small", b"PUT", "/py-put-small", request_size=100, response_size=100), + RequestCase("py-delete-empty", b"DELETE", "/py-delete-empty", status=204), + RequestCase("py-options-small", b"OPTIONS", "/py-options-small", response_size=100), + ] + concurrent_cases = [ + RequestCase("py-get-concurrent-large", b"GET", "/py-get-concurrent-large", response_size=LARGE_BODY_SIZE), + RequestCase("py-get-concurrent-small", b"GET", "/py-get-concurrent-small", response_size=100), + ] + large_cases = [ + RequestCase("py-get-large", b"GET", "/py-get-large", response_size=LARGE_BODY_SIZE), + RequestCase("py-post-large", b"POST", "/py-post-large", request_size=LARGE_BODY_SIZE, response_size=LARGE_BODY_SIZE), + RequestCase("py-put-large", b"PUT", "/py-put-large", request_size=LARGE_BODY_SIZE, response_size=LARGE_BODY_SIZE), + ] + + await run_requests(host, port, args.authority, args.server_name, sequential_cases) + await run_requests(host, port, args.authority, args.server_name, large_cases) + await run_concurrent_requests(host, port, args.authority, args.server_name, concurrent_cases) + await run_edge_cases(host, port, args.authority, args.server_name) + print("completed 18 Python HTTP/3 checks") + + +def main() -> None: + asyncio.run(async_main()) + + +if __name__ == "__main__": + main() diff --git a/tests/gold_tests/h3/replays/h3_active_timeout.replay.yaml b/tests/gold_tests/h3/replays/h3_active_timeout.replay.yaml new file mode 100644 index 00000000000..f7a39324fd1 --- /dev/null +++ b/tests/gold_tests/h3/replays/h3_active_timeout.replay.yaml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify HTTP/3 transaction cleanup on active timeout" + + server: + name: "server-h3-active-timeout" + + client: + name: "client-h3-active-timeout" + return_code: 1 + + ats: + name: "ts-h3-active-timeout" + startup_timeout: 60 + process_config: + enable_tls: true + enable_quic: true + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: "quic|http" + proxy.config.http.transaction_active_timeout_in: 1 + proxy.config.quic.no_activity_timeout_in: 0 + proxy.config.quic.server.stateless_retry_enabled: 0 + + remap_config: + - from: "https://example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + +sessions: +- protocol: + - name: http + version: 3 + - name: tls + sni: example.com + - name: udp + - name: ip + + transactions: + - client-request: + version: "3" + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-active-timeout ] + - [ uuid, h3-active-timeout ] + + server-response: + # Delay longer than the 1 second ATS active timeout configured by this replay. + delay: 2s + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "timeout-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrst" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] diff --git a/tests/gold_tests/h3/replays/h3_flow_control.replay.yaml b/tests/gold_tests/h3/replays/h3_flow_control.replay.yaml new file mode 100644 index 00000000000..0444a72eaef --- /dev/null +++ b/tests/gold_tests/h3/replays/h3_flow_control.replay.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify HTTP/3 flow control with small QUIC windows" + + server: + name: "server-h3-flow-control" + process_config: + verbose: false + other_args: "--poll-timeout 30000" + + client: + name: "client-h3-flow-control" + process_config: + verbose: false + other_args: "--poll-timeout 30000" + + ats: + name: "ts-h3-flow-control" + startup_timeout: 60 + process_config: + enable_tls: true + enable_quic: true + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: "quic|http3" + proxy.config.quic.initial_max_data_in: 4096 + proxy.config.quic.initial_max_stream_data_bidi_remote_in: 4096 + proxy.config.quic.server.stateless_retry_enabled: 0 + + remap_config: + - from: "https://example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + + log_validation: + traffic_out: + contains: + - expression: 'start HTTP/3 app \(ALPN=h3\)' + description: "ATS should negotiate HTTP/3" + +sessions: +- protocol: + stack: http3 + tls: + sni: example.com + + transactions: + - client-request: + version: "3" + headers: + fields: + - [ ":method", POST ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-flow-post-large ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, h3-flow-post-large ] + content: + size: 300000 + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 + + - client-request: + version: "3" + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-flow-get-large ] + - [ uuid, h3-flow-get-large ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 diff --git a/tests/gold_tests/h3/replays/h3_proxy_verifier.replay.yaml b/tests/gold_tests/h3/replays/h3_proxy_verifier.replay.yaml new file mode 100644 index 00000000000..817485859a5 --- /dev/null +++ b/tests/gold_tests/h3/replays/h3_proxy_verifier.replay.yaml @@ -0,0 +1,481 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify HTTP/3 client interop with Proxy Verifier across multiple connections" + + server: + name: "server-h3-proxy-verifier" + process_config: + verbose: false + other_args: "--poll-timeout 30000" + + client: + name: "client-h3-proxy-verifier" + process_config: + verbose: false + other_args: "--poll-timeout 30000" + + ats: + name: "ts-h3-proxy-verifier" + startup_timeout: 60 + process_config: + enable_tls: true + enable_quic: true + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: "quic|http3" + proxy.config.quic.initial_max_data_in: 1000000 + proxy.config.quic.initial_max_stream_data_bidi_remote_in: 1000000 + proxy.config.quic.server.stateless_retry_enabled: 0 + + remap_config: + - from: "https://example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + + log_validation: + traffic_out: + contains: + - expression: 'start HTTP/3 app \(ALPN=h3\)' + description: "ATS should negotiate HTTP/3" + +sessions: +- protocol: + - name: http + version: 3 + - name: tls + sni: example.com + - name: udp + - name: ip + + transactions: + - client-request: + version: "3" + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-get-empty ] + - [ uuid, h3-get-empty ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "0" ] + content: + size: 0 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "0", as: equal } ] + + - client-request: + version: "3" + await: h3-get-empty + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-get-small ] + - [ uuid, h3-get-small ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzAB" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzAB" + verify: { as: equal } + + - client-request: + version: "3" + await: h3-get-small + headers: + fields: + - [ ":method", HEAD ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-head-no-body ] + - [ uuid, h3-head-no-body ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + + - client-request: + version: "3" + await: h3-head-no-body + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-204-no-body ] + - [ uuid, h3-204-no-body ] + + server-response: + status: 204 + reason: No Content + headers: + fields: + - [ Content-Length, "0" ] + - [ X-H3-Status, no-content ] + + proxy-response: + status: 204 + + - client-request: + version: "3" + await: h3-204-no-body + headers: + fields: + - [ ":method", POST ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-post-small ] + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + - [ uuid, h3-post-small ] + content: + encoding: plain + data: "post-body-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqr" + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "post-body-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqr" + verify: { as: equal } + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "post-response-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "post-response-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn" + verify: { as: equal } + + - client-request: + version: "3" + await: h3-post-small + headers: + fields: + - [ ":method", PUT ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-put-small ] + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + - [ uuid, h3-put-small ] + content: + encoding: plain + data: "put-body-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrs" + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "put-body-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrs" + verify: { as: equal } + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "put-response-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmno" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "put-response-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmno" + verify: { as: equal } + + - client-request: + version: "3" + await: h3-put-small + headers: + fields: + - [ ":method", DELETE ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-delete-empty ] + - [ uuid, h3-delete-empty ] + + server-response: + status: 204 + reason: No Content + headers: + fields: + - [ X-H3-Status, delete-no-content ] + + proxy-response: + status: 204 + + - client-request: + version: "3" + await: h3-delete-empty + headers: + fields: + - [ ":method", OPTIONS ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-options-small ] + - [ uuid, h3-options-small ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Allow, "GET, HEAD, POST, PUT, DELETE, OPTIONS" ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "options-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrst" + + proxy-response: + status: 200 + headers: + fields: + - [ Allow, { value: "GET, HEAD, POST, PUT, DELETE, OPTIONS", as: equal } ] + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "options-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrst" + verify: { as: equal } + +- protocol: + - name: http + version: 3 + - name: tls + sni: example.com + - name: udp + - name: ip + + transactions: + - client-request: + version: "3" + headers: + fields: + - [ ":method", POST ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-post-empty ] + - [ Content-Type, text/plain ] + - [ Content-Length, "0" ] + - [ uuid, h3-post-empty ] + content: + size: 0 + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "0", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "0" ] + content: + size: 0 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "0", as: equal } ] + + - client-request: + version: "3" + await: h3-post-empty + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-get-large ] + - [ uuid, h3-get-large ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 + + - client-request: + version: "3" + await: h3-get-large + headers: + fields: + - [ ":method", PUT ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-put-large ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, h3-put-large ] + content: + size: 300000 + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 + +- protocol: + - name: http + version: 3 + - name: tls + sni: example.com + - name: udp + - name: ip + + transactions: + - client-request: + version: "3" + headers: + fields: + - [ ":method", POST ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-post-large ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, h3-post-large ] + content: + size: 300000 + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "300000", as: equal } ] + content: + size: 300000 diff --git a/tests/gold_tests/h3/replays/h3_server_for_go_client.replay.yaml b/tests/gold_tests/h3/replays/h3_server_for_go_client.replay.yaml new file mode 100644 index 00000000000..a7a475d8c3e --- /dev/null +++ b/tests/gold_tests/h3/replays/h3_server_for_go_client.replay.yaml @@ -0,0 +1,268 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + + blocks: + - request_base: &request_base + version: "1.1" + - empty_response: &empty_response + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, "0" ] + content: + size: 0 + - generated_100_response: &generated_100_response + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + content: + size: 100 + - generated_300k_response: &generated_300k_response + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + +sessions: +- transactions: + - client-request: + <<: *request_base + method: GET + url: /go-get-empty + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-get-empty ] + - [ X-H3-Test-Case, go-get-empty ] + + server-response: + <<: *empty_response + + - client-request: + <<: *request_base + method: GET + url: /go-get-small + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-get-small ] + - [ X-H3-Test-Case, go-get-small ] + + server-response: + <<: *generated_100_response + + - client-request: + <<: *request_base + method: HEAD + url: /go-head-no-body + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-head-no-body ] + - [ X-H3-Test-Case, go-head-no-body ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + + - client-request: + <<: *request_base + method: GET + url: /go-204-no-body + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-204-no-body ] + - [ X-H3-Test-Case, go-204-no-body ] + + server-response: + status: 204 + reason: No Content + headers: + fields: + - [ Content-Length, "0" ] + - [ X-H3-Status, no-content ] + + - client-request: + <<: *request_base + method: POST + url: /go-post-small + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + - [ uuid, go-post-small ] + - [ X-H3-Test-Case, go-post-small ] + content: + size: 100 + verify: { as: equal } + + server-response: + <<: *generated_100_response + + - client-request: + <<: *request_base + method: PUT + url: /go-put-small + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + - [ uuid, go-put-small ] + - [ X-H3-Test-Case, go-put-small ] + content: + size: 100 + verify: { as: equal } + + server-response: + <<: *generated_100_response + + - client-request: + <<: *request_base + method: DELETE + url: /go-delete-empty + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-delete-empty ] + - [ X-H3-Test-Case, go-delete-empty ] + + server-response: + status: 204 + reason: No Content + headers: + fields: + - [ X-H3-Status, delete-no-content ] + + - client-request: + <<: *request_base + method: OPTIONS + url: /go-options-small + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-options-small ] + - [ X-H3-Test-Case, go-options-small ] + + server-response: + <<: *generated_100_response + +- transactions: + - client-request: + <<: *request_base + method: GET + url: /go-get-concurrent-large + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-get-concurrent-large ] + - [ X-H3-Test-Case, go-get-concurrent-large ] + + server-response: + <<: *generated_300k_response + + - client-request: + <<: *request_base + method: GET + url: /go-get-concurrent-small + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-get-concurrent-small ] + - [ X-H3-Test-Case, go-get-concurrent-small ] + + server-response: + <<: *generated_100_response + +- transactions: + - client-request: + <<: *request_base + method: GET + url: /go-get-large + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ uuid, go-get-large ] + - [ X-H3-Test-Case, go-get-large ] + + server-response: + <<: *generated_300k_response + + - client-request: + <<: *request_base + method: POST + url: /go-post-large + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, go-post-large ] + - [ X-H3-Test-Case, go-post-large ] + content: + size: 300000 + verify: { as: equal } + + server-response: + <<: *generated_300k_response + + - client-request: + <<: *request_base + method: PUT + url: /go-put-large + headers: + fields: + - [ X-H3-Go-Client, quic-go ] + - [ X-H3-Reused-Header, stable-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, go-put-large ] + - [ X-H3-Test-Case, go-put-large ] + content: + size: 300000 + verify: { as: equal } + + server-response: + <<: *generated_300k_response diff --git a/tests/gold_tests/h3/replays/h3_server_for_python_client.replay.yaml b/tests/gold_tests/h3/replays/h3_server_for_python_client.replay.yaml new file mode 100644 index 00000000000..2ccf00799bd --- /dev/null +++ b/tests/gold_tests/h3/replays/h3_server_for_python_client.replay.yaml @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + + blocks: + - request_base: &request_base + version: "1.1" + - empty_response: &empty_response + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, "0" ] + content: + size: 0 + - generated_100_response: &generated_100_response + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + content: + size: 100 + - generated_300k_response: &generated_300k_response + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + content: + size: 300000 + +sessions: +- transactions: + - client-request: + <<: *request_base + method: GET + url: /py-get-empty + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-get-empty ] + - [ X-H3-Test-Case, py-get-empty ] + + server-response: + <<: *empty_response + + - client-request: + <<: *request_base + method: GET + url: /py-get-small + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-get-small ] + - [ X-H3-Test-Case, py-get-small ] + + server-response: + <<: *generated_100_response + + - client-request: + <<: *request_base + method: HEAD + url: /py-head-no-body + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-head-no-body ] + - [ X-H3-Test-Case, py-head-no-body ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "0" ] + + - client-request: + <<: *request_base + method: GET + url: /py-204-no-body + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-204-no-body ] + - [ X-H3-Test-Case, py-204-no-body ] + + server-response: + status: 204 + reason: No Content + headers: + fields: + - [ X-H3-Status, no-content ] + + - client-request: + <<: *request_base + method: POST + url: /py-post-small + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + - [ uuid, py-post-small ] + - [ X-H3-Test-Case, py-post-small ] + content: + size: 100 + verify: { as: equal } + + server-response: + <<: *generated_100_response + + - client-request: + <<: *request_base + method: PUT + url: /py-put-small + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "100" ] + - [ uuid, py-put-small ] + - [ X-H3-Test-Case, py-put-small ] + content: + size: 100 + verify: { as: equal } + + server-response: + <<: *generated_100_response + + - client-request: + <<: *request_base + method: DELETE + url: /py-delete-empty + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-delete-empty ] + - [ X-H3-Test-Case, py-delete-empty ] + + server-response: + status: 204 + reason: No Content + headers: + fields: + - [ X-H3-Status, delete-no-content ] + + - client-request: + <<: *request_base + method: OPTIONS + url: /py-options-small + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-options-small ] + - [ X-H3-Test-Case, py-options-small ] + + server-response: + <<: *generated_100_response + +- transactions: + - client-request: + <<: *request_base + method: GET + url: /py-get-concurrent-large + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-get-concurrent-large ] + - [ X-H3-Test-Case, py-get-concurrent-large ] + + server-response: + <<: *generated_300k_response + + - client-request: + <<: *request_base + method: GET + url: /py-get-concurrent-small + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-get-concurrent-small ] + - [ X-H3-Test-Case, py-get-concurrent-small ] + + server-response: + <<: *generated_100_response + +- transactions: + - client-request: + <<: *request_base + method: GET + url: /py-get-large + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-get-large ] + - [ X-H3-Test-Case, py-get-large ] + + server-response: + <<: *generated_300k_response + + - client-request: + <<: *request_base + method: POST + url: /py-post-large + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, py-post-large ] + - [ X-H3-Test-Case, py-post-large ] + content: + size: 300000 + verify: { as: equal } + + server-response: + <<: *generated_300k_response + + - client-request: + <<: *request_base + method: PUT + url: /py-put-large + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ Content-Type, application/octet-stream ] + - [ Content-Length, "300000" ] + - [ uuid, py-put-large ] + - [ X-H3-Test-Case, py-put-large ] + content: + size: 300000 + verify: { as: equal } + + server-response: + <<: *generated_300k_response + +- transactions: + - client-request: + <<: *request_base + method: GET + url: /py-edge-after-unknown + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-edge-after-unknown ] + - [ X-H3-Test-Case, py-edge-after-unknown ] + + server-response: + <<: *generated_100_response + +- transactions: + - client-request: + <<: *request_base + method: GET + url: /py-edge-after-reserved + headers: + fields: + - [ X-H3-Python-Client, aioquic ] + - [ X-H3-Reused-Header, stable-python-qpack-value ] + - [ uuid, py-edge-after-reserved ] + - [ X-H3-Test-Case, py-edge-after-reserved ] + + server-response: + <<: *generated_100_response diff --git a/tests/gold_tests/h3/replays/h3_sni.replay.yaml b/tests/gold_tests/h3/replays/h3_sni.replay.yaml index a4ca2bae174..55d23d08542 100644 --- a/tests/gold_tests/h3/replays/h3_sni.replay.yaml +++ b/tests/gold_tests/h3/replays/h3_sni.replay.yaml @@ -21,7 +21,7 @@ sessions: - protocol: stack: http3 tls: - sni: test_sni + sni: foo.com transactions: - client-request: @@ -31,7 +31,7 @@ sessions: - [ Content-Length, 0 ] - [:method, GET] - [:scheme, https] - - [:authority, example.com] + - [:authority, foo.com] - [:path, /path/test1] - [ uuid, has_sni ] server-response: @@ -55,7 +55,7 @@ sessions: - [ Content-Length, 0 ] - [:method, GET] - [:scheme, https] - - [:authority, example.com] + - [:authority, foo.com] - [:path, /path/test1] - [ uuid, no_sni ] server-response: diff --git a/tests/gold_tests/h3/replays/h3_stream_lifetime.replay.yaml b/tests/gold_tests/h3/replays/h3_stream_lifetime.replay.yaml new file mode 100644 index 00000000000..b263bea5277 --- /dev/null +++ b/tests/gold_tests/h3/replays/h3_stream_lifetime.replay.yaml @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify HTTP/3 stream lifetime handling with concurrent streams" + + server: + name: "server-h3-stream-lifetime" + process_config: + verbose: false + + client: + name: "client-h3-stream-lifetime" + process_config: + verbose: false + + ats: + name: "ts-h3-stream-lifetime" + startup_timeout: 60 + process_config: + enable_tls: true + enable_quic: true + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: "quic|http3|v_http3_trans" + proxy.config.quic.server.stateless_retry_enabled: 0 + + remap_config: + - from: "https://example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + + log_validation: + traffic_out: + contains: + - expression: 'start HTTP/3 app \(ALPN=h3\)' + description: "ATS should negotiate HTTP/3" + +sessions: +- protocol: + - name: http + version: 3 + - name: tls + sni: example.com + - name: udp + - name: ip + + transactions: + - client-request: + version: "3" + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-slow ] + - [ uuid, h3-slow ] + + server-response: + delay: 500ms + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "slow-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvw" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "slow-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvw" + verify: { as: equal } + + - client-request: + version: "3" + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-empty ] + - [ uuid, h3-empty ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "0" ] + content: + size: 0 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "0", as: equal } ] + + - client-request: + version: "3" + headers: + fields: + - [ ":method", GET ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-small ] + - [ uuid, h3-small ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "small-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuv" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "small-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuv" + verify: { as: equal } + + - client-request: + version: "3" + headers: + fields: + - [ ":method", POST ] + - [ ":scheme", https ] + - [ ":authority", example.com ] + - [ ":path", /h3-post ] + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + - [ uuid, h3-post ] + content: + encoding: plain + data: "post-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvw" + + proxy-request: + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "post-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvw" + verify: { as: equal } + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, text/plain ] + - [ Content-Length, "100" ] + content: + encoding: plain + data: "post-response-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn" + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "100", as: equal } ] + content: + encoding: plain + data: "post-response-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn" + verify: { as: equal } diff --git a/tests/gold_tests/timeout/active_timeout.test.py b/tests/gold_tests/timeout/active_timeout.test.py index 5f5d1d52ec9..214f06ebe67 100644 --- a/tests/gold_tests/timeout/active_timeout.test.py +++ b/tests/gold_tests/timeout/active_timeout.test.py @@ -60,7 +60,7 @@ tr3.MakeCurlCommand('-k -i --http2 https://127.0.0.1:{0}/file'.format(ts.Variables.ssl_port), ts=ts) tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Activity Timeout", "Request should fail with active timeout") - if Condition.HasATSFeature('TS_HAS_QUICHE') and Condition.HasCurlFeature('http3'): + if Condition.HasATSFeature('TS_USE_QUIC') and Condition.HasCurlFeature('http3'): tr4 = Test.AddTestRun("tr") tr4.MakeCurlCommand('-k -i --http3 https://localhost:{0}/file'.format(ts.Variables.ssl_port), ts=ts) tr4.Processes.Default.Streams.stdout = Testers.ContainsExpression( diff --git a/tests/gold_tests/timeout/quic_no_activity_timeout.test.py b/tests/gold_tests/timeout/quic_no_activity_timeout.test.py index 0b688c8b0a5..2f61dd0a132 100644 --- a/tests/gold_tests/timeout/quic_no_activity_timeout.test.py +++ b/tests/gold_tests/timeout/quic_no_activity_timeout.test.py @@ -16,7 +16,7 @@ Test.Summary = 'Basic checks on QUIC max_idle_timeout set by ts.quic.no_activity_timeout_in' -Test.SkipUnless(Condition.HasATSFeature('TS_HAS_QUICHE'), Condition.HasCurlFeature('http3')) +Test.SkipUnless(Condition.HasATSFeature('TS_USE_QUIC')) class Test_quic_no_activity_timeout: @@ -113,18 +113,19 @@ def run(self, check_for_max_idle_timeout=False): replay_keys="nodelays") test0.run() -test1 = Test_quic_no_activity_timeout( - "Test ts.quic.no_activity_timeout_in(quic max_idle_timeout) with a 5s delay", - no_activity_timeout_in=3000, # 3s `max_idle_timeout` - replay_keys="delay5s", - gold_file="gold/quic_no_activity_timeout.gold") -test1.run(check_for_max_idle_timeout=True) - -# QUIC Ignores the default_inactivity_timeout config, so the ts.quic.no_activity_timeout_in -# should be honor -test2 = Test_quic_no_activity_timeout( - "Ignoring default_inactivity_timeout and use the ts.quic.no_activity_timeout_in instead", - replay_keys="delay5s", - no_activity_timeout_in=3000, - extra_recs={'proxy.config.net.default_inactivity_timeout': 1}) -test2.run(check_for_max_idle_timeout=True) +if Condition.HasATSFeature('TS_HAS_QUICHE'): + test1 = Test_quic_no_activity_timeout( + "Test ts.quic.no_activity_timeout_in(quic max_idle_timeout) with a 5s delay", + no_activity_timeout_in=3000, # 3s `max_idle_timeout` + replay_keys="delay5s", + gold_file="gold/quic_no_activity_timeout.gold") + test1.run(check_for_max_idle_timeout=True) + + # QUIC Ignores the default_inactivity_timeout config, so the ts.quic.no_activity_timeout_in + # should be honored + test2 = Test_quic_no_activity_timeout( + "Ignoring default_inactivity_timeout and use the ts.quic.no_activity_timeout_in instead", + replay_keys="delay5s", + no_activity_timeout_in=3000, + extra_recs={'proxy.config.net.default_inactivity_timeout': 1}) + test2.run(check_for_max_idle_timeout=True) diff --git a/tests/pyproject.toml b/tests/pyproject.toml index d055f938c73..f976251df3b 100644 --- a/tests/pyproject.toml +++ b/tests/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "pyOpenSSL", "eventlet", + "aioquic==1.3.0", # To test stats_over_http prometheus exporter. "prometheus_client", @@ -62,4 +63,3 @@ dev = [ "pyflakes", ] - From a51cc7bea76a9992138666eef8e63c0bdafa7054 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 30 Jul 2026 09:04:49 -0500 Subject: [PATCH 08/11] TSHttpAltInfoQualitySet: add an autest (#13391) This adds an AuTest to verify the TSHttpAltInfoQualitySet plugin API. The test verifies alternate creation and selection through observable cached responses. Closes: #7205 (cherry picked from commit 799c4cc89a64972d93716e23ddba9ede099b50f5) --- .../cache/alternate-caching.test.py | 8 +- .../replay/alternate-caching-quality.yaml | 142 ++++++++++++++++++ tests/tools/plugins/CMakeLists.txt | 1 + tests/tools/plugins/http_alt_info_quality.cc | 101 +++++++++++++ 4 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 tests/gold_tests/cache/replay/alternate-caching-quality.yaml create mode 100644 tests/tools/plugins/http_alt_info_quality.cc diff --git a/tests/gold_tests/cache/alternate-caching.test.py b/tests/gold_tests/cache/alternate-caching.test.py index b6ec1350836..7f3b698c672 100644 --- a/tests/gold_tests/cache/alternate-caching.test.py +++ b/tests/gold_tests/cache/alternate-caching.test.py @@ -17,9 +17,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + Test.Summary = ''' Test the alternate caching feature. ''' -# Verify disabled negative revalidating behavior. +# Verify cache alternate replacement behavior. Test.ATSReplayTest(replay_file="replay/alternate-caching-update-size.yaml") + +# Verify TSHttpAltInfoQualitySet affects alternate selection. +tr = Test.ATSReplayTest(replay_file="replay/alternate-caching-quality.yaml") +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'http_alt_info_quality.so'), tr.Processes.ts_alt_quality) diff --git a/tests/gold_tests/cache/replay/alternate-caching-quality.yaml b/tests/gold_tests/cache/replay/alternate-caching-quality.yaml new file mode 100644 index 00000000000..06915997fd7 --- /dev/null +++ b/tests/gold_tests/cache/replay/alternate-caching-quality.yaml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +meta: + version: "1.0" + +autest: + description: 'Verify TSHttpAltInfoQualitySet affects alternate selection' + + dns: + name: 'dns_alt_quality' + + server: + name: 'server_alt_quality' + + client: + name: 'client_alt_quality' + + ats: + name: 'ts_alt_quality' + process_config: + enable_cache: true + + records_config: + proxy.config.cache.select_alternate: 1 + proxy.config.cache.limits.http.max_alts: 4 + proxy.config.http.cache.ignore_query: 1 + + plugin_config: + - "xdebug.so --enable=x-cache" + + remap_config: + - from: "http://example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + +sessions: +- transactions: + # Store the English response as the first alternate. + - all: { headers: { fields: [[ uuid, 1 ]]}} + client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /path/request?variant=English + delay: 100ms + headers: + fields: + - [ Host, example.com ] + - [ Accept-Language, English ] + - [ X-Debug, X-Cache ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + - [ Cache-Control, max-age=60 ] + - [ Content-Language, English ] + - [ X-Response-Variant, English ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Variant, { value: English, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # The language mismatch prevents reuse of the English alternate, allowing + # the French response to be stored as another alternate. + - all: { headers: { fields: [[ uuid, 2 ]]}} + client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /path/request?variant=French + delay: 100ms + headers: + fields: + - [ Host, example.com ] + - [ Accept-Language, French ] + - [ X-Debug, X-Cache ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + - [ Cache-Control, max-age=60 ] + - [ Content-Language, French ] + - [ X-Response-Variant, French ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Variant, { value: French, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Both languages are acceptable, so the plugin's query-based quality + # selects the older English alternate without contacting the origin. + - all: { headers: { fields: [[ uuid, 3 ]]}} + client-request: + method: "GET" + version: "1.1" + scheme: "http" + url: /path/request?variant=English + delay: 100ms + headers: + fields: + - [ Host, example.com ] + - [ Accept-Language, "English, French" ] + - [ X-Debug, X-Cache ] + + proxy-request: + expect: absent + + server-response: + status: 500 + reason: NOT USED + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Variant, { value: English, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] diff --git a/tests/tools/plugins/CMakeLists.txt b/tests/tools/plugins/CMakeLists.txt index a909cf59d1b..b7f18109ef0 100644 --- a/tests/tools/plugins/CMakeLists.txt +++ b/tests/tools/plugins/CMakeLists.txt @@ -24,6 +24,7 @@ add_autest_plugin(delay_txn_start delay_txn_start.cc) add_autest_plugin(emergency_shutdown emergency_shutdown.cc) add_autest_plugin(fatal_shutdown fatal_shutdown.cc) add_autest_plugin(hook_add_plugin hook_add_plugin.cc) +add_autest_plugin(http_alt_info_quality http_alt_info_quality.cc) add_autest_plugin(missing_mangled_definition missing_mangled_definition_c.c missing_mangled_definition_cpp.cc) add_autest_plugin(missing_ts_plugin_init missing_ts_plugin_init.cc) add_autest_plugin(server_packet_mark server_packet_mark.cc packet_mark_common.cc) diff --git a/tests/tools/plugins/http_alt_info_quality.cc b/tests/tools/plugins/http_alt_info_quality.cc new file mode 100644 index 00000000000..dcd180d4a56 --- /dev/null +++ b/tests/tools/plugins/http_alt_info_quality.cc @@ -0,0 +1,101 @@ +/** @file + + Select cache alternates by matching request query strings. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */ + +#include + +#include + +namespace +{ +constexpr char PLUGIN_NAME[] = "http_alt_info_quality"; + +std::string +get_query(TSMBuffer buffer, TSMLoc header) +{ + TSMLoc url; + int length = 0; + + if (TSHttpHdrUrlGet(buffer, header, &url) != TS_SUCCESS) { + return {}; + } + + const char *query = TSUrlHttpQueryGet(buffer, url, &length); + std::string result; + + if (query != nullptr) { + result.assign(query, static_cast(length)); + } + + TSHandleMLocRelease(buffer, header, url); + return result; +} + +int +select_alternate(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata) +{ + TSAssert(event == TS_EVENT_HTTP_SELECT_ALT); + + auto alt_info = static_cast(edata); + TSMBuffer client_buffer; + TSMBuffer cached_buffer; + TSMLoc client_header; + TSMLoc cached_header; + + if (TSHttpAltInfoClientReqGet(alt_info, &client_buffer, &client_header) != TS_SUCCESS) { + TSHttpAltInfoQualitySet(alt_info, 0.0F); + return 0; + } + if (TSHttpAltInfoCachedReqGet(alt_info, &cached_buffer, &cached_header) != TS_SUCCESS) { + TSHttpAltInfoQualitySet(alt_info, 0.0F); + TSHandleMLocRelease(client_buffer, TS_NULL_MLOC, client_header); + return 0; + } + + std::string client_query = get_query(client_buffer, client_header); + std::string cached_query = get_query(cached_buffer, cached_header); + + TSHttpAltInfoQualitySet(alt_info, client_query == cached_query ? 1.0F : 0.0F); + + TSHandleMLocRelease(client_buffer, TS_NULL_MLOC, client_header); + TSHandleMLocRelease(cached_buffer, TS_NULL_MLOC, cached_header); + return 0; +} +} // namespace + +void +TSPluginInit(int /* argc ATS_UNUSED */, const char ** /* argv ATS_UNUSED */) +{ + TSPluginRegistrationInfo info; + + info.plugin_name = PLUGIN_NAME; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] plugin registration failed", PLUGIN_NAME); + return; + } + + TSHttpHookAdd(TS_HTTP_SELECT_ALT_HOOK, TSContCreate(select_alternate, nullptr)); +} From a922e6f693e3d1af4aca87b1e2b7bf0e9cd2afa3 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 30 Jul 2026 09:04:03 -0500 Subject: [PATCH 09/11] Extend pipeline autest to verify HTTP/1.1 request framing (#13402) Traffic Server already handles these request shapes correctly; this patch adds no product code and changes no behavior. It only adds test coverage so that the currently correct handling cannot regress unnoticed. A proxy that disagrees with its origin about where a request body ends can be desynchronized from that origin. Two common shapes provoke this: pipelining a body-less POST (Content-Length: 0) ahead of a second request, and sending a request with conflicting Content-Length header fields. Traffic Server must keep such request boundaries intact. This extends the pipeline autest with two runs that exercise those shapes. This confirms that Traffic Server delivers the pipelined requests to the origin as two independent requests, so the second request cannot be folded into the first, and that it rejects the ambiguously-framed request rather than forwarding it. (cherry picked from commit 991524a53a46bd193c2c0a8470520e3095d5428f) --- tests/gold_tests/pipeline/pipeline.test.py | 116 ++++++++++- .../pipeline/request_framing_client.py | 129 ++++++++++++ .../pipeline/request_framing_server.py | 185 ++++++++++++++++++ 3 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 tests/gold_tests/pipeline/request_framing_client.py create mode 100644 tests/gold_tests/pipeline/request_framing_server.py diff --git a/tests/gold_tests/pipeline/pipeline.test.py b/tests/gold_tests/pipeline/pipeline.test.py index dea95511fb4..68217fd4cf9 100644 --- a/tests/gold_tests/pipeline/pipeline.test.py +++ b/tests/gold_tests/pipeline/pipeline.test.py @@ -19,7 +19,7 @@ from ports import get_port import sys -Test.Summary = '''Test pipelined requests.''' +Test.Summary = '''Test pipelined requests and HTTP/1.1 request framing.''' IP_ALLOW_CONTENT = ''' ip_allow: @@ -133,5 +133,119 @@ def _configure_client(self, tr: 'TestRun') -> 'Process': client.StartBefore(self._ts) +class TestRequestFraming: + """Verify Traffic Server preserves HTTP/1.1 request framing. + + A body-less POST (Content-Length: 0) immediately followed by a second + pipelined request must be delivered to the origin as two independent + requests, so the second request cannot be folded into the first. A request + with conflicting Content-Length header fields must be rejected rather than + forwarded. + """ + + _client_script: str = 'request_framing_client.py' + _server_script: str = 'request_framing_server.py' + _counter: int = 0 + + def __init__(self, mode: str) -> None: + """Configure a test run for the given request mode. + + :param mode: 'pipeline' for a body-less POST followed by a pipelined + request, or 'conflicting_cl' for a request with conflicting + Content-Length header fields. + """ + self._mode = mode + self._name = f'framing_{mode}_{TestRequestFraming._counter}' + TestRequestFraming._counter += 1 + + description = { + 'pipeline': 'Test a body-less POST followed by a pipelined request.', + 'conflicting_cl': 'Test a request with conflicting Content-Length headers.', + }[mode] + tr = Test.AddTestRun(description) + tr.TimeOut = 20 + self._configure_server(tr) + self._configure_traffic_server(tr) + self._configure_client(tr) + + def _configure_server(self, tr: 'TestRun') -> 'Process': + """Configure the recording origin server.""" + server = tr.Processes.Process(f'origin_{self._name}') + tr.Setup.Copy(self._server_script) + http_port = get_port(server, "http_port") + server.Command = f'{sys.executable} {self._server_script} 127.0.0.1 {http_port} ' + server.ReturnCode = 0 + server.Ready = When.PortOpenv4(http_port) + + if self._mode == 'pipeline': + # The origin must see exactly the two requests the client sent, with + # their boundaries intact. If ATS folded them together, the origin + # would see a single request or the second request's bytes inside + # the POST body. + server.Streams.All += Testers.ContainsExpression( + r'REQUEST_LINE: POST / HTTP/1.1', 'Origin should receive the POST as its own request.') + server.Streams.All += Testers.ContainsExpression( + r'REQUEST_LINE: GET /second HTTP/1.1', 'Origin should receive the GET as its own request.') + server.Streams.All += Testers.ContainsExpression( + r'ORIGIN_REQUEST_COUNT: 2', 'Origin should receive the second request as a distinct request.') + server.Streams.All += Testers.ExcludesExpression( + r'ORIGIN_REQUEST_COUNT: 3', 'Origin should receive exactly two requests, no more.') + # The second request's bytes must never appear inside the first + # (POST) request's body. + server.Streams.All += Testers.ExcludesExpression( + r"BODY:.*GET /second", 'The GET request must not appear inside the POST body.') + server.Streams.All += Testers.ExcludesExpression( + r"BODY:.*X-Marker", 'The second request header must not appear in the POST body.') + else: + # An ambiguously-framed request must be rejected by ATS before it + # ever reaches the origin. + server.Streams.All += Testers.ExcludesExpression( + r'REQUEST_LINE:', 'Origin must not receive an ambiguously-framed request.') + self._server = server + return server + + def _configure_traffic_server(self, tr: 'TestRun') -> 'Process': + """Configure ATS as a reverse proxy in front of the origin.""" + ts = tr.MakeATSProcess(f'ts_{self._name}', enable_cache=False) + self._ts = ts + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}/') + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + }) + return ts + + def _configure_client(self, tr: 'TestRun') -> 'Process': + """Configure the client that sends the framed request.""" + client = tr.Processes.Default + tr.Setup.Copy(self._client_script) + client.Command = ( + f'{sys.executable} {self._client_script} 127.0.0.1 {self._ts.Variables.port} ' + f'www.example.com {self._mode}') + client.ReturnCode = 0 + if self._mode == 'pipeline': + # Two independent responses must come back, one per request. + client.Streams.All += Testers.ContainsExpression( + r'STATUS_LINE_COUNT: 2', 'Client should receive two independent responses.') + client.Streams.All += Testers.ContainsExpression( + r'X-Origin-Response: first', 'Client should receive the response to the POST.') + client.Streams.All += Testers.ContainsExpression( + r'X-Origin-Response: second', 'Client should receive the response to the GET.') + else: + # The conflicting Content-Length request must be rejected with a + # 400, exactly one response must come back, and the second request + # must never be answered. + client.Streams.All += Testers.ContainsExpression( + r'HTTP/1.1 400', 'Client should receive a 400 for the ambiguous request.') + client.Streams.All += Testers.ContainsExpression(r'STATUS_LINE_COUNT: 1', 'Client should receive exactly one response.') + client.Streams.All += Testers.ExcludesExpression( + r'X-Origin-Response: second', 'The second request must not be answered.') + client.StartBefore(self._server) + client.StartBefore(self._ts) + + TestPipelining(buffer_requests=False) TestPipelining(buffer_requests=True) + +TestRequestFraming(mode='pipeline') +TestRequestFraming(mode='conflicting_cl') diff --git a/tests/gold_tests/pipeline/request_framing_client.py b/tests/gold_tests/pipeline/request_framing_client.py new file mode 100644 index 00000000000..292e44fb97b --- /dev/null +++ b/tests/gold_tests/pipeline/request_framing_client.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Send a request over a raw socket and print the responses received. + +Two request shapes are supported: a body-less POST (Content-Length: 0) followed +by a second pipelined request on the same connection, and a single request that +carries conflicting Content-Length header fields. +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import sys + + +def parse_args() -> argparse.Namespace: + """Parse the command line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("proxy_address", help="Address of the proxy to connect to.") + parser.add_argument("proxy_port", type=int, help="The port of the proxy to connect to.") + parser.add_argument("host", help="The Host header field value to use.") + parser.add_argument( + "mode", + nargs='?', + default='pipeline', + choices=['pipeline', 'conflicting_cl'], + help="Which request to send: a body-less POST followed by a pipelined " + "request, or a request with conflicting Content-Length headers.") + return parser.parse_args() + + +def build_request(mode: str, host: str) -> bytes: + """Build the raw request bytes for the given mode. + + :param mode: 'pipeline' for a body-less POST followed by a pipelined GET, or + 'conflicting_cl' for a request carrying conflicting Content-Length + header fields. + :param host: The Host header field value. + :returns: The raw request bytes. + """ + if mode == 'conflicting_cl': + # Two different Content-Length values are an ambiguous framing that a + # careful proxy must reject (RFC 9112 section 6.3) rather than forward, + # since a downstream server might frame the body differently. + return ( + f'POST / HTTP/1.1\r\n' + f'Host: {host}\r\n' + f'Content-Length: 0\r\n' + f'Content-Length: 38\r\n' + f'Connection: keep-alive\r\n' + f'\r\n' + f'GET /second HTTP/1.1\r\n' + f'Host: {host}\r\n' + f'X-Marker: second-request\r\n' + f'\r\n').encode() + + return ( + f'POST / HTTP/1.1\r\n' + f'Host: {host}\r\n' + f'Content-Length: 0\r\n' + f'Connection: keep-alive\r\n' + f'\r\n' + f'GET /second HTTP/1.1\r\n' + f'Host: {host}\r\n' + f'X-Marker: second-request\r\n' + f'\r\n').encode() + + +def main() -> int: + """Send the request and print the received responses.""" + args = parse_args() + + request = build_request(args.mode, args.host) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.connect((args.proxy_address, args.proxy_port)) + print(f'Connected to {args.proxy_address}:{args.proxy_port}') + print(f'Sending request ({len(request)} bytes):') + print(request) + sock.sendall(request) + + # The pipeline mode expects two responses; the conflicting_cl mode + # expects a single 400. + expected_responses = 1 if args.mode == 'conflicting_cl' else 2 + sock.settimeout(5.0) + responses = b"" + reached_expected = False + try: + while True: + data = sock.recv(4096) + if not data: + break + responses += data + # Each response terminates its header block with a blank line. + # Key off that rather than a trailing newline in the payload, + # since ATS-generated error bodies need not end in a newline. + if not reached_expected and responses.count(b'\r\n\r\n') >= expected_responses: + # Got the expected responses. Shorten the timeout so an + # unexpected extra response is still caught without waiting + # out the full initial timeout. + reached_expected = True + sock.settimeout(1.0) + except socket.timeout: + if not reached_expected: + print('Read timed out.') + + print('==== RESPONSES RECEIVED ====') + print(responses.decode(errors='replace')) + print('==== END RESPONSES ====') + print(f'STATUS_LINE_COUNT: {responses.count(b"HTTP/1.1")}') + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/pipeline/request_framing_server.py b/tests/gold_tests/pipeline/request_framing_server.py new file mode 100644 index 00000000000..cbe27f7fd60 --- /dev/null +++ b/tests/gold_tests/pipeline/request_framing_server.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""An origin server that records HTTP/1.1 request boundaries. + +The server parses each request's framing itself (headers, then a +Content-Length-delimited body) and prints what it received, so a test can verify +that the proxy delivered each request to the origin with its boundaries intact. +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import signal +import socket +import sys + + +def parse_args() -> argparse.Namespace: + """Parse the command line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("address", help="Address to listen on.") + parser.add_argument("port", type=int, help="The port to listen on.") + return parser.parse_args() + + +def get_listening_socket(address: str, port: int) -> socket.socket: + """Create a listening socket.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((address, port)) + sock.listen(1) + return sock + + +def recv_until(sock: socket.socket, buffer: bytes, delimiter: bytes) -> bytes: + """Read from the socket until the buffer contains the delimiter. + + :param sock: The socket to read from. + :param buffer: Bytes already read from the socket. + :param delimiter: The delimiter to read until. + :returns: The buffer, guaranteed to contain the delimiter, or all bytes read + before the socket closed. + """ + while delimiter not in buffer: + data = sock.recv(4096) + if not data: + break + buffer += data + return buffer + + +def response_for(path: str) -> bytes: + """Build the origin response for a given request target. + + :param path: The request target. + :returns: The raw response bytes. + """ + if path == '/second': + body = b'second response body\n' + return ( + b'HTTP/1.1 200 OK\r\n' + b'X-Origin-Response: second\r\n' + b'Content-Type: text/plain\r\n' + b'Content-Length: ' + str(len(body)).encode() + b'\r\n\r\n' + body) + body = b'first response body\n' + return ( + b'HTTP/1.1 200 OK\r\n' + b'X-Origin-Response: first\r\n' + b'Content-Type: text/plain\r\n' + b'Content-Length: ' + str(len(body)).encode() + b'\r\n\r\n' + body) + + +def handle_connection(sock: socket.socket) -> None: + """Read and record every request received on a single connection. + + :param sock: The accepted client socket. + """ + sock.settimeout(5.0) + buffer = b"" + request_count = 0 + while True: + try: + buffer = recv_until(sock, buffer, b'\r\n\r\n') + except socket.timeout: + print("Timed out waiting for a request.") + break + if b'\r\n\r\n' not in buffer: + print("Connection closed by peer.") + break + + header_bytes, _, rest = buffer.partition(b'\r\n\r\n') + header_text = header_bytes.decode(errors='replace') + lines = header_text.split('\r\n') + request_line = lines[0] + path = request_line.split(' ')[1] if len(request_line.split(' ')) > 1 else '' + + content_length = 0 + for line in lines[1:]: + name, _, value = line.partition(':') + if name.strip().lower() == 'content-length': + try: + content_length = int(value.strip()) + except ValueError: + content_length = 0 + + # Read the body, if any, according to Content-Length. + body = rest + timed_out = False + try: + while len(body) < content_length: + data = sock.recv(4096) + if not data: + break + body += data + except socket.timeout: + print("Timed out waiting for the request body.") + timed_out = True + if timed_out: + break + remainder = body[content_length:] + body = body[:content_length] + + request_count += 1 + print(f'---- ORIGIN REQUEST {request_count} ----') + print(f'REQUEST_LINE: {request_line}') + for line in lines[1:]: + print(f'HEADER: {line}') + print(f'BODY_LEN: {len(body)}') + print(f'BODY: {body!r}') + print(f'---- END ORIGIN REQUEST {request_count} ----') + print(f'ORIGIN_REQUEST_COUNT: {request_count}') + sys.stdout.flush() + + sock.sendall(response_for(path)) + + # Any bytes past this request's body belong to the next pipelined + # request on the connection. + buffer = remainder + + print(f'TOTAL_ORIGIN_REQUESTS: {request_count}') + sys.stdout.flush() + + +def main() -> int: + """Run the recording origin server until terminated by the test harness.""" + # AuTest terminates long-running processes with SIGINT. Register the handler + # explicitly because SIGINT may be inherited as ignored from the launcher. + signal.signal(signal.SIGINT, lambda *_: sys.exit(0)) + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + + args = parse_args() + try: + with get_listening_socket(args.address, args.port) as listening_sock: + print(f"Listening on {args.address}:{args.port}") + sys.stdout.flush() + while True: + conn, _ = listening_sock.accept() + with conn: + handle_connection(conn) + except (KeyboardInterrupt, SystemExit): + # SIGTERM from the test harness or a Ctrl-C is a clean shutdown. + pass + except OSError as e: + # An unexpected socket error (e.g. bind or accept failure) should be + # surfaced with a non-zero exit rather than masked as success. + print(f"Origin server error: {e}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e68c255c88aa995e72bfa7bdda75b4d7d6289702 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 30 Jul 2026 08:25:05 -0500 Subject: [PATCH 10/11] Fix PluginVC server connection cast (#13453) Plugin intercepts provide a PluginVC for the server connection, but HttpSM treats it as a UnixNetVConnection. This invalid downcast can trigger libc++ RTTI diagnostics and is undefined behavior. This keeps the connection at its NetVConnection base type, which provides both TLS service lookups needed at that point. Fixes: #8105 (cherry picked from commit 43e0b5dad4ae77afb38f5d5f90bc34c3ae07421a) --- src/proxy/http/HttpSM.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc index eaa71d56539..6c9e17d149b 100644 --- a/src/proxy/http/HttpSM.cc +++ b/src/proxy/http/HttpSM.cc @@ -6918,7 +6918,7 @@ HttpSM::attach_server_session() server_entry->vc_type = HttpVC_t::SERVER_VC; server_entry->vc_write_handler = &HttpSM::state_send_server_request_header; - UnixNetVConnection *server_vc = static_cast(server_txn->get_netvc()); + NetVConnection *server_vc = server_txn->get_netvc(); // set flag for server session is SSL if (server_vc->get_service()) { From 1427ebf71ad9abef22c8c4fd40b391caa935591a Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 30 Jul 2026 16:09:15 -0500 Subject: [PATCH 11/11] Fix JSONRPC server shutdown race (#13461) JSONRPC server shutdown can race with worker thread startup. When the worker starts after stop_thread(), it restores the running flag and polls a closed socket indefinitely. This causes test_jsonrpcserver and process shutdown to hang. This marks the socket server as running before the worker is created, so a concurrent stop cannot be overwritten. It also passes the owning server to the worker instead of relying on the mutable global server pointer. (cherry picked from commit 8b9a0075cc62cff394da104b0c9cfb9aff65faa3) --- include/mgmt/rpc/server/IPCSocketServer.h | 2 +- src/mgmt/rpc/server/IPCSocketServer.cc | 6 ++++-- src/mgmt/rpc/server/RPCServer.cc | 12 ++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/include/mgmt/rpc/server/IPCSocketServer.h b/include/mgmt/rpc/server/IPCSocketServer.h index 4ffe377d43c..7c45204b13f 100644 --- a/include/mgmt/rpc/server/IPCSocketServer.h +++ b/include/mgmt/rpc/server/IPCSocketServer.h @@ -144,7 +144,7 @@ class IPCSocketServer : public BaseCommInterface void close(); void late_check_peer_credentials(int peedFd, TSRPCHandlerOptions const &options, swoc::Errata &errata) const; - std::atomic_bool _running; + std::atomic_bool _running{false}; struct sockaddr_un _serverAddr; int _socket{-1}; diff --git a/src/mgmt/rpc/server/IPCSocketServer.cc b/src/mgmt/rpc/server/IPCSocketServer.cc index 5063f90a593..10e91d2e155 100644 --- a/src/mgmt/rpc/server/IPCSocketServer.cc +++ b/src/mgmt/rpc/server/IPCSocketServer.cc @@ -168,6 +168,10 @@ IPCSocketServer::init() return ec; } + // Set this before RPCServer creates the worker thread so an immediate stop + // cannot be overwritten when the worker eventually enters run(). + _running.store(true); + return ec; } @@ -201,8 +205,6 @@ IPCSocketServer::poll_for_new_client(std::chrono::milliseconds timeout) const void IPCSocketServer::run() { - _running.store(true); - while (_running) { // poll till socket it's ready. if (!this->poll_for_new_client()) { diff --git a/src/mgmt/rpc/server/RPCServer.cc b/src/mgmt/rpc/server/RPCServer.cc index eb121359169..6c8a55afa6a 100644 --- a/src/mgmt/rpc/server/RPCServer.cc +++ b/src/mgmt/rpc/server/RPCServer.cc @@ -59,13 +59,13 @@ RPCServer::~RPCServer() void * /* static */ RPCServer::run_thread(void *a) { - void *ret = a; - if (jsonrpcServer->_init) { - jsonrpcServer->_rpcThread = jsonrpcServer->_init(); + auto *server = static_cast(a); + if (server->_init) { + server->_rpcThread = server->_init(); } - jsonrpcServer->_socketImpl->run(); + server->_socketImpl->run(); Dbg(dbg_ctl, "Socket stopped"); - return ret; + return a; } void @@ -75,7 +75,7 @@ RPCServer::start_thread(std::function const &cb_init, std::function< _init = cb_init; _destroy = cb_destroy; - ink_thread_create(&_this_thread, run_thread, nullptr, 0, 0, nullptr); + ink_thread_create(&_this_thread, run_thread, this, 0, 0, nullptr); } void