From f211d9095186b4e7840b2dc31e7fc6a499c515ae Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Mon, 3 Aug 2026 14:52:21 -0500 Subject: [PATCH 01/10] net: fix and always maintain the per group peak connection count Two problems with the per group peak, which is reported as the "max" field of the connection tracker group dump. update_max_count() made a single compare_exchange_weak attempt with no retry, so a racing update, or a spurious failure of the weak form, silently discarded the sample. Retry until the value is stored or is no longer the largest. It was also only called when a maximum was configured. With metrics enabled and no configured maximum, the count was reserved and then discarded, so the peak stayed at zero. Pass the reserved count through in that case too. --- include/iocore/net/ConnectionTracker.h | 10 +++++++--- src/proxy/http/HttpSM.cc | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/include/iocore/net/ConnectionTracker.h b/include/iocore/net/ConnectionTracker.h index ae3691fdbe2..238746be522 100644 --- a/include/iocore/net/ConnectionTracker.h +++ b/include/iocore/net/ConnectionTracker.h @@ -489,9 +489,13 @@ ConnectionTracker::TxnState::clear() inline void ConnectionTracker::TxnState::update_max_count(int count) { - auto cmax = _g->_count_max.load(); - if (count > cmax) { - _g->_count_max.compare_exchange_weak(cmax, count); + auto cmax = _g->_count_max.load(std::memory_order_relaxed); + + while (count > cmax) { + if (_g->_count_max.compare_exchange_weak(cmax, count, std::memory_order_relaxed, std::memory_order_relaxed)) { + break; + } + // cmax was reloaded by the failed exchange; retry if we are still larger. } } diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc index 0d54ad376d7..4387b6d9b40 100644 --- a/src/proxy/http/HttpSM.cc +++ b/src/proxy/http/HttpSM.cc @@ -5921,7 +5921,9 @@ HttpSM::do_http_server_open(bool raw, bool only_direct) } else if (t_state.txn_conf->connection_tracker_config.server_min > 0 || t_state.http_config_param->global_connection_tracker_config.metric_enabled) { auto &ct_state = t_state.outbound_conn_track_state; - ct_state.reserve(); + // Feed the count through as well, otherwise the group's peak stays at zero whenever metrics + // are enabled without a configured maximum. + ct_state.update_max_count(ct_state.reserve()); } // We did not manage to get an existing session and need to open a new connection From 6ccb1b481489c755cf458338c7b16093290b50f0 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Wed, 5 Aug 2026 18:14:42 -0500 Subject: [PATCH 02/10] net: per server connection metrics via hidden and derived metrics Aggregate the per group connection counts into per hostname metrics, using hidden metrics as the inputs and a derived metric for the aggregation, so the work happens on the stat sync task rather than on every connection. metric_enabled becomes a level rather than a flag: 0 disables, 1 publishes only the per hostname aggregates, 2 also mirrors the per group metrics into the published store. The per group metrics themselves are always created in the hidden store, so changing the level at runtime only changes what is registered for publication and never has to move a metric between stores. Aggregates are registered only for the 'both' match type, the only one with more than one group per hostname; for 'host' the group name is already the bare hostname and the two would collide on a single name. current_connection_max is the largest current count among a hostname's groups, sampled, rather than a monotone peak, so it falls again as traffic drains and a maximum over time can be computed downstream. --- doc/admin-guide/files/records.yaml.en.rst | 29 ++ .../statistics/core/http-connection.en.rst | 58 ++++ include/iocore/net/ConnectionTracker.h | 54 +++- src/iocore/net/ConnectionTracker.cc | 43 ++- src/records/RecordsConfig.cc | 2 +- .../per_server_connection_max.test.py | 256 ++++++++++++++++-- 6 files changed, 408 insertions(+), 34 deletions(-) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index f076dabbafb..d34886df7ad 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2007,6 +2007,35 @@ Origin Server Connect Attempts the connection. Useful when the origin supports keep-alive, removing the time needed to set up a new connection from the next request at the expense of added (inactive) connections. +.. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_enabled INT 0 + :reloadable: + + Publish per upstream server connection metrics. These metrics are dynamically named, one set per + upstream server group or hostname, so the number of them scales with the number of distinct + upstream servers seen. See :ref:`per-server-connection-metrics`. + + ===== ====================================================================================== + Value Effect + ===== ====================================================================================== + ``0`` No per server connection metrics. + ``1`` Publish only the per hostname aggregate metrics. The per group metrics from which the + aggregates are computed exist internally but are not published. + ``2`` Publish the per hostname aggregates and the per group metrics. + ===== ====================================================================================== + + Level ``2`` can produce a very large number of metrics when the + :ts:cv:`match type ` includes the address or + port, since there is then one set per address and port rather than one per hostname. + +.. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_prefix STRING NULL + :reloadable: + + An optional prefix inserted into the per server connection metric names, between the fixed + ``proxy.process.http.per_server..`` portion of the name and the upstream server group + or hostname. Useful to distinguish metrics from separate + :ts:cv:`match ` configurations sharing the same + upstream. See :ref:`per-server-connection-metrics`. + .. ts:cv:: CONFIG proxy.config.http.connect_attempts_rr_retries INT 3 :reloadable: :overridable: diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index 7b3b23940f0..78a75e11f8f 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -201,6 +201,64 @@ HTTP Connection Current number of TCP connections for tunnels where the far end is the server, except for those counted by ``proxy.process.tunnel.current_server_connections_tls`` +.. _per-server-connection-metrics: + +Per Server Connection Metrics +----------------------------- + +Unlike the metrics above these do not have fixed names. They are created dynamically, one set per +upstream server group as defined by :ts:cv:`proxy.config.http.per_server.connection.match`, and, +when the match type is ``both``, one aggregate set per hostname. Whether any of them are published, +and at what granularity, is controlled by +:ts:cv:`proxy.config.http.per_server.connection.metric_enabled`. An optional +:ts:cv:`proxy.config.http.per_server.connection.metric_prefix` can be inserted into the names. + +Per group names are ``proxy.process.http.per_server..``, where ```` depends on +the match type: an IP address, an ``address:port`` pair, a hostname, or, for ``both``, +``.``. Per hostname names are +``proxy.process.http.per_server..``. Aggregates exist only for match type +``both``, because that is the only match type with more than one group per hostname; for match type +``host`` the group name is already the bare hostname, so an aggregate would carry the same name as +the single group it summarises. + +For a group, ```` is one of: + +current_connection + Gauge. The number of connections currently open to the group. + +total_connection + Counter. The total number of connections ever opened to the group. Never decreases. + +blocked_connection + Counter. The total number of connection attempts to the group blocked by + :ts:cv:`proxy.config.http.per_server.connection.max`. Never decreases. + +For a hostname aggregate, ```` is one of those three, each summed across the groups of that +hostname, plus: + +current_connection_max + Gauge. The largest ``current_connection`` value among the groups of that hostname at the moment + of sampling, so the maximum rather than the sum of the groups' current counts. This is useful + because :ts:cv:`proxy.config.http.per_server.connection.max` is enforced per group rather than + per hostname, so the busiest group is what determines whether connections are about to be + blocked. Like ``current_connection`` it rises and falls with traffic and is not a high-water + mark. There is no per group ``current_connection_max``; it exists only as a hostname aggregate. + +Every published per server metric is recomputed periodically, currently every 5 seconds, rather than +on every connection event, so a reader sees a value up to that interval old. This is true of the +hostname aggregates and, at level ``2``, of the published per group metrics as well: those are +mirrored from the internal ones by the same periodic mechanism, not written as connections open and +close. It applies to ``current_connection_max`` too, which reports the maximum across groups as of +the last sample rather than a running peak. To obtain the peak over a longer window, compute a +maximum over time from this gauge in the monitoring system. + +At :ts:cv:`metric_enabled ` level ``1`` the +per group metrics still exist internally, since the aggregates are computed from them, but are not +published. They can be listed with ``traffic_ctl metric match per_server --include-hidden``, which +reads them directly and so is not subject to the sampling delay above. That visibility is intended +for debugging and is not a stable interface: the existence, granularity and naming of the per group +metrics may change independently of the published aggregates. + HTTP/2 ------ diff --git a/include/iocore/net/ConnectionTracker.h b/include/iocore/net/ConnectionTracker.h index 238746be522..5041c278274 100644 --- a/include/iocore/net/ConnectionTracker.h +++ b/include/iocore/net/ConnectionTracker.h @@ -82,16 +82,36 @@ class ConnectionTracker MatchType server_match{MATCH_IP}; ///< Server match type. }; + /** Levels for the @c metric_enabled configuration. + * + * The per group metrics are always created, in the hidden metric store, whenever metrics are + * enabled at all. What varies by level is what gets published: + * - @c METRIC_LEVEL_NONE: no per group metrics are created and nothing is published. + * - @c METRIC_LEVEL_HOST: the per group metrics stay hidden; the per hostname aggregates + * (computed across the groups of a hostname by a @c Derived metric) are published. + * - @c METRIC_LEVEL_GROUP: as above, plus the per group metrics are also mirrored into the + * published store. + * + * Keeping the per group metrics in the hidden store at every level means changing the level at + * runtime is only a change of what is registered for publication, with no metric to migrate + * between the two stores. + */ + enum MetricLevel { + METRIC_LEVEL_NONE = 0, ///< No per server metrics. + METRIC_LEVEL_HOST = 1, ///< Only the per hostname aggregate metrics are published. + METRIC_LEVEL_GROUP = 2, ///< The per hostname aggregates and the per group metrics are published. + }; + /** Static configuration values. */ struct GlobalConfig { GlobalConfig() = default; GlobalConfig(GlobalConfig const &); GlobalConfig &operator=(GlobalConfig const &); - std::chrono::seconds client_alert_delay{60}; ///< Alert delay in seconds. - std::chrono::seconds server_alert_delay{60}; ///< Alert delay in seconds. - bool metric_enabled{false}; ///< Enabling per server metrics. - std::string metric_prefix; ///< Per server metric prefix. + std::chrono::seconds client_alert_delay{60}; ///< Alert delay in seconds. + std::chrono::seconds server_alert_delay{60}; ///< Alert delay in seconds. + MetricLevel metric_enabled{METRIC_LEVEL_NONE}; ///< Which per server metrics to publish. + std::string metric_prefix; ///< Per server metric prefix. swoc::IPRangeSet client_exempt_list; ///< The set of IP addresses to not block due client connection counting. mutable ts::bravo::shared_mutex client_exempt_list_mutex; ///< Protects client_exempt_list from concurrent access. }; @@ -145,7 +165,8 @@ class ConnectionTracker std::atomic _in_queue{0}; ///< # of connections queued, waiting for a connection. std::atomic _last_alert{0}; ///< Absolute time of the last alert. - // Recording data as metrics + // Recording data as metrics. These are always in the hidden metric store when created; see + // @c MetricLevel for how they are published. ts::Metrics::Gauge::AtomicType *_count_metric = nullptr; ts::Metrics::Counter::AtomicType *_count_total_metric = nullptr; ts::Metrics::Counter::AtomicType *_blocked_metric = nullptr; @@ -171,6 +192,20 @@ class ConnectionTracker std::time_t get_last_alert_epoch_time() const; static std::string metric_name(const Key &key, std::string_view fqdn, std::string metric_prefix); + /** Name of the metric which aggregates a value across all groups of a hostname. + * + * Only @c MATCH_BOTH groups have more than one group per hostname. For @c MATCH_HOST there is + * exactly one group per hostname, so an aggregate would be over a set of one, and + * @c Group::metric_name already returns the FQDN alone for that match type - identical to what + * this would return, so publishing both would collide on one name. + * + * @param key The group key. + * @param fqdn The full FQDN. + * @param metric_prefix The configured metric prefix. + * @return The metric name, or an empty string if @a key is not @c MATCH_BOTH. + */ + static std::string host_metric_name(const Key &key, std::string_view fqdn, std::string metric_prefix); + /// Release the reference count to this group and remove it from the /// group table if it is no longer referenced. void release(); @@ -433,6 +468,15 @@ ConnectionTracker::Group::metric_name(const Key &key, std::string_view fqdn, std return metric_prefix.empty() ? std::move(metric_name) : metric_prefix + "." + metric_name; } +inline std::string +ConnectionTracker::Group::host_metric_name(const Key &key, std::string_view fqdn, std::string metric_prefix) +{ + if (MATCH_BOTH != key._match_type) { + return {}; // Only MATCH_BOTH has more than one group per hostname to aggregate across. + } + return metric_prefix.empty() ? std::string(fqdn) : metric_prefix + "." + std::string(fqdn); +} + inline bool ConnectionTracker::TxnState::is_active() const { diff --git a/src/iocore/net/ConnectionTracker.cc b/src/iocore/net/ConnectionTracker.cc index 45ce7e60f07..e8d4fc1eee5 100644 --- a/src/iocore/net/ConnectionTracker.cc +++ b/src/iocore/net/ConnectionTracker.cc @@ -26,6 +26,8 @@ #include "records/RecCore.h" #include "swoc/IPAddr.h" +#include + using namespace std::literals; ConnectionTracker::TableSingleton ConnectionTracker::_inbound_table; @@ -156,7 +158,9 @@ Config_Update_Conntrack_Metric_Enabled(const char * /* name ATS_UNUSED */, RecDa auto config = static_cast(cookie); if (RECD_INT == dtype) { - config->metric_enabled = data.rec_int; + auto level = std::clamp(static_cast(data.rec_int), static_cast(ConnectionTracker::METRIC_LEVEL_NONE), + static_cast(ConnectionTracker::METRIC_LEVEL_GROUP)); + config->metric_enabled = static_cast(level); return true; } return false; @@ -440,11 +444,40 @@ ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::st { Metrics::Gauge::increment(net_rsb.connection_tracker_table_size); // only add metrics for server connections - if (_global_config->metric_enabled && direction == DirectionType::OUTBOUND) { + if (_global_config->metric_enabled != METRIC_LEVEL_NONE && direction == DirectionType::OUTBOUND) { std::string _metric_name = metric_name(key, fqdn, _global_config->metric_prefix); - _count_metric = Metrics::Gauge::createPtr("proxy.process.http.per_server.current_connection.", _metric_name); - _count_total_metric = Metrics::Counter::createPtr("proxy.process.http.per_server.total_connection.", _metric_name); - _blocked_metric = Metrics::Counter::createPtr("proxy.process.http.per_server.blocked_connection.", _metric_name); + // Per group metrics always live in the hidden store. metric_enabled controls what is published + // from them (see MetricLevel), not whether they exist. + _count_metric = Metrics::Gauge::createHiddenPtr("proxy.process.http.per_server.current_connection.", _metric_name); + _count_total_metric = Metrics::Counter::createHiddenPtr("proxy.process.http.per_server.total_connection.", _metric_name); + _blocked_metric = Metrics::Counter::createHiddenPtr("proxy.process.http.per_server.blocked_connection.", _metric_name); + + // Only MATCH_BOTH groups have siblings sharing a hostname to aggregate across. + std::string _host_metric_name = host_metric_name(key, fqdn, _global_config->metric_prefix); + if (!_host_metric_name.empty()) { + Metrics::Derived::add_source("proxy.process.http.per_server.current_connection." + _host_metric_name, + Metrics::MetricType::GAUGE, _count_metric, Metrics::Derived::Op::SUM); + Metrics::Derived::add_source("proxy.process.http.per_server.total_connection." + _host_metric_name, + Metrics::MetricType::COUNTER, _count_total_metric, Metrics::Derived::Op::SUM); + Metrics::Derived::add_source("proxy.process.http.per_server.blocked_connection." + _host_metric_name, + Metrics::MetricType::COUNTER, _blocked_metric, Metrics::Derived::Op::SUM); + // The largest current count among this hostname's groups, sampled. Deliberately taken over + // the instantaneous gauge rather than each group's all time peak, so the value falls again + // and a maximum over time can be computed by whatever scrapes it. + Metrics::Derived::add_source("proxy.process.http.per_server.current_connection_max." + _host_metric_name, + Metrics::MetricType::GAUGE, _count_metric, Metrics::Derived::Op::MAX); + } + + if (_global_config->metric_enabled >= METRIC_LEVEL_GROUP) { + // Mirror the per group metrics into the published store under their own name. A single + // source SUM is an identity: the published value always equals the hidden source. + Metrics::Derived::add_source("proxy.process.http.per_server.current_connection." + _metric_name, Metrics::MetricType::GAUGE, + _count_metric, Metrics::Derived::Op::SUM); + Metrics::Derived::add_source("proxy.process.http.per_server.total_connection." + _metric_name, Metrics::MetricType::COUNTER, + _count_total_metric, Metrics::Derived::Op::SUM); + Metrics::Derived::add_source("proxy.process.http.per_server.blocked_connection." + _metric_name, Metrics::MetricType::COUNTER, + _blocked_metric, Metrics::Derived::Op::SUM); + } if (dbg_ctl.on()) { swoc::LocalBufferWriter<256> w; diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index 1eec9cce822..f2663de8f62 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -404,7 +404,7 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.per_server.connection.min", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "[0-1]", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-2]$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_prefix", RECD_STRING, "", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index e7bad788ab3..9dc9b12f204 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -1,5 +1,6 @@ ''' -Verify the behavior of proxy.config.http.per_server.connection.max. +Verify the behavior of proxy.config.http.per_server.connection.max and the per server +connection metrics (proxy.config.http.per_server.connection.metric_enabled). ''' # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -22,6 +23,17 @@ Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) +# The per hostname aggregates are derived metrics, recomputed once per +# REC_RAW_STAT_SYNC_INTERVAL_MS (5000ms, src/records/P_RecDefs.h) on an ET_TASK thread. Nothing in +# records.yaml drives that interval, so a test has to sleep comfortably longer than one sync period +# before reading an aggregate rather than trying to configure a faster one. Reading too early +# silently compares against zeros. +_STAT_SYNC_WAIT_SECONDS: int = 6 + +# NOTE: assigning to a Streams attribute REPLACES any tester already set for that stream +# (TesterSet.Assign), so every assertion after the first on the same stream must use '+=' or it +# silently discards the earlier ones. + class PerServerConnectionMaxTest: """Define an object to test our max origin connection behavior.""" @@ -54,7 +66,10 @@ def _configure_trafficserver(self) -> None: 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http|conn_track', 'proxy.config.http.per_server.connection.max': self._origin_max_connections, - 'proxy.config.http.per_server.connection.metric_enabled': 1, + # Level 2 (METRIC_LEVEL_GROUP): the match here is 'port', which never has more than + # one group per hostname, so there is no aggregate to read and the per group + # metrics themselves have to be published to be checked below. + 'proxy.config.http.per_server.connection.metric_enabled': 2, 'proxy.config.http.per_server.connection.metric_prefix': 'foo', 'proxy.config.http.per_server.connection.match': 'port', }) @@ -64,16 +79,26 @@ def _configure_trafficserver(self) -> None: def _test_metrics(self) -> None: """Use traffic_ctl to test metrics.""" + group_name = f'foo.127.0.0.1:{self._server.Variables.http_port}' + tr = Test.AddTestRun("Check connection metrics") - tr.Processes.Default.Command = 'traffic_ctl metric match per_server' + # At level 2 the per group metrics are published by mirroring the hidden ones through a + # derived metric, so a sync tick has to pass before they carry a value. + tr.Processes.Default.Command = f'sleep {_STAT_SYNC_WAIT_SECONDS}; traffic_ctl metric match per_server' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.TimeOut = _STAT_SYNC_WAIT_SECONDS + 30 tr.Processes.Default.Streams.All = Testers.ContainsExpression( - f'per_server.total_connection.foo.127.0.0.1:{self._server.Variables.http_port} 4', - 'incorrect statistic return, or possible error.') - tr.Processes.Default.Streams.All = Testers.ContainsExpression( - f'per_server.blocked_connection.foo.127.0.0.1:{self._server.Variables.http_port} 1', - 'incorrect statistic return, or possible error.') + f'per_server.total_connection.{group_name} 4', 'incorrect statistic return, or possible error.') + tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + 'INVALID_INCOMING_DATA', 'The metric query must not be rejected.') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + f'per_server.blocked_connection.{group_name} 1', 'incorrect statistic return, or possible error.') + + # A 'port' match has one group per address:port and no hostname, so no aggregate should be + # registered for it at all. + tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + 'per_server.current_connection_max.', 'A non-"both" match type must not register a hostname aggregate.') def run(self) -> None: """Configure the TestRun.""" @@ -88,16 +113,28 @@ def run(self) -> None: class ConnectMethodTest: - """Test our max origin connection behavior with CONNECT traffic.""" + """Test our max origin connection behavior with CONNECT traffic. + + Also covers the two publication levels of proxy.config.http.per_server.connection.metric_enabled: + - 1 (METRIC_LEVEL_HOST): only the per hostname aggregate is published; the per group metrics + stay hidden and are visible only with --include-hidden. + - 2 (METRIC_LEVEL_GROUP): the per hostname aggregate is published, and the per group metrics + are also mirrored into the published store. + + The match here defaults to 'both' and there is exactly one group for this hostname, so the + aggregate is a trivial sum over that single group. MultiGroupAggregateTest below covers the + case where an aggregate genuinely spans more than one group. + """ _process_counter: int = 0 _client_counter: int = 0 - def __init__(self, max_conn) -> None: + def __init__(self, max_conn, metric_level=1) -> None: """Configure the server processes in preparation for the TestRun.""" + self._metric_level = metric_level self._configure_dns() self._configure_origin_server() - self._configure_trafficserver(max_conn) + self._configure_trafficserver(max_conn, metric_level) ConnectMethodTest._process_counter += 1 def _configure_dns(self) -> None: @@ -108,8 +145,8 @@ def _configure_origin_server(self) -> None: """Configure the httpbin origin server.""" self._server = Test.MakeHttpBinServer(f"server_{ConnectMethodTest._process_counter}") - def _configure_trafficserver(self, max_conn) -> None: - self._ts = Test.MakeATSProcess("ts2_" + str(max_conn)) + def _configure_trafficserver(self, max_conn, metric_level) -> None: + self._ts = Test.MakeATSProcess(f"ts2_{max_conn}_{metric_level}") self._ts.Disk.records_config.update( { @@ -119,7 +156,7 @@ def _configure_trafficserver(self, max_conn) -> None: 'proxy.config.diags.debug.tags': 'http|dns|hostdb|conn_track', 'proxy.config.http.server_ports': f"{self._ts.Variables.port} {self._ts.Variables.uds_path}", 'proxy.config.http.connect_ports': f"{self._server.Variables.Port}", - 'proxy.config.http.per_server.connection.metric_enabled': 1, + 'proxy.config.http.per_server.connection.metric_enabled': metric_level, 'proxy.config.http.per_server.connection.max': max_conn, }) @@ -136,17 +173,47 @@ def _configure_client_with_slow_response(self, tr) -> 'Test.Process': return p def _test_metrics(self, blocked) -> None: - """Use traffic_ctl to test metrics.""" + """Use traffic_ctl to test metrics, honoring the configured publication level.""" + host_name = 'www.this.origin.com' + group_name = f'{host_name}.127.0.0.1:{self._server.Variables.Port}' + tr = Test.AddTestRun("Check connection metrics") - tr.Processes.Default.Command = 'traffic_ctl metric match per_server; sleep 2' + tr.Processes.Default.Command = f'sleep {_STAT_SYNC_WAIT_SECONDS}; traffic_ctl metric match per_server' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.TimeOut = _STAT_SYNC_WAIT_SECONDS + 30 + + # The per hostname aggregate is published at every non-zero level. tr.Processes.Default.Streams.All = Testers.ContainsExpression( - f'per_server.total_connection.www.this.origin.com.127.0.0.1:{self._server.Variables.Port} 5', - 'incorrect statistic return, or possible error.') - tr.Processes.Default.Streams.All = Testers.ContainsExpression( - f'per_server.blocked_connection.www.this.origin.com.127.0.0.1:{self._server.Variables.Port} {blocked}', - 'incorrect statistic return, or possible error.') + f'per_server.total_connection.{host_name} 5', 'incorrect statistic return, or possible error.') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + f'per_server.blocked_connection.{host_name} {blocked}', 'incorrect statistic return, or possible error.') + + if self._metric_level >= 2: + # METRIC_LEVEL_GROUP additionally mirrors the per group metrics into the published store. + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + f'per_server.total_connection.{group_name} 5', 'The per group metric should be published at METRIC_LEVEL_GROUP.') + else: + # METRIC_LEVEL_HOST keeps the per group metrics hidden, so none of the three per group + # names may appear in a normal query. current_connection_max is not among them: it only + # ever exists as a hostname aggregate, never per group. + for counter in ('current_connection', 'total_connection', 'blocked_connection'): + tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + f'per_server.{counter}.{group_name} ', f'per_server.{counter}.{group_name} must stay hidden at level 1.') + + # The per group metrics must be visible with --include-hidden at either level. This is also + # the end to end test for that traffic_ctl option. + tr2 = Test.AddTestRun("Check hidden per group connection metrics") + tr2.Processes.Default.Command = 'traffic_ctl metric match per_server --include-hidden' + tr2.Processes.Default.ReturnCode = 0 + tr2.Processes.Default.Env = self._ts.Env + # No sleep needed: the hidden per group metrics are written directly on each connection, + # unlike the derived aggregates. + tr2.Processes.Default.Streams.All = Testers.ContainsExpression( + f'per_server.total_connection.{group_name} 5', + 'The per group metric should be visible with --include-hidden at any level.') + tr2.Processes.Default.Streams.All += Testers.ExcludesExpression( + 'INVALID_INCOMING_DATA', 'The --include-hidden query must not be rejected by the RPC decoder.') def run(self, blocked, gold_file) -> None: """Verify per_server.connection.max with CONNECT traffic.""" @@ -179,6 +246,149 @@ def run(self, blocked, gold_file) -> None: self._test_metrics(blocked) +class MultiGroupAggregateTest: + """Verify a per hostname aggregate that genuinely spans more than one group. + + The other tests here resolve a hostname to a single 127.0.0.1:port, so their "aggregate" is + trivially a set of one. Here two remap rules point at the same hostname ('multi.origin.com') + on two different origin ports, so under match 'both' the connection tracker creates two + distinct groups sharing one host aggregate. The two groups are given different concurrency so + the SUM and the MAX are distinguishable from each other. + + current_connection and current_connection_max are instantaneous gauges recomputed from the live + per group values every ~5s, so they rise and fall with traffic rather than remembering a peak. + Observing a non-zero value therefore requires holding connections open across a sync tick. The + most robust assertion, and the one that actually distinguishes this instantaneous behavior from + a monotone peak, is that both gauges return to 0 once traffic drains and another tick passes. + """ + + _process_counter: int = 0 + _client_counter: int = 0 + + # Concurrent slow requests per group. Deliberately different so SUM (5) and MAX (3) differ. + _group_a_concurrency: int = 2 + _group_b_concurrency: int = 3 + + # How long each request holds its connection open. Must comfortably exceed + # _STAT_SYNC_WAIT_SECONDS so a sync tick is guaranteed to land while the connections are still + # open. NOTE: httpbin clamps /delay/ to 10 seconds, so the effective hold is min(this, 10); + # the waits below are derived from this value and tolerate that clamp. + _hold_seconds: int = 12 + + def __init__(self) -> None: + """Configure the test processes in preparation for the TestRun.""" + self._configure_dns() + self._configure_origin_servers() + self._configure_trafficserver() + MultiGroupAggregateTest._process_counter += 1 + + def _configure_dns(self) -> None: + """Configure a nameserver for the test.""" + self._dns = Test.MakeDNServer(f"magg_dns_{MultiGroupAggregateTest._process_counter}", default='127.0.0.1') + + def _configure_origin_servers(self) -> None: + """Configure the two httpbin origins which stand in for two groups of one hostname.""" + self._server_a = Test.MakeHttpBinServer(f"magg_server_a_{MultiGroupAggregateTest._process_counter}") + self._server_b = Test.MakeHttpBinServer(f"magg_server_b_{MultiGroupAggregateTest._process_counter}") + + def _configure_trafficserver(self) -> None: + """Configure Traffic Server with two remap rules to the same hostname on different ports.""" + self._ts = Test.MakeATSProcess(f"magg_ts_{MultiGroupAggregateTest._process_counter}") + self._ts.Disk.records_config.update( + { + 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|dns|hostdb|conn_track', + 'proxy.config.http.per_server.connection.metric_enabled': 1, + 'proxy.config.http.per_server.connection.match': 'both', + }) + self._ts.Disk.remap_config.AddLines( + [ + f"map http://multi.origin.com/a/ http://multi.origin.com:{self._server_a.Variables.Port}/", + f"map http://multi.origin.com/b/ http://multi.origin.com:{self._server_b.Variables.Port}/", + ]) + + def _make_slow_client(self, tr, path) -> 'Test.Process': + """Configure a client which makes a slow request through one of the two remapped groups.""" + p = tr.Processes.Process(f'magg_client_{MultiGroupAggregateTest._client_counter}') + MultiGroupAggregateTest._client_counter += 1 + tr.MakeCurlCommand( + f"-v --fail -s -x 127.0.0.1:{self._ts.Variables.port} " + f"'http://multi.origin.com/{path}/delay/{MultiGroupAggregateTest._hold_seconds}'", + p=p, + ts=self._ts) + return p + + def _test_metrics_while_held(self) -> None: + """While the slow requests are still in flight, verify the live gauges reflect them.""" + total = MultiGroupAggregateTest._group_a_concurrency + MultiGroupAggregateTest._group_b_concurrency + group_max = max(MultiGroupAggregateTest._group_a_concurrency, MultiGroupAggregateTest._group_b_concurrency) + + tr = Test.AddTestRun("Check the host aggregate spans both groups while connections are held open") + tr.Processes.Default.Command = f'sleep {_STAT_SYNC_WAIT_SECONDS}; traffic_ctl metric match per_server' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.TimeOut = _STAT_SYNC_WAIT_SECONDS + 30 + tr.Processes.Default.Streams.All = Testers.ContainsExpression( + f'per_server.total_connection.multi.origin.com {total}', + 'The host aggregate total_connection should be the SUM across both groups ' + f'({MultiGroupAggregateTest._group_a_concurrency} + {MultiGroupAggregateTest._group_b_concurrency}).') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + f'per_server.current_connection.multi.origin.com {total}', + 'While held open, the host aggregate current_connection should be the SUM of the ' + 'currently open connections across both groups.') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + f'per_server.current_connection_max.multi.origin.com {group_max}', + 'While held open, current_connection_max should be the largest single group current ' + 'count (MAX), not the sum across the two groups.') + + def _test_metrics_after_drain(self) -> None: + """After traffic drains and a further sync tick passes, both live gauges must read 0. + + This validates the behavior the design exists to provide: an instantaneous gauge, unlike a + monotone peak, comes back down. + """ + tr = Test.AddTestRun("Check the host aggregate drains back to 0 after traffic stops") + # The slow requests are already _STAT_SYNC_WAIT_SECONDS old by now; wait for the rest of + # their hold time and then for another sync tick to observe the drop to 0. + wait = max(0, MultiGroupAggregateTest._hold_seconds - _STAT_SYNC_WAIT_SECONDS) + _STAT_SYNC_WAIT_SECONDS + tr.Processes.Default.Command = f'sleep {wait}; traffic_ctl metric match per_server' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.TimeOut = wait + 30 + tr.Processes.Default.Streams.All = Testers.ContainsExpression( + 'per_server.current_connection.multi.origin.com 0', + 'Once all connections close, the host aggregate current_connection must drain to 0.') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'per_server.current_connection_max.multi.origin.com 0', + 'Once all connections close, current_connection_max must also come back down to 0: it ' + 'is a live gauge, not a monotone peak.') + + def run(self) -> None: + """Drive concurrent traffic through both groups, then check the aggregate metrics.""" + tr = Test.AddTestRun() + tr.Processes.Default.StartBefore(self._dns) + tr.Processes.Default.StartBefore(self._server_a) + tr.Processes.Default.StartBefore(self._server_b) + tr.Processes.Default.StartBefore(self._ts) + + clients = [self._make_slow_client(tr, 'a') for _ in range(MultiGroupAggregateTest._group_a_concurrency)] + clients += [self._make_slow_client(tr, 'b') for _ in range(MultiGroupAggregateTest._group_b_concurrency)] + for p in clients: + tr.Processes.Default.StartBefore(p) + + # Let the slow requests connect and overlap before checking anything; they stay open for + # _hold_seconds from about this point. + tr.Processes.Default.Command = 'sleep 1' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.TimeOut = 30 + + self._test_metrics_while_held() + self._test_metrics_after_drain() + + PerServerConnectionMaxTest().run() -ConnectMethodTest(3).run(blocked=2, gold_file="gold/two_503_congested.gold") -ConnectMethodTest(0).run(blocked=0, gold_file="gold/two_200_ok.gold") +ConnectMethodTest(3, metric_level=1).run(blocked=2, gold_file="gold/two_503_congested.gold") +ConnectMethodTest(0, metric_level=2).run(blocked=0, gold_file="gold/two_200_ok.gold") +MultiGroupAggregateTest().run() From 6d4626cb2e49d22e115240b8c882310d7fce6f1a Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Mon, 17 Aug 2026 12:56:59 -0500 Subject: [PATCH 03/10] Make per_server.connection.metric_enabled overridable Enabling per upstream connection metrics registers several metrics per connection group, which is worth paying for on the origins being watched and not on the rest. A single global switch left no way to keep the metrics for most traffic while suppressing them for one mapping. Move metric_enabled from GlobalConfig to TxnConfig, thread the creating transaction's level into Group, and add TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED. metric_prefix stays global. The new key is appended to OVERRIDABLE_CONFIGS and to TSOverridableConfigKey so existing key values are unchanged. --- doc/admin-guide/files/records.yaml.en.rst | 13 +++ doc/admin-guide/plugins/lua.en.rst | 1 + .../functions/TSHttpOverridableConfig.en.rst | 1 + .../api/types/TSOverridableConfigKey.en.rst | 1 + include/iocore/net/ConnectionTracker.h | 27 ++++--- include/proxy/http/OverridableConfigDefs.h | 3 +- include/ts/apidefs.h.in | 1 + src/api/InkAPI.cc | 3 + src/iocore/net/ConnectionTracker.cc | 26 +++--- src/proxy/http/HttpSM.cc | 4 +- .../per_server_connection_max.test.py | 80 +++++++++++++++++++ 11 files changed, 136 insertions(+), 24 deletions(-) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index d34886df7ad..ad86bb24bdd 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2009,6 +2009,7 @@ Origin Server Connect Attempts .. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_enabled INT 0 :reloadable: + :overridable: Publish per upstream server connection metrics. These metrics are dynamically named, one set per upstream server group or hostname, so the number of them scales with the number of distinct @@ -2027,6 +2028,18 @@ Origin Server Connect Attempts :ts:cv:`match type ` includes the address or port, since there is then one set per address and port rather than one per hostname. + Because this is overridable, metrics can be enabled for the upstreams of interest and left off + for the rest, for example with :ref:`admin-plugins-conf-remap` on a specific mapping. + + The value is applied when a connection group is created. Where two mappings that disagree about + this setting resolve to the same group -- that is, the same key under + :ts:cv:`proxy.config.http.per_server.connection.match` -- the transaction that creates the group + determines its metrics, and later transactions do not change them. A group is discarded once its + connection count reaches zero, so the choice is made again the next time that upstream is + reopened. This affects only which metrics exist; enforcement of + :ts:cv:`proxy.config.http.per_server.connection.max` uses the group's own connection count and is + unaffected. + .. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_prefix STRING NULL :reloadable: diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index 0716e89c115..764ae44abe2 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -4746,6 +4746,7 @@ Http config constants TS_LUA_CONFIG_HTTP_SERVER_MIN_KEEP_ALIVE_CONNS TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH + TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DOWN_SERVER TS_LUA_CONFIG_HTTP_CONNECT_DOWN_POLICY diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 19e992d2599..b73b960853e 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -155,6 +155,7 @@ TSOverridableConfigKey Value Confi :enumerator:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` :enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH` :ts:cv:`proxy.config.http.per_server.connection.match` :enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX` :ts:cv:`proxy.config.http.per_server.connection.max` +:enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED` :ts:cv:`proxy.config.http.per_server.connection.metric_enabled` :enumerator:`TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED` :ts:cv:`proxy.config.http.post.check.content_length.enabled` :enumerator:`TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY` :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` :enumerator:`TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED` :ts:cv:`proxy.config.http.request_buffer_enabled` diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst index d9ae419c296..389a95c7455 100644 --- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst +++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst @@ -150,6 +150,7 @@ Enumeration Members .. enumerator:: TS_CONFIG_HTTP_ALLOW_HALF_OPEN .. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX .. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH +.. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED .. enumerator:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY .. enumerator:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES .. enumerator:: TS_CONFIG_SSL_CLIENT_SNI_POLICY diff --git a/include/iocore/net/ConnectionTracker.h b/include/iocore/net/ConnectionTracker.h index 5041c278274..9ad101c39fa 100644 --- a/include/iocore/net/ConnectionTracker.h +++ b/include/iocore/net/ConnectionTracker.h @@ -75,13 +75,6 @@ class ConnectionTracker /// String equivalents for @c MatchType. static const std::array(MATCH_BOTH) + 1> MATCH_TYPE_NAME; - /// Per transaction configuration values. - struct TxnConfig { - int server_max{0}; ///< Maximum concurrent server connections. - int server_min{0}; ///< Minimum keepalive server connections. - MatchType server_match{MATCH_IP}; ///< Server match type. - }; - /** Levels for the @c metric_enabled configuration. * * The per group metrics are always created, in the hidden metric store, whenever metrics are @@ -102,16 +95,23 @@ class ConnectionTracker METRIC_LEVEL_GROUP = 2, ///< The per hostname aggregates and the per group metrics are published. }; + /// Per transaction configuration values. + struct TxnConfig { + int server_max{0}; ///< Maximum concurrent server connections. + int server_min{0}; ///< Minimum keepalive server connections. + MatchType server_match{MATCH_IP}; ///< Server match type. + MetricLevel metric_enabled{METRIC_LEVEL_NONE}; ///< Which per server metrics to publish. + }; + /** Static configuration values. */ struct GlobalConfig { GlobalConfig() = default; GlobalConfig(GlobalConfig const &); GlobalConfig &operator=(GlobalConfig const &); - std::chrono::seconds client_alert_delay{60}; ///< Alert delay in seconds. - std::chrono::seconds server_alert_delay{60}; ///< Alert delay in seconds. - MetricLevel metric_enabled{METRIC_LEVEL_NONE}; ///< Which per server metrics to publish. - std::string metric_prefix; ///< Per server metric prefix. + std::chrono::seconds client_alert_delay{60}; ///< Alert delay in seconds. + std::chrono::seconds server_alert_delay{60}; ///< Alert delay in seconds. + std::string metric_prefix; ///< Per server metric prefix. swoc::IPRangeSet client_exempt_list; ///< The set of IP addresses to not block due client connection counting. mutable ts::bravo::shared_mutex client_exempt_list_mutex; ///< Protects client_exempt_list from concurrent access. }; @@ -176,8 +176,10 @@ class ConnectionTracker * @param key A populated @c Key structure - values are copied to the @c Group. * @param fqdn The full FQDN. * @param min_keep_alive The minimum number of origin keep alive connections to maintain. + * @param metric_enabled The metric level of the transaction that is creating this group. */ - Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive); + Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive, + MetricLevel metric_enabled = METRIC_LEVEL_NONE); ~Group(); /// Key equality checker. static bool equal(Key const &lhs, Key const &rhs); @@ -382,6 +384,7 @@ class ConnectionTracker static const MgmtConverter MIN_SERVER_CONV; static const MgmtConverter MAX_SERVER_CONV; static const MgmtConverter SERVER_MATCH_CONV; + static const MgmtConverter METRIC_ENABLED_CONV; protected: static GlobalConfig *_global_config; ///< Global configuration data. diff --git a/include/proxy/http/OverridableConfigDefs.h b/include/proxy/http/OverridableConfigDefs.h index 864461b1c8f..891012084a4 100644 --- a/include/proxy/http/OverridableConfigDefs.h +++ b/include/proxy/http/OverridableConfigDefs.h @@ -253,6 +253,7 @@ X(HTTP_CACHE_POST_METHOD, cache_post_method, "proxy.config.http.cache.post_method", INT, GENERIC) \ X(HTTP_CACHE_TARGETED_CACHE_CONTROL_HEADERS, targeted_cache_control_headers, "proxy.config.http.cache.targeted_cache_control_headers", STRING, TargetedCacheControlHeaders_Conv) \ X(SSL_CLIENT_CA_CERT_PATH, ssl_client_ca_cert_path, "proxy.config.ssl.client.CA.cert.path", STRING, NONE) \ - X(HTTP_CACHE_MAX_STALE_AGE_PERCENT, cache_max_stale_age_percent, "proxy.config.http.cache.max_stale_age_percent", INT, GENERIC) + X(HTTP_CACHE_MAX_STALE_AGE_PERCENT, cache_max_stale_age_percent, "proxy.config.http.cache.max_stale_age_percent", INT, GENERIC) \ + X(HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED, connection_tracker_config.metric_enabled, ConnectionTracker::CONFIG_SERVER_VAR_METRIC_ENABLED, INT, ConnectionTracker_METRIC_ENABLED_CONV) // clang-format on diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 8e110942b01..16e0a47beb3 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -919,6 +919,7 @@ enum TSOverridableConfigKey { TS_CONFIG_HTTP_CACHE_TARGETED_CACHE_CONTROL_HEADERS, TS_CONFIG_SSL_CLIENT_CA_CERT_PATH, TS_CONFIG_HTTP_CACHE_MAX_STALE_AGE_PERCENT, + TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED, TS_CONFIG_LAST_ENTRY, }; diff --git a/src/api/InkAPI.cc b/src/api/InkAPI.cc index 581dae89982..1b846d56318 100644 --- a/src/api/InkAPI.cc +++ b/src/api/InkAPI.cc @@ -7339,6 +7339,8 @@ _memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) -> typename std: case TS_CONFIG_##KEY: ret = &overridableHttpConfig->MEMBER; conv = &ConnectionTracker::MAX_SERVER_CONV; break; #define _CONF_CASE_ConnectionTracker_SERVER_MATCH_CONV(KEY, MEMBER) \ case TS_CONFIG_##KEY: ret = &overridableHttpConfig->MEMBER; conv = &ConnectionTracker::SERVER_MATCH_CONV; break; +#define _CONF_CASE_ConnectionTracker_METRIC_ENABLED_CONV(KEY, MEMBER) \ + case TS_CONFIG_##KEY: ret = &overridableHttpConfig->MEMBER; conv = &ConnectionTracker::METRIC_ENABLED_CONV; break; // Custom converter: Parses/formats host resolution preference strings. #define _CONF_CASE_HttpTransact_HOST_RES_CONV(KEY, MEMBER) \ @@ -7397,6 +7399,7 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr #undef _CONF_CASE_ConnectionTracker_MIN_SERVER_CONV #undef _CONF_CASE_ConnectionTracker_MAX_SERVER_CONV #undef _CONF_CASE_ConnectionTracker_SERVER_MATCH_CONV +#undef _CONF_CASE_ConnectionTracker_METRIC_ENABLED_CONV #undef _CONF_CASE_HttpTransact_HOST_RES_CONV #undef _CONF_CASE_TargetedCacheControlHeaders_Conv #undef _CONF_CASE_DISPATCH diff --git a/src/iocore/net/ConnectionTracker.cc b/src/iocore/net/ConnectionTracker.cc index e8d4fc1eee5..39478f897b1 100644 --- a/src/iocore/net/ConnectionTracker.cc +++ b/src/iocore/net/ConnectionTracker.cc @@ -72,10 +72,19 @@ const MgmtConverter ConnectionTracker::SERVER_MATCH_CONV{ } }}; +// Clamp on store so a plugin cannot leave an out of range level in the transaction config; the +// records reload path does its own clamping in Config_Update_Conntrack_Metric_Enabled. +const MgmtConverter ConnectionTracker::METRIC_ENABLED_CONV{ + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](void *data, MgmtInt i) -> void { + auto level = std::clamp(static_cast(i), static_cast(ConnectionTracker::METRIC_LEVEL_NONE), + static_cast(ConnectionTracker::METRIC_LEVEL_GROUP)); + *static_cast(data) = static_cast(level); + }}; + const std::array(ConnectionTracker::MATCH_BOTH) + 1> ConnectionTracker::MATCH_TYPE_NAME{ {"ip"sv, "port"sv, "host"sv, "both"sv} }; - // Make sure the clock is millisecond resolution or finer. static_assert(ConnectionTracker::Group::Clock::period::num == 1); static_assert(ConnectionTracker::Group::Clock::period::den >= 1000); @@ -155,7 +164,7 @@ Config_Update_Conntrack_Client_Alert_Delay(const char *name, RecDataT dtype, Rec bool Config_Update_Conntrack_Metric_Enabled(const char * /* name ATS_UNUSED */, RecDataT dtype, RecData data, void *cookie) { - auto config = static_cast(cookie); + auto config = static_cast(cookie); if (RECD_INT == dtype) { auto level = std::clamp(static_cast(data.rec_int), static_cast(ConnectionTracker::METRIC_LEVEL_NONE), @@ -272,7 +281,6 @@ ConnectionTracker::GlobalConfig::GlobalConfig(GlobalConfig const &other) { this->client_alert_delay = other.client_alert_delay; this->server_alert_delay = other.server_alert_delay; - this->metric_enabled = other.metric_enabled; this->metric_prefix = other.metric_prefix; // Lock the source to safely copy the exempt list. @@ -290,7 +298,6 @@ ConnectionTracker::GlobalConfig::operator=(GlobalConfig const &other) if (this != &other) { this->client_alert_delay = other.client_alert_delay; this->server_alert_delay = other.server_alert_delay; - this->metric_enabled = other.metric_enabled; this->metric_prefix = other.metric_prefix; // Lock both source and destination to safely copy the exempt list. // Lock in a consistent order to avoid deadlock (lock 'other' first, then 'this'). @@ -316,7 +323,7 @@ ConnectionTracker::config_init(GlobalConfig *global, TxnConfig *txn, RecConfigUp Enable_Config_Var(CONFIG_SERVER_VAR_MAX, &Config_Update_Conntrack_Max, config_cb, txn); Enable_Config_Var(CONFIG_SERVER_VAR_MATCH, &Config_Update_Conntrack_Match, config_cb, txn); Enable_Config_Var(CONFIG_SERVER_VAR_ALERT_DELAY, &Config_Update_Conntrack_Server_Alert_Delay, config_cb, global); - Enable_Config_Var(CONFIG_SERVER_VAR_METRIC_ENABLED, &Config_Update_Conntrack_Metric_Enabled, config_cb, global); + Enable_Config_Var(CONFIG_SERVER_VAR_METRIC_ENABLED, &Config_Update_Conntrack_Metric_Enabled, config_cb, txn); Enable_Config_Var(CONFIG_SERVER_VAR_METRIC_PREFIX, &Config_Update_Conntrack_Metric_Prefix, config_cb, global); } @@ -425,7 +432,7 @@ ConnectionTracker::obtain_outbound(TxnConfig const &txn_cnf, std::string_view fq if (loc != _outbound_table._table.end()) { zret._g = loc->second; } else { - zret._g = std::make_shared(Group::DirectionType::OUTBOUND, key, fqdn, txn_cnf.server_min); + zret._g = std::make_shared(Group::DirectionType::OUTBOUND, key, fqdn, txn_cnf.server_min, txn_cnf.metric_enabled); // Note that we must use zret._g's key, not the above key, because Key's // members are references to the Group's members. Thus the above key's // members are invalid after this function. @@ -434,7 +441,8 @@ ConnectionTracker::obtain_outbound(TxnConfig const &txn_cnf, std::string_view fq return zret; } -ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive) +ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive, + MetricLevel metric_enabled) : _direction{direction}, _hash(key._hash), _match_type(key._match_type), @@ -444,7 +452,7 @@ ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::st { Metrics::Gauge::increment(net_rsb.connection_tracker_table_size); // only add metrics for server connections - if (_global_config->metric_enabled != METRIC_LEVEL_NONE && direction == DirectionType::OUTBOUND) { + if (metric_enabled != METRIC_LEVEL_NONE && direction == DirectionType::OUTBOUND) { std::string _metric_name = metric_name(key, fqdn, _global_config->metric_prefix); // Per group metrics always live in the hidden store. metric_enabled controls what is published // from them (see MetricLevel), not whether they exist. @@ -468,7 +476,7 @@ ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::st Metrics::MetricType::GAUGE, _count_metric, Metrics::Derived::Op::MAX); } - if (_global_config->metric_enabled >= METRIC_LEVEL_GROUP) { + if (metric_enabled >= METRIC_LEVEL_GROUP) { // Mirror the per group metrics into the published store under their own name. A single // source SUM is an identity: the published value always equals the hidden source. Metrics::Derived::add_source("proxy.process.http.per_server.current_connection." + _metric_name, Metrics::MetricType::GAUGE, diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc index 4387b6d9b40..ef942ddd883 100644 --- a/src/proxy/http/HttpSM.cc +++ b/src/proxy/http/HttpSM.cc @@ -5891,7 +5891,7 @@ HttpSM::do_http_server_open(bool raw, bool only_direct) // See if the outbound connection tracker data is needed. If so, get it here for consistency. if (t_state.txn_conf->connection_tracker_config.server_max > 0 || t_state.txn_conf->connection_tracker_config.server_min > 0 || - t_state.http_config_param->global_connection_tracker_config.metric_enabled) { + t_state.txn_conf->connection_tracker_config.metric_enabled) { t_state.outbound_conn_track_state = ConnectionTracker::obtain_outbound(t_state.txn_conf->connection_tracker_config, std::string_view{t_state.current.server->name}, t_state.current.server->dst_addr); @@ -5919,7 +5919,7 @@ HttpSM::do_http_server_open(bool raw, bool only_direct) ct_state.update_max_count(ccount); } else if (t_state.txn_conf->connection_tracker_config.server_min > 0 || - t_state.http_config_param->global_connection_tracker_config.metric_enabled) { + t_state.txn_conf->connection_tracker_config.metric_enabled) { auto &ct_state = t_state.outbound_conn_track_state; // Feed the count through as well, otherwise the group's peak stays at zero whenever metrics // are enabled without a configured maximum. diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index 9dc9b12f204..49b7e22d685 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -388,7 +388,87 @@ def run(self) -> None: self._test_metrics_after_drain() +class MetricOverrideTest: + """Verify proxy.config.http.per_server.connection.metric_enabled is overridable per remap rule. + + Metrics are enabled globally at level 2 and one of the two remap rules turns them off with + conf_remap. The two rules point at different origin ports and the match is 'port', so each gets + its own group and the two decisions cannot influence each other. + """ + + def __init__(self) -> None: + """Configure the test processes in preparation for the TestRun.""" + self._dns = Test.MakeDNServer("dns_metric_override", default='127.0.0.1') + self._server_on = Test.MakeHttpBinServer("server_metric_on") + self._server_off = Test.MakeHttpBinServer("server_metric_off") + self._configure_trafficserver() + + def _configure_trafficserver(self) -> None: + """Configure Traffic Server to be used in the test.""" + self._ts = Test.MakeATSProcess("ts_metric_override") + self._ts.Disk.records_config.update( + { + 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|conn_track', + # Enabled globally; the second remap rule below opts out. + 'proxy.config.http.per_server.connection.metric_enabled': 2, + 'proxy.config.http.per_server.connection.match': 'port', + }) + self._ts.Disk.remap_config.AddLines( + [ + f'map http://metric-on.com/ http://127.0.0.1:{self._server_on.Variables.Port}/', + f'map http://metric-off.com/ http://127.0.0.1:{self._server_off.Variables.Port}/' + ' @plugin=conf_remap.so' + ' @pparam=proxy.config.http.per_server.connection.metric_enabled=0', + ]) + + def _test_metrics(self) -> None: + """Use traffic_ctl to verify which per server metrics exist.""" + on_group = f'127.0.0.1:{self._server_on.Variables.Port}' + off_group = f'127.0.0.1:{self._server_off.Variables.Port}' + + tr = Test.AddTestRun("Check that only the non-overridden remap has per server metrics") + tr.Processes.Default.Command = f'sleep {_STAT_SYNC_WAIT_SECONDS}; traffic_ctl metric match per_server' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.TimeOut = _STAT_SYNC_WAIT_SECONDS + 30 + tr.Processes.Default.Streams.All = Testers.ContainsExpression( + f'per_server.total_connection.{on_group} 1', 'The remap with metrics enabled should have per server metrics.') + # The group for the overridden remap must not exist at all, hidden or otherwise, so this + # also holds with --include-hidden below. + tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + f'per_server.total_connection.{off_group}', 'The remap with metrics disabled should have no per server metrics.') + + tr2 = Test.AddTestRun("The overridden remap has no hidden per server metrics either") + tr2.Processes.Default.Command = 'traffic_ctl metric match per_server --include-hidden' + tr2.Processes.Default.ReturnCode = 0 + tr2.Processes.Default.Env = self._ts.Env + tr2.Processes.Default.Streams.All = Testers.ContainsExpression( + f'per_server.total_connection.{on_group} 1', 'The enabled remap group should be present in the hidden store.') + tr2.Processes.Default.Streams.All += Testers.ExcludesExpression( + f'per_server.total_connection.{off_group}', 'No group should be created at all for the overridden remap.') + + def run(self) -> None: + """Configure the TestRun.""" + tr = Test.AddTestRun('Verify metric_enabled is overridable per remap rule') + tr.Processes.Default.StartBefore(self._dns) + tr.Processes.Default.StartBefore(self._server_on) + tr.Processes.Default.StartBefore(self._server_off) + tr.Processes.Default.StartBefore(self._ts) + tr.MakeCurlCommandMulti( + f"{{curl}} -v -s -H 'Host: metric-on.com' http://127.0.0.1:{self._ts.Variables.port}/get" + f" --next -v -s -H 'Host: metric-off.com' http://127.0.0.1:{self._ts.Variables.port}/get") + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.TimeOut = 30 + tr.StillRunningAfter = self._ts + + self._test_metrics() + + PerServerConnectionMaxTest().run() ConnectMethodTest(3, metric_level=1).run(blocked=2, gold_file="gold/two_503_congested.gold") ConnectMethodTest(0, metric_level=2).run(blocked=0, gold_file="gold/two_200_ok.gold") MultiGroupAggregateTest().run() +MetricOverrideTest().run() From 647e4d08937be64952cced979daff9b3f86c6870 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Mon, 17 Aug 2026 13:21:33 -0500 Subject: [PATCH 04/10] Shorten per_server_connection_max by driving the stat sync interval The derived metric assertions here each slept more than five seconds because Metrics::Derived::update_derived() runs from raw_stat_sync_cont, whose period comes from proxy.config.raw_stat_sync_interval_ms. A comment claimed nothing in records.yaml drove that interval; it does, so set it short in each ATS instance and cut the waits to match. The record is left startup only. Sleeps drop from ~43s to ~17s and the aggregate test's connection hold from 10s (httpbin's clamp of 12) to 6s. --- .../per_server_connection_max.test.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index 49b7e22d685..ab03f35ff6a 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -23,12 +23,22 @@ Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) -# The per hostname aggregates are derived metrics, recomputed once per -# REC_RAW_STAT_SYNC_INTERVAL_MS (5000ms, src/records/P_RecDefs.h) on an ET_TASK thread. Nothing in -# records.yaml drives that interval, so a test has to sleep comfortably longer than one sync period -# before reading an aggregate rather than trying to configure a faster one. Reading too early -# silently compares against zeros. -_STAT_SYNC_WAIT_SECONDS: int = 6 +# The per hostname aggregates are derived metrics. Metrics::Derived::update_derived() runs from +# raw_stat_sync_cont (src/iocore/eventsystem/RecProcess.cc), which is scheduled every +# proxy.config.raw_stat_sync_interval_ms. That record defaults to 5000ms, which would force every +# assertion here to sleep more than five seconds; each ATS instance below shortens it so the waits +# can be short instead. The record is startup only, so it has to be set in records.yaml rather than +# adjusted at runtime. Reading before a tick lands silently compares against zeros. +_STAT_SYNC_INTERVAL_MS: int = 500 + +# How long to wait before reading a derived metric. Several sync periods, to absorb ET_TASK +# scheduling jitter and the traffic_ctl round trip rather than racing the tick. +_STAT_SYNC_WAIT_SECONDS: int = 2 + +# The records.yaml settings every ATS instance in this file needs for the waits above to hold. +_STAT_SYNC_RECORDS: dict = { + 'proxy.config.raw_stat_sync_interval_ms': _STAT_SYNC_INTERVAL_MS, +} # NOTE: assigning to a Streams attribute REPLACES any tester already set for that stream # (TesterSet.Assign), so every assertion after the first on the same stream must use '+=' or it @@ -61,6 +71,7 @@ def _configure_trafficserver(self) -> None: self._ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') self._ts.Disk.records_config.update( { + **_STAT_SYNC_RECORDS, 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.diags.debug.enabled': 1, @@ -150,6 +161,7 @@ def _configure_trafficserver(self, max_conn, metric_level) -> None: self._ts.Disk.records_config.update( { + **_STAT_SYNC_RECORDS, 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.diags.debug.enabled': 1, @@ -271,9 +283,8 @@ class MultiGroupAggregateTest: # How long each request holds its connection open. Must comfortably exceed # _STAT_SYNC_WAIT_SECONDS so a sync tick is guaranteed to land while the connections are still - # open. NOTE: httpbin clamps /delay/ to 10 seconds, so the effective hold is min(this, 10); - # the waits below are derived from this value and tolerate that clamp. - _hold_seconds: int = 12 + # open. Well under the 10 second cap httpbin puts on /delay/, so no clamping applies. + _hold_seconds: int = 6 def __init__(self) -> None: """Configure the test processes in preparation for the TestRun.""" @@ -296,6 +307,7 @@ def _configure_trafficserver(self) -> None: self._ts = Test.MakeATSProcess(f"magg_ts_{MultiGroupAggregateTest._process_counter}") self._ts.Disk.records_config.update( { + **_STAT_SYNC_RECORDS, 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.diags.debug.enabled': 1, @@ -408,6 +420,7 @@ def _configure_trafficserver(self) -> None: self._ts = Test.MakeATSProcess("ts_metric_override") self._ts.Disk.records_config.update( { + **_STAT_SYNC_RECORDS, 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.diags.debug.enabled': 1, From 138ec1fd729e3f38eb5ad833d347d2a785545711 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Mon, 17 Aug 2026 13:49:00 -0500 Subject: [PATCH 05/10] Share one microDNS server across per_server_connection_max The file started five nameservers, one per test class, all configured identically with a wildcard 127.0.0.1 answer. Each process costs about five seconds when the test tears down, so four of them were 20 seconds of pure shutdown for no coverage. Use a single shared server, started by whichever run needs it first. Also note in the header comment that StillRunningAfter is a TesterSet, so it has the same '=' clobbers '+=' hazard already documented for Streams. --- .../per_server_connection_max.test.py | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index ab03f35ff6a..d55d606af31 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -42,7 +42,31 @@ # NOTE: assigning to a Streams attribute REPLACES any tester already set for that stream # (TesterSet.Assign), so every assertion after the first on the same stream must use '+=' or it -# silently discards the earlier ones. +# silently discards the earlier ones. The same applies to StillRunningAfter and the other process +# state checks: they are TesterSets too. + +# One microDNS server shared by every ATS instance here. All of them want the same thing, a +# wildcard answer of 127.0.0.1, and each extra process costs about five seconds when the test tears +# down, so this file uses a single server rather than one per test class. +_dns = Test.MakeDNServer("dns", default='127.0.0.1') +_dns_started: bool = False + + +def _use_shared_dns(tr) -> None: + """Make the shared nameserver available to a TestRun. + + Only the first run may start it: StartBefore is tracked per TestRun, so asking twice would try + to start an already running process. Later runs just assert it is still alive. + """ + global _dns_started + + if _dns_started: + # '+=': StillRunningAfter is a TesterSet like Streams, so '=' would discard any + # process check the caller has already set on this run. + tr.StillRunningAfter += _dns + else: + tr.Processes.Default.StartBefore(_dns) + _dns_started = True class PerServerConnectionMaxTest: @@ -59,7 +83,7 @@ def __init__(self) -> None: def _configure_dns(self) -> None: """Configure a nameserver for the test.""" - self._dns = Test.MakeDNServer("dns", default='127.0.0.1') + self._dns = _dns def _configure_server(self) -> None: """Configure the server to be used in the test.""" @@ -114,7 +138,7 @@ def _test_metrics(self) -> None: def run(self) -> None: """Configure the TestRun.""" tr = Test.AddTestRun('Verify we enforce proxy.config.http.per_server.connection.max') - tr.Processes.Default.StartBefore(self._dns) + _use_shared_dns(tr) tr.Processes.Default.StartBefore(self._server) tr.Processes.Default.StartBefore(self._ts) @@ -150,7 +174,7 @@ def __init__(self, max_conn, metric_level=1) -> None: def _configure_dns(self) -> None: """Configure a nameserver for the test.""" - self._dns = Test.MakeDNServer(f"dns_{ConnectMethodTest._process_counter}", default='127.0.0.1') + self._dns = _dns def _configure_origin_server(self) -> None: """Configure the httpbin origin server.""" @@ -230,7 +254,7 @@ def _test_metrics(self, blocked) -> None: def run(self, blocked, gold_file) -> None: """Verify per_server.connection.max with CONNECT traffic.""" tr = Test.AddTestRun() - tr.Processes.Default.StartBefore(self._dns) + _use_shared_dns(tr) tr.Processes.Default.StartBefore(self._server) tr.Processes.Default.StartBefore(self._ts) @@ -295,7 +319,7 @@ def __init__(self) -> None: def _configure_dns(self) -> None: """Configure a nameserver for the test.""" - self._dns = Test.MakeDNServer(f"magg_dns_{MultiGroupAggregateTest._process_counter}", default='127.0.0.1') + self._dns = _dns def _configure_origin_servers(self) -> None: """Configure the two httpbin origins which stand in for two groups of one hostname.""" @@ -380,7 +404,7 @@ def _test_metrics_after_drain(self) -> None: def run(self) -> None: """Drive concurrent traffic through both groups, then check the aggregate metrics.""" tr = Test.AddTestRun() - tr.Processes.Default.StartBefore(self._dns) + _use_shared_dns(tr) tr.Processes.Default.StartBefore(self._server_a) tr.Processes.Default.StartBefore(self._server_b) tr.Processes.Default.StartBefore(self._ts) @@ -410,7 +434,7 @@ class MetricOverrideTest: def __init__(self) -> None: """Configure the test processes in preparation for the TestRun.""" - self._dns = Test.MakeDNServer("dns_metric_override", default='127.0.0.1') + self._dns = _dns self._server_on = Test.MakeHttpBinServer("server_metric_on") self._server_off = Test.MakeHttpBinServer("server_metric_off") self._configure_trafficserver() @@ -466,7 +490,7 @@ def _test_metrics(self) -> None: def run(self) -> None: """Configure the TestRun.""" tr = Test.AddTestRun('Verify metric_enabled is overridable per remap rule') - tr.Processes.Default.StartBefore(self._dns) + _use_shared_dns(tr) tr.Processes.Default.StartBefore(self._server_on) tr.Processes.Default.StartBefore(self._server_off) tr.Processes.Default.StartBefore(self._ts) @@ -475,7 +499,7 @@ def run(self) -> None: f" --next -v -s -H 'Host: metric-off.com' http://127.0.0.1:{self._ts.Variables.port}/get") tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 30 - tr.StillRunningAfter = self._ts + tr.StillRunningAfter += self._ts self._test_metrics() From 770a7f01a09a46f66077019fbbda41f71c316a0c Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Mon, 17 Aug 2026 16:48:06 -0500 Subject: [PATCH 06/10] Split per server metric publication out of metric_enabled metric_enabled shipped in 10.2.0 as a plain on/off, where 1 published the per group metrics. Redefining 1 to mean "aggregates only, per group hidden" silently removed those metrics on upgrade, and for any match type other than 'both' left nothing published at all, since aggregates exist only for that match type. Restore metric_enabled to 0/1 with its original meaning and move the publication choice to a new overridable metric_aggregate: 0 publishes the per group metrics (the default, matching 10.2.0), 1 adds the hostname aggregates, 2 publishes only the aggregates. Value 2 falls back to publishing the per group metrics for a group that has no aggregate, so that combination cannot report nothing. --- doc/admin-guide/files/records.yaml.en.rst | 50 +++++++++++++--- .../statistics/core/http-connection.en.rst | 12 ++-- doc/admin-guide/plugins/lua.en.rst | 1 + .../functions/TSHttpOverridableConfig.en.rst | 1 + .../api/types/TSOverridableConfigKey.en.rst | 1 + include/iocore/net/ConnectionTracker.h | 55 ++++++++++-------- include/proxy/http/OverridableConfigDefs.h | 3 +- include/ts/apidefs.h.in | 1 + src/api/InkAPI.cc | 3 + src/iocore/net/ConnectionTracker.cc | 57 +++++++++++++----- src/records/RecordsConfig.cc | 4 +- .../per_server_connection_max.test.py | 58 ++++++++++--------- 12 files changed, 164 insertions(+), 82 deletions(-) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index ad86bb24bdd..d97380c36d0 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2011,22 +2011,20 @@ Origin Server Connect Attempts :reloadable: :overridable: - Publish per upstream server connection metrics. These metrics are dynamically named, one set per - upstream server group or hostname, so the number of them scales with the number of distinct - upstream servers seen. See :ref:`per-server-connection-metrics`. + Enable per upstream server connection metrics. These metrics are dynamically named, one set per + upstream server group, so the number of them scales with the number of distinct upstream servers + seen. See :ref:`per-server-connection-metrics`. ===== ====================================================================================== Value Effect ===== ====================================================================================== ``0`` No per server connection metrics. - ``1`` Publish only the per hostname aggregate metrics. The per group metrics from which the - aggregates are computed exist internally but are not published. - ``2`` Publish the per hostname aggregates and the per group metrics. + ``1`` Per server connection metrics are collected for each upstream server group. ===== ====================================================================================== - Level ``2`` can produce a very large number of metrics when the - :ts:cv:`match type ` includes the address or - port, since there is then one set per address and port rather than one per hostname. + What is published from them is controlled separately by + :ts:cv:`proxy.config.http.per_server.connection.metric_aggregate`, which by default publishes the + per group metrics themselves. Because this is overridable, metrics can be enabled for the upstreams of interest and left off for the rest, for example with :ref:`admin-plugins-conf-remap` on a specific mapping. @@ -2040,6 +2038,40 @@ Origin Server Connect Attempts :ts:cv:`proxy.config.http.per_server.connection.max` uses the group's own connection count and is unaffected. +.. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_aggregate INT 0 + :reloadable: + :overridable: + + Control what is published from the per server connection metrics enabled by + :ts:cv:`proxy.config.http.per_server.connection.metric_enabled`. Has no effect when that setting + is ``0``. + + A per hostname aggregate sums a counter across every group belonging to that hostname, and exists + only for :ts:cv:`match type ` ``both``, since that + is the only match type with more than one group per hostname. See + :ref:`per-server-connection-metrics`. + + ===== ====================================================================================== + Value Effect + ===== ====================================================================================== + ``0`` No aggregates. The per group metrics are published under their own names. + ``1`` Publish the per hostname aggregates and the per group metrics. + ``2`` Publish only the per hostname aggregates. The per group metrics from which they are + computed are collected but not published, which keeps the number of published metrics + proportional to hostnames rather than to groups. + ===== ====================================================================================== + + With value ``2``, a group that has no aggregate to belong to -- any match type other than + ``both`` -- has its per group metrics published anyway, since otherwise nothing at all would be + reported for it. + + Values ``0`` and ``1`` can produce a very large number of metrics when the match type includes the + address or port, since there is then one set per address and port rather than one per hostname. + + Like :ts:cv:`proxy.config.http.per_server.connection.metric_enabled`, this is applied when a + connection group is created, with the same consequence for mappings that disagree and resolve to + the same group. + .. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_prefix STRING NULL :reloadable: diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index 78a75e11f8f..6546dba56b5 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -208,9 +208,9 @@ Per Server Connection Metrics Unlike the metrics above these do not have fixed names. They are created dynamically, one set per upstream server group as defined by :ts:cv:`proxy.config.http.per_server.connection.match`, and, -when the match type is ``both``, one aggregate set per hostname. Whether any of them are published, -and at what granularity, is controlled by -:ts:cv:`proxy.config.http.per_server.connection.metric_enabled`. An optional +when the match type is ``both``, one aggregate set per hostname. Whether they are collected at all is +controlled by :ts:cv:`proxy.config.http.per_server.connection.metric_enabled`, and which of them are +published by :ts:cv:`proxy.config.http.per_server.connection.metric_aggregate`. An optional :ts:cv:`proxy.config.http.per_server.connection.metric_prefix` can be inserted into the names. Per group names are ``proxy.process.http.per_server..``, where ```` depends on @@ -246,14 +246,14 @@ current_connection_max Every published per server metric is recomputed periodically, currently every 5 seconds, rather than on every connection event, so a reader sees a value up to that interval old. This is true of the -hostname aggregates and, at level ``2``, of the published per group metrics as well: those are +hostname aggregates and of the published per group metrics alike: those are mirrored from the internal ones by the same periodic mechanism, not written as connections open and close. It applies to ``current_connection_max`` too, which reports the maximum across groups as of the last sample rather than a running peak. To obtain the peak over a longer window, compute a maximum over time from this gauge in the monitoring system. -At :ts:cv:`metric_enabled ` level ``1`` the -per group metrics still exist internally, since the aggregates are computed from them, but are not +At :ts:cv:`metric_aggregate ` value ``2`` +the per group metrics still exist internally, since the aggregates are computed from them, but are not published. They can be listed with ``traffic_ctl metric match per_server --include-hidden``, which reads them directly and so is not subject to the sampling delay above. That visibility is intended for debugging and is not a stable interface: the existence, granularity and naming of the per group diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index 764ae44abe2..9b52391cff2 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -4747,6 +4747,7 @@ Http config constants TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED + TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_AGGREGATE TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DOWN_SERVER TS_LUA_CONFIG_HTTP_CONNECT_DOWN_POLICY diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index b73b960853e..f8c0a814b52 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -155,6 +155,7 @@ TSOverridableConfigKey Value Confi :enumerator:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` :enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH` :ts:cv:`proxy.config.http.per_server.connection.match` :enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX` :ts:cv:`proxy.config.http.per_server.connection.max` +:enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_AGGREGATE` :ts:cv:`proxy.config.http.per_server.connection.metric_aggregate` :enumerator:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED` :ts:cv:`proxy.config.http.per_server.connection.metric_enabled` :enumerator:`TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED` :ts:cv:`proxy.config.http.post.check.content_length.enabled` :enumerator:`TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY` :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst index 389a95c7455..74897622a5e 100644 --- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst +++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst @@ -151,6 +151,7 @@ Enumeration Members .. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX .. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH .. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED +.. enumerator:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_AGGREGATE .. enumerator:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY .. enumerator:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES .. enumerator:: TS_CONFIG_SSL_CLIENT_SNI_POLICY diff --git a/include/iocore/net/ConnectionTracker.h b/include/iocore/net/ConnectionTracker.h index 9ad101c39fa..3abaa5a90cf 100644 --- a/include/iocore/net/ConnectionTracker.h +++ b/include/iocore/net/ConnectionTracker.h @@ -75,32 +75,38 @@ class ConnectionTracker /// String equivalents for @c MatchType. static const std::array(MATCH_BOTH) + 1> MATCH_TYPE_NAME; - /** Levels for the @c metric_enabled configuration. + /** Whether, and how, the per hostname aggregate metrics are published. * - * The per group metrics are always created, in the hidden metric store, whenever metrics are - * enabled at all. What varies by level is what gets published: - * - @c METRIC_LEVEL_NONE: no per group metrics are created and nothing is published. - * - @c METRIC_LEVEL_HOST: the per group metrics stay hidden; the per hostname aggregates - * (computed across the groups of a hostname by a @c Derived metric) are published. - * - @c METRIC_LEVEL_GROUP: as above, plus the per group metrics are also mirrored into the - * published store. + * This is independent of @c TxnConfig::metric_enabled, which decides only whether per server + * metrics exist for a group at all. The per group metrics are always created in the hidden metric + * store; what varies here is what gets published from them: + * - @c AGGREGATE_NONE: no aggregate. The per group metrics are published under their own names. + * This is the default and matches the behavior of releases that had no aggregate support. + * - @c AGGREGATE_GROUP: the per hostname aggregates are published, and so are the per group + * metrics they are computed from. + * - @c AGGREGATE_ONLY: the per hostname aggregates are published and the per group metrics stay + * hidden, which keeps the published metric count proportional to hostnames rather than to + * groups. Where a group has no aggregate to belong to -- see @c Group::host_metric_name, which + * only yields a name for match type @c MATCH_BOTH -- the per group metrics are published + * anyway, since otherwise nothing at all would be reported for that group. * - * Keeping the per group metrics in the hidden store at every level means changing the level at - * runtime is only a change of what is registered for publication, with no metric to migrate - * between the two stores. + * Keeping the per group metrics in the hidden store in every case means changing this at runtime + * is only a change of what is registered for publication, with no metric to migrate between the + * two stores. */ - enum MetricLevel { - METRIC_LEVEL_NONE = 0, ///< No per server metrics. - METRIC_LEVEL_HOST = 1, ///< Only the per hostname aggregate metrics are published. - METRIC_LEVEL_GROUP = 2, ///< The per hostname aggregates and the per group metrics are published. + enum MetricAggregate { + AGGREGATE_NONE = 0, ///< No hostname aggregate; the per group metrics are published. + AGGREGATE_GROUP = 1, ///< Hostname aggregates published, along with the per group metrics. + AGGREGATE_ONLY = 2, ///< Hostname aggregates published, per group metrics kept hidden. }; /// Per transaction configuration values. struct TxnConfig { - int server_max{0}; ///< Maximum concurrent server connections. - int server_min{0}; ///< Minimum keepalive server connections. - MatchType server_match{MATCH_IP}; ///< Server match type. - MetricLevel metric_enabled{METRIC_LEVEL_NONE}; ///< Which per server metrics to publish. + int server_max{0}; ///< Maximum concurrent server connections. + int server_min{0}; ///< Minimum keepalive server connections. + MatchType server_match{MATCH_IP}; ///< Server match type. + int metric_enabled{0}; ///< Whether per server metrics exist for a group. + MetricAggregate metric_aggregate{AGGREGATE_NONE}; ///< What is published, see @c MetricAggregate. }; /** Static configuration values. */ @@ -126,6 +132,7 @@ class ConnectionTracker static constexpr std::string_view CONFIG_SERVER_VAR_MATCH{"proxy.config.http.per_server.connection.match"}; static constexpr std::string_view CONFIG_SERVER_VAR_ALERT_DELAY{"proxy.config.http.per_server.connection.alert_delay"}; static constexpr std::string_view CONFIG_SERVER_VAR_METRIC_ENABLED{"proxy.config.http.per_server.connection.metric_enabled"}; + static constexpr std::string_view CONFIG_SERVER_VAR_METRIC_AGGREGATE{"proxy.config.http.per_server.connection.metric_aggregate"}; static constexpr std::string_view CONFIG_SERVER_VAR_METRIC_PREFIX{"proxy.config.http.per_server.connection.metric_prefix"}; /// A record for the outbound connection count. @@ -166,7 +173,7 @@ class ConnectionTracker std::atomic _last_alert{0}; ///< Absolute time of the last alert. // Recording data as metrics. These are always in the hidden metric store when created; see - // @c MetricLevel for how they are published. + // @c MetricAggregate for how they are published. ts::Metrics::Gauge::AtomicType *_count_metric = nullptr; ts::Metrics::Counter::AtomicType *_count_total_metric = nullptr; ts::Metrics::Counter::AtomicType *_blocked_metric = nullptr; @@ -176,10 +183,11 @@ class ConnectionTracker * @param key A populated @c Key structure - values are copied to the @c Group. * @param fqdn The full FQDN. * @param min_keep_alive The minimum number of origin keep alive connections to maintain. - * @param metric_enabled The metric level of the transaction that is creating this group. + * @param metric_enabled Whether the transaction creating this group wants per server metrics. + * @param metric_aggregate What that transaction wants published, see @c MetricAggregate. */ - Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive, - MetricLevel metric_enabled = METRIC_LEVEL_NONE); + Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive, int metric_enabled = 0, + MetricAggregate metric_aggregate = AGGREGATE_NONE); ~Group(); /// Key equality checker. static bool equal(Key const &lhs, Key const &rhs); @@ -385,6 +393,7 @@ class ConnectionTracker static const MgmtConverter MAX_SERVER_CONV; static const MgmtConverter SERVER_MATCH_CONV; static const MgmtConverter METRIC_ENABLED_CONV; + static const MgmtConverter METRIC_AGGREGATE_CONV; protected: static GlobalConfig *_global_config; ///< Global configuration data. diff --git a/include/proxy/http/OverridableConfigDefs.h b/include/proxy/http/OverridableConfigDefs.h index 891012084a4..8b18797494f 100644 --- a/include/proxy/http/OverridableConfigDefs.h +++ b/include/proxy/http/OverridableConfigDefs.h @@ -254,6 +254,7 @@ X(HTTP_CACHE_TARGETED_CACHE_CONTROL_HEADERS, targeted_cache_control_headers, "proxy.config.http.cache.targeted_cache_control_headers", STRING, TargetedCacheControlHeaders_Conv) \ X(SSL_CLIENT_CA_CERT_PATH, ssl_client_ca_cert_path, "proxy.config.ssl.client.CA.cert.path", STRING, NONE) \ X(HTTP_CACHE_MAX_STALE_AGE_PERCENT, cache_max_stale_age_percent, "proxy.config.http.cache.max_stale_age_percent", INT, GENERIC) \ - X(HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED, connection_tracker_config.metric_enabled, ConnectionTracker::CONFIG_SERVER_VAR_METRIC_ENABLED, INT, ConnectionTracker_METRIC_ENABLED_CONV) + X(HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED, connection_tracker_config.metric_enabled, ConnectionTracker::CONFIG_SERVER_VAR_METRIC_ENABLED, INT, ConnectionTracker_METRIC_ENABLED_CONV) \ + X(HTTP_PER_SERVER_CONNECTION_METRIC_AGGREGATE, connection_tracker_config.metric_aggregate, ConnectionTracker::CONFIG_SERVER_VAR_METRIC_AGGREGATE, INT, ConnectionTracker_METRIC_AGGREGATE_CONV) // clang-format on diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 16e0a47beb3..ab85d096de9 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -920,6 +920,7 @@ enum TSOverridableConfigKey { TS_CONFIG_SSL_CLIENT_CA_CERT_PATH, TS_CONFIG_HTTP_CACHE_MAX_STALE_AGE_PERCENT, TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_ENABLED, + TS_CONFIG_HTTP_PER_SERVER_CONNECTION_METRIC_AGGREGATE, TS_CONFIG_LAST_ENTRY, }; diff --git a/src/api/InkAPI.cc b/src/api/InkAPI.cc index 1b846d56318..47f2495d7d5 100644 --- a/src/api/InkAPI.cc +++ b/src/api/InkAPI.cc @@ -7341,6 +7341,8 @@ _memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) -> typename std: case TS_CONFIG_##KEY: ret = &overridableHttpConfig->MEMBER; conv = &ConnectionTracker::SERVER_MATCH_CONV; break; #define _CONF_CASE_ConnectionTracker_METRIC_ENABLED_CONV(KEY, MEMBER) \ case TS_CONFIG_##KEY: ret = &overridableHttpConfig->MEMBER; conv = &ConnectionTracker::METRIC_ENABLED_CONV; break; +#define _CONF_CASE_ConnectionTracker_METRIC_AGGREGATE_CONV(KEY, MEMBER) \ + case TS_CONFIG_##KEY: ret = &overridableHttpConfig->MEMBER; conv = &ConnectionTracker::METRIC_AGGREGATE_CONV; break; // Custom converter: Parses/formats host resolution preference strings. #define _CONF_CASE_HttpTransact_HOST_RES_CONV(KEY, MEMBER) \ @@ -7400,6 +7402,7 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr #undef _CONF_CASE_ConnectionTracker_MAX_SERVER_CONV #undef _CONF_CASE_ConnectionTracker_SERVER_MATCH_CONV #undef _CONF_CASE_ConnectionTracker_METRIC_ENABLED_CONV +#undef _CONF_CASE_ConnectionTracker_METRIC_AGGREGATE_CONV #undef _CONF_CASE_HttpTransact_HOST_RES_CONV #undef _CONF_CASE_TargetedCacheControlHeaders_Conv #undef _CONF_CASE_DISPATCH diff --git a/src/iocore/net/ConnectionTracker.cc b/src/iocore/net/ConnectionTracker.cc index 39478f897b1..db53c8de499 100644 --- a/src/iocore/net/ConnectionTracker.cc +++ b/src/iocore/net/ConnectionTracker.cc @@ -72,14 +72,20 @@ const MgmtConverter ConnectionTracker::SERVER_MATCH_CONV{ } }}; -// Clamp on store so a plugin cannot leave an out of range level in the transaction config; the -// records reload path does its own clamping in Config_Update_Conntrack_Metric_Enabled. +// Both of these clamp on store so a plugin cannot leave an out of range value in the transaction +// config; the records reload path clamps separately in its own update callbacks. const MgmtConverter ConnectionTracker::METRIC_ENABLED_CONV{ - [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, [](void *data, MgmtInt i) -> void { - auto level = std::clamp(static_cast(i), static_cast(ConnectionTracker::METRIC_LEVEL_NONE), - static_cast(ConnectionTracker::METRIC_LEVEL_GROUP)); - *static_cast(data) = static_cast(level); + *static_cast(data) = std::clamp(static_cast(i), 0, 1); + }}; + +const MgmtConverter ConnectionTracker::METRIC_AGGREGATE_CONV{ + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](void *data, MgmtInt i) -> void { + auto level = std::clamp(static_cast(i), static_cast(ConnectionTracker::AGGREGATE_NONE), + static_cast(ConnectionTracker::AGGREGATE_ONLY)); + *static_cast(data) = static_cast(level); }}; const std::array(ConnectionTracker::MATCH_BOTH) + 1> ConnectionTracker::MATCH_TYPE_NAME{ @@ -167,9 +173,21 @@ Config_Update_Conntrack_Metric_Enabled(const char * /* name ATS_UNUSED */, RecDa auto config = static_cast(cookie); if (RECD_INT == dtype) { - auto level = std::clamp(static_cast(data.rec_int), static_cast(ConnectionTracker::METRIC_LEVEL_NONE), - static_cast(ConnectionTracker::METRIC_LEVEL_GROUP)); - config->metric_enabled = static_cast(level); + config->metric_enabled = std::clamp(static_cast(data.rec_int), 0, 1); + return true; + } + return false; +} + +bool +Config_Update_Conntrack_Metric_Aggregate(const char * /* name ATS_UNUSED */, RecDataT dtype, RecData data, void *cookie) +{ + auto config = static_cast(cookie); + + if (RECD_INT == dtype) { + auto level = std::clamp(static_cast(data.rec_int), static_cast(ConnectionTracker::AGGREGATE_NONE), + static_cast(ConnectionTracker::AGGREGATE_ONLY)); + config->metric_aggregate = static_cast(level); return true; } return false; @@ -324,6 +342,7 @@ ConnectionTracker::config_init(GlobalConfig *global, TxnConfig *txn, RecConfigUp Enable_Config_Var(CONFIG_SERVER_VAR_MATCH, &Config_Update_Conntrack_Match, config_cb, txn); Enable_Config_Var(CONFIG_SERVER_VAR_ALERT_DELAY, &Config_Update_Conntrack_Server_Alert_Delay, config_cb, global); Enable_Config_Var(CONFIG_SERVER_VAR_METRIC_ENABLED, &Config_Update_Conntrack_Metric_Enabled, config_cb, txn); + Enable_Config_Var(CONFIG_SERVER_VAR_METRIC_AGGREGATE, &Config_Update_Conntrack_Metric_Aggregate, config_cb, txn); Enable_Config_Var(CONFIG_SERVER_VAR_METRIC_PREFIX, &Config_Update_Conntrack_Metric_Prefix, config_cb, global); } @@ -432,7 +451,8 @@ ConnectionTracker::obtain_outbound(TxnConfig const &txn_cnf, std::string_view fq if (loc != _outbound_table._table.end()) { zret._g = loc->second; } else { - zret._g = std::make_shared(Group::DirectionType::OUTBOUND, key, fqdn, txn_cnf.server_min, txn_cnf.metric_enabled); + zret._g = std::make_shared(Group::DirectionType::OUTBOUND, key, fqdn, txn_cnf.server_min, txn_cnf.metric_enabled, + txn_cnf.metric_aggregate); // Note that we must use zret._g's key, not the above key, because Key's // members are references to the Group's members. Thus the above key's // members are invalid after this function. @@ -442,7 +462,7 @@ ConnectionTracker::obtain_outbound(TxnConfig const &txn_cnf, std::string_view fq } ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::string_view fqdn, int min_keep_alive, - MetricLevel metric_enabled) + int metric_enabled, MetricAggregate metric_aggregate) : _direction{direction}, _hash(key._hash), _match_type(key._match_type), @@ -452,17 +472,19 @@ ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::st { Metrics::Gauge::increment(net_rsb.connection_tracker_table_size); // only add metrics for server connections - if (metric_enabled != METRIC_LEVEL_NONE && direction == DirectionType::OUTBOUND) { + if (metric_enabled && direction == DirectionType::OUTBOUND) { std::string _metric_name = metric_name(key, fqdn, _global_config->metric_prefix); - // Per group metrics always live in the hidden store. metric_enabled controls what is published - // from them (see MetricLevel), not whether they exist. + // Per group metrics always live in the hidden store. metric_aggregate controls what is + // published from them (see MetricAggregate), not whether they exist. _count_metric = Metrics::Gauge::createHiddenPtr("proxy.process.http.per_server.current_connection.", _metric_name); _count_total_metric = Metrics::Counter::createHiddenPtr("proxy.process.http.per_server.total_connection.", _metric_name); _blocked_metric = Metrics::Counter::createHiddenPtr("proxy.process.http.per_server.blocked_connection.", _metric_name); // Only MATCH_BOTH groups have siblings sharing a hostname to aggregate across. std::string _host_metric_name = host_metric_name(key, fqdn, _global_config->metric_prefix); - if (!_host_metric_name.empty()) { + bool const has_aggregate = !_host_metric_name.empty(); + + if (has_aggregate && metric_aggregate != AGGREGATE_NONE) { Metrics::Derived::add_source("proxy.process.http.per_server.current_connection." + _host_metric_name, Metrics::MetricType::GAUGE, _count_metric, Metrics::Derived::Op::SUM); Metrics::Derived::add_source("proxy.process.http.per_server.total_connection." + _host_metric_name, @@ -476,7 +498,10 @@ ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::st Metrics::MetricType::GAUGE, _count_metric, Metrics::Derived::Op::MAX); } - if (metric_enabled >= METRIC_LEVEL_GROUP) { + // AGGREGATE_ONLY suppresses the per group metrics to keep the published count proportional to + // hostnames. Without an aggregate to stand in for them there would be nothing at all reported + // for this group, so in that case publish them regardless. + if (metric_aggregate != AGGREGATE_ONLY || !has_aggregate) { // Mirror the per group metrics into the published store under their own name. A single // source SUM is an identity: the published value always equals the hidden source. Metrics::Derived::add_source("proxy.process.http.per_server.current_connection." + _metric_name, Metrics::MetricType::GAUGE, diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index f2663de8f62..f432e9c2481 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -404,7 +404,9 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.per_server.connection.min", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-2]$", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-1]$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_aggregate", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-2]$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http.per_server.connection.metric_prefix", RECD_STRING, "", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index d55d606af31..982f2fd1599 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -1,6 +1,7 @@ ''' Verify the behavior of proxy.config.http.per_server.connection.max and the per server -connection metrics (proxy.config.http.per_server.connection.metric_enabled). +connection metrics (proxy.config.http.per_server.connection.metric_enabled and +proxy.config.http.per_server.connection.metric_aggregate). ''' # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -101,10 +102,10 @@ def _configure_trafficserver(self) -> None: 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http|conn_track', 'proxy.config.http.per_server.connection.max': self._origin_max_connections, - # Level 2 (METRIC_LEVEL_GROUP): the match here is 'port', which never has more than - # one group per hostname, so there is no aggregate to read and the per group - # metrics themselves have to be published to be checked below. - 'proxy.config.http.per_server.connection.metric_enabled': 2, + # The match here is 'port', which has no hostname aggregate, so the per group + # metrics themselves are what gets checked below. That is what the default + # metric_aggregate of 0 publishes, so only metric_enabled is needed. + 'proxy.config.http.per_server.connection.metric_enabled': 1, 'proxy.config.http.per_server.connection.metric_prefix': 'foo', 'proxy.config.http.per_server.connection.match': 'port', }) @@ -117,8 +118,8 @@ def _test_metrics(self) -> None: group_name = f'foo.127.0.0.1:{self._server.Variables.http_port}' tr = Test.AddTestRun("Check connection metrics") - # At level 2 the per group metrics are published by mirroring the hidden ones through a - # derived metric, so a sync tick has to pass before they carry a value. + # The per group metrics are published by mirroring the hidden ones through a derived + # metric, so a sync tick has to pass before they carry a value. tr.Processes.Default.Command = f'sleep {_STAT_SYNC_WAIT_SECONDS}; traffic_ctl metric match per_server' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = self._ts.Env @@ -150,10 +151,11 @@ def run(self) -> None: class ConnectMethodTest: """Test our max origin connection behavior with CONNECT traffic. - Also covers the two publication levels of proxy.config.http.per_server.connection.metric_enabled: - - 1 (METRIC_LEVEL_HOST): only the per hostname aggregate is published; the per group metrics + Also covers the two aggregate-publishing modes of + proxy.config.http.per_server.connection.metric_aggregate: + - 2 (AGGREGATE_ONLY): only the per hostname aggregate is published; the per group metrics stay hidden and are visible only with --include-hidden. - - 2 (METRIC_LEVEL_GROUP): the per hostname aggregate is published, and the per group metrics + - 1 (AGGREGATE_GROUP): the per hostname aggregate is published, and the per group metrics are also mirrored into the published store. The match here defaults to 'both' and there is exactly one group for this hostname, so the @@ -164,12 +166,12 @@ class ConnectMethodTest: _process_counter: int = 0 _client_counter: int = 0 - def __init__(self, max_conn, metric_level=1) -> None: + def __init__(self, max_conn, metric_aggregate=2) -> None: """Configure the server processes in preparation for the TestRun.""" - self._metric_level = metric_level + self._metric_aggregate = metric_aggregate self._configure_dns() self._configure_origin_server() - self._configure_trafficserver(max_conn, metric_level) + self._configure_trafficserver(max_conn, metric_aggregate) ConnectMethodTest._process_counter += 1 def _configure_dns(self) -> None: @@ -180,8 +182,8 @@ def _configure_origin_server(self) -> None: """Configure the httpbin origin server.""" self._server = Test.MakeHttpBinServer(f"server_{ConnectMethodTest._process_counter}") - def _configure_trafficserver(self, max_conn, metric_level) -> None: - self._ts = Test.MakeATSProcess(f"ts2_{max_conn}_{metric_level}") + def _configure_trafficserver(self, max_conn, metric_aggregate) -> None: + self._ts = Test.MakeATSProcess(f"ts2_{max_conn}_{metric_aggregate}") self._ts.Disk.records_config.update( { @@ -192,7 +194,8 @@ def _configure_trafficserver(self, max_conn, metric_level) -> None: 'proxy.config.diags.debug.tags': 'http|dns|hostdb|conn_track', 'proxy.config.http.server_ports': f"{self._ts.Variables.port} {self._ts.Variables.uds_path}", 'proxy.config.http.connect_ports': f"{self._server.Variables.Port}", - 'proxy.config.http.per_server.connection.metric_enabled': metric_level, + 'proxy.config.http.per_server.connection.metric_enabled': 1, + 'proxy.config.http.per_server.connection.metric_aggregate': metric_aggregate, 'proxy.config.http.per_server.connection.max': max_conn, }) @@ -219,23 +222,23 @@ def _test_metrics(self, blocked) -> None: tr.Processes.Default.Env = self._ts.Env tr.Processes.Default.TimeOut = _STAT_SYNC_WAIT_SECONDS + 30 - # The per hostname aggregate is published at every non-zero level. + # The per hostname aggregate is published in both modes under test. tr.Processes.Default.Streams.All = Testers.ContainsExpression( f'per_server.total_connection.{host_name} 5', 'incorrect statistic return, or possible error.') tr.Processes.Default.Streams.All += Testers.ContainsExpression( f'per_server.blocked_connection.{host_name} {blocked}', 'incorrect statistic return, or possible error.') - if self._metric_level >= 2: - # METRIC_LEVEL_GROUP additionally mirrors the per group metrics into the published store. + if self._metric_aggregate == 1: + # AGGREGATE_GROUP additionally mirrors the per group metrics into the published store. tr.Processes.Default.Streams.All += Testers.ContainsExpression( - f'per_server.total_connection.{group_name} 5', 'The per group metric should be published at METRIC_LEVEL_GROUP.') + f'per_server.total_connection.{group_name} 5', 'The per group metric should be published at AGGREGATE_GROUP.') else: - # METRIC_LEVEL_HOST keeps the per group metrics hidden, so none of the three per group + # AGGREGATE_ONLY keeps the per group metrics hidden, so none of the three per group # names may appear in a normal query. current_connection_max is not among them: it only # ever exists as a hostname aggregate, never per group. for counter in ('current_connection', 'total_connection', 'blocked_connection'): tr.Processes.Default.Streams.All += Testers.ExcludesExpression( - f'per_server.{counter}.{group_name} ', f'per_server.{counter}.{group_name} must stay hidden at level 1.') + f'per_server.{counter}.{group_name} ', f'per_server.{counter}.{group_name} must stay hidden at AGGREGATE_ONLY.') # The per group metrics must be visible with --include-hidden at either level. This is also # the end to end test for that traffic_ctl option. @@ -337,6 +340,9 @@ def _configure_trafficserver(self) -> None: 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http|dns|hostdb|conn_track', 'proxy.config.http.per_server.connection.metric_enabled': 1, + # Aggregates only: the per group metrics stay hidden, which is what this test is + # about reading through the aggregate. + 'proxy.config.http.per_server.connection.metric_aggregate': 2, 'proxy.config.http.per_server.connection.match': 'both', }) self._ts.Disk.remap_config.AddLines( @@ -427,7 +433,7 @@ def run(self) -> None: class MetricOverrideTest: """Verify proxy.config.http.per_server.connection.metric_enabled is overridable per remap rule. - Metrics are enabled globally at level 2 and one of the two remap rules turns them off with + Metrics are enabled globally and one of the two remap rules turns them off with conf_remap. The two rules point at different origin ports and the match is 'port', so each gets its own group and the two decisions cannot influence each other. """ @@ -450,7 +456,7 @@ def _configure_trafficserver(self) -> None: 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http|conn_track', # Enabled globally; the second remap rule below opts out. - 'proxy.config.http.per_server.connection.metric_enabled': 2, + 'proxy.config.http.per_server.connection.metric_enabled': 1, 'proxy.config.http.per_server.connection.match': 'port', }) self._ts.Disk.remap_config.AddLines( @@ -505,7 +511,7 @@ def run(self) -> None: PerServerConnectionMaxTest().run() -ConnectMethodTest(3, metric_level=1).run(blocked=2, gold_file="gold/two_503_congested.gold") -ConnectMethodTest(0, metric_level=2).run(blocked=0, gold_file="gold/two_200_ok.gold") +ConnectMethodTest(3, metric_aggregate=2).run(blocked=2, gold_file="gold/two_503_congested.gold") +ConnectMethodTest(0, metric_aggregate=1).run(blocked=0, gold_file="gold/two_200_ok.gold") MultiGroupAggregateTest().run() MetricOverrideTest().run() From 5e7cc8ec93b9ed2686614a8f8673969ebef53b6c Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Mon, 17 Aug 2026 21:15:36 -0500 Subject: [PATCH 07/10] Don't clamp the per server metric configs in the API converters SDK_API_OVERRIDABLE_CONFIGS sets every overridable INT config to an arbitrary value and requires it to read back unchanged, so clamping in the converter fails the test. SERVER_MATCH_CONV right above these already made that tradeoff for the same reason; follow it. Range checking stays where it belongs: records.yaml validates the value and the reload callbacks clamp. Give MetricAggregate a fixed underlying type so that storing a value outside 0..2, which the API now permits, is defined rather than UB. Such a value publishes both the aggregate and the per group metrics. --- include/iocore/net/ConnectionTracker.h | 6 +++++- src/iocore/net/ConnectionTracker.cc | 15 +++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/include/iocore/net/ConnectionTracker.h b/include/iocore/net/ConnectionTracker.h index 3abaa5a90cf..99971478a11 100644 --- a/include/iocore/net/ConnectionTracker.h +++ b/include/iocore/net/ConnectionTracker.h @@ -93,8 +93,12 @@ class ConnectionTracker * Keeping the per group metrics in the hidden store in every case means changing this at runtime * is only a change of what is registered for publication, with no metric to migrate between the * two stores. + * + * The records layer validates and clamps this to 0..2. A plugin setting the overridable config + * directly is not clamped, see @c METRIC_AGGREGATE_CONV; any other value behaves as + * @c AGGREGATE_GROUP, publishing both the aggregate and the per group metrics. */ - enum MetricAggregate { + enum MetricAggregate : int { AGGREGATE_NONE = 0, ///< No hostname aggregate; the per group metrics are published. AGGREGATE_GROUP = 1, ///< Hostname aggregates published, along with the per group metrics. AGGREGATE_ONLY = 2, ///< Hostname aggregates published, per group metrics kept hidden. diff --git a/src/iocore/net/ConnectionTracker.cc b/src/iocore/net/ConnectionTracker.cc index db53c8de499..44c17cd7d46 100644 --- a/src/iocore/net/ConnectionTracker.cc +++ b/src/iocore/net/ConnectionTracker.cc @@ -72,20 +72,23 @@ const MgmtConverter ConnectionTracker::SERVER_MATCH_CONV{ } }}; -// Both of these clamp on store so a plugin cannot leave an out of range value in the transaction -// config; the records reload path clamps separately in its own update callbacks. +// Neither of these clamps, for the same reason as SERVER_MATCH_CONV above: the InkAPITest +// regression test requires an arbitrary integer to round trip through the setter and getter. The +// records paths do the range checking instead -- records.yaml validates the value and the reload +// callbacks below clamp -- so an out of range value is only reachable by a plugin that sets one +// deliberately. Both settings degrade safely if that happens: any non-zero metric_enabled enables +// metrics, and any metric_aggregate outside 0..2 publishes both the aggregate and the per group +// metrics, the same as AGGREGATE_GROUP. const MgmtConverter ConnectionTracker::METRIC_ENABLED_CONV{ [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, [](void *data, MgmtInt i) -> void { - *static_cast(data) = std::clamp(static_cast(i), 0, 1); + *static_cast(data) = static_cast(i); }}; const MgmtConverter ConnectionTracker::METRIC_AGGREGATE_CONV{ [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, [](void *data, MgmtInt i) -> void { - auto level = std::clamp(static_cast(i), static_cast(ConnectionTracker::AGGREGATE_NONE), - static_cast(ConnectionTracker::AGGREGATE_ONLY)); - *static_cast(data) = static_cast(level); + *static_cast(data) = static_cast(i); }}; const std::array(ConnectionTracker::MATCH_BOTH) + 1> ConnectionTracker::MATCH_TYPE_NAME{ From edb63d3f9e51a79a509b262a650d49e0008a9589 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Fri, 21 Aug 2026 12:28:29 -0500 Subject: [PATCH 08/10] Cover the metric_aggregate fallback with no host aggregate Every existing test that sets metric_aggregate to 2 uses match 'both', so the branch that publishes per group metrics when a match type has no hostname aggregate was unexercised: dropping it would have left the suite green while reporting nothing at all for those groups. --- .../per_server_connection_max.test.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index 982f2fd1599..05f2c5587c1 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -510,8 +510,79 @@ def run(self) -> None: self._test_metrics() +class AggregateOnlyWithoutHostAggregateTest: + """Verify metric_aggregate 2 still publishes per group metrics when there is no aggregate. + + metric_aggregate 2 (AGGREGATE_ONLY) normally leaves the per group metrics hidden and publishes + only the per hostname aggregate. That aggregate exists only under match 'both', which is the + only match type with more than one group per hostname (Group::host_metric_name returns empty + for the others). With match 'port' there is therefore nothing for the aggregate to stand in + for, so the per group metrics have to be published regardless, or level 2 would report nothing + at all for this group. + + Every other test in this file that sets metric_aggregate 2 uses match 'both', so without this + case a regression that dropped the fallback would leave the suite green. + """ + + def __init__(self) -> None: + """Configure the test processes in preparation for the TestRun.""" + self._dns = _dns + self._server = Test.MakeHttpBinServer("agg_only_no_host_server") + self._configure_trafficserver() + + def _configure_trafficserver(self) -> None: + """Configure Traffic Server for aggregates only against a match type with no aggregate.""" + self._ts = Test.MakeATSProcess("ts_agg_only_no_host") + self._ts.Disk.records_config.update( + { + **_STAT_SYNC_RECORDS, + 'proxy.config.dns.nameservers': f"127.0.0.1:{self._dns.Variables.Port}", + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|conn_track', + 'proxy.config.http.per_server.connection.metric_enabled': 1, + 'proxy.config.http.per_server.connection.metric_aggregate': 2, + # 'port' has no per hostname aggregate, which is the point of this test. + 'proxy.config.http.per_server.connection.match': 'port', + }) + self._ts.Disk.remap_config.AddLine(f'map http://agg-only.com/ http://127.0.0.1:{self._server.Variables.Port}/') + + def _test_metrics(self) -> None: + """Verify the per group metrics are published despite metric_aggregate 2.""" + group = f'127.0.0.1:{self._server.Variables.Port}' + + tr = Test.AddTestRun("Check the per group metrics are published when no aggregate exists") + tr.Processes.Default.Command = f'sleep {_STAT_SYNC_WAIT_SECONDS}; traffic_ctl metric match per_server' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.TimeOut = _STAT_SYNC_WAIT_SECONDS + 30 + # Published, not just hidden: this query does not pass --include-hidden. + tr.Processes.Default.Streams.All = Testers.ContainsExpression( + f'per_server.total_connection.{group} 1', + 'At metric_aggregate 2 with a match type that has no aggregate, the per group metric ' + 'must still be published.') + # The hostname never appears in a metric name under match 'port', so its absence confirms + # the published metric came from the per group fallback and not from an aggregate. + tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + 'per_server.total_connection.agg-only.com', 'No per hostname aggregate should exist for match "port".') + + def run(self) -> None: + """Drive one request through the origin, then check the metrics.""" + tr = Test.AddTestRun('Verify metric_aggregate 2 falls back to per group metrics') + _use_shared_dns(tr) + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.MakeCurlCommand(f"-v -s -H 'Host: agg-only.com' http://127.0.0.1:{self._ts.Variables.port}/get", ts=self._ts) + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.TimeOut = 30 + tr.StillRunningAfter += self._ts + + self._test_metrics() + + PerServerConnectionMaxTest().run() ConnectMethodTest(3, metric_aggregate=2).run(blocked=2, gold_file="gold/two_503_congested.gold") ConnectMethodTest(0, metric_aggregate=1).run(blocked=0, gold_file="gold/two_200_ok.gold") MultiGroupAggregateTest().run() MetricOverrideTest().run() +AggregateOnlyWithoutHostAggregateTest().run() From 7ec180057270eb6afc5a309c763f9fd957e3e427 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Tue, 25 Aug 2026 12:14:20 -0500 Subject: [PATCH 09/10] Document what a metric_aggregate reload cannot undo Publication can be added at runtime but never removed, so switching metric_aggregate to 2 does not reduce the number of published metrics on a running server, which is the one thing an operator sets it for. Also correct the claim that only match type 'both' has more than one group per hostname: MATCH_IP keys on the address alone, so several A records already produce several groups. The accurate reason an aggregate needs 'both' is that ip and port keys carry no hostname at all. Records the two ways overridable settings make an aggregate misleading: a group joins only if the mapping that first opened the upstream had aggregation enabled, and a hostname using both 'host' and 'both' publishes the group and the aggregate under one name. --- doc/admin-guide/files/records.yaml.en.rst | 28 ++++++++++++++----- .../statistics/core/http-connection.en.rst | 21 +++++++++++--- include/iocore/net/ConnectionTracker.h | 18 ++++++++---- src/iocore/net/ConnectionTracker.cc | 3 +- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index d97380c36d0..5e50150b0c4 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2033,8 +2033,13 @@ Origin Server Connect Attempts this setting resolve to the same group -- that is, the same key under :ts:cv:`proxy.config.http.per_server.connection.match` -- the transaction that creates the group determines its metrics, and later transactions do not change them. A group is discarded once its - connection count reaches zero, so the choice is made again the next time that upstream is - reopened. This affects only which metrics exist; enforcement of + connection count reaches zero, so *raising* the level of publication is picked up the next time + that upstream is reopened: enabling metrics, or enabling the aggregates, takes effect as upstreams + reconnect. Lowering it does not. Metrics are never retired once published, so disabling this + setting, or switching + :ts:cv:`proxy.config.http.per_server.connection.metric_aggregate` to ``2``, leaves the names that + are already published in place, frozen at their last sampled value, until |TS| is restarted. This + affects only which metrics exist; enforcement of :ts:cv:`proxy.config.http.per_server.connection.max` uses the group's own connection count and is unaffected. @@ -2046,10 +2051,10 @@ Origin Server Connect Attempts :ts:cv:`proxy.config.http.per_server.connection.metric_enabled`. Has no effect when that setting is ``0``. - A per hostname aggregate sums a counter across every group belonging to that hostname, and exists - only for :ts:cv:`match type ` ``both``, since that - is the only match type with more than one group per hostname. See - :ref:`per-server-connection-metrics`. + A per hostname aggregate sums a counter across every group belonging to that hostname that has + aggregation enabled, and exists only for + :ts:cv:`match type ` ``both``, since that is the + only match type whose group key carries the hostname. See :ref:`per-server-connection-metrics`. ===== ====================================================================================== Value Effect @@ -2070,7 +2075,16 @@ Origin Server Connect Attempts Like :ts:cv:`proxy.config.http.per_server.connection.metric_enabled`, this is applied when a connection group is created, with the same consequence for mappings that disagree and resolve to - the same group. + the same group. A group joins its hostname's aggregate only if the mapping that first opened that + upstream had aggregation enabled, so mappings that disagree for one hostname produce an aggregate + that covers only part of it. + + The reload is one-directional for the same reason given under + :ts:cv:`proxy.config.http.per_server.connection.metric_enabled`. Raising the value takes effect + as upstreams reconnect, but moving to ``2`` does not hide per group metrics that are already + published, and moving from ``1`` to ``0`` does not stop the hostname aggregates from publishing. + Reducing the number of published metrics therefore requires a restart, which matters most for + ``2``, the value chosen specifically to bound that number. .. ts:cv:: CONFIG proxy.config.http.per_server.connection.metric_prefix STRING NULL :reloadable: diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index 6546dba56b5..47075ce6fa9 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -217,9 +217,10 @@ Per group names are ``proxy.process.http.per_server..``, where ` the match type: an IP address, an ``address:port`` pair, a hostname, or, for ``both``, ``.``. Per hostname names are ``proxy.process.http.per_server..``. Aggregates exist only for match type -``both``, because that is the only match type with more than one group per hostname; for match type -``host`` the group name is already the bare hostname, so an aggregate would carry the same name as -the single group it summarises. +``both``, because that is the only match type whose group key carries the hostname. An ``ip`` or +``port`` group is keyed on the address alone and is shared by every hostname that resolves to it, so +there is no single hostname to aggregate it under. For match type ``host`` the group name is already +the bare hostname, so an aggregate would carry the same name as the single group it summarises. For a group, ```` is one of: @@ -234,7 +235,7 @@ blocked_connection :ts:cv:`proxy.config.http.per_server.connection.max`. Never decreases. For a hostname aggregate, ```` is one of those three, each summed across the groups of that -hostname, plus: +hostname which have aggregation enabled, plus: current_connection_max Gauge. The largest ``current_connection`` value among the groups of that hostname at the moment @@ -244,6 +245,18 @@ current_connection_max blocked. Like ``current_connection`` it rises and falls with traffic and is not a high-water mark. There is no per group ``current_connection_max``; it exists only as a hostname aggregate. +Because :ts:cv:`proxy.config.http.per_server.connection.metric_aggregate` is overridable, a group +joins its hostname's aggregate only if the mapping that first opened that upstream had aggregation +enabled. Mappings that disagree for one hostname therefore produce an aggregate over part of it: the +sums cover a subset of the groups and ``current_connection_max`` takes its maximum over that same +subset, with nothing in the metric to indicate it. Keeping the setting uniform across the mappings +for a hostname avoids this. + +Because :ts:cv:`proxy.config.http.per_server.connection.match` is also overridable, one hostname can +use match type ``host`` on one mapping and ``both`` on another. The ``host`` group and the hostname +aggregate are then published under the same name and merged into a single metric that carries both, +so a hostname should use one match type throughout. + Every published per server metric is recomputed periodically, currently every 5 seconds, rather than on every connection event, so a reader sees a value up to that interval old. This is true of the hostname aggregates and of the published per group metrics alike: those are diff --git a/include/iocore/net/ConnectionTracker.h b/include/iocore/net/ConnectionTracker.h index 99971478a11..e18f6aa66c6 100644 --- a/include/iocore/net/ConnectionTracker.h +++ b/include/iocore/net/ConnectionTracker.h @@ -208,10 +208,18 @@ class ConnectionTracker /** Name of the metric which aggregates a value across all groups of a hostname. * - * Only @c MATCH_BOTH groups have more than one group per hostname. For @c MATCH_HOST there is - * exactly one group per hostname, so an aggregate would be over a set of one, and - * @c Group::metric_name already returns the FQDN alone for that match type - identical to what - * this would return, so publishing both would collide on one name. + * Only @c MATCH_BOTH keys carry both a hostname and an address, so it is the only match type + * whose groups can be gathered by hostname at all. @c MATCH_IP and @c MATCH_PORT key on the + * address alone and one such group is shared by every hostname resolving to it, so there is no + * single hostname to aggregate it under. For @c MATCH_HOST there is exactly one group per + * hostname, so an aggregate would be over a set of one, and @c Group::metric_name already + * returns the FQDN alone for that match type - identical to what this would return, so + * publishing both would collide on one name. + * + * Note that reasoning holds within a single match type. @c TxnConfig::server_match is + * overridable, so one hostname can be @c MATCH_HOST on one mapping and @c MATCH_BOTH on + * another, and then that group's own published name and this aggregate name are the same + * string and are merged into one derived metric. * * @param key The group key. * @param fqdn The full FQDN. @@ -488,7 +496,7 @@ inline std::string ConnectionTracker::Group::host_metric_name(const Key &key, std::string_view fqdn, std::string metric_prefix) { if (MATCH_BOTH != key._match_type) { - return {}; // Only MATCH_BOTH has more than one group per hostname to aggregate across. + return {}; // Only MATCH_BOTH keys carry the hostname needed to gather groups under it. } return metric_prefix.empty() ? std::string(fqdn) : metric_prefix + "." + std::string(fqdn); } diff --git a/src/iocore/net/ConnectionTracker.cc b/src/iocore/net/ConnectionTracker.cc index 44c17cd7d46..85519616328 100644 --- a/src/iocore/net/ConnectionTracker.cc +++ b/src/iocore/net/ConnectionTracker.cc @@ -506,7 +506,8 @@ ConnectionTracker::Group::Group(DirectionType direction, Key const &key, std::st // for this group, so in that case publish them regardless. if (metric_aggregate != AGGREGATE_ONLY || !has_aggregate) { // Mirror the per group metrics into the published store under their own name. A single - // source SUM is an identity: the published value always equals the hidden source. + // source SUM combines nothing, but the published value is still a sample: it is whatever + // the last derived tick read, and it reads 0 from creation until that first tick. Metrics::Derived::add_source("proxy.process.http.per_server.current_connection." + _metric_name, Metrics::MetricType::GAUGE, _count_metric, Metrics::Derived::Op::SUM); Metrics::Derived::add_source("proxy.process.http.per_server.total_connection." + _metric_name, Metrics::MetricType::COUNTER, From eca982740ea80a97523b1e689ca1bb878db2cb6b Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Tue, 25 Aug 2026 12:14:20 -0500 Subject: [PATCH 10/10] Drive the stat sync interval in per_server_metric_enabled The per group metric this asserts is now published by a derived mirror on the raw_stat_sync_interval_ms tick. Against the 5000ms default the existing six second wait leaves at most one second of margin, so shorten the interval rather than racing it under ASan on shared CI. --- .../origin_connection/per_server_metric_enabled.test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/gold_tests/origin_connection/per_server_metric_enabled.test.py b/tests/gold_tests/origin_connection/per_server_metric_enabled.test.py index 510265c3df9..30091ecac51 100644 --- a/tests/gold_tests/origin_connection/per_server_metric_enabled.test.py +++ b/tests/gold_tests/origin_connection/per_server_metric_enabled.test.py @@ -36,6 +36,12 @@ class PerServerMetricEnabledTest: _replay_file: str = 'per_server_metric_enabled.replay.yaml' _keep_alive_timeout: int = 2 + # The per group metric asserted below is published by mirroring the internal one through a + # derived metric, refreshed every proxy.config.raw_stat_sync_interval_ms. The 5000ms default + # would leave at most one second of margin inside this test's wait, so shorten the interval + # rather than racing it. The record is startup only and so has to be set in records.yaml. + _stat_sync_interval_ms: int = 500 + def __init__(self) -> None: """Configure the test processes in preparation for the TestRun.""" self._configure_server() @@ -51,6 +57,7 @@ def _configure_trafficserver(self) -> None: self._ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') self._ts.Disk.records_config.update( { + 'proxy.config.raw_stat_sync_interval_ms': self._stat_sync_interval_ms, 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http_ss|conn_track', 'proxy.config.http.per_server.connection.metric_enabled': 1,