From f6ee0a8f81c0e569c4c923492133e92ff88bd98f Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 16 Jul 2026 15:43:07 +0200 Subject: [PATCH] aggregator-head-lag: drop 'unless changes==0' from series queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: the guard was designed for modern 15s-cadence data, where changes([15m]) works cleanly. On backfilled historical data compacted to ~1 sample/hour, changes([15m]) returns 0 even though the gauge is healthy — the 30D view lost the first 8 days of data to false-drops (all three providers marked DATA MISSING for a whole week). Real-time silence is still surfaced by the live_activity scalar (pill + banner + downtime band), which runs against fresh scrape cadence and stays accurate. The chart series now shows every point Prom has, flat during old outages instead of gapped. --- benchmarks/aggregator-head-lag.yml | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index f3eb9cd8..816c18bb 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -161,14 +161,14 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="mobula"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) - # `unless ... == 0` drops the sample when the gauge saw no fresh - # event in the last 15 min. Without this, the head_lag gauge stays - # frozen at its last value during a full WS outage and the chart - # draws a flat line where the reader would expect a gap. The - # per-series match means a partial outage on one chain leaves the - # other chains' points untouched (aggregate line stays continuous - # when at least one chain is still receiving events). - series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 + # No `unless changes == 0` guard on the historical series: it + # false-drops points on backfilled data (Prom-compacted to ~1 + # sample per hour, so `changes[15m]` returns 0 even though the + # gauge was healthy at the time). The 30D view was losing the + # first ~8 days of data to this. Live-feed silence is still + # surfaced by the `live_activity` scalar below (badge + banner + # + downtime pill), which runs against fresh 15s-cadence data. + series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 # Total gauge-value changes on this aggregator across chains and # regions in the last 15min. Non-zero = fresh events arrived; 0 = # every subscription has been silent for 15+ minutes (the gauge is @@ -178,13 +178,13 @@ providers: regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 - slug: codex name: Codex @@ -197,18 +197,18 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="codex"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 - slug: geckoterminal name: GeckoTerminal @@ -221,16 +221,16 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000