From 25b556eaf3b947502a73d0dcc3aedbbd9a18fba4 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 30 Apr 2026 20:07:23 +0200 Subject: [PATCH] central prometheus: shared infra/, strip per-harness ops --- CONTRIBUTING.md | 7 +- README.md | 61 +- harnesses/README.md | 31 +- harnesses/aggregator-head-lag/Makefile | 151 -- harnesses/aggregator-head-lag/README.md | 97 +- harnesses/aggregator-head-lag/assets/logo.png | Bin 65169 -> 0 bytes .../cmd/script/update_metrics_calls.sh | 24 - .../aggregator-head-lag/docker-compose.yml | 82 - .../aggregator-head-lag/grafana-entrypoint.sh | 29 - .../aggregator-head-lag/grafana/Dockerfile | 34 - .../disabled/mobula_pulse_vs_fasttrade.json | 595 ----- .../dashboards/disabled/pulse_vs_codex.json | 661 ------ .../disabled/quote_api_latency.json | 916 -------- .../grafana/dashboards/head_lag.json | 945 -------- .../grafana/grafana-entrypoint.sh | 29 - .../provisioning/dashboards/dashboard.yml | 13 - .../provisioning/datasources/prometheus.yml | 10 - .../monitoring/alert_rules.yml | 127 - .../monitoring/alertmanager.yml | 36 - .../disabled/quote_api_latency.json | 819 ------- .../grafana/dashboards/head_lag.json | 773 ------- .../provisioning/dashboards/dashboard.yml | 13 - .../provisioning/datasources/prometheus.yml | 10 - .../monitoring/prometheus.yml | 21 - .../aggregator-head-lag/prometheus/Dockerfile | 14 - .../prometheus/alert_rules.yml | 223 -- .../prometheus/alertmanager.yml | 36 - .../prometheus/prometheus.staging.yml | 26 - .../prometheus/prometheus.yml | 27 - harnesses/bridge-monitor/README.md | 99 +- harnesses/bridge-monitor/deploy/DEPLOY.md | 158 -- .../deploy/alertmanager/Dockerfile | 15 - .../deploy/alertmanager/alertmanager.yml.tmpl | 41 - .../deploy/alertmanager/entrypoint.sh | 19 - .../deploy/alertmanager/templates.tmpl | 19 - .../bridge-monitor/deploy/docker-compose.yml | 80 - .../bridge-monitor/deploy/grafana/Dockerfile | 19 - .../dashboards/bridge-latency.json | 2044 ----------------- .../dashboards/bridge-pricing.json | 1619 ------------- .../provisioning/dashboards/dashboard.yml | 12 - .../provisioning/datasources/prometheus.yml | 9 - .../deploy/prometheus/Dockerfile | 12 - .../deploy/prometheus/alert_rules.yml | 130 -- .../deploy/prometheus/prometheus.yml | 19 - infrastructure/README.md | 83 + infrastructure/prometheus/Dockerfile | 19 + infrastructure/prometheus/prometheus.yml | 46 + 47 files changed, 265 insertions(+), 9988 deletions(-) delete mode 100644 harnesses/aggregator-head-lag/Makefile delete mode 100644 harnesses/aggregator-head-lag/assets/logo.png delete mode 100755 harnesses/aggregator-head-lag/cmd/script/update_metrics_calls.sh delete mode 100644 harnesses/aggregator-head-lag/docker-compose.yml delete mode 100755 harnesses/aggregator-head-lag/grafana-entrypoint.sh delete mode 100644 harnesses/aggregator-head-lag/grafana/Dockerfile delete mode 100644 harnesses/aggregator-head-lag/grafana/dashboards/disabled/mobula_pulse_vs_fasttrade.json delete mode 100644 harnesses/aggregator-head-lag/grafana/dashboards/disabled/pulse_vs_codex.json delete mode 100644 harnesses/aggregator-head-lag/grafana/dashboards/disabled/quote_api_latency.json delete mode 100644 harnesses/aggregator-head-lag/grafana/dashboards/head_lag.json delete mode 100755 harnesses/aggregator-head-lag/grafana/grafana-entrypoint.sh delete mode 100644 harnesses/aggregator-head-lag/grafana/provisioning/dashboards/dashboard.yml delete mode 100644 harnesses/aggregator-head-lag/grafana/provisioning/datasources/prometheus.yml delete mode 100644 harnesses/aggregator-head-lag/monitoring/alert_rules.yml delete mode 100644 harnesses/aggregator-head-lag/monitoring/alertmanager.yml delete mode 100644 harnesses/aggregator-head-lag/monitoring/grafana/dashboards/disabled/quote_api_latency.json delete mode 100644 harnesses/aggregator-head-lag/monitoring/grafana/dashboards/head_lag.json delete mode 100644 harnesses/aggregator-head-lag/monitoring/grafana/provisioning/dashboards/dashboard.yml delete mode 100644 harnesses/aggregator-head-lag/monitoring/grafana/provisioning/datasources/prometheus.yml delete mode 100644 harnesses/aggregator-head-lag/monitoring/prometheus.yml delete mode 100644 harnesses/aggregator-head-lag/prometheus/Dockerfile delete mode 100644 harnesses/aggregator-head-lag/prometheus/alert_rules.yml delete mode 100644 harnesses/aggregator-head-lag/prometheus/alertmanager.yml delete mode 100644 harnesses/aggregator-head-lag/prometheus/prometheus.staging.yml delete mode 100644 harnesses/aggregator-head-lag/prometheus/prometheus.yml delete mode 100644 harnesses/bridge-monitor/deploy/DEPLOY.md delete mode 100644 harnesses/bridge-monitor/deploy/alertmanager/Dockerfile delete mode 100644 harnesses/bridge-monitor/deploy/alertmanager/alertmanager.yml.tmpl delete mode 100644 harnesses/bridge-monitor/deploy/alertmanager/entrypoint.sh delete mode 100644 harnesses/bridge-monitor/deploy/alertmanager/templates.tmpl delete mode 100644 harnesses/bridge-monitor/deploy/docker-compose.yml delete mode 100644 harnesses/bridge-monitor/deploy/grafana/Dockerfile delete mode 100644 harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-latency.json delete mode 100644 harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-pricing.json delete mode 100644 harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/dashboard.yml delete mode 100644 harnesses/bridge-monitor/deploy/grafana/provisioning/datasources/prometheus.yml delete mode 100644 harnesses/bridge-monitor/deploy/prometheus/Dockerfile delete mode 100644 harnesses/bridge-monitor/deploy/prometheus/alert_rules.yml delete mode 100644 harnesses/bridge-monitor/deploy/prometheus/prometheus.yml create mode 100644 infrastructure/README.md create mode 100644 infrastructure/prometheus/Dockerfile create mode 100644 infrastructure/prometheus/prometheus.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 794f5313..4e406eca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,9 +28,10 @@ docs/ Methodology, ADRs, style guide 1. **Open an issue** with the [πŸ“Š Propose a benchmark template](https://github.com/OpenChainBench/OpenChainBench/issues/new?template=new-benchmark.yml). Sketch the metric, providers, methodology, hosting plan β€” get feedback before you write code. The issue lands in the `Requested` column of the roadmap. 2. **Write the spec.** Drop a YAML at `benchmarks/.yml`. The format is described in [`benchmarks/README.md`](./benchmarks/README.md) and validated by `src/lib/spec-schema.ts`. -3. **Build the harness** at `harnesses//`. Anything that pushes Prometheus metrics with the labels your spec references. See [`harnesses/README.md`](./harnesses/README.md) for the contract and the existing Go harnesses as reference implementations. -4. **Open a PR** referencing the issue (`Closes #N`). CI runs `pnpm validate` (schema lint), `pnpm typecheck`, `pnpm lint`, and `pnpm build`. Once green and reviewed, merge β†’ site picks up the spec on next ISR cycle (≀60 s). -5. **Wire the harness on Railway** (maintainer task). Light harnesses run on the project's shared Railway. Heavier harnesses (wallets, signing) are hosted by the contributor and push to a publicly-reachable Prom endpoint. +3. **Build the harness** at `harnesses//`. A harness is a data producer only β€” it exposes `/metrics` over HTTP with the metric names and labels your spec references. See [`harnesses/README.md`](./harnesses/README.md) for the full contract and the existing Go harnesses as reference implementations. +4. **Append a scrape job** to [`infrastructure/prometheus/prometheus.yml`](./infrastructure/prometheus/prometheus.yml) so the shared Prometheus picks up your harness. Format documented in [`infrastructure/README.md`](./infrastructure/README.md). +5. **Open a PR** referencing the issue (`Closes #N`). CI runs `pnpm validate` (schema lint), `pnpm typecheck`, `pnpm lint`, and `pnpm build`. Once green and reviewed, merge β†’ site picks up the spec on next ISR cycle (≀60 s). +6. **Wire the harness on Railway** (maintainer task). Light harnesses run on the project's shared Railway. Heavier harnesses (wallets, signing) are hosted by the contributor and expose `/metrics` on a publicly-reachable URL β€” the central Prometheus scrapes it identically. ## Local development diff --git a/README.md b/README.md index d4f57788..437f45fb 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,13 @@ benchmarks/ Spec files β€” one YAML per published benchmark └── README.md Spec format reference + submission guide harnesses/ The runners that produce the metrics -β”œβ”€β”€ aggregator-head-lag/ Go service: WebSocket monitor for indexation lag -β”œβ”€β”€ bridge-quote/ Go service: 4-bridge quote latency + fees -β”œβ”€β”€ bridge-fee/ Go service: cost-percent comparison +β”œβ”€β”€ aggregator-head-lag/ Go service: WebSocket monitor (exposes :2112/metrics) +β”œβ”€β”€ bridge-monitor/ Go service: 4-bridge quote loop + execution (:9090/metrics) └── README.md Contract for new harnesses +infrastructure/ Shared services every harness depends on +└── prometheus/ Single shared Prometheus that scrapes all harnesses + src/ Next.js 16 site (App Router, ISR, Tailwind v4) β”œβ”€β”€ app/ Pages β€” overview, benchmarks index, [slug] reports β”œβ”€β”€ components/ time-series-chart, ledger-table, region-grid, … @@ -34,34 +36,38 @@ docs/ Methodology, ADRs, style guide ## How a benchmark gets data ``` -[harness] ──── push metrics ────▢ [Prometheus] - β–² β”‚ - β”‚ β”‚ PromQL queries - β”‚ runs 24/7 on Railway β”‚ (defined in YAML) - β”‚ β–Ό - β”‚ [benchmarks/.yml] - β”‚ β”‚ - β”‚ β”‚ resolved server-side - β”‚ β”‚ at request time - β”‚ β–Ό - β”‚ [Next.js site] ── ISR 60s - β”‚ β”‚ - β”‚ β–Ό - β”‚ openchainbench.xyz/benchmarks/ - └──── source code lives in harnesses//, deployed from this repo + Railway (OpenChainBench project) + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ + β”‚ harnesses/aggregator-head-lag β†’ :2112/metrics ─┐ β”‚ + β”‚ harnesses/bridge-monitor β†’ :9090/metrics ─┼─▢ prometheus + β”‚ harnesses/ β†’ :????/metrics β”€β”˜ (scrape every 15s, + β”‚ 365d retention) + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ HTTPS public URL + β”‚ /api/v1/query + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Next.js site β”‚ + β”‚ on Vercel β”‚ + β”‚ ISR 60s β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό + openchainbench.xyz/benchmarks/ ``` -The harness is the source of truth: it calls real provider endpoints, measures latency / cost / success, and pushes Prometheus metrics with the labels declared in the spec. The site never fakes numbers β€” if the harness stops emitting, the affected percentiles disappear from the page rather than fall back to placeholders. +The harnesses are data producers β€” they call real provider endpoints, measure latency / cost / success, and expose Prometheus metrics on `/metrics`. A single shared Prometheus scrapes every harness over Railway's internal DNS and aggregates everything into one queryable instance. The Next.js site queries that single Prometheus URL declared in each YAML spec via the standard Prometheus HTTP API (`/api/v1/query` and `/api/v1/query_range`); ISR caches the response on Vercel's edge for 60 s. ## Architecture | Layer | Where it runs | Why | |---|---|---| | Site (Next.js, ISR) | Vercel | Static pages with 60s revalidate, edge cache | -| Prometheus | Railway | Time-series DB, 24/7 | +| Prometheus | Railway | Single shared instance scraping every harness | | Harnesses (Go) | Railway | Long-running WebSockets, schedulers, on-chain signing | -Vercel and Railway are intentionally split: Vercel can't host long-lived WebSocket connections or sign on-chain transactions; Railway can't serve a globally cached Next.js site at the same cost. They communicate over HTTPS β€” the site queries Prom URLs declared in each YAML spec. +Vercel and Railway are intentionally split: Vercel can't host long-lived WebSocket connections or sign on-chain transactions; Railway can't serve a globally cached Next.js site at the same cost. They communicate over HTTPS β€” the site queries the shared Prometheus URL declared in each YAML spec. ## Running the site locally @@ -80,15 +86,15 @@ pnpm build # production build ## Running a harness locally -Each harness has its own README with run instructions. They are independent Go programs (one per benchmark) that you can build with `go run ./cmd/...` or via the included Dockerfile. +Each harness is a standalone Go binary that exposes `/metrics` on a documented port (`:2112` for aggregator, `:9090` for bridge). They have no Prometheus / Grafana dependencies β€” that lives in [`infrastructure/`](./infrastructure/) and is shared. ```bash cd harnesses/aggregator-head-lag cp .env.example .env # fill in API keys -docker-compose up -d # local Prom + monitor + Grafana +go run ./cmd/script/ # or: docker build -t hh . && docker run -p 2112:2112 hh ``` -Set `prom_url` in the corresponding YAML to your local Prom (`http://localhost:9090`) to render the site against your own data. +To render the site against your local harness, run a local Prometheus scraping `localhost:` (the [`infrastructure/prometheus/README.md`](./infrastructure/prometheus/README.md) has notes) and point the corresponding YAML's `prom_url` at it. ## Adding a benchmark @@ -96,10 +102,11 @@ Full guide in [CONTRIBUTING.md](./CONTRIBUTING.md). Short version: 1. **Open an issue** with the [πŸ“Š Propose a benchmark template](https://github.com/OpenChainBench/OpenChainBench/issues/new?template=new-benchmark.yml). Sketch the metric, providers, methodology β€” get feedback before you build. Want to brainstorm first? Use [Discussions β†’ Ideas](https://github.com/OpenChainBench/OpenChainBench/discussions/categories/ideas) instead. 2. **Write the spec** at `benchmarks/.yml`. Format documented in [`benchmarks/README.md`](./benchmarks/README.md), validated by `src/lib/spec-schema.ts`. -3. **Build the harness** in `harnesses//`. Any language works as long as it pushes Prometheus metrics with the labels your spec references. See the existing harnesses as reference. -4. **Open a PR.** CI runs schema validation, typecheck, lint, and build. Once green and merged: the site picks up the new spec automatically; a maintainer wires the harness into Railway (one-time setup per benchmark). +3. **Build the harness** in `harnesses//`. Any language works as long as it exposes `/metrics` over HTTP with the metric names and labels your spec references. The harness is a data producer only β€” no Prometheus, Grafana, or Alertmanager packaging. +4. **Add a scrape entry** to `infrastructure/prometheus/prometheus.yml` so the shared Prometheus picks up your harness. +5. **Open a PR.** CI runs schema validation, typecheck, lint, and build. Once merged: the site picks up the new spec automatically; a maintainer creates the Railway service for the harness and redeploys Prometheus to apply the new scrape job. -Hosting trade-off: light harnesses (one HTTP poll loop, no secrets) can be deployed onto the OpenChainBench Railway. Harnesses that hold wallets, sign transactions, or otherwise represent capital must run from infra owned by the contributor β€” they push metrics to a public Prom endpoint and the site queries it the same way. +Hosting trade-off: light harnesses (one HTTP poll loop, no secrets) are deployed onto the OpenChainBench Railway. Harnesses that hold wallets, sign transactions, or otherwise represent capital are hosted by the contributor β€” they expose `/metrics` on a publicly-reachable URL and the central Prometheus scrapes it the same way as project-hosted harnesses. ## Editorial conventions diff --git a/harnesses/README.md b/harnesses/README.md index a9b6e3aa..dbd1c4cb 100644 --- a/harnesses/README.md +++ b/harnesses/README.md @@ -1,52 +1,50 @@ # Harnesses -Each subdirectory holds the runner for one or more benchmarks β€” the long-running process that calls providers, measures latency / cost / success, and pushes metrics to Prometheus. The OpenChainBench site queries that same Prometheus and renders the data. +Each subdirectory holds the runner for one or more benchmarks β€” the long-running process that calls providers, measures latency / cost / success, and exposes Prometheus metrics on `/metrics`. The shared OpenChainBench Prometheus (see [`/infrastructure/prometheus`](../infrastructure/prometheus)) scrapes those endpoints, and the site queries Prometheus over HTTPS. ``` harnesses/ -β”œβ”€β”€ aggregator-head-lag/ Bench β„– 001 β€” WebSocket monitor (Go) -└── bridge-monitor/ Bench β„– 002 + β„– 003 β€” quote loop + execution (Go) +β”œβ”€β”€ aggregator-head-lag/ Bench β„– 001 β€” WebSocket monitor (Go), exposes :2112/metrics +└── bridge-monitor/ Bench β„– 002 + β„– 003 β€” quote loop + execution (Go), exposes :9090/metrics ``` A single harness can serve multiple benchmarks when the same set of measurements is consumed by more than one spec β€” `bridge-monitor` is the canonical example, producing both `bridge_quote_latency_ms` (read by `bridge-quote-latency.yml`) and `bridge_cost_percent` (read by `bridge-fee.yml`). ## Contract -A harness can be written in any language. The contract it must satisfy is small: +A harness is a **data producer**, nothing more. It must: | Concern | Requirement | | --- | --- | | Inputs | Read provider API keys / wallet keys from env vars, never commit them | -| Loop | Run continuously and push the same metric set every iteration | +| Loop | Run continuously and update the same metric set every iteration | | Metric names | Match the names referenced in the matching `benchmarks/.yml` exactly | | Labels | Include `provider` (or equivalent) and `region` at minimum; chain/route labels encouraged | -| Push target | Scrape endpoint exposed for Prometheus, or push to a remote-write endpoint | +| Endpoint | Expose `/metrics` over HTTP on a documented port (e.g. `:2112`, `:9090`). No auth β€” the central Prometheus scrapes over Railway's internal network | | Timeouts | Documented; failures fail closed (counted toward success rate, excluded from latency aggregates) | -| Reproducibility | README explains how to run it locally with one command | +| Reproducibility | README explains how to run locally with one command | | License | MIT, same as the rest of the repo | -## Subdirectory layout +A harness does **not** ship its own Prometheus, Grafana, Alertmanager, or `docker-compose.yml`. The shared infrastructure handles all of that. -A harness is expected to ship at minimum: +## Subdirectory layout ``` harnesses// -β”œβ”€β”€ README.md What it measures, providers, labels, env vars, how to run +β”œβ”€β”€ README.md What it measures, providers, labels, env vars, port β”œβ”€β”€ Dockerfile Container image for the runner β”œβ”€β”€ .env.example Every env var the runner reads, with placeholders └── … Source files in whatever language fits ``` -If the harness ships a full local stack (Prometheus + Grafana + Alertmanager) it should include a `docker-compose.yml` and a `Makefile` for the common targets (`make run`, `make logs`, `make stop`). The two existing harnesses do. - ## Hosting Two paths exist for getting a harness into production: -1. **OpenChainBench Railway.** Light harnesses (one HTTP loop, no wallets, no signing) can be deployed onto the project's shared Railway. A maintainer wires the service after the PR is merged. -2. **Contributor-hosted.** Harnesses that hold wallets, sign transactions, or otherwise represent capital must run from infrastructure owned by the contributor. They push metrics to a publicly-reachable Prometheus endpoint and the site queries it the same way as the project-hosted harnesses. +1. **OpenChainBench Railway.** Light harnesses (one HTTP loop, no wallets, no signing) are deployed onto the project's shared Railway. A maintainer creates the service after the PR is merged and adds a corresponding scrape entry in [`infrastructure/prometheus/prometheus.yml`](../infrastructure/prometheus/prometheus.yml). +2. **Contributor-hosted.** Harnesses that hold wallets, sign transactions, or otherwise represent capital must run from infrastructure owned by the contributor. They expose `/metrics` on a publicly-reachable URL and the central Prometheus scrapes it the same way as project-hosted harnesses (the scrape job uses a public hostname instead of `*.railway.internal`). -Either way the YAML spec's `prom_url` decides which Prometheus the site reads from β€” the data path is identical. +Either way the YAML spec's `prom_url` always points to the shared OpenChainBench Prometheus β€” the contributor's hosting choice is invisible to the site. ## Submitting a new harness @@ -55,4 +53,5 @@ See [`/CONTRIBUTING.md`](../CONTRIBUTING.md) for the full submission flow. Short 1. Open an issue with the new-benchmark template (sketch the metric, providers, methodology). 2. Write the spec at `benchmarks/.yml`. 3. Build the harness here at `harnesses//`. -4. Open a PR. +4. Append a scrape entry to `infrastructure/prometheus/prometheus.yml`. +5. Open a PR. diff --git a/harnesses/aggregator-head-lag/Makefile b/harnesses/aggregator-head-lag/Makefile deleted file mode 100644 index 7c1b2c68..00000000 --- a/harnesses/aggregator-head-lag/Makefile +++ /dev/null @@ -1,151 +0,0 @@ -# ============================================================================ -# Aggregator Latency Monitor with Grafana Dashboard -# ============================================================================ - -BINARY_NAME = latency_monitor -BINARY_PATH = bin/monitor -GO_FILES = ./cmd/script - -.PHONY: help -help: - @echo "Aggregator Latency Monitor - Grafana Dashboard" - @echo "==============================================" - @echo "" - @echo "Commands:" - @echo " make run - Start everything (Grafana + All monitors in background)" - @echo " make pulse - Start Mobula Pulse monitor only (foreground)" - @echo " make stop - Stop all services" - @echo " make logs - Follow monitor logs" - @echo " make status - Show status of all services" - @echo " make build - Build Go binary" - @echo " make clean - Stop services and remove binaries/logs" - @echo " make destroy - Remove everything including volumes (asks confirmation)" - @echo "" - @echo "Dashboard Access:" - @echo " Grafana: http://localhost:3000 (admin/admin)" - @echo " Prometheus: http://localhost:9090" - @echo " Metrics: http://localhost:2112/metrics" - @echo "" - -.PHONY: deps -deps: - @echo "πŸ“¦ Downloading dependencies..." - @go mod tidy - @go mod download - @echo "βœ“ Dependencies ready" - @echo "" - -.PHONY: build -build: deps - @echo "πŸ”¨ Building $(BINARY_NAME)..." - @mkdir -p bin - @go build -o $(BINARY_PATH) $(GO_FILES) - @echo "βœ“ Build complete: $(BINARY_PATH)" - @echo "" - -.PHONY: start-grafana -start-grafana: - @echo "πŸ“Š Starting Grafana + Prometheus stack..." - @docker-compose up -d - @echo "βœ“ Grafana stack running" - @echo " β†’ Grafana: http://localhost:3000 (admin/admin)" - @echo " β†’ Prometheus: http://localhost:9090" - @echo "" - -.PHONY: run -run: build start-grafana - @echo "πŸš€ Starting Aggregator Latency Monitors in background..." - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo " β†’ Starting monitors (CoinGecko, Mobula Pulse, Mobula Trade, Codex)..." - @./$(BINARY_PATH) > monitor.log 2>&1 & echo $$! > monitor.pid - @sleep 2 - @if [ -f monitor.pid ]; then \ - echo " βœ“ Monitors started (PID: $$(cat monitor.pid))"; \ - else \ - echo " ❌ Failed to start monitors"; \ - fi - @echo "" - @echo "βœ“ All monitors running" - @echo "βœ“ Monitoring: CoinGecko, Mobula (Pulse + Trade), Codex" - @echo "βœ“ Chains: Solana, BNB, Base, Monad" - @echo "βœ“ Metrics: http://localhost:2112/metrics" - @echo "βœ“ Logs: make logs" - @echo "βœ“ Stop: make stop" - @echo "" - -.PHONY: down -down: - @echo "πŸ›‘ Stopping services..." - @if [ -f monitor.pid ]; then \ - echo " β†’ Stopping monitors (PID: $$(cat monitor.pid))..."; \ - kill $$(cat monitor.pid) 2>/dev/null || true; \ - rm -f monitor.pid; \ - fi - @echo " β†’ Killing all monitor processes..." - @pkill -9 -f "bin/monitor" 2>/dev/null || true - @echo " β†’ Stopping Docker containers..." - @docker-compose down 2>/dev/null || true - @docker stop prometheus grafana 2>/dev/null || true - @docker rm prometheus grafana 2>/dev/null || true - @echo "βœ“ All services stopped (volumes preserved)" - -.PHONY: stop -stop: down - -.PHONY: clean -clean: down - @echo "🧹 Cleaning binaries and logs..." - @rm -f $(BINARY_PATH) $(BINARY_NAME) monitor.log - @echo "βœ“ Clean complete" - -.PHONY: logs -logs: - @if [ -f monitor.log ]; then \ - tail -f monitor.log; \ - else \ - echo "❌ No log file found. Is the monitor running?"; \ - echo " Run 'make run' first"; \ - fi - -.PHONY: status -status: - @echo "πŸ“Š Service Status:" - @echo "" - @if [ -f monitor.pid ] && kill -0 $$(cat monitor.pid) 2>/dev/null; then \ - echo " βœ“ Monitors: Running (PID: $$(cat monitor.pid))"; \ - else \ - echo " βœ— Monitors: Stopped"; \ - fi - @if docker-compose ps | grep -q "Up"; then \ - echo " βœ“ Grafana: Running (http://localhost:3000)"; \ - echo " βœ“ Prometheus: Running (http://localhost:9090)"; \ - else \ - echo " βœ— Grafana: Stopped"; \ - echo " βœ— Prometheus: Stopped"; \ - fi - @echo "" - -.PHONY: destroy -destroy: - @echo "⚠️ WARNING: This will remove all containers, volumes, and binaries!" - @echo "⚠️ All Grafana dashboards and Prometheus data will be lost!" - @echo "" - @read -p "Are you sure? [y/N] " -n 1 -r; \ - echo; \ - if [[ $$REPLY =~ ^[Yy]$$ ]]; then \ - echo "πŸ—‘οΈ Destroying everything..."; \ - if [ -f monitor.pid ]; then kill $$(cat monitor.pid) 2>/dev/null || true; rm -f monitor.pid; fi; \ - pkill -9 -f "bin/monitor" 2>/dev/null || true; \ - docker-compose down -v 2>/dev/null || true; \ - rm -f $(BINARY_PATH) $(BINARY_NAME) monitor.log monitor.pid; \ - echo "βœ“ Everything destroyed"; \ - else \ - echo "❌ Cancelled"; \ - fi - -.PHONY: pulse -pulse: - @echo "πŸš€ Starting Mobula Pulse V2 Monitor..." - @go run ./cmd/pulse/*.go - -.DEFAULT_GOAL := help diff --git a/harnesses/aggregator-head-lag/README.md b/harnesses/aggregator-head-lag/README.md index 77575161..1b282f1c 100644 --- a/harnesses/aggregator-head-lag/README.md +++ b/harnesses/aggregator-head-lag/README.md @@ -16,6 +16,16 @@ The delta is exported as a Prometheus gauge (`head_lag_seconds`), labelled by ag **Tracked aggregators**: GeckoTerminal Β· Mobula Β· Codex **Supported chains**: Solana Β· Ethereum Β· BNB Chain Β· Base +## Where the data goes + +This harness is a **data producer only** β€” it exposes `/metrics` on port `2112`. The shared OpenChainBench Prometheus (see [`/infrastructure/prometheus`](../../infrastructure/prometheus)) scrapes that endpoint over Railway's internal DNS: + +``` +aggregator-head-lag.railway.internal:2112 ──► prometheus.railway.internal ──► public site +``` + +A contributor running locally just needs `/metrics` reachable from their Prom scraper β€” nothing else. + ## Metrics produced ``` @@ -32,36 +42,28 @@ quote_api_latency_milliseconds_{bucket,sum,count}{...} histogram metadata_coverage_total / metadata_coverage_success{...} counter ``` -The Prometheus URL declared in the YAML spec is what the OpenChainBench site queries β€” the harness pushes here and the site reads from the same instance. - ## Run locally -Prerequisites: Go 1.24+, Docker, API keys for the aggregators you want to track. +Prerequisites: Go 1.24+, API keys for the aggregators you want to track. ```bash cp .env.example .env # Fill in the keys (any missing key disables that aggregator's monitor) -docker-compose up -d +go run ./cmd/script/ ``` -Access: +`/metrics` will be exposed on `http://localhost:2112/metrics`. To exercise the OpenChainBench site against your local data, point the YAML spec's `prom_url` at a local Prometheus that scrapes this endpoint (the `infrastructure/prometheus` config has notes on how to do this). -- Grafana: (admin / admin) -- Prometheus: -- Metrics endpoint: +Or run via Docker: -If the YAML spec at `benchmarks/aggregator-head-lag.yml` is repointed at `http://localhost:9090`, the OpenChainBench site rendered with `pnpm dev` will display your local data instead of production. +```bash +docker build -t aggregator-head-lag . +docker run --rm --env-file .env -p 2112:2112 aggregator-head-lag +``` ## Run on Railway -The deployed services live in this repo at `harnesses/aggregator-head-lag/`. To deploy: - -1. Create three services on Railway from this repo, root directory `harnesses/aggregator-head-lag/`: - - **monitor** β€” uses `Dockerfile` - - **prometheus** β€” root `harnesses/aggregator-head-lag/prometheus`, uses `Dockerfile` - - **alertmanager** β€” Docker image `prom/alertmanager:latest`, mounts `monitoring/alertmanager.yml` -2. Set environment variables on the monitor service (see `.env.example`). -3. Point the corresponding YAML's `prom_url` at the public Prometheus URL. +This service is deployed from the OpenChainBench repo, root directory `harnesses/aggregator-head-lag/`. Set the env vars listed below, and the shared Prometheus will pick it up via DNS automatically. ## Environment variables @@ -72,57 +74,36 @@ The deployed services live in this repo at `harnesses/aggregator-head-lag/`. To | `DEFINED_SESSION_COOKIE` | Defined.fi session cookie (for Codex). Auto-scraped if absent. | optional | | `MOBULA_WS_URL` | Override the Mobula WS endpoint | optional | | `MONITOR_REGION` | Label written on every metric (e.g. `us-east`) | recommended | -| `GF_SECURITY_ADMIN_PASSWORD` | Grafana admin password (Docker only) | recommended | A monitor with no key for a given aggregator is skipped cleanly β€” the harness will run for whatever providers it can authenticate with. ## Project layout ``` -cmd/script/ Single Go binary: all monitors + Prometheus exporter - β”œβ”€β”€ main.go Entrypoint, supervises goroutines per aggregator - β”œβ”€β”€ config.go Env-var loader - β”œβ”€β”€ metrics.go Prometheus metric definitions + registration - β”œβ”€β”€ head_lag_monitor.go WebSocket monitor: on-chain ↔ feed timestamp delta - β”œβ”€β”€ mobula_*.go Mobula REST + WS monitors - β”œβ”€β”€ codex_*.go Codex REST monitor +cmd/script/ Single Go binary β€” all monitors + Prometheus exporter + β”œβ”€β”€ main.go Entrypoint, supervises goroutines per aggregator + β”œβ”€β”€ config.go Env-var loader + β”œβ”€β”€ metrics.go Prometheus metric definitions + HTTP /metrics handler + β”œβ”€β”€ head_lag_monitor.go WebSocket monitor: on-chain ↔ feed timestamp delta + β”œβ”€β”€ mobula_*.go Mobula REST + WS monitors + β”œβ”€β”€ codex_*.go Codex REST monitor β”œβ”€β”€ geckoterminal_monitor.go - β”œβ”€β”€ quote_api_monitor.go Mobula swap quoting latency + β”œβ”€β”€ quote_api_monitor.go Mobula swap quoting latency β”œβ”€β”€ metadata_coverage_monitor.go - β”œβ”€β”€ proxy.go HTTP transport + retries - └── log_buffer.go In-memory ring buffer (last N log lines) - -monitoring/ Local docker-compose stack - β”œβ”€β”€ prometheus.yml - β”œβ”€β”€ alert_rules.yml - β”œβ”€β”€ alertmanager.yml - └── grafana/ Datasource + dashboards provisioning - -prometheus/ Production Prometheus (Railway service) - β”œβ”€β”€ Dockerfile - β”œβ”€β”€ prometheus.yml - β”œβ”€β”€ prometheus.staging.yml - └── alert_rules.yml - -grafana/ Production Grafana (Railway service) - β”œβ”€β”€ Dockerfile - β”œβ”€β”€ grafana-entrypoint.sh - β”œβ”€β”€ dashboards/ - └── provisioning/ - -Dockerfile Monitor binary (multi-stage Go build) -docker-compose.yml Local dev stack -Makefile `make run`, `make logs`, `make stop`, `make clean` + β”œβ”€β”€ proxy.go HTTP transport + retries + └── log_buffer.go In-memory ring buffer (last N log lines) + +Dockerfile Multi-stage Go build +.env.example Documented env vars ``` ## Adding an aggregator 1. Create `cmd/script/_monitor.go` mirroring an existing file (e.g. `geckoterminal_monitor.go`). -2. Implement the WebSocket / REST loop and call `RecordLatency("", chain, latencyMs)` (or the appropriate metric registrar in `metrics.go`). +2. Implement the WebSocket / REST loop and call the appropriate `metrics.go` recorder. 3. Add the API key field to `Config` in `config.go` and to `.env.example`. 4. Start the monitor goroutine in `main.go`. 5. Update the YAML spec at `benchmarks/aggregator-head-lag.yml` to reference the new aggregator under `providers:` and add its query under `prometheus.providers.`. -6. Add a Grafana panel if you want it visualised internally. ## Troubleshooting @@ -130,17 +111,11 @@ Makefile `make run`, `make logs`, `make stop`, `make clean` # Are metrics being produced? curl http://localhost:2112/metrics | grep head_lag_seconds -# Is Prometheus scraping? -open http://localhost:9090/targets # all targets should be UP - -# Restart the stack -docker-compose down && docker-compose up -d --build - -# Tail monitor logs -docker-compose logs -f monitor +# Tail logs (Docker) +docker logs -f ``` -If a specific aggregator silently emits nothing: most often the API key is missing or rate-limited. Check the monitor logs for `[AGGREGATOR][skip]` lines. +If a specific aggregator silently emits nothing: most often the API key is missing or rate-limited. Check the logs for `[AGGREGATOR][skip]` lines. ## License diff --git a/harnesses/aggregator-head-lag/assets/logo.png b/harnesses/aggregator-head-lag/assets/logo.png deleted file mode 100644 index e40972bdec57aac376b0d99672528992e35aaf19..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65169 zcmafb1yq#l_cdMW&-O@@Z4T93$-6065gmef31Jd0}N)BC0HwfP| zsJDOLe=XL%vv8RAea~}ZpMCb3AVqnJD|pxOP*6~=NJ)w*qo80xP*6~9aInB{4BmPs zfd8RBSC+VsQrHV!K|!HGkrESCb=3Rz=~7L6)A6~7gCfIsW!ix7Td#eUX_bkhUtk9W zMn}K!+oq>fREoM^9x@ilkin3l42e_Z559ljUR~;wZ*83B`MK#gQ>RDff@@V9NhkOE zek6@_bxjRJ3!VD*x^)h8?mj1N@9gX(VkDwLC8F^v!(i9CDw!NYF6oK!@1Gi``XalH zq|%2(l5*)Nl-R2H7k>r*@PGYyVW9Fs-d|!MkV4yiD`BU3lLR*s<-dQLJ;5h(kQu#* z`j0FBc`r1K7y_^u*)~yGd%sDBe_gj-wB9GejPu4H|KKIdAkrqv7=G(S42?nL|M&9> zV;R=BNSHp{X5IaD4!oG7aevZF74xshIh6CPHghL7`SHgG2@$2w z)k{?7zxp3fMuUa1d^@y(=0gYr0UMemk>cz9QCm;`i0Qs(VQBQ2~_e6hO z?jJ8~8G!*(B+}fN{xHgazVLzq1Efp;uk-oi_eR9&bDo|5X-WhoYVD5|N}1u`wx&gz ziDAhpD2NfRW|{0@0%Xb)J8G9`c2V4Th^N^z`gxgVTKY z_E1QvahLQ5lOFFiLhb)~>%SkVpAiu4xggVvq6J>CqTkDb4@!AjqpO z!*Ig6eqY0@4rkNQ)C}!!f7k4b5_X<=`ChdY8o2~`@=yxp7lHqW-FI>OZPWY0Z%F+< zh|9Q}r*X6*SO3dbsB!vj5$LMwQvZyi&-5ZMEq7aeiK6~6p2$7$$pk&NZ&d#<@qfP{ zz(bv;#J=;WEVAnPlK&T8Zs~u1`ak~;5$?$r?(-l3((%!9JeJ{xbZlhw4H!?$(z_CC@ zs{iASI8ng7w2cSvR;Jxg_m#-H`CfZd&qBRvJdyCSj(-#;3B^W!Qxc1w9~4z zF&wd6jb~APAL+b^qrgxl$CT>$_H>T=+(tX}=#P;6vx18xqogvAl|74D+h#=C&mk-s z6Zx++^dMzKMm2TH|CvU9dY%X)p@B_+>~*<)zu!Jga}10fct)mlAKd90&Sujvoc3Kk z0|7f~FA*xb>GOX(SRpcd;a$7??}GieV-_(1<&EY;Tf!&$|B7%)QO^pLoa?_nK9UDj zABtUfepcQ6ic6+k^P`2pE{|nadFT(Jyw3cH+I7?G-m@=a9rl|D zQFmeqGy9vDk&fl4;2Dm-k%Xl_m%{rzLoS&+_tVQMCngtEd|A&#t}>gM`&HjcyHtzx zJ$^Vm`&w#@fwTG85(CSN?r+9K!EKK6Y)gAGUSXXOqadUe1?4mfoOs?R81+`VO*F&b zrxE#f&kb;vo0KIUS5m+=N?niaa`fwJZ)<<-uJhOYKo=PocVlmLNF`g?rs+vX^bLk5 zcTgklxSI1S+Wb9KyxtR#2{;yCrWVo(LT5s=q7-BPY)g>&IsLX0L;Y)iB)s6H1vw7) z6G2z)`Ps>rvS%tc8d!>I=*~}im#PbiAw1o{Lls+h$2J#eSHy$bCS7he zWYqJDOZmMbt&Gd#gwX&9Wi&EOUrxJax6%+|Ocd%VJ_0Fq6KC_XwBxt0oPXwFv4r|} z5kj|q9Pg>173$aN-BNu|z-zby3!I2lq(K^CGY+J=+S$EqI~ZQCz>cii(SU;Ro!9&? z_p@n-Uom+xrMW7f?G<_C)XjXQgApbE8I3>vqn;Q|z!ZV|F!|QkI*&TlLLIFa>l4f% zB}8|`Fi9x8@aB0(aY9wUoM+<(TfY0>dHUp+Ke;uYM#tIwpW!GXR0S8U)R7+4!V^g-`XZ zhW)v-yg`4uD5u@Lq#9dmmi@|?)QSeYf9Fxo7!h^#_Q_h zV)LLhkv98@Ise!j$u#bYk6OGBVm2V4@#0*TkTDC*5hh)qtkwYjn2xr+zd9t-8cw;U z@IZ9L?1=w6S%_2&V@~$Fj}~9)B&gl&`!T*!;T}rVZ%=1?7--)8)4f3wNF=V@{UWpQ zy<$2Z#6)g-JLaXU_lgGFu7j%(vgR=&jfO*X2=Kv`w>Xf#X}U1-yARiEcI5Y{YtN3& zb9AcgG8V!@Y(RRAc~owevmQ>tBk#QbjXa_7L)yQK!7B`*qj}g5z)X8n(!~v2AULD@+?hVC z9G7sLBTy7jTCBV0MxU%YK_yd6+;uMcwEwEidRmXv>0t8Zg7+m>8Vkx6^8V&cp_&&<@ zbgNCd&fQH5l-Lxwm74yC($Wl3AFNWdpC>}P zDB<2w^b)zf*3-4Row3YMuHf)N8p&?CZ~2P8wgl<=R2(!+IOdHYcV#&3EJ`HIad|If#u4E*Hnwrsd%3*^4GZBaw48B+5G^j_xd;*Gs@|*jZdG(;*7_w2)L>?hK zX!*T3?w{W}+#k0_Vg_Zn{WIQ`jF?mNKrWYu23_%YN?i`D7Upx*^5RYRO6dq^YKeya z0u_+;q5d(N@ad+{>ET?2a@kX*9KBC)2SMNp7eEVS7y~f?5$IiSf5S&zCpc+}EB}wy z*^>%9_ikm$1?@8`4gNq|BHZr3;gN_4NGUit_b%MUE06%dFgkQm4sa{oxSNH+ybp$d z_cTA_d4?!mjhgj*uwagN!Q=d7S~*{ngPh~BFUd_sn%uX_tRc8ISjCQ;H=SFNB*dJ5 zEahK2^!CeWr}n*}wnl7!)WQ^^^u4)QlAS2eb7XQ$m>u zwMxZa@mRBhzE(0>m>0b@!)g+)oGDGJu`pby%d$37lmHTb)vas(`QSrpNtl(zu5()~ zOAn{tH8nQw05M0*45n+^yc%{4zdRSCsL#X9B1@ zF|X;+m=EViUFR}`T|Mj*K3An!C1#S31^3O%8W4&vxEmP?Km@2xRK2pzwV#>n24iNBqsoZD=2x;m7XY0~p5JX3*;9=0&` z$>n9+LgeQ`TY`*y8))n;HEBK9EpZLE%Y1h!u|2em!=Bti;Yi8Ku~c{$@0*{RNE?e9{mod8_pVIa9^ z^~8KjQr=_z@jwlPILF$PWv2GfL?R<6q(>KrAUouUkw8NF`P!3p6=jRDGAVh1t!5pf zFlu4F-NvVbMVBUniAvkmA1ya;@Il_*yB_;z12=%%e2Z~4SaaxjM z+e#i#eu)V)g;Wv=(I=~4c74f|S?FL&i;ZK^$fdvsT~80h^Yu+=S^l!HH_aR~0|UG3 z-jK$EFR^@Np>8!^Xi3st)6PVWCmAm{rf9?M*Rud5AGA><`<*F%*JvDH5whu3J%`Y6 z^diY)ZT!~@XWPc#83>qw&-%i~0(m@Ren>iAeGh!3Dg(4rnYT5IMFg44oOVpK0GKsw zh}xo>1yrLQ=7`(jOn@<>-s`J_Mvn3g4V!tkx>M!OIF{_kXbfMx(6LL?1mS1LYYSgZ z(|SnfkPv)(NJBp-J33mWztnyAvxFURNfyFX=T+vatyYQ~rZ)4G{4d%;Ozap@%=QCp zV)(^${11O&Ah1T8Pz^pf{BSTsgfpi-nMXBXYakL$T!boc6@U?(5YLTppR0EswNF$% zZ(7wi8gF^#1j9o@J`G(5wa)ApoA#eUy2#Wh(&K#bP18XJ0gyr@m-(<{*mW-Def;F* zN^(zaY#RawG*-D3m=fG!ExvRD{8x1XgJHtt>2&R3pL}xIMTdYyfyS7m4H&`Di!DFp@ru>5vY0KSGAvh$MX zU5M4;foz*?jOcsxP_bEb3U4(Nhl8}y1qV$r=`uO4d^Dh$0K89!d|>{kBha9}MEd$m zCd1VUd#Iy$^d1sPcv2#poYBvFn;Fe39?hina-NGl$z7*MJ1TfP2p{}hM4e=yoXujA zDZnl&U#4J^di`77N)c3o^MHs>u;+pF_qJTqzYQWT)r`0gpHje4k(UkR+ZeX8bYnjO zoA)#iF^Hg)fKpKDyC(OVkE;gtML(xC6z17{`_H5PSzaRzFQO5T7DQu|k>Eu%Vk4uG zj7eb9H@!*H`qN&#j{RqG=FIw-hoCL%S9$VkJ*x@LNgilu>t_@M5^vFQtG@M4HOtT!yH25py) zhlEHr|9U0;hzI@y^=6*>80pBpwW2y9*{mnwybtx97DxfvqxGm))C9Pp&V)niB8lI_Su5&u$Y37Xn~L#mEt%^8x>T* zh|A=h4903fh<3;8xbmojKK6j{=|mz^g9dNrE}`B22j0ggyDRhx!awZy2Siq^WPE8^ ztjIlaA3y9)4(k4&3akt6#1l!sC%kzXF}O~OdN zqHWwK41{y(ApPB%LM8HMO>5-h(;L9w_1OVeutg5^ zJl}ydmX?>^E*Zlp7Y&jz+)B{tZtcUc>$eabXQE@DTqy8$|Di8~1Z z83}%R4$djZ5rhB5H$@)IV#5fE%b>ce>`l0zeBW-Tk1!f>YP!PnJ{be>A#niu;xj_r&@$H{YqJst6JPlVuL0)Z-lAvN<7KRl801^~XiS2~x z_2M3mkkQ+@dhGZ4(f^DNSAO4$67z1!bk6B!kRM z70>om3z^q&*GAzW4dl zb-JIom5Q7Zg9ch-A0T+?D!YY2>XU<=q&AA*T7Zo-H<@~=fk16`fGd-PJ!)OITCSBk z@0qDxRv>=uadz|tz%K{5(3Y6e+g<=hS+Y^AC=m})y_uPy1djgi0fV#pP*jhD(1wqC zfdCt|@9GRS1ME6$?crlEn+VZyobjt_#(y-NpMU#kK1N|ajAtrxp(}xfoB!pysvSu0 za+BR61k0yr!u;gu!rQ>2*lNT79tAQYBUVGq^*Pk>2F=>f&VMr&K+M$H?rKf}CSNA+ zNKhLXkVAqOc`^yCYRUVf3~hHG$PWM_V31e{^aGXeRE|6L!pS)93#$1ZVp}Pbl{S_r zWyzB7)hNdjQ(`MoQFcE3XH+&^L}dj)w#(5^crTD`47jXN#CTEN*{$ppyI{WW9$j}T z-jks+GM6aMVWCd#zI;VQF{B`{|gA;Z&v{b4+#ORYkOjL(3M#a_ZR)t(<>xHzI)AzQf` zf5rK%zH+%UIGZg3HXFzuFnsWlW65CsvlLv%@(Zf2)?BF-rsd9ZctD3hx_MesyV(Fd zGY>syaRiyk7lmV`mSF5j`pCDs!x!u=sbZ!x)&&T5`Eaw~rQ6zq9$|wEW-*Y&_Ui%G z4o`6?vEg?sn(P$s3vl>SL%nzS!Ym_modm%yh0Zv&&1bKDMJ9!R?8dCU4Q%q7jvFWO&;R7S7hUqC4Nf|g zds-=l`H_$LDP{c!TO_3r{(iNC_jM`!IGAaq_Ho!JJYMs*?GWvy(5G}iCq6)*4P$zg)7@PH320IL8KH=ywU-^AYHDyc4=zCF8=WSmhD#-+$ z%Yv0`TFFgece1uYXD3l=w;&%??%Sq0eTrQ`;cJXVv3?N~Lugw~^U&n51_%+S56Qp9q{CWSyR(^&% zh!e}1MnA^YC3t>em=#^#owfKNNX9GuCV)o^NX}Uzdu&D)q!@tRPu8av7x}L9& zEgQD4{-}U=+&obkHtPNIr$Ipc_gP1mM|;wvDcM75ipzReJTMkpd#;n!SP>VNUS-u8 zwdB~#50bB9H)^@_PRz!Bgt9yXrIlKA(6S%IMS~*Z)>?xPwr)hQRkGtPhTw)tU35tm z15jJGJ@{^~~ zl%R3G7Lfly((5NT2UJoE9WGf(>T?y-?kwkL2X#Yt&I#!sK5fbCy2tqKV!Up|i7KQP z+wrce_TOB`-?&9j82~IQFC_W^P;{G0EqEPCGzwsKuKu&?|55&b zzNj}rwnOgcKVBwrna9s6|D0SV^@O@cxqTliFLnRn{A?&UF7No|_c1WTxE%!=5}6>P z8#-}AJ*TUAMV&}~2=K3V`(I7h)QKglz?4BBlid<9>UsO#i@`Ld$UGih0($6BTZCCz zV$xqq;KeO)2B7RKtPU5jsO75K4lz%2z(H?8E7P#z_`q9pk3%w22(02a={Eqf0dSftCEp*=d2Pai|D=+zI zHiRwyV9G_o{|W^oD2^iqjFEN;dd@j^>YyDOQgab~{sYODKmGhx`XS7_SMWebW7|2( z32;rw63eWT6?&PHZ!qMRmINLe`-?QUeXYF_@c7K{Miv3Iq)Xy>;=hWv=m&VZuM@8& zyTZ9zc%7*Qv+QDv3X|Kuy}1OIkl+?%GUPFePA z!*_^8yo+^w2 zPmCEEm__xh;O<9WNO2I-Q1HjZ_eP2-v42l(IlR46M-%Na!;&b8 z4*_pA4;W#Mqm#Y0Z0=19xym~#MS6k@_GSRm)$H{0e|*t$y{eCYLjW)rRf9$P`d|oP z5;oz4i}%9=EH*+)w1ubg#EtD1h%CD6ZY5-iy ztW#-|N@TdDS>ssJoy?<<>T%}epuCg+p*I>iRa){17dn1A?flqh4#oe5yt>f-=o94{ zN86TcaPX`5)^f5_Uq#Dl>NLH8GEgiS0A54(jHq(0usYVAZiXSQAbU54hi=++N3Lo< zTILk9ZEOLctoGXYD`y~1%A=)AmBPo(D+?Ou*}RM-sO4AY{F4iqArGAfTm9G62Ol<5 zIaOMHy{oPivl~F|9;qVFD<0&xL7Kw(^3|zh939U10?*E&$0sGnd_QVEG0Gtw}m2>u{+OYP^p-}I5d=Ggq;%muOsnHjdv0e z`*JTex27?WU=Q$V>gT9TIh*tTVJmM#Se50w!JX7l>7HSe?Gh%!Li{}hhP9qg zj({(GJPziz8k+#k7#@~8{zu`j#@x9LajCq|v+S4qMY#fDB483DZxiM+NHi}G>qhqC zM@z!zD@r~2j9@}t@tztE{AL9JL4wM?VeN$=rc}kgZ-!KjbxqRlW2~Cvo4(8t;w)(Z z<(0V&>^Q$_;Do$;R&8qs-PE2`)!ywKSo{oBOLP|oibon~Wi;In9VN$MZ5TN@(SqCO z2mqLjto9BeDAX-M>0fY5HqVQYw4}$DtT+%X40SkL)Y9PW?XCE~ zAN$G9R97G}-9yBTvc*!adB%Qjp4nFnS;}hPmUlZl#5?@(n!NXNUB>~9m?IT5qei;a z7Nm)5@dfnPYA19m8mZB>Zy*Oq3uOcHDzkIdm;oD&l1S^tlw{F12(5yyL142Uz4K%1 z40K}E@4*bnpz(KmVROanYwIk&4gGoCbMr6+Xa^q6)l!WKtS33C&^cotY=6@bdOT*e zlR+WD2l+y{jo843K-=A=mq81tA*%emRy@$aL*H#L&e{DY?~LFq7zMvK4+dCJeM5%{ zTcXt^&~az14*^&DWV`uUg1Q3`0Yx@~dxI)wDtG>*Gcy5dh?>0~KZnAkYP0IL)cg8N zj!L$xb-iQyaXuZk7m_!1yG*lEV|blg5ymlN--HbjT2&S-xcV_AY4i-FnISpa(-%eX z)mtOBI*lf1ZVIl4b7pS6yu21ioS)pF%7-I$XPIzvqfiF6%0{I~n^?ggy92At_p=Z< zHj{Ya4q4^R4lDx??~R_>e;ZYzjBMiB1XYqvJml&XblxRKGF4U=NO6B1KJ_$_^xYF8UTW{@vANTOxtucY?{hDHg#gmItJfXGx5TGx6 zBOj!dEMVgf7;O0#IuUQw!CE($Hqp?{Saj}S=e^Y%PujxOnDCKw% z%5_t`c#-W74rCFhvlSRDT4x0_FUX*n+8-@SzN`@5h+o&}tjlPd@r{w^lbSrNZ{_Dk zSV~1+<4I|p5GeJF)3>vm+zzo=9O-gk*=ZXsqMGH$toRWD><-Y5AhZ-hMf~svv#niME zGKVrUOPGH(M=Xd)xztFCsnp0K6)2Oql2N(gTP9C>DX??+(r~Z>Rc1$THh%Z+29S#S zY@Vioy=K$@gG!qyuz?MfL0IGSFF}aQ@@+#|lnVcJ4@tI~&HMW*My;eLI--DZ9qggf zPSe^ZSD+bsGEf2L*JeA{=AsOFD)*?&K?$V5r2)e%28?^!%wD4Dt}<_h--X6g@K~t^C?z)1);q93 z*OU5hjTGQSyj|z|=m|%nC}Vya+VDxh8WT!(iH@@V)~4-Y^Q_v?AgOMJHJFI|B=82r z(V*moMGmB#?&W6w@`iB>s_dqRZC)zM{zsb)A>vfoOVPw=R~w@|*Mfs+6gxcHxR9FR z#W4#nFE5ojoOHlqUn7@J@Ys6K2cg;1w*GVjXeONsAwP-e`scSmBi6ZhP+W!e9R}7U zqOq=AZ_icBi)}&)N6*^CG1wug(`SHHZ&HNVH8L+WGt#HZ^8-lm?zYv#E+mg*!heaK z6({4`f^Rk+;V3tU{>djj@g#8)BZ_-ZFJ_`a-ee+~G6~_{nogZ4N1HW7Up^g7vA|w; zkWir(YR{BZd@?T7{up6Uxy@rXD2a&!q?m6EJ_rVq#D3>|+>BENs7*VE8~`3VrMzMI zPA$X(lNvbQuk|gB;K)kv{w!SYQ<)POz{dt!az_ksyql|MX2##12{RPfL$&6Vr&yB3 ziMDT}d^vx-p(&}j2&M?#J5J;novwg$M#^9}MLH+c#J<|1JR_V&=@a()48|UAn%Fkd zTX7nCQDCMj9&Fwp?gI+727xsO?#5b^RtvcI)2uC>OWv`wWuN&sYKT+WpFIKtrV2h> zKYKtX;$D84WrI5b1}&T>-DG;4~y@T4^#b`jj<`P6rI=vT9N_ z%`7>N`H^s{Tte6u#>6{g!kHT1v-V6B^ci(ukJLP}_yIOtPgql-LL)S8>Uzl4ex1!5j0FI`QyQGzwbg90MWI zEnF!NOWADmMtR{_h~Em30|%DZsoS@OC$n%;G89U_p{fDIP(xEy&rKI3CF6)%FF%H=7U}7ljT9DK zW=_G8$H>@MlvPt3w1zSq@bJR6e^dR(cN4uWRV%SpV26NT9Ng1mXl|3VPsfk@q)TD-k&a7E_b<19IDG=ci0&bgk-Cq1aZh4A%ob>;l?36naUg zybW9E@#lJ{%}S)0_i%G2YUX;z9AeD8NUeg+Mj1-ME*?y>r5K|X zhl`qW-16*E29Q$-^ud@I_mbn~;@R3gzh&9Ay%E^O6b*<%^n2M7Wyap7-o3fiKz)OJ z?zYS}d=tz_5BCS(K<3b~A@7?=97|E{fCfx0`+EFV5X#7%wEEWcM}^v8(JBp|Si_1y zApwt`!ETw50xCeZxeX3_z?||85GBH7$7-gGKci$gaYQD$EyYmBJP_V9_d;hGW+Nj@ zf1WDnGU#tPJIIqOsfVz=Axvno+sRTC3;Js2>Nvkk*Q5us<9)Lw-mpFaYiZ6LNPBkrc4HC5T%f^QAOn%+sY?x5YqXpy_k+A z1TRrmK#E9^pmGv>0N)jVy9yIYe+EO}#31C>wMBmi)?8LQ!YQmOMR`qaUX$V~I5A+y z<0YFbg43$V-(pY)@{a+0*mDeN8eac;(2xxch$u;RZ1A}QrAz>?&^sDx<3kp1EUP@) znH&}YCsM!I(?a@P{;hdg5~FvQJ%|HmUErd1#&T}`^z{xhAQ9S5zJ)9kONP7on7Se) zH#so@EYl|C!9Cvpp7&T}CW|r!Cs0ch=*h@oHtEi{NVZmlPQKkmwsEV49kt#GHQt3R zDwF`M;6I9)sE|l*6TBOL7$TxtH{{R?(g>v)YZrq{^bQN&63W$w9Y8E+3^cOOv^XAYV%XT> z^onL6X*nQa@xDeU5ds!EIvavsuFPz@F1>zvKOqQhUXEs~dlEEI%q;5pff$T1JR<7d zN?eUnqYlY-aassd2wn)qxM3D)`Y9|x|v2_n&&0a;) z4~uLmB%mrka)ohg?X!{<-v*{O(BIJxR zK--g(Ob(0kMk(V}8P)NzfdYLt5EBpW0jgXkiPJdLKGE+7y{C#eviUGWv{8GIz%Dn* zCR&5r&lGdq>ET(3mSM$Jfqs0Aj|vL|{bqq3(4)|tIDITm7xm}@T9a2B({&#E1(n6~ zhSfQ^23IWm)4kwH0Tf}5Q*7UW@6#Uvis*)JO?L5u(aBr+CcDYEjLQTFc;8`T>^4Z?aE>=p?>hgJL7Ht$eMnu!81w`w?2jvRAniVpoiS<)-U%L|va2W3g{ zwx!)Pt0mPzx8^Jxn$ia+`!>IDbX`0D1)E$l1=0qe6~;Tda8+KiVyW=F5YY_aL))wT zMlD#psor)m-ixahb2MM?PGu`llLOkG0@_W%?gjcLWC2ypa~vRPr~y`ZvYA{zK5Nr^ zqfjD%nRvW68D*+jJkn?a@(iF`JI;WlO;@VIuk@C`L{jW6hU3q0klP4*pc7svd30|Xh6ka?lhniEt*0+tO-5z|@Bcpoo4 z4w7pQ(I;53)H7 zONBJ#VAQNoi%BoR4naR$-dz7o;`NmdI`QC)%VoG)wjuPBLwzY|*gZqmZKx^A=vbE( zC&diDG-s6!i@NzzLukN}jF zKpcZGUwpb0K0NvvbUbQf$x%hHQe{K-xLEEQM)6Sy1-4{mV?-5Ks4p?8qcXm{VY7yz zg%A6~2DfC;2onlS2ZSAiFsdr5e@6tgYX#rND->3&RlS>C2FpVW0EyQMDcGQ#gD>#U zhFT%=ifW^Mu<>(EU2m!9KOG!-Kgt?p-2`TD(iK*dI(t%pwNs;~HSN9EhFUlDK>cRe z;_h!?_b zHE|Qkun+ist8s@xvJ}z^<<0bMTSyCy@LuIjhoft}HZ}Q|Ovy_T2hoq28sGAgCpFzZ z1`w(fWrQ!LWS>KQdt$yL^}C)F$ic)L(t#Rq5gTcAuT;Ztm2#UkD=8+e%)?@vaut>j zZwP}~Eje!Izt;l^)WBbhIT+o%26h&qU^9y#K#BVsJ({8$NIzcyg;51$VwcUI*b;a) zWdhMQ-orZgABVTmhrIteMgVgTJAyX`V4S&|w6{IX&*h^E zM-Q)7la@04_a4Px2PHsng5M_U==$gGjmT3BM?0E(yB3LmURaqhJ?P=&g*KQ4x#iiy zt|a+S3kLF_u5-1zoPKrr{V!eyxL~d15F`j=pc=|$0`vGd&_7|0vs`+lUHzgUAE?WK z>nuo*0ZI_4*=3MIbmg~VxEjU-#=RI1IIH6pe`^6wk!c3RHnJ%na5yNgJL%wJ6D5#6 z1`8e<9%cY`8E`;0tMj!h3RQEIfj*SIa*QF5C6UTVujH}c=|SCj9I=9c(%47KHPNt@ zyg(nwa&K39c3#m+hfDI4Un^>A&wj0_O>sv`SQ^?44QV7B?YSZB?h&-YjZ3+)3(&bc zu9vN+e*!>R4HRg`E7$Kue&`e8Cm?)}p2Ly6Y)<2oFepjJ9kKq%piylkyjW^D-Jz{{ zu_P!@t`w^P{bPAHKmKyzOT%t0Fu5$F@-4CQYv6($gAYy2*LLq*1;h?D77!t5*gf%~ zXs}3xFm3*I)YI<@#E`c{wt&DS11NtAJ6maYG86Q2Q`CT<&tYgXz3L1Y1Swv>K@SyO zN_8n@aUxD4G(w8(ye2CzDh=ly3D@ld< zSZ3+EyT3|4RBSNWCDtm(>PwH*;EtdmwD-`4>&Sh>Nj7r0i={Wu zRpMkZYZjB=F}K=EwRxYq@qJSd+?ey1wMDzGw^q88TWx9o*!+Cyfxgc$b&Nxog4POb zGsa5DA(?e^^jjWSfsp(nQK5BPY_#4PXiao+lM;QsXIF=?y1`E1U>Ih`#YW%*_M%>< zP{IlOWiWhU5fXh~E~TzP6LXhBXKSmG>}#!Cb&BhDhxA+I*h=tVBF0){HXR+lQyNrj zwiaO`$`(qPB2Wax&31eZxL#FEbtG6Cx^CbzqWd%1vPo~x1i`{Zj^Z2dF<(HMg_E%Az!ZYmZEx$T{egK=#a>pkpVMI@yt6{+&16My3?l&I47jid~kvj zXf0-Ce*)AJ-iXLy?--&L}seiX5y{MzhG@C)07*1IeOkn38R{08IGX!Q?g@ zUixf+kurM{IdVTW*oIg<%V!%*jmFy*UU%Z?}iI5 zy$U2@cTpc*a|e0o`iQ!(+@d*d~LPG8E+H5%>< zJYSUkb5ezyjI?wAJFwlFn~Walr1W0#P17;Xr~2v0O$Iq+GvkF#6$t?3L(9XN4E8^D z_g{^hfGP4(<;ah?pO}Rrk_dG0AKs6AG2tvNZ~GZ!RxJ(vcj^3ALT`7XuQk;6@-9L5 znDWIF73{Lot6nXG>fv7?oV7M=UH2nvm=1QkL{A}+omL~keq@o|zW(Aw%B)vmlBPR~ z*qjG0jf0D}Z_nZ8Uh4t|k0nMK@`(4{7j2*Hj5dSQ+qtgKwC|bbOsG1!GJI#!sov<* zCFfqe2gB*DJasQ*M?h9M7w38E7UsQPsa4?}n-K1U@{}58LvzCX223jTQh^qr^S6jx`|gN?&g15zG%GEtW(B5IW4~ z<`qgG)ugYFLH!}44CW-eZOyib=EXOV(*;<2^9C+)a$C@Pay}+}$Fq>qFeYDq^yGD0 zPR|o!kcfKBU0;u(yRZ3A@9}F@*e`+V1aX5PeNnxxZCL6DsxPSuG#AcHIt4 zx8gLO-t-3;0wLEL(rA9IHIUoa(G5)j(c`Ng6pBfty75VyC|x>49_PsbR>h@d*rB;& z47&YepBg|efz1WO=UiVsrD}%*D^^rm>LelWeU_g7UUN8peryAp%01x9{fqE z)u+wIORHc$W+gwIr@_|8x>DF(Y)= z#(t*9n%3bQK}5FRp>D;1gUm~+e%+1!jr65_cB4HP*VltZ)yVW{1JYyOpRR>w3&ohk zr5KZOWfxV~sp56qOh0y<`oQGzK$v4C9XgfSwsTJs063o0DNa#Z|8?>KUfWn-?bm(H8fdwJ26#1!025{bRlE++FJB9L*mws5wF= z<<4XSU_5ggEF=U8ui?iR!Ix_uipYSWnbBXN;Fdi;*ho`u{gu*=vq&H;!@3nV<5z6$ zZFUL({O9yIJB_`w`pZy!(8u~8R`?*m5`$jdKT`Y4ohELUtv^cV$4?wZRuPlSr|oO4 zKjAfFL>`4pJ-!=Gu6*`dJV<<{Q<#FLezfL_b1C}#aHK2YXJ0R?eJJ{JB41mybFC}P zhlbxkMtc;|LWhrUZ`Zh@4Y+7pAPgIr2J050D?}H=6ecqU}e`r zsv!wd-t$jN;WdUf0PyvZel_;vYnA-RR{MOy1zzN=Tv^TQdb2N3UC4o4d?g3MmT2t0 zB}$iXCCtW`ej)3nPDJVbNry{)YIGt7k1ad;OgKuMfDRXc65hdDqY+zZWx3OP^rR!f zH&1pd)gA-TVnhXs^jkXf_`ou`9l@Cw(3;+j*2N$^4R*DMr*wKO z#A##z93C~BTaD^Nfqsb1OgKV2DGAi+_c80;zL$>-`s>>@-nlBEp#t{AbcQaXb-?yu zXM?Y`9f(d?0(pIQfvkchz!*+~P1*lCp+R0l|Mwff>b{x)X-4%rTqfOPMfn$;x%gM$ zAgMsb%{YZ0><935@PjBWzc+7>q@~?sZhm`9G@ z8V~~Y#JGS4kzsz?uj9aNu)jXZLV8P85gv^b|J~meOz%>IJxhXw=x271ey`KyU3vx3 z1e_ebXnU26o~#K-Hnm$|B8u~;t^91fbH6$j=S*Vxwa*+6iFEo~k;6(bzj`s2(s}Jy ztD@n95H)%2z*6$@?Ge~PlsVb|t}FHUSBJc6Mj+7nZP*sky?8XuF-F3vjIBH^$^BU) zLbcJB3&7;LP8B1dG`6pLg$!adK31!)wNZf}ZqUE^lnYf2$>13HShR6*##QU^wOAbkypKvV zxgSvPf*EZU(DkWZV_yv1KnNIjqZNFqp?B>7^$vT?6PH)3G>vRux+Zw*Yc+}@{HuuPv zy>fLGc^*_aO=uvfQneGo%<@qYq8=y(i&>auEmG4R0KU_`;w_Oh$p`7SNUihDf*7?P z_-ZG&U5Xn5W?Vug=5J)w`b0*-;{bv^dhKIS-j&iFxhcQp!7!JI%q(V62|yaqwV={M0>}` z%w~#Mu1FQF!@!)$j<8nz4m}}--XDuY0-QRNR`pPrjRLD%z(BuD+ z_TKSS|L^~}Eh>_gjAWKQB0I{;CL=k>u8gc>g{;cTmTVctp(A8wXUi%(vPV`Vl2qU8 zDZO63zn}N#_xak7#-F90_JUm8|n!)+bE z^>}ZzM7VG0@74NVSLKwClZ+0btlQ~1 z3v!R=hkb{hmHcji8BY)(FLC~a#cHMI}r>%XZY4e!5O;HmDLP8lU&h0q1@5TO%i!dB8t``Ugdcdj1~yqeHh%)*7mG(ontN-%r`s=R z9|{J%n})i3fhA3W+oam#+6KJg9Gk`1+Wn_sH0>^tM%y>1VN)oUX<479~`_FYOaOc&PR)1{QCC0X~PbB9>BP00!Dx@VWrqFcH zeuECwGkLcAOFsDQ)qgH|y2a{Chx5z&D|G`(GHm(!UaxGPeShyPQR_Po@Wi*XmrT&$ zE5_Nd(+&b^zo98ssA_Q8=vryOWuVpN!<|BANpHUyueEAYgUpmO5 zRd0{lj^MIyR3A$#b&i?T0@&wTt)bI#s`L8$_A&nb0x_s&!ZYP0Yw(>I$Hzw*C3Hbm zG^_mbPoohr`8}mQPcSAvJ||_&BY{cd6Tob((Uu4O2AW`ELCs;1SB!LhkpH;DrAIu5 zZ=+oi4}-+@*n;2RJR>`VbR5!tiQCds{quc#X0fHRP$`!J9H!?kRn4=||GP*u=RvRH z@^96Ds&XBI)>qcO2LHi9DP5h~|H&2Z#@&f?HDmv6cgiyWeYs0dUneFWaX0FPQ&?mk zpK(>&JzA5;swbOxA4Ay;VldO%o$13!I*U6HB;!UNyca5O! zSgS7J&S%|`v2nf?e!UB^I>gL_aw1NZud=DgQqDyru>_#Q4V<00zK>N~2bR1zB;=u| zmhPiG&%w6T=Un;GJ=5=$VX-Oewj?|6t0`9Mspfjs9chZy5HwbVj*`DaqSUkZ5x`;5 z1Ay$9?*(Emxg$NX+Tg4G)joP_Y}2;&W7XV9xfjm&>rTBxXBH!p(cH=q8MU71hGEV{ zT-BP?Q>hov-vqN>M}wcbvy|X#S20}g$L20nnsSwwLwL%i4zN9EgI3w0 z3Vqe2+91Lvh3W2|lhS#Sn8QE;au}8pH)Ui%KkIThP5jc*L4a9?f;L$<^TG1ivE=t< z+@NxU9SiYvgf}LUa1`0Uav8YOOwqyeohHg;xIr zQF8elvTP;+N*@s|{O6e(c1+JhH_vAOi_b1QsTibRN#-o{vYS4sXIS%+`0YdV)w^Gv z$AuCBHh3j4UyxyS^wRkhXjAUyKO0R52Uu>eEZ{#eYYQPyAOnmg$&QBxy_@lPm;TWa z?`p6k@Sc>?;*|jE0~X*wh~z4aXVgDV9^<+E@>>(7CK4&IqJj>XhX)la>lHT=B8b;I zi)c2(4yj#(+>NZkKSkilqoImsoGphO98=*$V%*;3453^eDyrV4{e$c8FIcg|4=A{u zJI+4KeQ%Ely+fxgV*uOVUlsKh`SJolQD9YOvtdlvH@45mr zh;p}FT#v7B9*iV<0yoc}y@6m#_p=NAwA7F&Ni?_b~V-~ZY=v#?EdIlgHu*7fCBWAE{ zQ6x*tZ!q^k%zkG!+#h9BBAi+jla-r!xJSRPo5&wdsD|m8>()K9@SuZtZT#w;A72Fq zd1Fh(Zy-=wA^P=Y9}Na#jU}hp2NF?9k3wpp0pb6A-)wUcOL)NIY+XDx)^BG1yM`DK zHy!(&1U=S1c-~ul|A~?roCcSNVYNhmecZ-_>Ud`l1r_1GZfIk`GdsbWc!~dmbO=W9 z_$Te6tgoo@l@rf$Rp{)j1WCM?o?{#4Yjq^O^G`&9F%6RQ1Jh)`a8_3?)(Db6 z@B0?fKBB1K>VoaTl?Q$wo4q5^R^!hq?h_!w6eLZ;-xdVpx31EMwT9qebs>c5m;U*Z zV`@EYi%uyOMTIQj#31A#K*~69y*xzH!uTpC@nSJoMXd>LY&O}|ORs>SgWRl`3uw|- zuxyAcHiwHJl1Gnc&iG;epHaY3knoJph5sL^2LEB7JBaCGB<}wc#q!@Ho#0O)%GSHm zX&q9&I+4r>+XH?!+J_9!b_g54wLGK>tO2TMgDx~8GApnb|8KHPgt}L@TZ&syW0R_ zOLLmS@s2mVholn#+Ki|_L4L*T#B?OS0eqkFC0BH-rSE|KyX+KRl=^ONK2lnR zRF}n-hAe}mfD3?{SVXA|5iqMKibBX1f0i*EPGGe_*)cf}3`D&-`^g}cG_Gu zCLeo$R+~>%p7im6O|iqrpJlG>5#W=gG%z*{G)bNjsw3*s`&z!@E)GGbI|64+YpY`z z>BkO{HM)xVKVb4whNy?LceYoY?#;g@zo08OhoqW`-0F8k(a0SwdLrXAE&mb9%^Bym z8?gm{8SYM@E30m?o!x{ap3?yN>ir^&$bekXAt$}6;-p1C;VEaMYveLJ8DkKPsQ(7P zZ!Rghmd}8au@}nZcd28QGCPy_NQl7hYcOi$fPJn?@0$r99}10@bwAq2C>HxouUtLW z*~}#sS|>z41<5I_Q1FXB<3m@<)rB#YvIeMi3vgV-~S2? zpOWHusbRTvtn=(r$_J=22rR<-5!LG+Xbp=zT^D$aKR6Y!x9b^P+B^G9vMy<_%=OOk z2k1@?EJrbs{brMu?^b{<$1nw`b-Ii1&$)uz?qX2n9Quu5uV(Hq03);j;};5DiG@U5+iXkX9Nlk`F?tq&fyd`KFCGDhFZ#tQyS^GcfTd1r8SB{p+%a~ zovX^i+Hb3tb2?ou2atlrwpUaWHE+OQVI9B-ikvl`d_yt=lA_1|TYgoFP1jC^>E8~! z>A61+JMIZ2DN-dw=Es*Sh^ew_%8#--!PqSkNz4N?8yjfVQ5|s3LXG%9<`CP;bVn_e zmLI!=;2f!>65s;5vfUSy|CBV>T!P++qj+h_@A4__K_Uyb{mt>U+zKOEc7oLjC=dK9 z+u*#l#W#gd?U&*r@rDkZh4DL&4?8W_cV_ROtAx)=rF`_2RcMCCP&MA}bgdR1pATUj zO84Ep9C6{NN6yKLvw`%&P8wApb4ao3ka%Z**;!=Z*zHarIp7q(W3t@+eVkrjOBVL= z-3qmRI!sZ7X7E_|PR;l9`t|dNGt>#~v+c209>GPETD|wVv+}15d(2JC_j-CioE3VwmUQnw-86WiyedBPx%+(6& z3jW<-iA;y^CjWdaQZZHcrsvZC{>?dTgK4W~+zLqai+fC6h=;f zdC%?LL+?#Eh-=j=iR%>DHfyW{12LSN@9#g~Y7-uq4-p>vLwM?1!Uw@+ZNLzV?XnQE z@oW&k0!y%UQL_3uQs(}!_hk2ZiZv2-8i*HJDIIzDZQw!INUmcSl_X0Ubh zVmM5y1G*C{I1$CJA9Xw-RnrQfyd@erL_-{?!nvg27zdZzwI36Z@TU6PUU#DCFV^ze z49>5k<)l&TN96uy9uY~9Con|uf7#O9Tv_%!C>-7wX0e90tvl`22K8$vKSh;tHp#oX2lhe*8bgbQl>%0b@ zzVm9F^QzE`cy`_qO>+|z;Y-_D2AAOoE?5=LMtDh!x}cZDdIxZ0CmtEhq=krTrd0j} zG4K*SX}6xYh1z3*25!9~43lv(acX~RNi|+vNEutDR}SNcfvi^!P})?-$WRXPCRZ$V zoV-^mPPt%ESZN)D96^H;2bnFvp#I`0M^H}qkj}#S$5&TsH=+Yc78z+&qkt0fnAYyS z0G5&%jLKG(0n3mm{Ti}iW%a1Bg8Kg>XDmkj-z2dzlW#0Kt^wKqh&uMD4tYb_pJ zGG2`X+-RqgBI=i99s^4KlvWmOWGM8){%^Z`@D;k=^5}+&``DX8lhThb({de~Z#MtM zTz>m--*vQHVD->I8TqsLvF-7mJq~ynx4JujilG@6J20>vizD?%rP2$dPd*!n(w(4m znh&i|sNhZ8dDQWZcPeL1zLL*4xz)1!bEp!>Xw2+VyqkwXB%}zycWOBM=5u}uL`W*) z%on0I(|b!ILu$pW-tG)YT+n=X>hj#D-l{4Poj)_DV4k`!3&1WTF+)rIRqx@izr$|x z9Y|1)4TQzkT_^JZnNgt`mT%=H8k6t~tiKuLmTc(*6fUA=)IU<@Fpl}aj6S)AO2{|p zSg^KGHGnE$9TTbT&hXC(eVcyf-xF$3HDm3&g;9UDC4Y}QO??IQHOJ|1V*o09f0Op@ zyf8wPi$B+*i)wJgYCx37{^`^a$y^a*HgIQsc;$g36@(ipV5^oe-s$pagA$)gG$ewE zcXF#F-hh|iV06k@x@t2YZZ*mFtW~jKq`U|Y=!y;5%uhfGcSWo>JNenT98<&lW31y} zIKRBK3(2k}bl0=KL($^6A|?jW6aBpSDMy+f5{&UVPKzm%c?5GF`okdN-Ha_jL@54g z(UWW+d!GaX8Oet|#T$D#DcilxBgfl86k0jl!rc?-Ryl@>rEYxqgbTlLcC;wNS4)2F zC#U1rDiTd=ZoRuHtvyLf{~SbfONp-`1G9Z)q$D-;)0uQAak@Jr2uh@4S5c{uk%5-| zT0WO^4QxxY*fqsnfF!iai=x&U^}(p_PHm~>@Owv)n*OyT_K>}#@%e@%+5X|qyuN7( z`+v9N#!2t1LO1T`{pL4LqexT8xUnV5QvxW%rJj@ z5N;EJM13H+%W4OScDIwu+DP3Tb0=4|*~F|sE%YpnKS=_O;YZcHRXR_PAQA>Y!apm4 zr1D?|Tz{{??i`dg9A0v{# z4djcC_s^Ui;)zZw{anr(W7YY8+4C;Mz?=7`)AtMOE%fM{fjlU>iJ{OjdFb+VRLW^4 zl+z6nw)capcPn)+{==|VBo>s_HlEFF`>&VErRqFj1CZB?RUEjz>Bl8Z=ePZ^WI;8e zHcNX;ghOgCSs@(6wyZ771Ty1}nWk(TAv7Pup$O;Se!87rr9_y?AlSo6xV}9P1sEqp zM9X7~!GWM9M{as&RfL-RwD=See%cu*tG%&EQL{tj@slJR60MhC?>fH`Y;8l*=^-v} zGcG@Gv|@ai^b)b!_u$VCH0u1LE=EeQLZGOs<1L?wGWZE^dz3ibut){GanXIOyU^t2 z$FhhQ6_D!rDq#4EGDrfdBq}Bi-olIHIZ{|FpJ!rJDS>A2E8XuAs!zipNgSLL)1N8#@`{Y|NO?1I4+Y~l?iP#`sj=WDlK zB9~@ElI)<7R!RdU_R50W+f=*D1WGNhk{xB(90W43AP3BjW20c8GjK4wk-@~4B8GZ- zBbtims(_Ut)Wf{^0u4slz&&aC1Z?6AJp|W5LUCjjO60x?wctOMF&)~1tI~C@q<<(~#KdVeAqkjJ#UO$AOL5X^d@2Ol-T8Lr$5ood7RO(e zhr0U(WppBet{tNGG1M1UApH|_f*4He#VGf2G{zL@ITM!wJ^a{*&jnm%%hN?qvdaA| z0~`x9p3ua7zmX-FDO@+&q-UT%E+TK>t~PHlE~?%^*t`X~P0>JNa%od=lk`v%#lv;! zUUL{@YE}*^6hPSDm(p2eL7%}mBsMIS*iGG2*_|k?)U5UTNmg~5>%c&y+F2WO)g@>k zR}LD;ohuqcZi3(JKW?*;Ed$P}Zh3TcKBRMR>5=Wz(Le1^46TQ5d`0&`h<9Sv(KRUN zbNyO(h;KsC2SVo%sm}S7pqO-SKmYJk@)%1HuU@blD^UK&E0=cg`&wGConh=+gF9)5uT9cdJ z?>@+&(J*C%18X8iHdfFjejd(OP!$&iG(xeJz7Q|Sbw&vCUt+VNDzCys_06f-T^<)# zrM=;5-_ApE*70h2o1S}ImI3m+_u(3o9&|(bYhQz~Yx3^CUE^6971qW3EfIgSg7_F$ zy^_~-_Nh<~dJUvRvWS${AMgN%zeAP3CeHXfq-5rRU~z=g#FwP&gEuB#Dr3FTW_kQ2~d0yqz+3qQS%owJGl_Ud_^!_LpNznaNN!OIgWYt_ud^z~&U^bT0djs~BXt z-`z(!4$6?TLuzHnHS6G@YDh`-0*s7E`g0{jdgMU5G*|CNu6C9#s1qqU+TJ7z>v`_c z4;dZsVX26$p_vuLVh>2|4lUqGT|;6E4dEVe^PG)VD()P2e0_8YV-NDOf$u3Eq$C;y zO>$?9t6m~`49$A=lC6;PlSp-DeJGDO*;3)RDNh@W83detFd~E*9DCUrzZ|;@Ac=76 zuoHPqpzN~qmWuJ1o!!R{Xf92*D7NHBPJ|#umB2T9&zuYYG!BEu3SC2od&Fi?`118DFNDvTc;p_VG-V`RiZ6{k3(TapE*#y394O&& zxS;Ag6VEFcw{!mZY)jUS>+Q-0m7H5G{zVTwwOB>0$rJrThu~MWFf&Df6pGI)wXbP0LBtvTD$lh|E?|R0U;?a5(r`qSCP1rj5 zanVm3f&)3Kw3VP(a~bffAOxs07Q&`F;W3C}$^uh^ndFQ>`3gHYrPQ5Zw zXDUwE%i%!9oQs;ibVQQ_>9EFq+qgLbc#EyAtVjZ+YawtM8E?~U4QV8#atCCWe>eAf z37e{)yMBWwYO#H8MZtUU@wVr-Sf{wa0;eM;ex{b6nT~ll_w!AP#lq*@tqLa0d8d36EZFW_Ci;i}jX@T1fWa3BwS5mWjM8 zPL|oM=RmYEY#`M^-3juc!ytc)v1K+l$lsCDO;%7~pf-)E)DdY)_R^hDnEQ|!A@4h$ zxt(poPm0oWt-qbl$+bOj^x#)5H_N8!Rn|-j3c;JhBkn#LDtTyK?QVIqXEZWxaB(0% z;r*@YD(3dk^P+3hZ66;$eS?#W%8_kB6?`@iu(`uz-vtdJivY*#&Z8WEHx3X@9 z4%)Bkz7|@kxmM@9Zr%dB!Je$yV(*iSQfk|VSUYjQI-Qf>uH{O|mUh;XbzirUn^rl> zkh_Sp^&wN^CY3jnB_#*8?&mHy3QS|>2HT)<=ltC5bXiTddEv<@&r*tfbHWkIAsDEZ zcy4?r=S6IZ_EUcdM!BgOq+wjHyT+R+*lI$557XZPs-}*@pLR&l`3fS%aeYY`29|b?Q|q~2CZ~Y+Ei-F zbE_!jZP-@+e0G#g^|)5!9f`IQ-oJiO)UTGnS$&Y5&BmoO=Ox~GvhHx*koBDZ?wD*; zMG`54)*A29ImLzrCp==asa>F@LbI-K{L+F%L>sKcDRJwNUv+tCb*S(ZGJvbfB$E_AUD#Rp!HreaY*?=1`7bF? z??kd*$p3{NQfQ8xtj8Vb;2M1Y@+RfVa1p*^QOsGGqZS0vcSy=1p#`d}OW{gX8FE0+ zX>=R6T%}pIdPvs=^H&sN7U7balWKX)4WG0`NafZ24Ska_gN*N~NmNSo8ArW7+@%@? zy)0&NR_!S`JxeDq4lq)GS6RK11CFPKS9`uLdUULM`WR#VXc1*}U{m}_F|9t_uU{Xf z8Ab;51ZF?ZXYwJr#myx{K47hFr~n`5;i22L--y7JsGQ`Gj7Qv$R-OC6BKLiw{Wz;q zL=d#zAt_0s3H$$a-OzmQut8nrs292OWc^^GS>;7_<{SKFxv>=44{j`-Q11`J8-o8D zhpXSH4ON9AY(li(vLG%cL9RbOWwFlp@WxR!t1O+!N=$E<+xVFsDGuvlZtTw78bq<2 zb;D_=k)oHm{h_*epox!_Q@qM`?wIcN?@L{5ivIZqZuQA@Q~G)&fKn@k|{{L?mBQ{1ysl_H>(Q;%5q7F0QGwFpR zSeQS3oKLd!;Y<5=e5DS_3&xC?&62cQd<@JCTBvq1Rr4bez~!hXc#K1N9q+$>!NUm| z$(@|R%@+Ut&;R<)Nlsic(Dx;OH{buO{=LU~%fIoTQ}$r~uRr;*tnwj`753~%IfGWy4Debm$!VYHqS=REy5Ib69;%xqVjacP>mjG%I{5z_r<+L5o;&5c^F*1e975i}&yPV?0~9K()nEh^a9g>0Ldfz^ zDV^!kI0Oc>T#Zjk;u=VwK>ga8A#Y2+P!~mkF9)XR8e7l2V)I6=Bbe;$Y*}aL;^m)X zCSqP+ZtQ>^aSFE7cD7HsvsPhAylwRcOxmDC{P{P(W8j{<)7j+>A_bhNU+f7emgA?P zPu2katB^wpg=t4 z7`9Y3Ct{Gg{3x_Qw{Ga++M@5%?CtigHxw|Z;`BB~gGr(m{A-&EyBUOJ>kVi~q1CZV zA^gCPl}Mp5G)%pB$Xx=W?~Fntlf#rAo+N~xgEGN+R&6X(f3%eazaTSA+Vh>C-e4#;9oi?Q6y;;-e=CkW}9rC+j#sr7EDu zwz{S3eR-)WCllWR2bk>7N%`HdPWqUMWkq+_3s-c$&%&Tg7`$e)U6D#fixie&zaZM2 z3V?4;rrKMB1tcK-f5=TCQ4MX=1Gu|9z?7z-`!mTF>kpG`!e#=I(C!|^<3aVfwYe>Y zkARE1dFJYijL9n?SlO7n2o*@Nl|?9bGo5xcqFtgfC7kA?^{#JCcnQ8Ce`x-1-D&s* z=lL}=3x6%*&v5*Dbghv@r6@6oRajYrUj z-5Hy>IqYZ#B7*}8FjZO0gyk3p@qw>K#Lh9C&nD20Kfta(30Xn3##13g6}DYZFK)2w z4f0~h+62O;tLBIejF+L1yJA~HCdr&t=l$Fek3nobs7JKf(FIQx7th|lemu|7FW!9{{8Jb>R+Xm1ov#M?+YnWYoleMgnRQ1?vD(}F3;rZ z@P@%`+DRMFHMA)D?xqHmddhAqa4JhXJu}H zE|ta(CXsQOs%U%-n?Sk5aK4(zn%>m<#LHzsoz7{J4q=+Yar~xC$b88SDG{rVIG%*W z7lyhQSh$LpdQZ`%D=FPAwtw7bTYWaJlkx z!AI?)XIP$FoMe)W9>&Emxk&$l_p+XIeld4CARq^9(2jdMPQw*WtWv-BHeWzAO;K0| zEGOON9L6F6@eW4!vC}`x!FLrE7spmx(0Y$1R*T=H`UEm|FW7t%r#13&lvA?QBLEr6 z1)^ObBRB&h=%(jbYr|6{&|E;N|Ef+D?b2(Zd(si=<^PKG#pL!1Oem;_jY$R*Drv}f zckVG0Q2~lfgkI|JbLNE51acrPxcX=uq?@lNK4(`(<#iu|9S4P*ch=F@GecU|FQdgS zls#g0>Uz`rJoXt}oLd@0-krl>9d?#npv*uj_GFOJwI!vRSiHT!SrI04c9*&;lmx3g zk~O)RJ7%PdE!q-Tt|u@z;e5vUMQKBsiU%T4X(elfY3uA^o61&ZG3S$wH;9hyN>%N~ z-fRsbYimnbyrzdmf;Bxvo3As;)Lr<#Qc@-VZZIO8|1#9Nu6`1|X9|osDio&-Q+L%a z59ncSXpu)-;1o>XYZp^lH2t+pTfH+K7WGZ41Img9s6P23PKEM%41R^3#v2g})+A;i zZRV*J4WoXY#U9IA!!Idxc7j87rvhPe2n_jeZf5S2NbKQvej8MxU z$t&b?(5=+*PV?}Y9{Ofr;>iwlBY7`mu~bObsSopDPEk-BqCZ9&r7IcS6U0TqJz$1cQwC)s)yp(d6ozSRvSw`bV-SHE71rezm=cb-am$edb>Xb5oW2hVKJuFg%d8>q7f8#rFP0p}J-IGbseA3w7MUj|o~r~dgH zC=0m?^k-Q1;twB_1n9nF@--b6ZsTzwO&P}*5z^w8ceZ6;_!x;dTR%7_FHhn`rqr`BUWKq&0?7B;zH3Rd zwF71)Yk_*&aStNzc}C)qkoS=PQ`Cu5_o(54$e3}xe_-8>W{&ySw6h*ERqyNlf3Mu& zu)Jf12t6h3LT3VbH5tR?n3$=_^h%h8*)pQ6 zeA3?dv@rP(m)RRZ1epS=NbP^!evMlfetnA^{t-wx;RFLo3=*zhdmp{Qo(CI_*!9S{ zH_MMMV~Y=VAGKu0r3aNq?~*_TzV3dgVX^Dmty4J)mwcdbE4rE{=EO3ElR_KhL6N3< z7hJ=yVbqgnE3V)52e{OG5_E@&tlqwT+q9vOmzSsS67Qs#D{;*B&BidZ_wZeq0^?1> z0LdO?AOvBb$Jx{XMRj)-y-4u>YeoiF8;0_s4=uk5>YOn2MfI`)_UyqZ#`_MbQUHTs z-K*veTWXh}6;WTQF1F%3}W zAMHy}VVR1`lgKbn_%S$?+jdg2uY&#!$^5in5+ZeT8#v#f*UH@Z&t7AC*7N0yYp!XX z%*t5h6@nuIj#0Uf_g{6=;-$BY@Twc63$5-v#}_JQuX@x$9a-oA?9+{ttD!@m1aw=t zXH+o-MhnaJ6B;Kkk4%5_gP@?}hXr^g0<9gq-GojJ)$zvS4b-i@un*xbT675wC9fqZ1^dVPK>1M^Rq^ma;m7< zo!j4w)oD;P-U$<9q#Ibzl~m$%N(hggH7Ye8-wJG@ZslKwQpvCC!^g+0hTkz+nhvbo zf3C?dBD5^p-gx5)ExB~RsX&2uBSreD6K#b(ntuLzoz&8z*rcLST%iWtsQhJlAq~6RAf0z~zW=qe~n9`0pD6 zOwFgrK!@HxTD#T~&=@}*E>-}w1ORvoB$SPSK^cwJc}TTD zHmjO_$EFu_wbQ@XPMcE$!SrUmLP)gGP-SYLgnoIYSi&XPj+{XZ2dc&mg;jOOoFhD3 zGLaoM3R5aup6fsFLM9%LyPE}zmf@+Pe;b?J9jShqye+jw96fzfgVskD=`g9Pd#56t zD{$zahK7ns8%ef53$to-;pTT;d|%FAdbVeoZbl_|Enm#@cgq}d<$tuyM@tHJisX=o zS&0wX$;iy*+PrEKY=477!-95YIl#$2X`rbLvk)qpOqnc#ko-SuH}cH|Yp~&&>!Y)r z*o%uPpu&2{AMusH|BYlLP^K`VBPZgli!`9XRStQJIOs8L@YOlS8<(eiyAmA_&;GqeqdTc|DlT+wsyu)8NG#_*Vz!!rK;Q3xru* zpt}1CHiolqWK#*x5lxs)Yf)ZXFdBiRv0NHVdiqOVN~yF6j2OGj&H;}w##r&@-r$fx zXXvXsN7@rKdpB@>$m^h@?l1alg|1HEUch!F@Ta>D`M&3&zQ96StfD#ji|g;*Wr&4_ zHBUJ-oN$P8Axr4iFW>$v$UFK7@asf*vvDEhPII2uL|SIm0Y*bf+0_TjBXWND#4>O2 zt)-P(2fpnRz6wWw)<}~|jK02xT;A(_3KPS?%sByi7DJD)St?+s< zYoN3h2H!z;nSZCY3 z7~~%ZsxA_aiih-tTbVHVo0`dY$E@$%2jZQakfZEvsBsDS`&yOduHE)427!-Tn@8lb zOUPA>9(QDI&vP^D8Fs8qMcb zeFv@e^~QOmHc3T|?@A4GT9reN1-!MHEctvtoR&8imo%R;Qc5thnr7NsRZeS&e3{t@ z3IZrEY{C6~*nS%VRb;a9@E|XJMU^r1eWNc#J4bQHy*gLFsUl0b-a=0wc>^XF(jnpf zTCd#fqZd7Z(ql0j0w-Vc2J?M#%bn} zM&quroq*s_F{R^md#%iP^NnX9flpa1td(r?TVcBQXOh{D_)f|n4fR)H2AaI7htzQi zPSH4K7xZ$t7Sepbl`h9DP%S?qet%fVpd2>!-kg{nEID|44Z zJ6fdmvWus`QF^PD%Qb%CC-~yz_2YDpg;0W=5iSgTGRcysV&{0mM!5e}%;T1gWey|t z1FKaE@dJyadlv}fitww@kKS*l;}Da1UuXKhrJA`WAm*pYQ8syTmQAA@>i`&+8T1!g z8M3&ViWaahA&kBEmHQ`#)^Md%7#GmTg)Qzrq(Cf?Q4HetAn)}~^06WvSP}+t4C>gO zY?}G$dS7+gr5Y=IgL0%3fg1U1bqmw8Apb#-T`jI%tg1m4?Kd6~n-*-x)rNG(n6jBQ zZR1kfyG!~1?8KzBG(H)(hRxiKnFKI~dtv2$UAD|+%|3fVJ*0tk@L~GGpBfniX9OSS zpt`f)|Fa|WLtLDqVVxANjPeUU!LK4Z{hAt|uv^K(L*gP78R(FyoBuRuwD{!js*wNe zL4`~wIeD+!Ir-;tzz8*~H^SWacIiz6rkr?p$Pd@bHq&l`@7E7DvdtA=U%X<{W zdq;thrvtoF6&0vNDRT2Ify1j+(T%NqS|uv+#^IA<^~BetI*`G2elSYrkf_HanMW{x zGB}lP=iZ&_?7D+3sF^Oq8-SPVUXg{e%b11 zXuENxGy6-2@lh8&s0Y7hG|nRULsdo9mZyN002n0tR$(@M2DBeoh4uGM`mY|ONVWreO6s3aCU%_ve_uTPluXbL~oJXkMRU6fU@li{Z%*G^T3 zoYTZsxp?ZHvn>`W&~Sge_3ZwU3+IQ(P6*%#3=t63)c)8u+Ay8V+z=iILUcLYay}nH zu2nrFaIeF)>Im|;;HMw%{_?4SAOY8N`OhCRJLw%1&$*P@{}Y#0H0+yba)RTa>YY0^ z5Pd)7Wh;RJz<^MZp{HJND`ZT=byKC$rAHd-BOO)P_~j{ zp*<|~9j0l7j^=zqDi@3oP-KjLja3ebfPjyOp10vD*7;$AS*s|;@87OWGbTVDUYXv< zYDwoVOE^xBW$bK;rG5OQL~?Bs$K7exRbJED0Z=sP!-^$*x|2i>0~j#(U@2vY*!T23 zNx2u-Wys!iZUBH2(iB!qNGT1Vy-`_GZme-V3j$pQ18~ilF=q5Omg*Va(jg)h?FS&6 zI1D;KoV|-P0O!2zx`HN_yf_~6%>Ccu?y}|@Jevp`B0BW$geg0DECyfqRd~Se^Fp&& zYIkQ*aK^q7-_->OjiaYdMFHBiS@crNrl;Mt@_?p+hdqI(ED(3`Wb17HV$3xMBq3M$qIj$ z5`AtYH@(!d(12!)RA`3@7HsV7XEmQph^jUtaCJY4-e1+7L&fw1iJgM`N|@_w(HXO% zj*o!|(cLF+D&L?Y(+9SqTuU69moLLZIg4H1lz8K!l< zI>#r&@(&1L8+R3S%*SbFLNXeD6P_6Pa&SrCYTMs)&W}WVSB^2BAFFz}X^4!dk})w> zI|?LL_zE6Qh^0D}SX-j)-XkdUC0Jo!f8s54g}I0_0yfuPYI{9paW$8_T}d` z7m0I=i9)#1;Rjm;2c055Fk`coa;W)N>VAPc@G0lvu|I5^nB%b)!vU$M@Qa-zA1|5L zO`+%s66#%t7_-S=KR^A-zSELrd&cyF33Q=Ou~w>wuUFs=&qlL~H5Eai@2{d`2BwuTp4Kl=5nkT^W3DU z*UPFMH;-J%@7!?4Ml_aqQLW(EZ@)GS!zZ3fF?gQU7*qyjFE!*DlO&1SV@g+ZMen~q z4}*x#+((AogAmMBS9`sJgaCPWZr;4fVunnYBjI(7&pi%9O-@Nc@|d{|4TtWz=#_rk zZ9CH%jyJ2q2-&;GS`WIAg4csrKs-U|LPLmMQBCSCSpPx?A_!79W~q~TGRMgna)oDU z{$h9k8%eBqX#J-p>+q$Cv&3W7H{uw{v0?3W)<|8h?*-pgLn z;YZT$rXXS{5d1i=L1U?E-;&@3aIwrRAbKay-C#zD(Rc6OnZZEci!H!!DbI;%MN@z3 z1ij)VP?zQG$WKi(J3*)gw>P}DS{U84i^D}({a>J!^}2nTgmyA4uyG5R8JOiwDpv&| zz`!!^12DNu8UMZ^fW3HXtgy&#v4iI2R>=y>!ozcN)3+KCZOU!y&RYFEnaBy09()r0 zfsK0ijzYa?_b9YFt_scy+K`O|WKHNH=H0BY5RZ8jxB5`;--|Eq-qt?OmGavS) z|ADk=P-j2MNwss+$n5K-4~D^e)0lmMQpvF?jR!CZMs*{ku^FDa-DB#Ac!ip996D?O zv;o(!%E7wpj%)8X=xOEt_;rKUX-c0CX#t}(8qNwI>Zdcq@0>x%v+lWPQohm`p2DsT zzY%=z=B(PqPt@>Jg`5%X0FTlK5Ua>S;C<7t2qx*1is2`o2*ueg>sUcGph;wIn8i^! z?Jy5HgP|FezJ8@{tJgFaa~J(gWg=bt%%WEEIb*4d6k+7=X3)&G)H$!sg&x{ph#4=A z+d$()CQ#59p=O^^2|7FuC*U+xs7Q7agy2e|VCbV3Y3Tt z92E0-9*`jV1d93qO$eTE+JI57dL8;yz+CW`>017P;CzK32KUNIb9b<&$-*CIfRos9 z#46#NxZdCvwLCWN!41$ieBxc^7-jVjXo)`ngYq<}n`<#ZyUzgq-x!R+=Mv>ENy$=t7yxiW$OVg670a?V z+R^bCe2Ka8Qs^bIh9!#C7aEX(cQ?kXO5ISpi~@6;1kJU#(sP4F+W<7<_z80Vbr_1> zSnau4yt+~L-T^^&EWi0c=O6%VY*0*^q#%eZZ)gBkV4}e0_cQ%g1S+ zPj8yp3`S@Hj@u}BHTC*I7_*+66_ipeH>jJB>tcn-`}RBS%nVhzIj%oM>l4ZUBw9Vm zRyddeZ3Z~3WQF6Iro?hZ)tgr<4|=l9OTKhP)$nTjFug-r?f>E(diUc2a8)L_%*zH z^>UHhgm1*&p_K*;uA6%>5K|+g@vGwwpMt@YCBLIGkzLz*Lwmy?B04|uzdXWf>5qS* zuX5ooyd2?KLCCP_25@N~O6>UxlpU2fNb{$6o(#VQ-G>JdSj|pCh?EXKpOgd!x*oar zNa~rh@zL+qOv22XhQwP2{6X3Y28?!uw1z&8V{m(K08(S-|L_qXBe=dnsaQUL=! z*72KxzDo%LJB+dQ&uvP_<%7-0`An&i*=1{VLA%J%%er+2ZdL&YqlAFWZjNiyJPd<>Lb)N8r(zig zpFpEH_4fm|eW$8mnIC=;8Tl&+PE+iVDShk=0+0dds$~eQNj$=uaF%bL#|0j0)N20r z1>EvlHXvPKZ#jFh!Okr7VWLjXv2j~*xVN-9TtDy}1a?Huvq@cFqoAwpL^L$W3NcXw zJ9~SBGL6+VOYvy=B}X))-XH_IUhmUXO&bw*$E+}mzenbKf!NeVn}4>~0)vy$U5a%! zN59tj>oH_JaQ6m)<3cIvE0UDjfMe>~lO)oKp3DHe&pZKFvAxf6Y7LK!VJ}Cc=<=;~ zLQjD9QN(0wF7cj43h^SMxKs}9)iB(l?F;)KJrLMHn0<`+{n5L~0L7!VDzsUi?)SbPcfS<-@ui>r_eY-_u#Xy|!9WdYqT?y@58xEjwJi<09nv;*M;!;&m^L|@op>!1{)qB=W zgHCX?oFgNpKL(9%H#{%@kTkadG+Nj=aYbr@ax}}J6YK#{cfn{} z+K`vR2~gY0LJZ0zT-1YJ8tKqhZ~xv8(NJx|{~j$9pwAEdL5dXo_Cu1!63oBB$`noH zW_3$ksCU3~@VShf3Z7T(AEWu&P(Ri^b468qR)!}?2NncDJhLDuco?3&*P;7&aWucQ zG~?1xA-Sac8Wk@75e5%!UC`_K(ayYhuKo`7fQ*q8^2unft}m{Gx2T^|9vBNFV=a<^G6)5Ht9Ty+t(3MM=&EA1W$`IQ;dj^?RUb4qXD_v0HIMrD3e%S%cG9Dju z5?~ky-#S4J+@_im?tNz*=wHvINA)4I5>1Yd>2Jf(FIgCOQ!vf--5dmg;B@ASk*xvU zsCi!)Z$%hkxgDfu3=)1^H~#&Oi`niAL8sou?XZ%v7#JSN%O0$?jT$ieIMW*foh94M zw<$HOD%+Bvi(HJw2`Mt|0UZPU`^9Wm)Hw2x1Cb27MEDq(DpU50S*@0RR&avjH!E21 zQE$J5F?rbh)J@$<+ILTE@2*0fv$^)Zc%S#?gnQencg|dOzh8Vw01eBbbimaP27D8~2Kb+>nw?qP95Upiy zNCUna84AV}*cp9J&hH2Vnp?wp7P=-}3CCeFnms%V_%TtE?hYhxr(T^?Mp>Q8OdWxn zA8KG3^>&HOf4lSpN5Ve>MyXTV6dAiQh$6&ryYC&p_I?Qro8*_;$WSJzZq9>pto8P1 zsDB&pWALex@7dPF>_t{*hU+?(b!q6c#Vl8VfKQCg=--OX&wAE255i-ur)qSj0(pym z6Ed1m@$$#ge%3&+uzcD`fyu159sJtWdka~Ce?Agnfo~?x|f-FmH*@Mz-ORv)$ z#K&GfAS&_Lm}Y5h0E6-*_SM?)fvB^^rT6w1VxQ5Q$@mXO?|ks3JL*vt7Em-zpQW*YeuK5Q|GoC-Vii;(j;TJZw5FUv%MQz}BLY83| zIi$>dHSqvdl{bAE07u81b}SIQ6SIL$`wgz8{UAc^X`HI(Z{A?-1kH9{FfaH(2UqZjg?eu4 z;1PW>udC=zlctBph4ai*Yt>F|b77|WeiLkI`Xi}M;OlFD{8QGn?$nb3bMiGsox=*7 z5KuzbuKyE)S{oreS19{hiS^NQw?9a+`rBIvhP0l14k?)otu8CWf^D&!kq4_0lh=UY z2nw{uanY+xcOyg|KgMZqw@0M)i;y}e^7|>lK@+-Xi#5c)IZQ#M0~#mx$CKOY)uJj0 z{C-eQga(iiBBd$_QMr~=`NpIFPix-+&UM>{%^#WBE1N`7X0~KT8Bv7H?6OxOWR;a! zRwU6tcA1I4mAx89)*l&Jk&%SH>t|HY^SSB;c(}hwcODM9yMD1ocZ!Z z>6J6W?OXCN@6FMtf`ae>ne-yohdd(#PRX~&=Myuhz=5e<;G9iLmx{zw2pnV6%xWzG z*4?b9ytFxg)L?3#Kk_94!VOJ=%u1KSBDe~B2{DvHvFMhXrloc=q6SMk;HZRzzGC2O zw=XD2$k^THKk49FUtbtJb{SoXUr6W&JQpM%p*e)n2mnw`{<+4(dS^X)MYJJ3K#RAb zbPPS=;ju6d6Dv^VE=se-x{z4-ggk@JGz`KfeyC1cOR7CiT4?W+pxXXaUK$i$;6L)<25->V+!H z(;e6A+Iz5HPsB!J=U4>Hg0e+8vRM8n#m~`vI)dHoB;RM%>{*#QJkm6rLancgLk+Ec zkq4+u7PxpK7{%@Y)FS7P!jnD}u@i>QcTGc?nFb(?#P#q2YVbYv7 z%naxJ99S;s)OZf4$*!O2ONb#W+e_SY3%;DKT*Cd2_4N7rHdh)zC$HN!=eaRG2~Ym7 zvNXs9;Ep9&A4cId?zPBx`F==d=V&vt5i=Bo`MRo~KTpTHzR>?t*?SQZh0C3xzgN+Z zdZhv!xaE^y)*GaJw<4@Rjut8*EwHkN;4FkCdr)jJT8W{}AUZz&07wt-Kw?V*GK(X5 zH3VKT3>E`bHkC>5O=rcJ!*}7qJkqps3w+B=#rg)ch6R?bv^A4ZNsw3TW?fL4YeVCr zmtwacOd?r4&n)nul{B$dNpDbN4*6)16J<<+JsE|RX=PLRXepoO2GU^Ns}+yBQh=NS z>1khtVCD5(m29u0y}ZjjNvht$9YnUCW}*Jt`-=$zO@?pQJfrzOT4ii)F0ss#)gIg#{TxHCKqfR$W<=`$MU1eCfoyLky8 z_d@?x-XFvfD4G9{!Pvo$wo13=aB3k5w9uYG|B8dBJdaU-TqG|Y$Knc(cDHTbdgQ^M zAW@267yno<*uf)-Dea`<{K>#3QhG@nX&f-P1#qiB;A9qea?m_2TzQdoO)X7H-7Ok z40|JTmKb!qv5N|@v!{>fO20{mQYzp;l`=Bmib81ro~~I4Df9rKO`@Ke5dLg|9ipaN z>*%vn_lseAd-Q_`Gn3!$+GY~RAl;)E$ZnolNs5i-h&}5n@dyNb`?-RWTqp3s;ktm- zqn|a)taIZZJ&mNPIvJF<$pt_*L|bLNHjY3`gK7u;dUWjPW$iu0a9v#nREd3>?&yei z#JF!11~3Kz7gGT+nXtrj?n6xs`8bd@QHrz0z4;8dOm*1d*#b-!3-IGb{AvJKEb%TT8$>o*3 zB+-MG#i76cM-DPp>d{d{CeFMM!8TA33>#>5J^cTc6vD}^WPHvjI|tT8jIfRSju>|L z76PBming@+<1kt`2qght8Go>j$9dDE&{Fboj1F3Xi)58u(t_SY92ceK^lh^sA9uy@ zHlgAJte>$lslHEZUUp^#@xO}$YQ3UYfafB#W#|v1*FMsF9cG5SnqbY;UGkr5y7#7p zj#;m6ZU5j#bj=3n#%mHTk9o%i!O>uF^;&X1Z~F5uE6RIHhLTr}#nql%+J*_r4}kFd1i^Ku6-05lLz1iD=el4HoN~_K&^{`!~ot0gC zX(?}#RbdXjlsp3vO7DuoG_+?5NrSAPWnBm6187pXEDpxr0nVzNSzXbC!@v>8AWb0< z$b@N)5<93}Ynbpxfm^vB9QL*F|0g>``#9v5;C2y5feY^D`~YkIEkk23H8*JZ(>+lP zN9n53hwXPt;>X6tfNRSmLMTXgdQUrNGH*-0E3@I%xq(fyy6K@*>FhIT z8%;dWFDS?}$m3=Upv%r__no!XwusQF1EY%?r~n!sA)OnL$rYYc)yFS-M0#U`;nr)( zI9?qIHQ*Pxw=(zUaPG9v8B|I153)$p1DJ*f)YDUgpoiA}NThzM$KPq%FAPDps6<;G z^#fO=ugK%~Ro$^Vw&=zNS|w`7wTKsdi|g;Ifw;~21qpjV(4=q&y!Ig2JF^v=n;$U@ z*^|eONrYP^mP-jR|T0k>25`CNxjgo^*nsdBywsZ7# zl@3}{LPV?DJyzN6=`(P9?kjuo9X( z`6L4{G$jph&KEnuV8Pcc&0ynU3O@Z~UUqahm~r1=_!1z2LxaZpApDow8|o`8FklUa zHEk(XndyXdNg8zh%Ubc6=L{Um@v^nEP=E~uwer0_ZXg5t8@Wp#f3|l9Paj1{PUF(R znaHkCAY=kq;kNCh^Bw1*3lCendd}mEyWC$`25JBZ9-}SVa~xs75k(bld>pT?Jg5Y7 zwVK(SBS5bfHu}6%0R%pCSUdW)CAa4g+K}ujbGIDS*Lj07dgRU~4f_n|gGhZ)yAZo= zbSn2|4|Ykuzv!b!Va=01ougn(tr) zb~!^cxG|Ez)XQMRjq#O)Wec_Ls@FU!jNVj74ZoN|Qj{VR(!Kf<^ffb^A`lvj_r^37I z!i!5M`oKs+yNWu0CBYrTUsUmkJJZm;{Us}KhbxqdadyOKZ{M?8^Z+e&={-o{QNPw3 zPrGI5r+vD@Ob`Xg9<-g-2CR~zMN~SeK=3_po2FJpQwzHA=U34QD>hFG!2y^hlJztx zjQb7(A_vQLunW9Sk#_hqbVcfGe^rl1O4a`wOQ1P>_Uu^e23Ubo3lOeCu`mju_duzC zAQ&PMfwe8?r@GH=WdrP)c)RK|ksbA!vCpc`QLS`bpa@fS5%3$7kUZ0PNCe!-(~gN+ zm_6-Tw>Cm7gE-G5d1K)?%gPe5-0u#_y_&|I$G0rAyumKHQ&e)T=p<~H?f40D@9p$ z;cy%W5@BK{v@R@|j;!S0+9EdgUQ%lEre){y*nr+r!;(VbQ{a^>f^}s0xCOY=9WAkX z%ccC*waY1)iz~P#3)4&Zx5z5!HLJ_D);l9&_MdRoADpB}-&2&aG^N;0O?PKFefxYC zrYO67F4K5$k`UCQTyG-`^dVv-9+CJ4XLTAZ+NmQaPa+{_rqa67HzYCu5o=RzWUJm9 zJbq;EHJZLN^Q2;!9SN_r zP;u7#5c$M6XuF0O;_5jC%R)U|giBg9ei7&eOX<}>yAiPebaLsohaE=t4lm2S&zubn z!XK`E$Yj4bRAFo4F^Y${Y(&oj@4yLEx7LJAib=c&E;w-$`^E+SrSh_tdbD%QJQR!W z-8qkGUfao=+}>{Tu8obTEr(MXIclV1kcaRJ=ZanU%yJ^4M`SqPxOkYZ@s{^mzjcGd zc}AjHw242?kS~AS3yp`1&>RA(=5@arWksbPkBvZo)Zl0k>-#Mj*n1n&(pl}7=vNJr z zA$Dx<#{K@faS-e60{UfEEavpRCybWqd#-2Nz9fXzowKx(1KkO;gegSQF0&)UqM2!u zj0Tg0(yAG4BG@Q#?%`neirjVE2Gz|J{K=`-W8WWXY`x4q(SuDc!BCfmqb&uk1kD6c zr$ff^ZaIjM+4SeNWF4|Q(y6$l0fKI~o>oH1JNoAPFxA5`+CrqQVIHiJqB1uv)gjbV z=laDtd=j2vLwO`13GSyt`j+QJ?fNh?Hivs77%Ry8ugC&(&%7p0?egUZ;#EgQL_{8I z!@d6uAg}JI3El#uVG@^5&ZYq%m7v0c@dtP3A=G17r`5R(CKG z49v1=;?H1jKKE40o%XeG?GxdzdbbRuG(vO zr;r%|D0e&wyhI^CNR;+0TBl;k@Pxj?<)?RYaE7JNYnIPkAU*BP0-|{klOTC!#7E{f zU0pC#TzU@VSd{N$#?{)rg%6p}m6vKw3{R#A!S-R^_Jc0@r*#P@+3_>kw+UOFZi@LT zl>>?4xV@>%{aav3ufHA1uUXIm&AxFKK{sd^*FjGNm(I{h&aiyg=erTwkV?<^)|NX( zCc(v;+;1M6R6-j-1L$OA@J)hF>`rR~o&)3q929G|`ER&bR?LYP%xcVEw8WoCz8yuF z2Qs+EH&cxYCtD8TAqn8;aah+~%t>hfFnAITzI9a&UlLmjN*;s6+n!bjxOTiVpMV|E z&_A}*j6l0Ly{(nR!Za-7#=RhAUTU2Zyt4Zk+}-eRsE(g*f}ylJq#Ed$c{uEq=5Pr()q4I}sJe|SS)(Jt+fP*!h`I$m z{F?niS)VkonT`f6(_pp^@2K5aVj+g_K77%U55%X-)77Ba;^p{bm7JN^dn5*q(JUgUIhF=UGuJ7u z#IFMIteIyhv6M0hqo5N-?HP0;f*gw&$Z%P@Fj8c zw&KS_#WH|eE@tRzM?<#`g+~|@K0*@!>1v21^pENMgXY4_2L&5}e?j>nQ%k|yGRnkX zf-uW`7DRW<2!U-osGMTa=79j6Arn}P^SLmArVAG+&3uA}7GZW7Z6EbRQkd`|zo7d; zT3|Xhk7TEB^^cw$`2kwaCX15Lu*;T6jrQVj{;*DdKK4H03?NwZrO5k@DM!0@2y{Y5SH_TB6HzKJoB)FiA|_~g>*r}1lxDgfy6sg+_R7IK zb+9C%9eV(EKUq=(JxXIBE$a6$!Ndq6l8(fMTA4 zwP#fD#WWp8`HJBfl_@hpuvKOs0$;7Z3}CY*f5g2x*0LtTBwE+01*}K>Ph~H#Og>4O zY?~((5=%Q!xlp!k!Q5xWWXV#c7#x~yfUtE2HPH5!IZEVmweK78$SEkZqk#f)x5C7G z2~RuU*l6c=Y{wc=~bL5?~YPj@;E_DWeC;*9a43(Nc{mRF=UDrD~-E68bjy1AUkwxH`f~g zE3_SHcX}#NxHaZfq}`^;L8pohy;7^E8-=7PSp2C-2F?@O`*#MOzKzlN0a+N4&Zk0i zd{bjlJ~j>B#h-5ikXTEqS)L89IY@SthY||f=K|~&qYyQgEHFR5Dwc* zt9A@bY4(HaFkG<_=^8hOfy|=+F20~QTOGdebLwLk6q)|g0Vy`p8?M7##w*}sHTDih zKoG!u>E`8)xA}F8yn-Y=7-toz*;cN++IIIRd@Mlkr%Zl?n>(BzQ}G`?4BXnC9!3WG zAMFcWseiOD%<%=jH|(}ACMa{%oFJi){7RC~nx2sc&2vyM&%9#=o*)u)8U*k7480SH zx?&%^SKfqql>pp!*PO50x84`m{2t3r_Q$L9!2|;y)v&(Q~SVt0KZEj@`TOc(u70$J8(1xp8*=` zqnd2vLTxuNl}A~Htw=~%Wd?&|HU=7gH!Q-GMckoL=3*$gC+`suJFuXa1v&RycO<*& zVWQC)L;ZD+=Q5RRRfqOcYogisJWmo^Dd0)$XtyDlFBP`JlBjRw2d~oNORIYL+o=o> zJE^quBxxFKA1PZR65$>0WR0rVuMZ!@!00W0l*JIa62TiEwJt7b3VAh zVe{?@Au_{#Y<8pXaM8L|QQGimBuYA=k5GP;E^xO=AvDz>P$SY*^7^_r0%2;}sjzONRF(v^NrALWc#YQ1>KP`Fw7nZYp?RDX9P z-x|+0D!5tor$Y{m6d73+ zEB@?q1Hx08xor9(NA6bY;sd~Yocf5UkZ?jcgEgj-4igE_;*+K%$m|GZSo2L~&A?D_ z>^a0j1<Wyu8W1-KYkTdu%DvLPD7UnyY9T=**FcktBJd+1f`sf@ zw)ztUg*C6xnZt5_ld+*+(!#JZMVMb+^w*7$Vj-R}fC0-^`))nq6bEY=kjbiiRwOVV zMjfMf=7Y(Mg({Roc}7xM&k!()zTW&rS79d7gkY@(KdnnyXG6;S>J?r+mn7#di~wR2 zfSb@BS3cqdf?yuq8TxW#4}#1+H1%NYbv#x4cv*iG-Pxb{+CQ)DfyYb0Yd|2ztO}|) zuN<7w#H%z&DVJ$07Y)$y^+XHIknnfEUMn=!dQtNV?0xQVjUgi!7_EUhWnnN?8kj^2>D+$v!sxXq3{lm}W?3DB zrfDr;Xt>}9761g~3(2Ei4A8H|{N?0#^4vY2i@{{9cYy2~LcFmy-zq`s{kY@d&JE6S zq>U}Fzi}w+G*4J4IJx^UK&T@7YNv|pTXq%OL1`)q#^G z23B`4Gy@E7aq}54o$@-l39zjb$VL>v60i00n&BF-T;*ZD%M~yH*DSuK(;G0#gb5!3 zLg~ocEf~&+Zi+b^H$MxW(^p_ttR{?xr5CokJ1aO_jh#r)A_RKx^M>6FA=HIfwR)5{+2+|ZfU zCaMO&LMFQG|krjeP%2Yrp;EzS+0l6=9JxR@QA@()`1U)=}qn|_`IQV>-Jc9q-3lKw8Fi$;Q+C%X3X0r@4$JsaFmJrqz`+);K zh$4X__QDXT*3l*-*X|jwa1Baihd50JJMyzRZBkA?kW_^G4a^1KTwC z9w(SKUJJll%DezFkQ3-`#}JnkT>Hg|8+$5+)B_rB(u$1-yrQueBn5Qdy?_50D9mQx zaTZc)4rc^h$dvS}5gEL?1lgkkyujPTI+>c-2#pcG|C`-9fEgqw#M*EZ{TgxmrnK;L z#+DN=m%6P>&wMV!`)iXZ-h&i2UE-_s3^_Sk*eY4fk?#yrsH0DBFOiLBctJjE3S-E| znv%fm`_g9bq!9ET=><$>ECNJG*tr!yab5lwVI@GFnP zwFmH&g0jl57rAzBH2^*H4YFmTBqbdD|I%W09|6w7bjPVEX$%R zdn7264vcP?Mq_m3v&T%3y`!c9-=TY0 z2R*2RNjUiHViu8jQ5#>nGw`Li*T=hHP8s5Onqydg0e9b8ZOLLYo2*-03=BLei{?kF zSwU4d6`CRx!C@`ULG&ZtVSKX>e1QGDZiAznHQv85^^3S_0)9g7GuqQ5)z#c}(EQ=n zE7yYeBFzPYHwIJSSk+ZLomtL|8UOVAsKBBM5e4O=;>~T(73?E^{wOv1yOs!^n@D9q z#AWj~VIgrGQ~hg;|8wV~5!&vKdpMolw*aHLodU%=wX#eDf@9Avt#eCDa+iQUcL$E% zes$xkaV7|+<3BgS>^Xm68-~4{%{tPpmB2&R4eG}`(43(3T_44t?#X3rOBRdbNni7V zlSz8;)~~H}qkcoGPq)u@vFYH0RDK5nKhe2ws&%(G+%N)shl5R1vOVv|r~dY442QW}LvmWX|i^H(mS5dPK( zTBHP~l^t99R=Ed97pk&}Iz=1lt{2g{vSadaNXW%QoiARvzy{o}jG%t8r{BmU{88MY z!W}vh3?kk!{p#F+YX>0A#l!bC8wHPAnS(rFm<))_lnRpIt7-_of%+(XV7^f?&(l7+;LZJm}-t;;^u6ppfLHPJ5)^e z8*+X>hJp>DfmGazBbIPVlaHDe!sfz=VXt|vkU!sf_^Ag-S1=Qo{tBSvmr(!$%d3Xt zzr{VyhO9ED;LzaeACrs}DKB&P&9drig$DPb&(*i4Y zyNL*&r7$TO&?4bPom$glL*W-%GdY6SeygCGsMFE!CE@J51pCCiPL^gN z6CLoHY}c&|NC5Wn`Yf;`?ATyOr0S>bTrPld88}rXwNet^4ebvF_xyktW8o01e9~$! zjNOjAxalh0g4YVM0lV6}r1+#=pC%{4TZ@AA^p7P0gulQ%mjJk5GUrc{9KiIhC(f0D z1jxAoERrf!(o=TKt#Ho@bSeP6af(rBf?e`o?C`$JcgR>tskk+$#~ydkow)o#QY%Yx z@i*cty^Ht?Tq!!|a5qJEDH2FkOJ89N_sPSW@eu~tGSRs7s7#cM%OK@n8BV%7Rb+T~RGZn+^(?1obK@S=TSQX(COjdkz_ep`*gpTb&lV{_)>mUidd(*BH zrp}V?+fV6BenO0;j^7=xHzWjGDFKY2tTFRT1pnjx0(&hrn@Zab1UuP5t()~hu*y5A*XQypnuHV{R!vBz?f3cFjfjyh!QOTS4J}qHqa>t6rCd*W$>Ykssjs&;pu_EnnTk$Cf=*qYSM7B(Sn>U@^-k z>DqqyptwIIW3@1z^Y|Hm{_&+8w*lfwlIi}J&&jExnrdVNavMBD-ZJ4s&s7ujG&P5! z_*`P&G~OHXygJ9Jya0!N(+xkodwjYuVew})FxE>!{}y?-Ho!GN`B=X(GTTiDyRLBFBbmg!IdIOeKN)0$HN--0`lt~nQ&9ba4~t#x zS9jH(sikgJFi~;n5gxV1@cb0G37vfC|7CSziu-rH|^9Y)L=8 z@+C`=s#hh&^)ia)E}#1jQ$l|^)t+*ujX%OX3b)pXcHJ-cb(4%@MhB*M4q;r>L<1xG z3vKhgHM5CrFk>ELIW%#$F*YK5T6T_@ez0(iQ;JpEL*Oo>n8tAw;|;^vV3zWcGa`;dE=?SDgic0$hOEImQThUNR*vR_z9d;ENO+)M;m(b4*)|L z2j~!;DrPA^*cqq}=Ob)s#(lQ5Y-I1K1MU=_<7C3y^`hRhmm`!>qs0~BZ3=GDRfC;v z$F4ueoqXU_^|rVDV#%p&(m9|C@l{N7AakS6Boy+S3S23^_uIutzuToU6$f8}V~iNi zNC7oiEn8HwtDbgxr54^STc696{!XTnk#i{V*6@vKJe?n+3PNOZtu)b|>1K%ihqpkq zslu@fPY9^_jhPVHL`)P08pIr)i$DR#06pV2cqh(n!RTvgke0%-*st9jm@P@z+*2MM;o@6YvWHs)7_W(#AaK_s=2{hRUxcXXAh)RUn9-r~2?&ct64e zW6IAk1TqJaNFG$IekPDFF5Ni^Z0ajRF0;!X3XukoZS0>-vag_CWc#$-nL&ny7&q4f z8kGhRN&zqsu~brY6gMAKyM1{#77TTs_NK6rfTJAb?nn(R^K(wP4`vmU6Gm}e+Xv!! zou)!Kl!qZ}4Nt>bkVG@%C93rk%9^6gybHI`IAj*dRC0}FK)N?iLmbi5TM-F$p;?2Q z+gDr0FJHbOhVV;AWEMzj#=X_(vPm&|i|bFpA04N}uAjq=bUEG>nSy8D=SR0)xzIN`T&$>vdEs8I96*wPa5f&Oll97=hqjwk`$oXBD64^^9z*=uC zGbu^fx?RUj``X^$q8`OM=iei1!NJGaHWsTa|G_(sl#A9SLIiek?SJ4ea8k*MLZ`)r-Ya z{SgFvB;8Rvdp+=GUgH_|3H(@wd<6WE~rUi1s8{8xg=)M%&e(8vutH zyObfLD8(NENv&e7!RfH%;Ay}$SbR5^lqo)_oICeT;rRKVvZrb?Xazj(EJKq1y$-TU z{vm6aAA{YmNyL@#b|VSc*%;!)U56PSCC)FBOybdJ-qOGsw-_o5){w~n=XVBXW4;53 zHnntCshsiea}LpGbBIkyn{t_wVS{~MwM-`;RamLZnMHa`na z5ZOEMqF%z|JFSQV{UCf$A|GyNG|`KD$n5|p8A@Aa6n|YtG(=^;pa@<_1JeRwq3_wCB-duxew%FPHTsGu?M^P` zyppXa+9W^7k+q1Jpr}V@OZ901&-$zv_lYOLG{Br-H*|Oa`O9})4i8*qf!-6#R1Gpw zKL^6{Es(5Qr~|7CA-(!+?Q-plX~1} zEEc?PZNi-)Mx>x>42C0GkbeKFdyugADeFqjZo!RgqU0TXD=$>}+US9^89P3jZ3V zap)fwCTI68Oj6+KsN??Ixa9JmDHT~>T>NXZB8td>oWCJ4wd}xRAK~7%i{rfyZFA2R zn8Kylu%#7=E?2~D;R@&;Y-v{iJL;p5-@h|01Bx1dG7|^03%q>rJ=5%q|Hii>K)U{D ztif^DDf)FDT^zBxt5K-~^KqjJZ48Ul_2WplP0DKz$zl^15z%>VDCgL9vKRJz4b^#R zc7f}_pajf>)$Pgg)9^;05$aL?%*bmGiR#Uri)%WmYh0=}f*C|Uwf|fPkvR-f8s0Qi z1$X?G3&2~5FY{9F1KaYHgk(4~SUWy9{Rf53~1J!XAka|;OJc20@`n|COYwp5{ z=*3|W%_!3D4*cWA4;Rl92CdUv1#l|xt}s@S!M8IaSWom6xzf)0RftOY;~1BYb^2Y+ zt8~a3K4kdlccyKM+J^88w9#@pt~50aR6GS>(x{@U8VIP1zPG-2yGXAr%tagpliAxK zT^ApaLC~fpk)QRUR1M)@{>AoG(=TBYGsWoz2rWe7k2%E?ZDusH1u^Y$KTudP5YZk# zx{@bROt0DAzMe57WDzSw)z$eVtCQ(&;CyMosfiI+_fS0%zZ2_ zqc;eZa(edT3nZ@}BX;uE4+(7oS7;KGTU#O)V7NWvq>CL7Q!Zoy@4&!Zl3?hlgwuUk zx0L$r#UILKWKagh2n|R%C2LrxXr`3GuK{a@28<~(?iCDz*me;TX!$+Qe?R_qaLqXY zzV^4lIB1qqOklpRb`?fqQGH(n?k~sWH0a?Dk|@yLN0tM56qdXFq&sh+On+@$?}@6f zJBU6X;o{`EgtT=bxZO%FI3V*0U4N>{#>he|0|P}Ptuhg_vVtfXQZcHnP&XljSIkhT{(Ppb4J$bT=q7(Ucy_1r_BZ}mrrjs zKKcXVDhE=kASTzzHh&;raI3~PgX3jw^F8jz@UlLAK6p-{#Hqpi2${^f>DyB*4dn!3 z9R(?{y~+;D8YO!Uoonl*g{4ydnrxltuM@6M9=(jNQ5PG+1(dJIvhp}S6#)~rwAW|E z+hN~sp3nc8dUEX^8NNRf-(<~<4|ms5-rsijoWPF^bnv&wY{1!}0QgUR*P^ZS<~2j` zHb4{eS7ITkPx_8C{o|G-xU>h!}L}CF}fDQS!f|^d#()6F^XK{yne{gb0W@<|AF%j|73K{9+lgS8pbIE^X8Rtv_|4Y1>3kU zi#pANhv#vSagJ0T{4Kx z`ili8^PLNViBYz~`39lZt;oJiD$6K4|JTFLM7$l+xF~y(TZ%^`M`i2>vzkF%ppc5R zK_1n}jK4oW=R=)&Q4^Qcn?Mn{}6e4OSFczur>VyU?{3@Icg;NGAF)QAY|!E%jml*u8N;zzi*JzrWr;LSJSTHso^ z2TxK2wUB!3;|Ata4>{&bi4~6?`A=m0OO?cx&_uK+u9NKz=;OSsWU4on1$PG}p&?*^ zr0O>6e}4;8?_!Ty^MUJ-A87t+U}Qt?s@fKtI`i!p!ZOUib^hvOTH1)=IQM#HQ5CqM zJI&^nTE%4TI-;|@(0-yqao&Pd9x*&xx&LLSY6y*-sBa*$|0ihPF%j!C64X~w-vp#o|~LN_`-Ljs@#H*w&nI&Z&I<*&h` zGmM@nMOsckbHD%qM8%&8JULVJ8qnly?d-UuL>pE?sAon;5D&96aTir(`H6M#7uTQs z!EmT3qICB$9OEzt_km|+V_KGpv$!`{ACn*rCgcF#jl^%b5VDQi)(5;^Ymr>r3X0drXI-L?vMOgU3?H1k$4d2AdpU z6bd!Lk_4yAH4+8n+UVi|=tIn-GR;&!g64715@zUGf8mEXy%Ju7BhB|5NYEEtg67Rm zmrHtuksv4QLuk-4YOs(Ym%|nar`y{D9`I0OHsL~PN4kFRDp?Lx<)b!-Er^3d%XRMr z{g30O;1?hhC-$jm2O~`iq|P>vUvV{dkdz?Z`2Lmu8?$N6lU8Vygq^gnE;H+T22~%& zEi}+wi91hyPjMX7alKb%tx}eTbg_``VD{0g0m>Z-lUWaRE%;fmzmz#Fy(>xXu_MEHBZ?U6vP1 zx2SROYl;-Q$|MtqvDRVGaD>{PhU~)G{yz9vLkUJDB?Sr?+;zNR&@V|VL2TGluj7Io zqcfU+4X+Jqa3L2)DEM&z?i&9hZf844tvmQQpj^%AMmL1L1}0}f|NRJqLifTOZQ|I@ zI@S1lohl*gbS!5%5l1Nv5r&7*tEGqkAON2-p6f)=q$gNZ%>$$$Y)N`g7;bep7kZ9n z+%*sq8ziP_c(lpoaN7bkU)gBUi`pfM^F0R^_hK{mBiTH|a#VwojvkfalG+B+eGS=$ zK9VdneYAsP3S@Nfky<7I+XGh%y0CpT-MEV8r{{xaS>?tS{av8cE%I#9KwPBIE1H>+Wn1mxd>OHoa z!#8B#TCTsl0dNoD>}W!3nZZtQohW%DS6%+>`EvzGHyg@VAZrRlOo$`FlWZS2Im*g) zb^+dDWbf^H6qTW%rT}!I{3&nH(m;#CPH(SRNXu)lbeLcgPg{@WF;sZz;QaPXiQ1n{ z7euInPuh1Qte|AED}o$di3g^Rzea|A(0V#<*i5I725?a7%3)`{dy>JHxl)u~MvqNn z$99fbsax^su{71jGe^Vm8sgBXOSqRP@C0*M&Bf>(@)>hTj-~Lg#ABV^ykR`+-YPcwHSveLtb~nfEh* zvuYP)t(m9qqKwXue9zDtar@enn|zmEkYhTPz&Xo~gq6Wo*S;DERu}64cRB~X@C;*v z0s1b^MnHosKs*hrm6pmC+W4XqZ~*5eEu|;h-gH-(*L%$(;i3p0$QGM0SZ(5wCG{%S z7&ZZ$pX^;cyPsAYxnqnxJ$Wkzg4>uyaa?j)EPnz5flr696&+%NP?1#lXKiHT$erS3 zH8diDnOH+IIU}U-n5Cksicdrsy6(}#Tx6?N%`V2TjEdV=4uO_LF!0-w9s^GbJu<)( zNbjwn-MRO)`S7JEo7Xcg78Ui0EY+Le!nJ|VYi2oh2DuUr;=Z-^%iJ5em=lclx*NHQ z+uB;9$hBa@A8c;el=zPgoVieh>}j^+vlql917&+t$yUW(C-1_5$IH;j=7y7+WL2m$ zg-%`9$F>>{^S2igqGsScH8iXX1>@iDr@0-)n>vO(v{S)n-XnbC3qUpyHn3U5)Cd91 zRJ=F7@vt8js*tK+KG`P}XCpU#fad``mcpOji=4kNAW7qs2oWikxrZ z##CuNRy^kH;E)GQWJQPyj`5L%t6lz$fOS2{`UzWy7XdmlPfM=RQ2#os$X*tT4@J-8Z|Ka`Xhp~lVh2oIiKAVNU=%iOh4qsL!m^_LTc3`vG7(9|rOAJ=ngDxfYE1 zk*xK2JI{|D-h@HJDqh3E#-Qbg+zgf{NROoy-3jVE`{(hVWhalTuK^kZ7qpIqWhG4$ zA3l6Yt*4E`79&nMb+dtM8UJ@wl+28ULTMyy#rcTKj6O~fHnWw zc1XJ?AP#AEB9mtj|7GX~l9YFZ`li#FnSwc>5U8Rgd^gK7CJDHWaah5wTvETpF6qZ} zAeTY`>8wqvv9nUvtNrY&EeGiOe5bl8UuWg47->cW!bEgI*Ri0H7q-X*;bBSr=pkbS@96wiNm6 z(YFGX-cPJAnMXc2oZ_*1>L@?|`3GW-4zU`Lzap|{Ngf<-WYZFSrYo2)tyO0d-|84w zW$N?p3{%$sQo89+o_N#zKo&E4C-MZHN|=o*VX>%|(2h$^kn59LAeR zz?1P3(PDFgD@^)*aDPQK{D3Rmhiv>D#HK*w=Qz_~nwJ3hSu1#b+e1-YG5Z7@R$QR| zYz1Zv)5>MBxMI^INZJndq|mRN1Cq1o`CvLT*@UNEoV}nxX-BL`FN5cd!+~$0fjkcW zn0GxYX)8y8irqGKA!_e28-1}Ql|V25sn!X)4wdj#;K6K(_jFRhU-b9inan12v& z4XIn6$aaAaXE36G*xp<|fAJTbC1*;VGrVuWOMuYyIYyZ!2KpYjAAyGT9B`_WW=jln zasY6MHkW`~ViSa*C*a(m<#5Fz3SI|i0$ehUhKcFV-Ag`~*GX6`x4%78V1+oPLjhf; zf&pko=q1rg*MuvQ-y%sY3;`V=(U>u4f1rsB-bD(7?FU~bCo2exG>Uu8;kf<={p7O{ z@FoCRqh-cG`>VWi<(;aK_xhqYM}^!RY+L*7Tp;{kfbl89=A%j0sT6M0lWhiZP@ZNn$`axsy8qb~E~Gu6vYEryxpJW5GfkNN}V@w%HeAyXxgKWZM@;TI5T2e$Nim zol!lt8#0{oWj|2i!w-I$bGf+Xjr@Hk>;$4|zv?Q%+?S*$XKvS z(j9Pq=-NasAP(p{MVe!5eGRbwMFXe6fb$-}f&FbZpg4Po417QU&y^$x9$T%ruBQ&( zj2g2n2LTFkx%y#K*A8wBAk6kuMz3kLMCgstoMXbZRUGg*G2{Cnaj~HlT8eEXtfV{p z#u`Hp& zwDS2LUWAHYUYIm2TM@&R`@jP%tj;%u2KM9{BouHn ziANHF9P$~McIVgo#C5UXkmEoHXXU&4gQLJ(Gj0mSf$MDXX^LjIvg!nVq#30<^r$6t zmltm!*Xxh^=k)n4Hl+lati!qFZVJ7OH{JZm>)X zxMhwS-A@DE{JsI`7U$Y7E)J*rq4i#M-_%ry>h z-c;AO{;-V5*_aG{d9ZfbY`K)achV zh=O2oNaWgm#=6@=$l2^nNAP^4X#V$_aVE1T+CP;E>UAJ?8~*SVM|IO6n!x_VGK@2f zj!=Pmb)xF$0dyB8Wf7>S0HK7gD1c65zUsty|ji_0)+6^=v!LL7A_&eqY}^y>hhDdFFBD^bBb7FIbKlcq{( z1V+si5xwZ-2so4h&AqaHHetPDuBbKkh<7GFEDLg1>;Yt36HG}(CAXZbrdigU^xu!p zvbAuSsU$v2VIzS zNnyraQZvVKe0X0&!?_5!kglm*WNz;#AZX*g=_r+O8L9GVPs{n*z5sPza}Xg_fbmp( zQFbdF?$_4DyVPUCh28oL32#*FrWxsl5-{o-q4R9Dx_?Q2Tw~Q@qtIXf;;X*b78K8L z4Dt~6PEr#<>~nhgJJZlI7rPI8wf)h7mdO1>70HSVT*rBE4Y@IlgDdmSNCW=mh(Ak3 zj59>*a7e#1HEZUa5h6`FShKg6OO|~C)`cq@4AA1 zsr>*K9Lhosy*^BC{m(6A+%ymOo6F)W0Yq!HCf?NE9tUyy0oONcmVnUf1 zHl8e-C<+f>3@4%g2SEO9Q9Y*07_!S-OL|Zuxz1cXf9eRN5=34`n5bz;NRr!=R!^UD z%c33eAsR*>w0r|z#%*)NJ{Jw|vBSd)Q&)KV<(P4ZKe~p)(gDuSq@?%hO2qwO*r4h8 zd`a@ww)Y`G3(6mn3Vg8+#6d2Gt!Q_NK)hDAeK?II8FqFqVtbbqa`857P|p#nPk(OYJ4{X{sdmkM`D9VM3xIZ9^(rK(X8nY&4CL zrN<5+M6S~YYT&E10}t=kCHXs2w|`f1W?ZskY*rtn3HE_ciRaei-*L37_=8X2 zMuoI$sl!lPau443LtU()6-!^C;bQ@VL6H&O65i_t$Gkz^)J}pL(kstwzwbU001;1h z7l;`!d=XrgG*V)3@C7p3q+*OgdK359v!ZM1vSM8y$1yd57plB}^oiP$q)x`scO4Sm z_|df`8Rer<0(g?##Uz5Te(X*$4>CVhEz^X`KQ;@dP4Ph5Ds(VM8z z732We%QJX?oR<24ypo;Ge^dkN=p}zYWn4FVcOwTIz@FLLj}lmP%YkKJ?^DARCJ6}% zD~t{mK?Uc9Oy%@=onHB#r->`jEqv0JA|d>Nr+ix)vK~M(+Ak=5>d=VPgbN|wNCLrK z=-G}R@%XOaj&~>_=L~^5x@0c|+t^bUL*n+hfx-HBZPW}!KbyXRDB(;TH^;BsicShb ze}p>%gAgr&O)3OLF6Gh>tgkj`*UIB%cswo|)fpvNboY~1=*sV07=9eozV_LLjw3Qz zka}h6xNLdC$~B6$0eBP;q{sCyI)<;pz<^MkEv8k-?^gxo@Q@Qy5y>a*ej@MwT4D%t z?6|ruD;(`Oh#ubqAY}1iec}^vK9wCcVXUfYzk8SD<(I;H3b(p0?E)(g4Y^8Ar&AT@ z0Eg!_lY~n>rzwmRnrf<@N9t}n{BZfuUJ$fCXs=)G>KB7j%k8F$rZ2Ma9>c;rF;InG zlZxVZHeEi&`v5NB-1Oozr{BR{4R%L;P*N}(`7Lati@FIt-K_5bU~$z>%gH(ZGRYaT z6lSA@C5sMVPgJO8pn4C58&n3<=eRpnrS~E**g*imhn> z2HzC7Co1>2@Uj!kG9?^sQ>vJS?DkiHKOpb zJ}SdEnH`7a@(km)EI2WF<9F5XoDHJ`h>@|q=S;*uZszZ6kfneSL_oZM{r|l7oqJj&juvO5-@CtmeqO}}yJuT< zdh)Ro0q0`~xtw;|+5dcQfB!1KTu6uvB=Qu{fvCz6OwF0+Er0Iv&)-tv1z^g#p4a~Q z(7*qNOULFc>-P80{@4BanZOxPvws%veEToL0zUSbe|(5Te#}o`_r3k|zV<&>4Hq1M j{rlzq=NE0Fe&G;C>O?d?yd;VR|2wOsp_nIc8t{Jrc+!dU diff --git a/harnesses/aggregator-head-lag/cmd/script/update_metrics_calls.sh b/harnesses/aggregator-head-lag/cmd/script/update_metrics_calls.sh deleted file mode 100755 index bd84baa2..00000000 --- a/harnesses/aggregator-head-lag/cmd/script/update_metrics_calls.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Auto-update all RecordX calls to include region parameter - -FILES="head_lag_monitor.go codex_rest_monitor.go mobula_rest_monitor.go quote_api_monitor.go geckoterminal_monitor.go moralis_rest_monitor.go metadata_coverage_monitor.go" - -for file in $FILES; do - if [ -f "$file" ]; then - echo "Updating $file..." - # Use perl for multiline regex - perl -i -pe 's/RecordHeadLag\(([^)]+)\)/RecordHeadLag($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordRESTLatency\(([^)]+)\)/RecordRESTLatency($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordRESTError\(([^)]+)\)/RecordRESTError($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordQuoteAPILatency\(([^)]+)\)/RecordQuoteAPILatency($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordQuoteAPIError\(([^)]+)\)/RecordQuoteAPIError($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordHeadLagError\(([^)]+)\)/RecordHeadLagError($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordCodexBlockNumber\(([^)]+)\)/RecordCodexBlockNumber($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordMetadataCoverage\(([^)]+)\)/RecordMetadataCoverage($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordMetadataLatency\(([^)]+)\)/RecordMetadataLatency($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordPoolDiscoveryLatency\(([^)]+)\)/RecordPoolDiscoveryLatency($1, config.MonitorRegion)/g' "$file" - perl -i -pe 's/RecordPoolDiscoveryError\(([^)]+)\)/RecordPoolDiscoveryError($1, config.MonitorRegion)/g' "$file" - fi -done - -echo "βœ“ All files updated" diff --git a/harnesses/aggregator-head-lag/docker-compose.yml b/harnesses/aggregator-head-lag/docker-compose.yml deleted file mode 100644 index 7536daee..00000000 --- a/harnesses/aggregator-head-lag/docker-compose.yml +++ /dev/null @@ -1,82 +0,0 @@ -services: - monitor: - build: - context: . - dockerfile: Dockerfile - container_name: latency_monitor - ports: - - "2112:2112" - environment: - - COINGECKO_API_KEY=${COINGECKO_API_KEY} - - MOBULA_API_KEY=${MOBULA_API_KEY} - - DEFINED_SESSION_COOKIE=${DEFINED_SESSION_COOKIE} - networks: - - monitoring - restart: unless-stopped - - prometheus: - image: prom/prometheus:latest - container_name: prometheus - ports: - - "9090:9090" - volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - - ./monitoring/alert_rules.yml:/etc/prometheus/alert_rules.yml - - prometheus_data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--web.enable-lifecycle' - networks: - - monitoring - restart: unless-stopped - depends_on: - - monitor - - alertmanager: - image: prom/alertmanager:latest - container_name: alertmanager - ports: - - "9093:9093" - volumes: - - ./monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml - command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--storage.path=/alertmanager' - networks: - - monitoring - restart: unless-stopped - - grafana: - image: grafana/grafana:latest - container_name: grafana - ports: - - "3000:3000" - volumes: - - ./monitoring/grafana/provisioning:/etc/grafana/provisioning - - ./monitoring/grafana/dashboards:/dashboards-source:ro - - ./grafana-entrypoint.sh:/grafana-entrypoint.sh:ro - entrypoint: ["/bin/sh", "/grafana-entrypoint.sh"] - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin} - - GF_USERS_ALLOW_SIGN_UP=false - - GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL:-http://localhost:3000} - - GF_INSTALL_PLUGINS=grafana-clock-panel - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/head_lag.json - networks: - - monitoring - depends_on: - - prometheus - restart: unless-stopped - -networks: - monitoring: - driver: bridge - -volumes: - prometheus_data: diff --git a/harnesses/aggregator-head-lag/grafana-entrypoint.sh b/harnesses/aggregator-head-lag/grafana-entrypoint.sh deleted file mode 100755 index ccf63ca7..00000000 --- a/harnesses/aggregator-head-lag/grafana-entrypoint.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -echo "=== GRAFANA ENTRYPOINT SCRIPT STARTING ===" -echo "Working directory: $(pwd)" -echo "User: $(whoami)" - -# Create dashboards directory -mkdir -p /var/lib/grafana/dashboards - -# Debug: print environment variables -echo "Checking environment..." -env | grep -E '(RAILWAY|HIDE_QUOTE)' || echo "No RAILWAY/HIDE_QUOTE variables found" - -# Copy dashboards from source -if [ "$HIDE_QUOTE_DASHBOARD" = "true" ]; then - echo "HIDE_QUOTE_DASHBOARD=true - hiding Quote API Latency Benchmark dashboard" - cp /dashboards-source/head_lag.json /var/lib/grafana/dashboards/ -else - echo "Copying all dashboards" - cp /dashboards-source/*.json /var/lib/grafana/dashboards/ -fi - -echo "Dashboards copied:" -ls -la /var/lib/grafana/dashboards/ - -echo "=== GRAFANA ENTRYPOINT SCRIPT COMPLETE ===" - -# Start Grafana with default entrypoint -exec /run.sh diff --git a/harnesses/aggregator-head-lag/grafana/Dockerfile b/harnesses/aggregator-head-lag/grafana/Dockerfile deleted file mode 100644 index eb500db9..00000000 --- a/harnesses/aggregator-head-lag/grafana/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM grafana/grafana:latest - -USER root - -# Cache buster - update this to force rebuild: v7 -ARG CACHE_BUST=7 - -# Copy provisioning configs and dashboards source (we're in grafana folder) -COPY provisioning /etc/grafana/provisioning -COPY dashboards /dashboards-source - -# Copy entrypoint script -COPY grafana-entrypoint.sh /grafana-entrypoint.sh -RUN chmod +x /grafana-entrypoint.sh - -# Set permissions -RUN chown -R grafana:root /etc/grafana/provisioning /dashboards-source - -USER grafana - -# Environment variables -ENV GF_AUTH_ANONYMOUS_ENABLED=true -ENV GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer -ENV GF_SECURITY_ADMIN_USER=admin -ENV GF_SECURITY_ADMIN_PASSWORD=admin -ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/head_lag.json - -# Prometheus datasource URL (overridden by staging deployment) -ENV PROMETHEUS_URL=http://prometheus.railway.internal:9090 - -EXPOSE 3000 - -# Use custom entrypoint -ENTRYPOINT ["/bin/sh", "/grafana-entrypoint.sh"] diff --git a/harnesses/aggregator-head-lag/grafana/dashboards/disabled/mobula_pulse_vs_fasttrade.json b/harnesses/aggregator-head-lag/grafana/dashboards/disabled/mobula_pulse_vs_fasttrade.json deleted file mode 100644 index 373e2f06..00000000 --- a/harnesses/aggregator-head-lag/grafana/dashboards/disabled/mobula_pulse_vs_fasttrade.json +++ /dev/null @@ -1,595 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2000 - }, - { - "color": "orange", - "value": 5000 - }, - { - "color": "red", - "value": 10000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Pulse.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Fast-Trade.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean", "last", "max", "min"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "pool_discovery_latency_milliseconds{aggregator=\"mobula\"}", - "legendFormat": "[{{region}}] Pulse V2 - {{chain}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "fast_trade_latency_milliseconds{aggregator=\"mobula\"}", - "legendFormat": "[{{region}}] Fast-Trade - {{chain}}", - "range": true, - "refId": "B" - } - ], - "title": "Mobula: Pulse V2 (Discovery) vs Fast-Trade (Swap Indexation)", - "description": "Pulse V2 measures pool discovery latency (on-chain creation β†’ Mobula indexation). Fast-Trade measures swap indexation latency (on-chain swap β†’ WebSocket receipt).", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2000 - }, - { - "color": "orange", - "value": 5000 - }, - { - "color": "red", - "value": 10000 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 3, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["last"], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "fast_trade_latency_milliseconds{aggregator=\"mobula\"}", - "legendFormat": "[{{region}}] {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Fast-Trade - Current Swap Indexation Latency", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2000 - }, - { - "color": "red", - "value": 5000 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["last"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "pool_discovery_latency_milliseconds{aggregator=\"mobula\",chain=\"solana\"}", - "legendFormat": "[{{region}}] Pulse V2", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "fast_trade_latency_milliseconds{aggregator=\"mobula\",chain=\"solana\"}", - "legendFormat": "[{{region}}] Fast-Trade", - "refId": "B" - } - ], - "title": "Solana - Pulse V2 vs Fast-Trade", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2000 - }, - { - "color": "red", - "value": 5000 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["last"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "pool_discovery_latency_milliseconds{aggregator=\"mobula\",chain=\"base\"}", - "legendFormat": "[{{region}}] Pulse V2", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "fast_trade_latency_milliseconds{aggregator=\"mobula\",chain=\"base\"}", - "legendFormat": "[{{region}}] Fast-Trade", - "refId": "B" - } - ], - "title": "Base - Pulse V2 vs Fast-Trade", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2000 - }, - { - "color": "red", - "value": 5000 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["last"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "pool_discovery_latency_milliseconds{aggregator=\"mobula\",chain=\"ethereum\"}", - "legendFormat": "[{{region}}] Pulse V2", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "fast_trade_latency_milliseconds{aggregator=\"mobula\",chain=\"ethereum\"}", - "legendFormat": "[{{region}}] Fast-Trade", - "refId": "B" - } - ], - "title": "Ethereum - Pulse V2 vs Fast-Trade", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2000 - }, - { - "color": "red", - "value": 5000 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["last"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "pool_discovery_latency_milliseconds{aggregator=\"mobula\",chain=\"bnb\"}", - "legendFormat": "[{{region}}] Pulse V2", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "fast_trade_latency_milliseconds{aggregator=\"mobula\",chain=\"bnb\"}", - "legendFormat": "[{{region}}] Fast-Trade", - "refId": "B" - } - ], - "title": "BNB - Pulse V2 vs Fast-Trade", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": ["mobula", "pulse", "fast-trade", "comparison"], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Mobula: Pulse V2 vs Fast-Trade Comparison", - "uid": "mobula_pulse_vs_fasttrade", - "version": 0, - "weekStart": "" -} diff --git a/harnesses/aggregator-head-lag/grafana/dashboards/disabled/pulse_vs_codex.json b/harnesses/aggregator-head-lag/grafana/dashboards/disabled/pulse_vs_codex.json deleted file mode 100644 index 8ced6186..00000000 --- a/harnesses/aggregator-head-lag/grafana/dashboards/disabled/pulse_vs_codex.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Direct comparison of Pulse vs Codex head lag on monitored pools", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (seconds)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*pulse.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull", "mean", "max"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{aggregator=~\"pulse|codex\",chain=~\"$chain\"}", - "legendFormat": "{{chain}} | {{aggregator}}", - "range": true, - "refId": "A" - } - ], - "title": "Pulse vs Codex - Head Lag Comparison", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Shows which provider is faster (negative = Pulse faster, positive = Codex faster)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": true, - "axisColorMode": "text", - "axisLabel": "Delta (seconds)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": -1 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(head_lag_seconds{aggregator=\"pulse\",chain=~\"$chain\"} - ignoring(aggregator) head_lag_seconds{aggregator=\"codex\",chain=~\"$chain\"})", - "legendFormat": "{{chain}} - Pulse vs Codex delta", - "range": true, - "refId": "A" - } - ], - "title": "Latency Delta: Pulse - Codex (negative = Pulse faster)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "color-text" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Chain" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Provider" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 3, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Mean (5m)" - } - ] - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "head_lag_seconds{aggregator=~\"pulse|codex\",chain=~\"$chain\"}", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Current" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg_over_time(head_lag_seconds{aggregator=~\"pulse|codex\",chain=~\"$chain\"}[5m])", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Mean5m" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max_over_time(head_lag_seconds{aggregator=~\"pulse|codex\",chain=~\"$chain\"}[5m])", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Max5m" - } - ], - "title": "Current Stats - Pulse vs Codex", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "deployment": true, - "instance": true, - "job": true, - "region": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "aggregator": "Provider", - "chain": "Chain" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "Chain": { - "aggregations": [], - "operation": "groupby" - }, - "Provider": { - "aggregations": [], - "operation": "groupby" - }, - "Value #Current": { - "aggregations": ["lastNotNull"], - "operation": "aggregate" - }, - "Value #Max5m": { - "aggregations": ["lastNotNull"], - "operation": "aggregate" - }, - "Value #Mean5m": { - "aggregations": ["lastNotNull"], - "operation": "aggregate" - } - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": { - "Chain": 0, - "Provider": 1, - "Value #Current (lastNotNull)": 2, - "Value #Max5m (lastNotNull)": 4, - "Value #Mean5m (lastNotNull)": 3 - }, - "renameByName": { - "Value #Current (lastNotNull)": "Current", - "Value #Max5m (lastNotNull)": "Max (5m)", - "Value #Mean5m (lastNotNull)": "Mean (5m)" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 3 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 4, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{aggregator=\"pulse\",chain=~\"$chain\"}", - "instant": true, - "legendFormat": "{{chain}}", - "refId": "A" - } - ], - "title": "Pulse - Current Head Lag", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 3 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 5, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{aggregator=\"codex\",chain=~\"$chain\"}", - "instant": true, - "legendFormat": "{{chain}}", - "refId": "A" - } - ], - "title": "Codex - Current Head Lag", - "type": "gauge" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": ["pulse", "codex", "comparison", "head-lag"], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": ["All"], - "value": ["$__all"] - }, - "hide": 0, - "includeAll": true, - "label": "Chain", - "multi": true, - "name": "chain", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "solana", - "value": "solana" - }, - { - "selected": false, - "text": "base", - "value": "base" - }, - { - "selected": false, - "text": "bnb", - "value": "bnb" - } - ], - "query": "solana,base,bnb", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Pulse vs Codex - Head Lag Comparison", - "uid": "pulse_vs_codex", - "version": 1, - "weekStart": "" -} diff --git a/harnesses/aggregator-head-lag/grafana/dashboards/disabled/quote_api_latency.json b/harnesses/aggregator-head-lag/grafana/dashboards/disabled/quote_api_latency.json deleted file mode 100644 index d0e12537..00000000 --- a/harnesses/aggregator-head-lag/grafana/dashboards/disabled/quote_api_latency.json +++ /dev/null @@ -1,916 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 500 - }, - { - "color": "red", - "value": 2000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*jupiter.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*kyberswap.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*paraswap.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*openocean.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain!=\"arbitrum\"}[1m])) by (le, provider, chain))", - "legendFormat": "{{provider}} - {{chain}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "Quote API Latency Comparison - All Providers (P50)", - "description": "Median Quote API response time - Mobula vs competitors (30s polling interval)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "orange", - "value": 700 - }, - { - "color": "red", - "value": 1500 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 2, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket[5m])) by (le, provider))", - "legendFormat": "{{provider}}", - "range": true, - "refId": "A" - } - ], - "title": "Latest Quote API Latency (P50) by Provider", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*jupiter.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain=\"solana\"}[1m])) by (le, provider))", - "legendFormat": "{{provider}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "🌞 Solana Quote APIs - Mobula vs Jupiter", - "description": "Solana swap quote latency comparison", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain=\"base\"}[1m])) by (le, provider))", - "legendFormat": "{{provider}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "πŸ”΅ Base Quote APIs - Mobula vs Competitors", - "description": "Base chain swap quote latency comparison", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain=\"ethereum\"}[1m])) by (le, provider))", - "legendFormat": "{{provider}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "⟠ Ethereum Quote APIs - Competitors Only", - "description": "Ethereum chain swap quote latency (Mobula not deployed)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "hue", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["sum"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(increase(quote_api_errors_total{chain!=\"arbitrum\"}[5m])) by (provider, chain)", - "legendFormat": "{{provider}} - {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Quote API Errors (Last 5min)", - "description": "Number of errors per provider and chain", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 95 - }, - { - "color": "red", - "value": 99 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "100 * sum(rate(quote_api_status_codes_total{status_code=\"200\"}[5m])) by (provider) / sum(rate(quote_api_status_codes_total[5m])) by (provider)", - "legendFormat": "{{provider}}", - "range": true, - "refId": "A" - } - ], - "title": "Quote API Success Rate by Provider", - "description": "Percentage of successful (200) responses", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 9, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{provider=\"mobula\",chain!=\"arbitrum\"}[1m])) by (le, chain))", - "legendFormat": "Mobula - {{chain}} (P50)", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(quote_api_latency_milliseconds_bucket{provider=\"mobula\",chain!=\"arbitrum\"}[1m])) by (le, chain))", - "legendFormat": "Mobula - {{chain}} (P95)", - "range": true, - "refId": "B" - } - ], - "title": "Mobula Quote API Latency by Chain (P50 & P95)", - "description": "Mobula swap quote latency across supported chains", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": ["quote-api", "swap", "latency", "mobula", "jupiter", "benchmark"], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Quote API Latency Benchmark", - "uid": "quote_api_latency", - "version": 0, - "weekStart": "" -} diff --git a/harnesses/aggregator-head-lag/grafana/dashboards/head_lag.json b/harnesses/aggregator-head-lag/grafana/dashboards/head_lag.json deleted file mode 100644 index c325ba7c..00000000 --- a/harnesses/aggregator-head-lag/grafana/dashboards/head_lag.json +++ /dev/null @@ -1,945 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds Behind", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "orange", - "value": 60 - }, - { - "color": "red", - "value": 300 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#1F78B4", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#1F78B4", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D03F", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#F4D03F", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#27AE60", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#27AE60", - "value": null - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "mean", - "last", - "max", - "min" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg_over_time(head_lag_seconds[1m])", - "legendFormat": "[{{region}}] {{aggregator}} - {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Head Lag (Estimated Seconds Behind)", - "description": "Estimated time in seconds the aggregator is behind the blockchain head. Calculated using average block time per chain.", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "#1F78B4" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg_over_time(head_lag_seconds{aggregator=\"mobula\"}[1m])", - "legendFormat": "[{{region}}] {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Mobula - Current Head Lag (Seconds)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "#F4D03F" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 4, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg_over_time(head_lag_seconds{aggregator=\"codex\"}[1m])", - "legendFormat": "[{{region}}] {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Codex - Current Head Lag (Seconds)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "#27AE60" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 5, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg_over_time(head_lag_seconds{aggregator=\"geckoterminal\"}[1m])", - "legendFormat": "[{{region}}] {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "GeckoTerminal - Current Head Lag (Seconds)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto", - "spanNulls": true - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#1F78B4", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#1F78B4", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D03F", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#F4D03F", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#27AE60", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#27AE60", - "value": null - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "avg_over_time(head_lag_seconds{region=\"eu-west\"}[1m])", - "legendFormat": "[{{region}}] {{aggregator}} - {{chain}}", - "refId": "A" - } - ], - "title": "EU West - Head Lag", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto", - "spanNulls": true - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#1F78B4", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#1F78B4", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D03F", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#F4D03F", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#27AE60", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#27AE60", - "value": null - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "avg_over_time(head_lag_seconds{region=\"us-east\"}[1m])", - "legendFormat": "[{{region}}] {{aggregator}} - {{chain}}", - "refId": "A" - } - ], - "title": "US East - Head Lag", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto", - "spanNulls": true - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#1F78B4", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#1F78B4", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D03F", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#F4D03F", - "value": null - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#27AE60", - "mode": "fixed" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#27AE60", - "value": null - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "avg_over_time(head_lag_seconds{region=\"sgp\"}[1m])", - "legendFormat": "[{{region}}] {{aggregator}} - {{chain}}", - "refId": "A" - } - ], - "title": "Singapore - Head Lag", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "head-lag", - "indexation", - "blockchain", - "sync" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Head Lag Monitor - Blockchain vs Aggregator Sync", - "uid": "head_lag_monitor", - "version": 0, - "weekStart": "" -} \ No newline at end of file diff --git a/harnesses/aggregator-head-lag/grafana/grafana-entrypoint.sh b/harnesses/aggregator-head-lag/grafana/grafana-entrypoint.sh deleted file mode 100755 index ccf63ca7..00000000 --- a/harnesses/aggregator-head-lag/grafana/grafana-entrypoint.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -echo "=== GRAFANA ENTRYPOINT SCRIPT STARTING ===" -echo "Working directory: $(pwd)" -echo "User: $(whoami)" - -# Create dashboards directory -mkdir -p /var/lib/grafana/dashboards - -# Debug: print environment variables -echo "Checking environment..." -env | grep -E '(RAILWAY|HIDE_QUOTE)' || echo "No RAILWAY/HIDE_QUOTE variables found" - -# Copy dashboards from source -if [ "$HIDE_QUOTE_DASHBOARD" = "true" ]; then - echo "HIDE_QUOTE_DASHBOARD=true - hiding Quote API Latency Benchmark dashboard" - cp /dashboards-source/head_lag.json /var/lib/grafana/dashboards/ -else - echo "Copying all dashboards" - cp /dashboards-source/*.json /var/lib/grafana/dashboards/ -fi - -echo "Dashboards copied:" -ls -la /var/lib/grafana/dashboards/ - -echo "=== GRAFANA ENTRYPOINT SCRIPT COMPLETE ===" - -# Start Grafana with default entrypoint -exec /run.sh diff --git a/harnesses/aggregator-head-lag/grafana/provisioning/dashboards/dashboard.yml b/harnesses/aggregator-head-lag/grafana/provisioning/dashboards/dashboard.yml deleted file mode 100644 index 332c8f34..00000000 --- a/harnesses/aggregator-head-lag/grafana/provisioning/dashboards/dashboard.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: 1 - -providers: - - name: 'Aggregator Latency Dashboards' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: false - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: true diff --git a/harnesses/aggregator-head-lag/grafana/provisioning/datasources/prometheus.yml b/harnesses/aggregator-head-lag/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 30d5cc21..00000000 --- a/harnesses/aggregator-head-lag/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: ${PROMETHEUS_URL} - uid: prometheus - isDefault: true - editable: false diff --git a/harnesses/aggregator-head-lag/monitoring/alert_rules.yml b/harnesses/aggregator-head-lag/monitoring/alert_rules.yml deleted file mode 100644 index e08b8a73..00000000 --- a/harnesses/aggregator-head-lag/monitoring/alert_rules.yml +++ /dev/null @@ -1,127 +0,0 @@ -groups: - - name: aggregator_latency_alerts - interval: 30s - rules: - # Missing metrics alerts - - alert: MissingMobulaMetrics - expr: absent(rest_api_latency_milliseconds_count{aggregator="mobula"}) - for: 5m - labels: - severity: critical - aggregator: mobula - alert_type: missing_metrics - annotations: - summary: "Mobula metrics are missing" - description: "No REST API metrics received from Mobula for 5 minutes. Check API key and monitor status." - - # Stale metrics alerts (>20 minutes without update) - - alert: MobulaEthereumStaleMetrics - expr: (time() - timestamp(rest_api_latency_milliseconds_count{aggregator="mobula",chain="ethereum"})) > 1200 - for: 2m - labels: - severity: warning - aggregator: mobula - chain: ethereum - alert_type: stale_metrics - annotations: - summary: "Mobula Ethereum metrics are stale" - description: "Mobula Ethereum REST API hasn't responded in over 20 minutes. Check API connectivity." - - - alert: MobulaBaseStaleMetrics - expr: (time() - timestamp(rest_api_latency_milliseconds_count{aggregator="mobula",chain="base"})) > 1200 - for: 2m - labels: - severity: warning - aggregator: mobula - chain: base - alert_type: stale_metrics - annotations: - summary: "Mobula Base metrics are stale" - description: "Mobula Base REST API hasn't responded in over 20 minutes. Check API connectivity." - - # Latency spike alerts - - alert: MobulaEthereumLatencySpike - expr: | - ( - rate(rest_api_latency_milliseconds_sum{aggregator="mobula",chain="ethereum"}[5m]) / - rate(rest_api_latency_milliseconds_count{aggregator="mobula",chain="ethereum"}[5m]) - ) > 500 - for: 3m - labels: - severity: warning - aggregator: mobula - chain: ethereum - alert_type: latency_spike - annotations: - summary: "Mobula Ethereum latency spike" - description: "Mobula Ethereum average latency is {{ $value | humanize }}ms over the last 5 minutes (threshold: 500ms)" - - - alert: MobulaBaseLatencySpike - expr: | - ( - rate(rest_api_latency_milliseconds_sum{aggregator="mobula",chain="base"}[5m]) / - rate(rest_api_latency_milliseconds_count{aggregator="mobula",chain="base"}[5m]) - ) > 500 - for: 3m - labels: - severity: warning - aggregator: mobula - chain: base - alert_type: latency_spike - annotations: - summary: "Mobula Base latency spike" - description: "Mobula Base average latency is {{ $value | humanize }}ms over the last 5 minutes (threshold: 500ms)" - - # Instant spike detection for Mobula (any single request >10s) - - alert: MobulaInstantLatencySpike - expr: | - rest_api_latency_milliseconds{aggregator="mobula"} > 10000 - labels: - severity: warning - aggregator: mobula - alert_type: instant_spike - annotations: - summary: "Mobula instant latency spike on {{ $labels.chain }}" - description: "Mobula {{ $labels.chain }} single request took {{ $value | humanize }}ms (>10s). Brief spike detected." - - # Extreme latency spikes (>5x normal) - - alert: MobulaExtremeLatencySpike - expr: | - ( - rate(rest_api_latency_milliseconds_sum{aggregator="mobula"}[5m]) / - rate(rest_api_latency_milliseconds_count{aggregator="mobula"}[5m]) - ) > 1000 - for: 1m - labels: - severity: critical - aggregator: mobula - alert_type: extreme_latency - annotations: - summary: "Mobula EXTREME latency spike on {{ $labels.chain }}" - description: "Mobula {{ $labels.chain }} latency is {{ $value | humanize }}ms (>1000ms). Possible service degradation." - - # Error rate alerts - - alert: HighRESTErrorRate - expr: | - ( - rate(rest_api_errors_total[5m]) / - (rate(rest_api_errors_total[5m]) + rate(rest_api_latency_milliseconds_count[5m])) - ) > 0.1 - for: 5m - labels: - severity: warning - alert_type: high_error_rate - annotations: - summary: "High REST API error rate for {{ $labels.aggregator }} {{ $labels.chain }}" - description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 10%)" - - # Service availability - - alert: CodexServiceDown - expr: up{job="latency_monitor"} == 0 - for: 2m - labels: - severity: critical - alert_type: service_down - annotations: - summary: "Latency monitor service is down" - description: "The aggregator latency monitor has been down for 2 minutes. No metrics are being collected." diff --git a/harnesses/aggregator-head-lag/monitoring/alertmanager.yml b/harnesses/aggregator-head-lag/monitoring/alertmanager.yml deleted file mode 100644 index 77b9e263..00000000 --- a/harnesses/aggregator-head-lag/monitoring/alertmanager.yml +++ /dev/null @@ -1,36 +0,0 @@ -global: - resolve_timeout: 5m - -route: - group_by: ['alertname', 'chain', 'aggregator'] - group_wait: 10s - group_interval: 30s - repeat_interval: 4h - receiver: 'slack-webhook' - -receivers: - - name: 'slack-webhook' - webhook_configs: - - url: 'https://agent-slack-production.up.railway.app/webhook/grafana' - send_resolved: true - -inhibit_rules: - # Inhibit warning alerts if critical alert is firing - - source_match: - severity: 'critical' - target_match: - severity: 'warning' - equal: ['alertname', 'chain', 'aggregator'] - - # Inhibit stale metrics alerts if service is down - - source_match: - alert_type: 'service_down' - target_match: - alert_type: 'stale_metrics' - - # Inhibit stale metrics if missing metrics alert is firing - - source_match: - alert_type: 'missing_metrics' - target_match: - alert_type: 'stale_metrics' - equal: ['aggregator'] diff --git a/harnesses/aggregator-head-lag/monitoring/grafana/dashboards/disabled/quote_api_latency.json b/harnesses/aggregator-head-lag/monitoring/grafana/dashboards/disabled/quote_api_latency.json deleted file mode 100644 index 16392e30..00000000 --- a/harnesses/aggregator-head-lag/monitoring/grafana/dashboards/disabled/quote_api_latency.json +++ /dev/null @@ -1,819 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 500 - }, - { - "color": "red", - "value": 2000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*jupiter.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*kyberswap.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*paraswap.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*openocean.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain!=\"arbitrum\"}[1m])) by (le, provider, chain))", - "legendFormat": "{{provider}} - {{chain}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "Quote API Latency Comparison - All Providers (P50)", - "description": "Median Quote API response time - Mobula vs competitors (30s polling interval)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "orange", - "value": 700 - }, - { - "color": "red", - "value": 1500 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 2, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket[5m])) by (le, provider))", - "legendFormat": "{{provider}}", - "range": true, - "refId": "A" - } - ], - "title": "Latest Quote API Latency (P50) by Provider", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*jupiter.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain=\"solana\"}[1m])) by (le, provider))", - "legendFormat": "{{provider}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "\ud83c\udf1e Solana Quote APIs - Mobula vs Jupiter", - "description": "Solana swap quote latency comparison", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{chain=\"base\"}[1m])) by (le, provider))", - "legendFormat": "{{provider}} (P50)", - "range": true, - "refId": "A" - } - ], - "title": "\ud83d\udd35 Base Quote APIs - Mobula vs Competitors", - "description": "Base chain swap quote latency comparison", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "hue", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["sum"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(increase(quote_api_errors_total{chain!=\"arbitrum\"}[5m])) by (provider, chain)", - "legendFormat": "{{provider}} - {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Quote API Errors (Last 5min)", - "description": "Number of errors per provider and chain", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 95 - }, - { - "color": "red", - "value": 99 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "100 * sum(rate(quote_api_status_codes_total{status_code=\"200\"}[5m])) by (provider) / sum(rate(quote_api_status_codes_total[5m])) by (provider)", - "legendFormat": "{{provider}}", - "range": true, - "refId": "A" - } - ], - "title": "Quote API Success Rate by Provider", - "description": "Percentage of successful (200) responses", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 9, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(quote_api_latency_milliseconds_bucket{provider=\"mobula\",chain!=\"arbitrum\"}[1m])) by (le, chain))", - "legendFormat": "Mobula - {{chain}} (P50)", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(quote_api_latency_milliseconds_bucket{provider=\"mobula\",chain!=\"arbitrum\"}[1m])) by (le, chain))", - "legendFormat": "Mobula - {{chain}} (P95)", - "range": true, - "refId": "B" - } - ], - "title": "Mobula Quote API Latency by Chain (P50 & P95)", - "description": "Mobula swap quote latency across supported chains", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": ["quote-api", "swap", "latency", "mobula", "jupiter", "benchmark"], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Quote API Latency Benchmark", - "uid": "quote_api_latency", - "version": 0, - "weekStart": "" -} diff --git a/harnesses/aggregator-head-lag/monitoring/grafana/dashboards/head_lag.json b/harnesses/aggregator-head-lag/monitoring/grafana/dashboards/head_lag.json deleted file mode 100644 index 02606a47..00000000 --- a/harnesses/aggregator-head-lag/monitoring/grafana/dashboards/head_lag.json +++ /dev/null @@ -1,773 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds Behind", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "orange", - "value": 60 - }, - { - "color": "red", - "value": 300 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["median", "lastNotNull", "max", "min"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Median", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{chain!=\"ethereum\"}", - "legendFormat": "{{aggregator}} - {{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Head Lag (Estimated Seconds Behind)", - "description": "Estimated time in seconds the aggregator is behind the blockchain head. Calculated using average block time per chain.", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{aggregator=\"mobula\",chain!=\"ethereum\"}", - "legendFormat": "{{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Mobula - Current Head Lag (Seconds)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 4, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{aggregator=\"codex\",chain!=\"ethereum\"}", - "legendFormat": "{{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "Codex - Current Head Lag (Seconds)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 5, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "head_lag_seconds{aggregator=\"geckoterminal\",chain!=\"ethereum\"}", - "legendFormat": "{{chain}}", - "range": true, - "refId": "A" - } - ], - "title": "GeckoTerminal - Current Head Lag (Seconds)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "head_lag_seconds{region=\"eu-west\",chain!=\"ethereum\"}", - "legendFormat": "{{aggregator}} - {{chain}}", - "refId": "A" - } - ], - "title": "EU West - Head Lag", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "head_lag_seconds{region=\"us-west\",chain!=\"ethereum\"}", - "legendFormat": "{{aggregator}} - {{chain}}", - "refId": "A" - } - ], - "title": "US West - Head Lag", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Seconds", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "lineWidth": 2, - "showPoints": "auto" - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*codex.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*geckoterminal.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "head_lag_seconds{region=\"singapore\",chain!=\"ethereum\"}", - "legendFormat": "{{aggregator}} - {{chain}}", - "refId": "A" - } - ], - "title": "Singapore - Head Lag", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": ["head-lag", "indexation", "blockchain", "sync"], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Head Lag Monitor - Blockchain vs Aggregator Sync", - "uid": "head_lag_monitor", - "version": 0, - "weekStart": "" -} diff --git a/harnesses/aggregator-head-lag/monitoring/grafana/provisioning/dashboards/dashboard.yml b/harnesses/aggregator-head-lag/monitoring/grafana/provisioning/dashboards/dashboard.yml deleted file mode 100644 index 49b448da..00000000 --- a/harnesses/aggregator-head-lag/monitoring/grafana/provisioning/dashboards/dashboard.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: 1 - -providers: - - name: 'Aggregator Latency Dashboards' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: true diff --git a/harnesses/aggregator-head-lag/monitoring/grafana/provisioning/datasources/prometheus.yml b/harnesses/aggregator-head-lag/monitoring/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 40a18e69..00000000 --- a/harnesses/aggregator-head-lag/monitoring/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: ${PROMETHEUS_URL:-http://prometheus:9090} - uid: prometheus - isDefault: true - editable: false diff --git a/harnesses/aggregator-head-lag/monitoring/prometheus.yml b/harnesses/aggregator-head-lag/monitoring/prometheus.yml deleted file mode 100644 index bd7cf041..00000000 --- a/harnesses/aggregator-head-lag/monitoring/prometheus.yml +++ /dev/null @@ -1,21 +0,0 @@ -global: - scrape_interval: 5s - evaluation_interval: 5s - -# Load alert rules -rule_files: - - '/etc/prometheus/alert_rules.yml' - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: ['alertmanager:9093'] - -scrape_configs: - - job_name: 'latency_monitor' - static_configs: - - targets: ['monitor:2112'] - labels: - app: 'aggregator_latency_monitor' - environment: 'production' diff --git a/harnesses/aggregator-head-lag/prometheus/Dockerfile b/harnesses/aggregator-head-lag/prometheus/Dockerfile deleted file mode 100644 index a422384f..00000000 --- a/harnesses/aggregator-head-lag/prometheus/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM prom/prometheus:v2.49.1 - -USER root - -# Pick which prometheus.yml to bake in (prometheus.yml for prod, prometheus.staging.yml for staging). -ARG PROMETHEUS_CONFIG=prometheus.yml - -COPY ${PROMETHEUS_CONFIG} /etc/prometheus/prometheus.yml -COPY alert_rules.yml /etc/prometheus/alert_rules.yml - -EXPOSE 9090 - -# --enable-feature=expand-external-labels: lets global.external_labels read ${ENVIRONMENT} from env -CMD ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--web.enable-lifecycle", "--web.enable-admin-api", "--enable-feature=expand-external-labels"] diff --git a/harnesses/aggregator-head-lag/prometheus/alert_rules.yml b/harnesses/aggregator-head-lag/prometheus/alert_rules.yml deleted file mode 100644 index dd89ab5e..00000000 --- a/harnesses/aggregator-head-lag/prometheus/alert_rules.yml +++ /dev/null @@ -1,223 +0,0 @@ -groups: - - name: aggregator_latency_alerts - interval: 30s - rules: - # Missing metrics alerts - - alert: MissingMobulaMetrics - expr: absent(rest_api_latency_milliseconds_count{aggregator="mobula"}) - for: 5m - labels: - severity: critical - aggregator: mobula - alert_type: missing_metrics - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula metrics are missing' - description: "No REST API metrics received from Mobula for 5 minutes. Check API key and monitor status." - - # Stale metrics alerts (>20 minutes without update) - - alert: MobulaBaseStaleMetrics - expr: (time() - timestamp(rest_api_latency_milliseconds_count{aggregator="mobula",chain="base"})) > 1200 - for: 2m - labels: - severity: warning - aggregator: mobula - chain: base - alert_type: stale_metrics - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula Base metrics are stale' - description: "Mobula Base REST API hasn't responded in over 20 minutes. Check API connectivity." - - # Latency spike alerts - - alert: MobulaSolanaLatencySpike - expr: | - ( - rate(rest_api_latency_milliseconds_sum{aggregator="mobula",chain="solana"}[5m]) / - rate(rest_api_latency_milliseconds_count{aggregator="mobula",chain="solana"}[5m]) - ) > 2000 - for: 3m - labels: - severity: warning - aggregator: mobula - chain: solana - alert_type: latency_spike - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula Solana latency spike' - description: "Mobula Solana average latency is {{ $value | humanize }}ms over the last 5 minutes (threshold: 2000ms)" - - - alert: MobulaBaseLatencySpike - expr: | - ( - rate(rest_api_latency_milliseconds_sum{aggregator="mobula",chain="base"}[5m]) / - rate(rest_api_latency_milliseconds_count{aggregator="mobula",chain="base"}[5m]) - ) > 2500 - for: 3m - labels: - severity: warning - aggregator: mobula - chain: base - alert_type: latency_spike - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula Base latency spike' - description: "Mobula Base average latency is {{ $value | humanize }}ms over the last 5 minutes (threshold: 2500ms)" - - - alert: MobulaBNBLatencySpike - expr: | - ( - rate(rest_api_latency_milliseconds_sum{aggregator="mobula",chain="bnb"}[5m]) / - rate(rest_api_latency_milliseconds_count{aggregator="mobula",chain="bnb"}[5m]) - ) > 2500 - for: 3m - labels: - severity: warning - aggregator: mobula - chain: bnb - alert_type: latency_spike - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula BNB latency spike' - description: "Mobula BNB average latency is {{ $value | humanize }}ms over the last 5 minutes (threshold: 2500ms)" - - # Instant spike detection for Mobula (any single request >10s) - - alert: MobulaInstantLatencySpike - expr: | - rest_api_latency_milliseconds{aggregator="mobula"} > 10000 - labels: - severity: warning - aggregator: mobula - alert_type: instant_spike - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula instant latency spike on {{ $labels.chain }}' - description: "Mobula {{ $labels.chain }} single request took {{ $value | humanize }}ms (>10s). Brief spike detected." - - # Error rate alerts - - alert: HighRESTErrorRate - expr: | - ( - rate(rest_api_errors_total[5m]) / - (rate(rest_api_errors_total[5m]) + rate(rest_api_latency_milliseconds_count[5m])) - ) > 0.1 - for: 5m - labels: - severity: warning - alert_type: high_error_rate - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}High REST API error rate for {{ $labels.aggregator }} {{ $labels.chain }}' - description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 10%)" - - # Head lag (WebSocket latency) alerts - per-chain thresholds - # NOTE: alerts use fixed-cardinality gauges now (no tx_hash label) to avoid stale-series spam. - # Sustained 30s window avoids single-spike firings (reconnect bursts, transient network jitter). - - alert: MobulaHeadLagSpikeBase - expr: mobula_head_lag_detailed_seconds{chain="base"} > 2.5 - for: 30s - labels: - severity: warning - aggregator: mobula - alert_type: head_lag_spike - app: aggregator_latency_monitor - chain: base - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula head lag spike on Base' - description: | - **Mobula WebSocket Latency Spike - Base** - - **Latency:** {{ $value }}s (threshold: 2.5s, sustained β‰₯30s) - β€’ Mobula Processing: {{ with query (printf "mobula_processing_lag_seconds{chain=\"base\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}{{ . | first | value }}s{{ end }} - β€’ Network: {{ with query (printf "mobula_network_lag_seconds{chain=\"base\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}{{ . | first | value }}s{{ end }} - - β€’ Pool: {{ $labels.pool_address }} - β€’ Region: {{ $labels.region }} - β€’ Latest tx: {{ with query (printf "mobula_last_tx_hash{chain=\"base\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}https://basescan.org/tx/{{ . | first | label "tx_hash" }}{{ end }} - - - alert: MobulaHeadLagSpikeSolana - expr: mobula_head_lag_detailed_seconds{chain="solana"} > 2 - for: 30s - labels: - severity: warning - aggregator: mobula - alert_type: head_lag_spike - app: aggregator_latency_monitor - chain: solana - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula head lag spike on Solana' - description: | - **Mobula WebSocket Latency Spike - Solana** - - **Latency:** {{ $value }}s (threshold: 2s, sustained β‰₯30s) - β€’ Mobula Processing: {{ with query (printf "mobula_processing_lag_seconds{chain=\"solana\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}{{ . | first | value }}s{{ end }} - β€’ Network: {{ with query (printf "mobula_network_lag_seconds{chain=\"solana\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}{{ . | first | value }}s{{ end }} - - β€’ Pool: {{ $labels.pool_address }} - β€’ Region: {{ $labels.region }} - β€’ Latest tx: {{ with query (printf "mobula_last_tx_hash{chain=\"solana\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}https://solscan.io/tx/{{ . | first | label "tx_hash" }}{{ end }} - - - alert: MobulaHeadLagSpikeBNB - expr: mobula_head_lag_detailed_seconds{chain="bnb"} > 2.5 - for: 30s - labels: - severity: warning - aggregator: mobula - alert_type: head_lag_spike - app: aggregator_latency_monitor - chain: bnb - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Mobula head lag spike on BNB Chain' - description: | - **Mobula WebSocket Latency Spike - BNB Chain** - - **Latency:** {{ $value }}s (threshold: 2.5s, sustained β‰₯30s) - β€’ Mobula Processing: {{ with query (printf "mobula_processing_lag_seconds{chain=\"bnb\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}{{ . | first | value }}s{{ end }} - β€’ Network: {{ with query (printf "mobula_network_lag_seconds{chain=\"bnb\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}{{ . | first | value }}s{{ end }} - - β€’ Pool: {{ $labels.pool_address }} - β€’ Region: {{ $labels.region }} - β€’ Latest tx: {{ with query (printf "mobula_last_tx_hash{chain=\"bnb\",region=\"%s\",pool_address=\"%s\"}" $labels.region $labels.pool_address) }}https://bscscan.com/tx/{{ . | first | label "tx_hash" }}{{ end }} - - # Missing head lag metrics (no data for 5 minutes = monitor down) - - alert: MissingHeadLagMetrics - expr: absent(head_lag_seconds) - for: 5m - labels: - severity: critical - alert_type: missing_head_lag - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Head lag metrics missing' - description: "No head_lag_seconds metrics received for 5 minutes. Check if monitors are running." - - # Per-aggregator staleness: fires when one provider stops pushing data - # (e.g. Codex WS disconnect) while others keep running β€” global absent() - # would NOT catch this. - - alert: AggregatorHeadLagStale - expr: (time() - timestamp(head_lag_seconds)) > 300 - for: 1m - labels: - severity: warning - alert_type: head_lag_stale - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}{{ $labels.aggregator }} head_lag stale on {{ $labels.chain }} ({{ $labels.region }})' - description: | - **{{ $labels.aggregator }}** hasn't pushed a head_lag sample for **{{ $labels.chain }} / {{ $labels.region }}** in over 5 minutes. - - Likely cause: WebSocket disconnected, JWT expired, proxy IP banned, or auth cookie rotated. - - Check the monitor logs for `[HEAD-LAG][{{ $labels.aggregator | toUpper }}]` errors. - - # Service availability - - alert: CodexServiceDown - expr: up{job="latency_monitor"} == 0 - for: 2m - labels: - severity: critical - alert_type: service_down - app: aggregator_latency_monitor - annotations: - summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Latency monitor service is down' - description: "The aggregator latency monitor has been down for 2 minutes. No metrics are being collected." diff --git a/harnesses/aggregator-head-lag/prometheus/alertmanager.yml b/harnesses/aggregator-head-lag/prometheus/alertmanager.yml deleted file mode 100644 index 77b9e263..00000000 --- a/harnesses/aggregator-head-lag/prometheus/alertmanager.yml +++ /dev/null @@ -1,36 +0,0 @@ -global: - resolve_timeout: 5m - -route: - group_by: ['alertname', 'chain', 'aggregator'] - group_wait: 10s - group_interval: 30s - repeat_interval: 4h - receiver: 'slack-webhook' - -receivers: - - name: 'slack-webhook' - webhook_configs: - - url: 'https://agent-slack-production.up.railway.app/webhook/grafana' - send_resolved: true - -inhibit_rules: - # Inhibit warning alerts if critical alert is firing - - source_match: - severity: 'critical' - target_match: - severity: 'warning' - equal: ['alertname', 'chain', 'aggregator'] - - # Inhibit stale metrics alerts if service is down - - source_match: - alert_type: 'service_down' - target_match: - alert_type: 'stale_metrics' - - # Inhibit stale metrics if missing metrics alert is firing - - source_match: - alert_type: 'missing_metrics' - target_match: - alert_type: 'stale_metrics' - equal: ['aggregator'] diff --git a/harnesses/aggregator-head-lag/prometheus/prometheus.staging.yml b/harnesses/aggregator-head-lag/prometheus/prometheus.staging.yml deleted file mode 100644 index a5c329de..00000000 --- a/harnesses/aggregator-head-lag/prometheus/prometheus.staging.yml +++ /dev/null @@ -1,26 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - # Staging external label triggers [STAGING] prefix in alert summaries via templating. - external_labels: - environment: 'staging' - -# Load alert rules (shared with production; [STAGING] prefix is conditional via $externalLabels) -rule_files: - - '/etc/prometheus/alert_rules.yml' - -# Staging Alertmanager (separate from production) -alerting: - alertmanagers: - - static_configs: - - targets: ['alertmanager-staging.railway.internal:9093'] - -# Staging scrape targets β€” 3 regions, same pattern as production -scrape_configs: - - job_name: 'monitor-staging' - static_configs: - - targets: - - 'agg-staging-eu.railway.internal:2112' - - 'agg-staging-us.railway.internal:2112' - - 'agg-staging-sgp.railway.internal:2112' - metrics_path: /metrics diff --git a/harnesses/aggregator-head-lag/prometheus/prometheus.yml b/harnesses/aggregator-head-lag/prometheus/prometheus.yml deleted file mode 100644 index f9a4dfce..00000000 --- a/harnesses/aggregator-head-lag/prometheus/prometheus.yml +++ /dev/null @@ -1,27 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - # Expanded at startup via --enable-feature=expand-external-labels. - # Unset => empty value => alert rule templates treat it as "not staging" and don't prefix. - external_labels: - environment: '${ENVIRONMENT}' - -# Load alert rules -rule_files: - - '/etc/prometheus/alert_rules.yml' - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: ['alertmanager.railway.internal:9093'] - -scrape_configs: - - job_name: 'monitor' - static_configs: - - targets: - - 'agg-eu-west.railway.internal:2112' - - 'alert-reflection.railway.internal:2112' - - 'aggregator-latency-benchmark.railway.internal:2112' - metrics_path: /metrics - diff --git a/harnesses/bridge-monitor/README.md b/harnesses/bridge-monitor/README.md index 14691d38..872031c9 100644 --- a/harnesses/bridge-monitor/README.md +++ b/harnesses/bridge-monitor/README.md @@ -16,12 +16,15 @@ Two loops run simultaneously inside one Go binary: 1. **Quote loop** (free, every 5 minutes) β€” fetches a quote from every bridge for every route Γ— every notional ($5 / $50 / $300). Records quote latency, fees, slippage, estimated time. All four bridges benchmarked here. 2. **Execution loop** (paid, fixed UTC times) β€” actually broadcasts real transactions through Mobula + Relay + Li.Fi. Measures quote β†’ broadcast β†’ settlement end-to-end latency and revert rates. Debridge is excluded from execution. -On top of those: +Plus daily P&L cron, wallet balance metrics, pre-flight tier simulation, and per-tier Slack notifications. -- **Wallet balance metrics** refreshed every 5 minutes -- **Alertmanager β†’ Slack** with balance / gas / consecutive-failure / stale-data alerts -- **Daily P&L cron** at 09:30 UTC (per-chain wallet diff vs 24 h ago) with tier viability health check -- **Pre-flight tier simulation** β€” a tier whose full cycle can't complete is skipped cleanly with a single Slack message +## Where the data goes + +This harness is a **data producer only** β€” it exposes `/metrics` on port `9090` (overridable via `METRICS_PORT`). The shared OpenChainBench Prometheus (see [`/infrastructure/prometheus`](../../infrastructure/prometheus)) scrapes that endpoint: + +``` +bridge-monitor.railway.internal:9090 ──► prometheus.railway.internal ──► public site +``` ## Test routes @@ -61,51 +64,35 @@ Capital circulates through the triangle β€” only bridge fees + gas are consumed. | `bridge_output_usd` | USD received on destination | | `bridge_quote_success` | 1 on success, 0 on error | | `bridge_success_total` / `bridge_reverts_total` / `bridge_errors_total` | Outcome counters | -| `bridge_consecutive_failures` | Streak length (resets on success) β€” drives paging alert | +| `bridge_consecutive_failures` | Streak length (resets on success) | ### Wallet state | Metric | Description | | --- | --- | | `wallet_balance_usd{chain,token}` | Current USD per leg, refreshed every 5 min | -| `wallet_balance_last_update_timestamp_seconds` | Last refresh timestamp (drives staleness alert) | - -The site queries these via the YAML specs β€” `bridge-quote-latency.yml` reads `bridge_quote_latency_ms_*`, `bridge-fee.yml` reads `bridge_cost_percent`. +| `wallet_balance_last_update_timestamp_seconds` | Last refresh timestamp | -## Alerts - -### Bridge performance / reliability - -- `HighBridgeQuoteLatency` β€” p95 > 2 s for 5 m (warn) -- `HighBridgeExecutionLatency` β€” p95 > 2 min for 5 m (warn) -- `HighBridgeFees` β€” fees > 5 % for 15 m (warn) -- `HighBridgeRevertRate` β€” > 10 % for 5 m (critical) -- `BridgeConsecutiveFailures` β€” β‰₯3 fails in a row (critical) β€” bridge-side or integration broken -- `BridgeMonitorDown` β€” monitor unreachable 2 m (critical) - -### Wallet / capital - -- `LowTriangleBalance` β€” any USDC/USDT leg < $50 (warn) -- `CriticalTriangleBalance` β€” leg < $10 (critical) -- `LowGasBalance` β€” ETH/SOL < $3 (warn) -- `WalletBalanceStale` β€” no refresh for 20 min (warn) +The site reads these via the YAML specs β€” `bridge-quote-latency.yml` reads `bridge_quote_latency_ms_*`, `bridge-fee.yml` reads `bridge_cost_percent`. ## Run locally ```bash cp .env.example .env # Fill in API keys, wallet keys (or leave dry-run defaults) -docker-compose -f deploy/docker-compose.yml up -d +go run ./cmd/monitor/ ``` -Access: +`/metrics` will be exposed on `http://localhost:9090/metrics`. + +If `EXECUTION_MODE=dry-run` (default) the monitor only fetches quotes β€” no transactions, no fees, no wallets needed. Quote-loop metrics still flow normally. -- Prometheus: -- Grafana: (anonymous viewer) -- Monitor metrics: -- Alertmanager: +Or via Docker: -If `EXECUTION_MODE=dry-run` (default) the monitor only fetches quotes β€” no transactions, no fees, no wallets needed. Quote-loop metrics still flow into Prometheus normally. +```bash +docker build -t bridge-monitor . +docker run --rm --env-file .env -p 9090:9090 bridge-monitor +``` ## Execution modes @@ -121,18 +108,9 @@ EXECUTION_MODE=single-test TEST_AMOUNT_USD=0.50 go run ./cmd/monitor/ EXECUTION_MODE=production go run ./cmd/monitor/ ``` -## Deploy on Railway - -Four services from this repo, all rooted at `harnesses/bridge-monitor/`: +## Run on Railway -| Service | Root | Dockerfile | Public | Env vars | -| --- | --- | --- | --- | --- | -| **bridge-monitor** | `harnesses/bridge-monitor` | `./Dockerfile` | no | wallet keys, API keys, `EXECUTION_MODE`, `SLACK_WEBHOOK_URL` | -| **prometheus** | `harnesses/bridge-monitor/deploy/prometheus` | `./Dockerfile` | yes (read-only) | none | -| **alertmanager** | `harnesses/bridge-monitor/deploy/alertmanager` | `./Dockerfile` | no | `SLACK_WEBHOOK_URL` | -| **grafana** | `harnesses/bridge-monitor/deploy/grafana` | `./Dockerfile` | yes | none | - -Internal networking: `bridge-monitor.railway.internal:9090`, `prometheus.railway.internal:9090`, `alertmanager.railway.internal:9093`. Detailed instructions in [`deploy/DEPLOY.md`](./deploy/DEPLOY.md). +This service is deployed from the OpenChainBench repo, root directory `harnesses/bridge-monitor/`. Set the env vars listed below; the shared Prometheus will pick it up via DNS automatically. ## Project layout @@ -140,7 +118,7 @@ Internal networking: `bridge-monitor.railway.internal:9090`, `prometheus.railway cmd/monitor/ Main binary β€” quote loop, execution, scheduler β”œβ”€β”€ main.go β”œβ”€β”€ config.go Env loading - β”œβ”€β”€ metrics.go Prometheus metric definitions + β”œβ”€β”€ metrics.go Prometheus metric definitions + HTTP /metrics handler β”œβ”€β”€ scheduler.go Fixed UTC time scheduling β”œβ”€β”€ executor.go Per-route balance check, exec orchestration β”œβ”€β”€ cycle_sim.go Pre-flight viability simulation @@ -158,13 +136,8 @@ cmd/monitor/ Main binary β€” quote loop, execution, schedu cmd/rebalance/ Manual one-shot rebalance tool (unused in prod) -deploy/ - β”œβ”€β”€ docker-compose.yml Local full stack - β”œβ”€β”€ prometheus/ Scrape config + alert_rules.yml + Dockerfile - β”œβ”€β”€ alertmanager/ envsubst entrypoint + Slack template + Dockerfile - └── grafana/ Dashboards + datasource provisioning + Dockerfile - -Dockerfile Monitor binary (multi-stage Go build) +Dockerfile Multi-stage Go build +.env.example Documented env vars ``` ## Bridge integration details @@ -207,7 +180,7 @@ Dockerfile Monitor binary (multi-stage Go build) | Arbitrum | USDT | $0+ (R2 auto-fills before R3 needs it) | R3 source | | Arbitrum | ETH | $5 | Gas (~$0.10/TX) | -Each bridge runs the triangle sequentially and settles before the next bridge starts, so the **peak capital need on any leg = 1Γ— tier** (not 3Γ—). Pre-flight `SimulateTriangleCycle` checks this threshold before every scheduled tier β€” non-viable tiers skip cleanly with a single Slack message. +Pre-flight `SimulateTriangleCycle` checks the 1Γ— threshold before every scheduled tier β€” non-viable tiers skip cleanly with a single Slack message. ## Environment variables @@ -224,7 +197,8 @@ Each bridge runs the triangle sequentially and settles before the next bridge st | `TEST_AMOUNT_USD` | for `single-test` | Default $1 | | `MAX_DAILY_SPEND_USD` | no | Safety cap, default $10 | | `MONITOR_REGION` | yes | Metric label | -| `SLACK_WEBHOOK_URL` | yes if alerts | Used by both monitor and alertmanager | +| `METRICS_PORT` | no | Default 9090 | +| `SLACK_WEBHOOK_URL` | optional | Direct Slack notifs from the monitor | | `SOLANA_RPC` / `BASE_RPC` / `ARB_RPC` | no | Override public RPCs | ## Safety features @@ -237,29 +211,12 @@ Each bridge runs the triangle sequentially and settles before the next bridge st 6. **EVM revert detection** β€” TX receipt inspected. 7. **Gas price +50 % bump** on Arbitrum β€” handles rapid base-fee variance. -## Estimated monthly cost (production mode) - -Based on $0.50 single-test validation: - -| Tier | Fees/cycle | Cycles/mo | Monthly | -| --- | --- | --- | --- | -| Triangle $5 daily (9 TX) | ~$0.48 | 30 | $14.40 | -| Triangle $50 Mon+Thu | ~$1.62 | 8.5 | $13.80 | -| Triangle $300 weekly Mon | ~$4.66 | 4.3 | $20.04 | -| R4 TRUMP $5 weekly | ~$0.60 | 4.3 | $2.60 | -| **On-chain subtotal** | | | **~$51/mo** | -| Railway (4 services) | | | **~$25/mo** | -| **Total** | | | **~$76/mo** | - -Variance up to ~$90/mo during gas spikes. - ## Adding a bridge 1. Create `cmd/monitor/_bridge.go` mirroring an existing integration (e.g. `relay_bridge.go`). 2. Implement quote, execute, status methods conforming to the `Bridge` interface. 3. Add API key field to `Config` and `.env.example`. 4. Register the bridge in `main.go`'s bridge list. -5. Update Grafana dashboards if you want it visualised internally. ## License diff --git a/harnesses/bridge-monitor/deploy/DEPLOY.md b/harnesses/bridge-monitor/deploy/DEPLOY.md deleted file mode 100644 index 78e9a882..00000000 --- a/harnesses/bridge-monitor/deploy/DEPLOY.md +++ /dev/null @@ -1,158 +0,0 @@ -# Railway Deployment Guide - -## Architecture - -4 Railway services from the GitHub repo: - -``` -miniapps/bridging-latency/ -β”œβ”€β”€ Dockerfile β†’ Service "bridge-monitor" -└── deploy/ - β”œβ”€β”€ prometheus/Dockerfile β†’ Service "prometheus" - β”œβ”€β”€ grafana/Dockerfile β†’ Service "grafana" - └── alertmanager/Dockerfile β†’ Service "alertmanager" -``` - ---- - -## Deployment Steps - -### 1. Service: bridge-monitor - -**Railway Config:** -- **Root Directory**: `miniapps/bridging-latency` -- **Dockerfile Path**: `./Dockerfile` -- **Port**: 9090 - -**Environment Variables:** -```env -MOBULA_API_KEY=your_key -WALLET_EVM_ADDRESS=0x... -WALLET_SOL_ADDRESS=... -EXECUTION_MODE=dry-run -MONITOR_REGION=railway-prod -``` - -**Service Name:** `bridge-monitor` - ---- - -### 2. Service: prometheus - -**Railway Config:** -- **Root Directory**: `miniapps/bridging-latency/deploy/prometheus` -- **Dockerfile Path**: `./Dockerfile` -- **Port**: 9090 - -**Service Name:** `prometheus` - -**Networking:** Accessible via `prometheus.railway.internal:9090` - ---- - -### 3. Service: grafana (Public Dashboard) - -**Railway Config:** -- **Root Directory**: `miniapps/bridging-latency/deploy/grafana` -- **Dockerfile Path**: `./Dockerfile` -- **Port**: 3000 - -**Service Name:** `grafana` - -**Networking:** -- βœ… **Generate public domain** (publicly accessible dashboard) -- Internal: `grafana.railway.internal:3000` - ---- - -### 4. Service: alertmanager - -**Railway Config:** -- **Root Directory**: `miniapps/bridging-latency/deploy/alertmanager` -- **Dockerfile Path**: `./Dockerfile` -- **Port**: 9093 - -**Service Name:** `alertmanager` - -**Networking:** `alertmanager.railway.internal:9093` - ---- - -## Recommended Order - -1. **prometheus** (no dependencies) -2. **alertmanager** (no dependencies) -3. **bridge-monitor** (scrapes to prometheus) -4. **grafana** (datasource to prometheus) - ---- - -## Verification - -### 1. Bridge Monitor -```bash -curl https://.railway.app/metrics -``` - -### 2. Prometheus -- Go to **Status β†’ Targets** -- Verify `bridge-monitor` is **UP** - -### 3. Grafana (Public Dashboard) -- Dashboard displays without login (anonymous auth) -- Graphs show data after 2-3 minutes - ---- - -## Local Development - -Run the full stack locally: -```bash -cd deploy -docker-compose up -d -``` - -Access: -- **Prometheus**: http://localhost:9091 -- **Grafana**: http://localhost:3000 -- **Monitor Metrics**: http://localhost:9090/metrics -- **Alertmanager**: http://localhost:9093 - ---- - -## Key Points - -βœ… **Service Names** must be exactly: `bridge-monitor`, `prometheus`, `grafana`, `alertmanager` -βœ… Railway internal URLs: `.railway.internal` -βœ… Grafana has **anonymous auth** = public access without login -βœ… Prometheus scrapes bridge-monitor every 15s -βœ… Quote tests run every **5 minutes** -βœ… Execution tests run at fixed UTC times (10:00 UTC) - ---- - -## Estimated Cost - -- **Railway Hobby:** $5/service Γ— 4 = **$20/month** -- **Execution costs:** ~$100/month (bridge fees for real transactions) - ---- - -## Troubleshooting - -### Prometheus not scraping bridge-monitor -```bash -railway logs prometheus -``` -Verify `bridge-monitor.railway.internal:9090` is accessible. - -### Grafana shows "No Data" -- Wait 2-3 minutes for data collection -- Check datasource: Settings β†’ Data Sources β†’ Prometheus -- URL should be: `http://prometheus.railway.internal:9090` -- Test query: `up` - -### Service crashes on startup -- Check logs: `railway logs ` -- Verify port matches (9090, 3000, 9093) -- Verify Dockerfile path is correct diff --git a/harnesses/bridge-monitor/deploy/alertmanager/Dockerfile b/harnesses/bridge-monitor/deploy/alertmanager/Dockerfile deleted file mode 100644 index b35cdf77..00000000 --- a/harnesses/bridge-monitor/deploy/alertmanager/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM prom/alertmanager:latest - -USER root - -# Newer prom/alertmanager images are busybox-based (no apk). We use plain sed in -# entrypoint.sh to substitute SLACK_WEBHOOK_URL β€” no envsubst dependency needed. - -COPY alertmanager.yml.tmpl /etc/alertmanager/alertmanager.yml.tmpl -COPY templates.tmpl /etc/alertmanager/templates.tmpl -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -EXPOSE 9093 - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/harnesses/bridge-monitor/deploy/alertmanager/alertmanager.yml.tmpl b/harnesses/bridge-monitor/deploy/alertmanager/alertmanager.yml.tmpl deleted file mode 100644 index f7d4eeea..00000000 --- a/harnesses/bridge-monitor/deploy/alertmanager/alertmanager.yml.tmpl +++ /dev/null @@ -1,41 +0,0 @@ -global: - resolve_timeout: 5m - slack_api_url: '${SLACK_WEBHOOK_URL}' - -templates: - - '/etc/alertmanager/templates.tmpl' - -route: - receiver: 'slack-warnings' - group_by: ['alertname', 'chain'] - group_wait: 30s - group_interval: 5m - repeat_interval: 6h - routes: - - matchers: - - severity = "critical" - receiver: 'slack-criticals' - group_wait: 10s - repeat_interval: 1h - -receivers: - - name: 'slack-warnings' - slack_configs: - - channel: '#bridge-bench' - send_resolved: true - title: '{{ template "slack.title" . }}' - text: '{{ template "slack.text" . }}' - - - name: 'slack-criticals' - slack_configs: - - channel: '#bridge-bench' - send_resolved: true - title: ':rotating_light: CRITICAL β€” {{ template "slack.title" . }}' - text: '{{ template "slack.text" . }}' - -inhibit_rules: - - source_matchers: - - severity = "critical" - target_matchers: - - severity = "warning" - equal: ['alertname', 'chain'] diff --git a/harnesses/bridge-monitor/deploy/alertmanager/entrypoint.sh b/harnesses/bridge-monitor/deploy/alertmanager/entrypoint.sh deleted file mode 100644 index 73ca47ff..00000000 --- a/harnesses/bridge-monitor/deploy/alertmanager/entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -set -e - -# Alertmanager does not expand env vars in its YAML by itself. Substitute the -# SLACK_WEBHOOK_URL placeholder at container start using sed (busybox-safe β€” no -# envsubst dependency). The Slack webhook URL contains only /, alphanumerics and -# colons so | as the sed delimiter is safe. -if [ -z "$SLACK_WEBHOOK_URL" ]; then - echo "⚠️ SLACK_WEBHOOK_URL not set β€” alerts will be dropped by Slack receiver." -fi - -sed "s|\${SLACK_WEBHOOK_URL}|${SLACK_WEBHOOK_URL}|g" \ - /etc/alertmanager/alertmanager.yml.tmpl \ - > /etc/alertmanager/alertmanager.yml - -exec /bin/alertmanager \ - --config.file=/etc/alertmanager/alertmanager.yml \ - --storage.path=/alertmanager \ - --log.level=info diff --git a/harnesses/bridge-monitor/deploy/alertmanager/templates.tmpl b/harnesses/bridge-monitor/deploy/alertmanager/templates.tmpl deleted file mode 100644 index c4a344ed..00000000 --- a/harnesses/bridge-monitor/deploy/alertmanager/templates.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -{{ define "slack.title" }} -{{- if eq .Status "firing" -}} - [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }} -{{- else -}} - [RESOLVED] {{ .CommonLabels.alertname }} -{{- end -}} -{{ end }} - -{{ define "slack.text" }} -{{ range .Alerts -}} -*{{ .Annotations.summary }}* -{{ .Annotations.description }} -{{ if .Labels.bridge }}β€’ bridge: `{{ .Labels.bridge }}`{{ end }} -{{ if .Labels.from_chain }}β€’ route: `{{ .Labels.from_chain }} β†’ {{ .Labels.to_chain }}`{{ end }} -{{ if .Labels.chain }}β€’ chain: `{{ .Labels.chain }}`{{ end }} -β€’ severity: `{{ .Labels.severity }}` -β€’ started: {{ .StartsAt.Format "2006-01-02 15:04:05 UTC" }} -{{ end }} -{{ end }} diff --git a/harnesses/bridge-monitor/deploy/docker-compose.yml b/harnesses/bridge-monitor/deploy/docker-compose.yml deleted file mode 100644 index d9e8fb95..00000000 --- a/harnesses/bridge-monitor/deploy/docker-compose.yml +++ /dev/null @@ -1,80 +0,0 @@ -version: '3.8' - -services: - monitor: - build: - context: .. - dockerfile: Dockerfile - container_name: bridge-monitor - restart: unless-stopped - env_file: - - ../.env - ports: - - "9090:9090" - networks: - - monitoring - - prometheus: - image: prom/prometheus:latest - container_name: bridge-prometheus - restart: unless-stopped - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--storage.tsdb.retention.time=365d' - - '--web.enable-lifecycle' - - '--web.enable-admin-api' - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - - ./prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml - - prometheus_data:/prometheus - ports: - - "9091:9090" - networks: - - monitoring - - alertmanager: - build: - context: ./alertmanager - dockerfile: Dockerfile - container_name: bridge-alertmanager - restart: unless-stopped - environment: - - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL} - volumes: - - alertmanager_data:/alertmanager - ports: - - "9093:9093" - networks: - - monitoring - - grafana: - image: grafana/grafana:latest - container_name: bridge-grafana - restart: unless-stopped - environment: - - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin} - - GF_USERS_ALLOW_SIGN_UP=false - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - - GF_AUTH_BASIC_ENABLED=false - - GF_AUTH_DISABLE_LOGIN_FORM=true - volumes: - - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards - - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources - - grafana_data:/var/lib/grafana - ports: - - "3000:3000" - depends_on: - - prometheus - networks: - - monitoring - -networks: - monitoring: - driver: bridge - -volumes: - prometheus_data: - alertmanager_data: - grafana_data: diff --git a/harnesses/bridge-monitor/deploy/grafana/Dockerfile b/harnesses/bridge-monitor/deploy/grafana/Dockerfile deleted file mode 100644 index 81553e8e..00000000 --- a/harnesses/bridge-monitor/deploy/grafana/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM grafana/grafana:latest - -USER root - -# Copy provisioning configs (we're in grafana folder) -COPY provisioning /etc/grafana/provisioning - -# Set permissions -RUN chown -R grafana:root /etc/grafana/provisioning - -USER grafana - -# Environment variables -ENV GF_AUTH_ANONYMOUS_ENABLED=true -ENV GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer -ENV GF_SECURITY_ADMIN_USER=admin -ENV GF_SECURITY_ADMIN_PASSWORD=admin - -EXPOSE 3000 diff --git a/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-latency.json b/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-latency.json deleted file mode 100644 index ea8ad0a2..00000000 --- a/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-latency.json +++ /dev/null @@ -1,2044 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 2000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_quote_latency_ms_sum[24h]) / max_over_time(bridge_quote_latency_ms_count[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} ({{amount_usd}}$) avg", - "refId": "A" - } - ], - "title": "Quote Latency (p50/p95)", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "currencyUSD" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "bridge_fees_usd", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} ({{amount_usd}}$)", - "refId": "A" - } - ], - "title": "Bridge Fees (USD)", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "bridge_fees_percent", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} ({{amount_usd}}$)", - "refId": "A" - } - ], - "title": "Bridge Fees (%)", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true - }, - "mappings": [], - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "rate(bridge_success_total[5m])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} success", - "refId": "A" - }, - { - "expr": "rate(bridge_reverts_total[5m])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} reverts", - "refId": "B" - }, - { - "expr": "rate(bridge_errors_total[5m])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} errors", - "refId": "C" - } - ], - "title": "Success/Revert/Error Rate", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 120000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_execution_latency_ms_sum[24h]) / max_over_time(bridge_execution_latency_ms_count[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} ({{amount_usd}}$) avg", - "refId": "A" - } - ], - "title": "Execution Latency (All)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 20, - "panels": [], - "title": "πŸ“Š Execution by Amount Tier", - "type": "row" - }, - { - "datasource": "Prometheus", - "description": "Daily at 10:00 UTC", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "points", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 8, - "showPoints": "always", - "spanNulls": false, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 120000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 25 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_execution_latency_ms_sum{amount_usd=\"5\"}[24h]) / max_over_time(bridge_execution_latency_ms_count{amount_usd=\"5\"}[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} avg", - "refId": "A" - } - ], - "title": "⚑ $5 Execution (Daily)", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "Monday & Thursday at 10:00 UTC", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "points", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 8, - "showPoints": "always", - "spanNulls": false, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 120000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 25 - }, - "id": 22, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_execution_latency_ms_sum{amount_usd=\"50\"}[24h]) / max_over_time(bridge_execution_latency_ms_count{amount_usd=\"50\"}[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} avg", - "refId": "A" - } - ], - "title": "πŸ’° $50 Execution (Mon/Thu)", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "Monday at 10:00 UTC (weekly)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "points", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 8, - "showPoints": "always", - "spanNulls": false, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 120000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_execution_latency_ms_sum{amount_usd=\"300\"}[24h]) / max_over_time(bridge_execution_latency_ms_count{amount_usd=\"300\"}[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} avg", - "refId": "A" - } - ], - "title": "πŸ’Ž $300 Execution (weekly Mon)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 30, - "panels": [], - "title": "🎯 End-to-End Latency by Tier", - "type": "row" - }, - { - "datasource": "Prometheus", - "description": "Quote start β†’ funds received", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "points", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 8, - "showPoints": "always", - "spanNulls": false, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 180000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 34 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_e2e_latency_ms_sum{amount_usd=\"5\"}[24h]) / max_over_time(bridge_e2e_latency_ms_count{amount_usd=\"5\"}[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} avg", - "refId": "A" - } - ], - "title": "⚑ $5 E2E Latency", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "Quote start β†’ funds received", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "points", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 8, - "showPoints": "always", - "spanNulls": false, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 180000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 34 - }, - "id": 32, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_e2e_latency_ms_sum{amount_usd=\"50\"}[24h]) / max_over_time(bridge_e2e_latency_ms_count{amount_usd=\"50\"}[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} avg", - "refId": "A" - } - ], - "title": "πŸ’° $50 E2E Latency", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "Quote start β†’ funds received", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "points", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 8, - "showPoints": "always", - "spanNulls": false, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 180000 - } - ] - }, - "unit": "suffix: ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 34 - }, - "id": 33, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_e2e_latency_ms_sum{amount_usd=\"300\"}[24h]) / max_over_time(bridge_e2e_latency_ms_count{amount_usd=\"300\"}[24h])", - "legendFormat": "{{bridge}} {{from_chain}}β†’{{to_chain}} avg", - "refId": "A" - } - ], - "title": "πŸ’Ž $300 E2E Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 40, - "panels": [], - "title": "πŸ“ˆ Execution Counts by Tier", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 43 - }, - "id": 41, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "sum(max_over_time(bridge_success_total{amount_usd=\"5\"}[24h]))", - "legendFormat": "Success", - "refId": "A" - } - ], - "title": "⚑ $5 Executions", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 43 - }, - "id": 42, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "sum(max_over_time(bridge_success_total{amount_usd=\"50\"}[24h]))", - "legendFormat": "Success", - "refId": "A" - } - ], - "title": "πŸ’° $50 Executions", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 43 - }, - "id": 43, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "sum(max_over_time(bridge_success_total{amount_usd=\"300\"}[24h]))", - "legendFormat": "Success", - "refId": "A" - } - ], - "title": "πŸ’Ž $300 Executions", - "type": "stat" - }, - { - "id": 50, - "type": "row", - "title": "πŸͺ™ HyperCore Deposit Benchmark (Arb USDC β†’ HL Perps)", - "collapsed": false, - "panels": [], - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - } - }, - { - "id": 51, - "type": "timeseries", - "datasource": "Prometheus", - "title": "HyperCore Cost % (Mobula vs Relay vs LiFi)", - "description": "Total cost as % of trade size on Arb USDC β†’ HyperCore deposit. Lower is better. Mobula's $0 service fee shines on small tickets where Relay/LiFi take a flat $1.20 (24% on $5).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true - }, - "mappings": [], - "unit": "percent", - "decimals": 3 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "bridge_cost_percent{to_chain=\"HyperCore\"}", - "legendFormat": "{{bridge}} ${{amount_usd}}", - "refId": "A" - } - ] - }, - { - "id": 52, - "type": "timeseries", - "datasource": "Prometheus", - "title": "HyperCore Quote Latency", - "description": "Quote API latency for HyperCore deposit. Measures only the quote step (broadcast not included for Phase 1 quote-only).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 5, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "scaleDistribution": { - "type": "log", - "log": 2 - } - }, - "mappings": [], - "unit": "ms", - "decimals": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "max_over_time(bridge_quote_latency_ms_sum{to_chain=\"HyperCore\"}[24h]) / max_over_time(bridge_quote_latency_ms_count{to_chain=\"HyperCore\"}[24h])", - "legendFormat": "{{bridge}} ${{amount_usd}} avg", - "refId": "A" - } - ] - }, - { - "id": 53, - "type": "timeseries", - "datasource": "Prometheus", - "title": "HyperCore Fees USD (absolute)", - "description": "Absolute fee paid in USD per provider per tier. Mobula stays flat at gas-only (~$0.10), Relay charges $1.20 flat regardless of amount, LiFi has no route below $50.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 5, - "lineWidth": 2, - "pointSize": 6, - "showPoints": "always", - "spanNulls": true - }, - "mappings": [], - "unit": "currencyUSD", - "decimals": 2 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 59 - }, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "bridge_cost_usd{to_chain=\"HyperCore\"}", - "legendFormat": "{{bridge}} ${{amount_usd}}", - "refId": "A" - } - ] - }, - { - "id": 54, - "type": "stat", - "datasource": "Prometheus", - "title": "HyperCore Quote Success Rate (24h)", - "description": "Fraction of HyperCore quotes returning successfully over 24h. LiFi will show partial coverage (no route <$50). Debridge skipped (chain not supported).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 0.95 - } - ] - }, - "unit": "percentunit", - "decimals": 2 - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 59 - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "vertical", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "avg by (bridge, amount_usd) (avg_over_time(bridge_quote_success{to_chain=\"HyperCore\"}[24h]))", - "legendFormat": "{{bridge}} ${{amount_usd}}", - "refId": "A" - } - ] - } - ], - "refresh": "30s", - "schemaVersion": 27, - "style": "dark", - "tags": [ - "bridge", - "latency", - "monitoring" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Bridge Latency Benchmark", - "uid": "bridge-latency", - "version": 0 -} \ No newline at end of file diff --git a/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-pricing.json b/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-pricing.json deleted file mode 100644 index 5453d0a8..00000000 --- a/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/bridge-pricing.json +++ /dev/null @@ -1,1619 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [ - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": true, - "title": "Latency Dashboard", - "tooltip": "View execution latency metrics", - "type": "link", - "url": "/d/bridge-latency" - } - ], - "panels": [ - { - "type": "row", - "title": "\ud83c\udfc6 Provider Ranking \u2014 $5 / $50 / $300 (change Route above)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 10, - "collapsed": false - }, - { - "datasource": "Prometheus", - "type": "table", - "title": "Main Ranking Table (one row per provider)", - "description": "One row per provider. Columns show cost in USD & % for each amount. Select a route via the 'Route' variable above. Sorted by $5 cost ascending.", - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 11, - "targets": [ - { - "expr": "bridge_cost_usd{from_token=~\"$route\",amount_usd=\"5\"}", - "refId": "A", - "format": "table", - "instant": true, - "legendFormat": "" - }, - { - "expr": "bridge_cost_percent{from_token=~\"$route\",amount_usd=\"5\"}", - "refId": "B", - "format": "table", - "instant": true, - "legendFormat": "" - }, - { - "expr": "bridge_cost_usd{from_token=~\"$route\",amount_usd=\"50\"}", - "refId": "C", - "format": "table", - "instant": true, - "legendFormat": "" - }, - { - "expr": "bridge_cost_percent{from_token=~\"$route\",amount_usd=\"50\"}", - "refId": "D", - "format": "table", - "instant": true, - "legendFormat": "" - }, - { - "expr": "bridge_cost_usd{from_token=~\"$route\",amount_usd=\"300\"}", - "refId": "E", - "format": "table", - "instant": true, - "legendFormat": "" - }, - { - "expr": "bridge_cost_percent{from_token=~\"$route\",amount_usd=\"300\"}", - "refId": "F", - "format": "table", - "instant": true, - "legendFormat": "" - }, - { - "expr": "bridge_estimated_time_ms{from_token=~\"$route\",amount_usd=\"300\"}", - "refId": "G", - "format": "table", - "instant": true, - "legendFormat": "" - } - ], - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "bridge", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "__name__ 1": true, - "__name__ 2": true, - "__name__ 3": true, - "__name__ 4": true, - "__name__ 5": true, - "__name__ 6": true, - "__name__ 7": true, - "job 1": true, - "job 2": true, - "job 3": true, - "job 4": true, - "job 5": true, - "job 6": true, - "job 7": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance 4": true, - "instance 5": true, - "instance 6": true, - "instance 7": true, - "region 1": true, - "region 2": true, - "region 3": true, - "region 4": true, - "region 5": true, - "region 6": true, - "region 7": true, - "from_chain 1": true, - "from_chain 2": true, - "from_chain 3": true, - "from_chain 4": true, - "from_chain 5": true, - "from_chain 6": true, - "from_chain 7": true, - "to_chain 1": true, - "to_chain 2": true, - "to_chain 3": true, - "to_chain 4": true, - "to_chain 5": true, - "to_chain 6": true, - "to_chain 7": true, - "from_token 1": true, - "from_token 2": true, - "from_token 3": true, - "from_token 4": true, - "from_token 5": true, - "from_token 6": true, - "from_token 7": true, - "to_token 1": true, - "to_token 2": true, - "to_token 3": true, - "to_token 4": true, - "to_token 5": true, - "to_token 6": true, - "to_token 7": true, - "amount_usd 1": true, - "amount_usd 2": true, - "amount_usd 3": true, - "amount_usd 4": true, - "amount_usd 5": true, - "amount_usd 6": true, - "amount_usd 7": true, - "service 1": true, - "service 2": true, - "service 3": true, - "service 4": true, - "service 5": true, - "service 6": true, - "service 7": true, - "service 8": true, - "service 9": true, - "service": true - }, - "indexByName": { - "bridge": 0, - "Value #A": 1, - "Value #B": 2, - "Value #C": 3, - "Value #D": 4, - "Value #E": 5, - "Value #F": 6, - "Value #G": 7 - }, - "renameByName": { - "bridge": "Provider", - "Value #A": "$5 Cost USD", - "Value #B": "$5 Cost %", - "Value #C": "$50 Cost USD", - "Value #D": "$50 Cost %", - "Value #E": "$300 Cost USD", - "Value #F": "$300 Cost %", - "Value #G": "Est Time ms" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "$5 Cost USD", - "desc": false - } - ] - } - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto" - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "Cost USD" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "decimals", - "value": 3 - }, - { - "id": "custom.displayMode", - "value": "color-background" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 2 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Cost %" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.displayMode", - "value": "color-background" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 5 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Est Time ms" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Provider" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "type": "row", - "title": "\ud83c\udf69 Cost Distribution \u2014 $5", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 100, - "collapsed": false - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$5 \u2014 USDC (Sol) \u2192 USDC (Base)", - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 14 - }, - "id": 101, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"5\",from_chain=\"Solana\",to_chain=\"Base\",from_token=\"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$5 \u2014 USDC (Base) \u2192 USDT (Arb)", - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 14 - }, - "id": 102, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"5\",from_chain=\"Base\",to_chain=\"Arbitrum\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$5 \u2014 TRUMP (Sol) \u2192 BRETT (Base)", - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 14 - }, - "id": 103, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"5\",from_token=\"6p6xgHyF7AeE6TZkSmFsko444wqoP15icUSqi2jfGiPN\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "type": "row", - "title": "\ud83c\udf69 Cost Distribution \u2014 $50", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 200, - "collapsed": false - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$50 \u2014 USDC (Sol) \u2192 USDC (Base)", - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 201, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"50\",from_chain=\"Solana\",to_chain=\"Base\",from_token=\"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$50 \u2014 USDC (Base) \u2192 USDT (Arb)", - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 24 - }, - "id": 202, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"50\",from_chain=\"Base\",to_chain=\"Arbitrum\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$50 \u2014 TRUMP (Sol) \u2192 BRETT (Base)", - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 24 - }, - "id": 203, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"50\",from_token=\"6p6xgHyF7AeE6TZkSmFsko444wqoP15icUSqi2jfGiPN\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "type": "row", - "title": "\ud83c\udf69 Cost Distribution \u2014 $300", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 300, - "collapsed": false - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$300 \u2014 USDC (Sol) \u2192 USDC (Base)", - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 34 - }, - "id": 301, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"300\",from_chain=\"Solana\",to_chain=\"Base\",from_token=\"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$300 \u2014 USDC (Base) \u2192 USDT (Arb)", - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 34 - }, - "id": 302, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"300\",from_chain=\"Base\",to_chain=\"Arbitrum\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "datasource": "Prometheus", - "type": "piechart", - "title": "$300 \u2014 TRUMP (Sol) \u2192 BRETT (Base)", - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 34 - }, - "id": 303, - "targets": [ - { - "expr": "bridge_cost_usd{amount_usd=\"300\",from_token=\"6p6xgHyF7AeE6TZkSmFsko444wqoP15icUSqi2jfGiPN\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "options": { - "pieType": "donut", - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*mobula.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#228be6" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*relay.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#fd7e14" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*lifi.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#40c057" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(?i).*debridge.*" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#7950f2" - } - } - ] - } - ] - } - }, - { - "type": "row", - "title": "\ud83d\udcc8 Cost Evolution Over Time (selected route + amount)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 400, - "collapsed": true, - "panels": [ - { - "datasource": "Prometheus", - "type": "timeseries", - "title": "Cost USD Over Time", - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 401, - "targets": [ - { - "expr": "bridge_cost_usd{from_token=~\"$route\",amount_usd=\"$amount\"}", - "legendFormat": "{{bridge}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10 - } - } - } - } - ] - } - ], - "refresh": "30s", - "schemaVersion": 27, - "style": "dark", - "tags": [ - "bridge", - "pricing", - "ranking" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "USDC Sol \u2192 Base", - "value": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v" - }, - "hide": 0, - "includeAll": false, - "label": "Route (for ranking table)", - "multi": false, - "name": "route", - "options": [ - { - "selected": true, - "text": "USDC Sol \u2192 Base", - "value": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v" - }, - { - "selected": false, - "text": "USDC Base \u2192 USDT Arb", - "value": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913" - }, - { - "selected": false, - "text": "TRUMP Sol \u2192 BRETT Base", - "value": "6p6xgHyF7AeE6TZkSmFsko444wqoP15icUSqi2jfGiPN" - } - ], - "query": "USDC Sol \u2192 Base : EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v, USDC Base \u2192 USDT Arb : 0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913, TRUMP Sol \u2192 BRETT Base : 6p6xgHyF7AeE6TZkSmFsko444wqoP15icUSqi2jfGiPN", - "skipUrlSync": false, - "type": "custom" - }, - { - "current": { - "selected": true, - "text": "5", - "value": "5" - }, - "hide": 0, - "includeAll": false, - "label": "Amount USD (for time series)", - "multi": false, - "name": "amount", - "options": [ - { - "selected": true, - "text": "5", - "value": "5" - }, - { - "selected": false, - "text": "50", - "value": "50" - }, - { - "selected": false, - "text": "300", - "value": "300" - } - ], - "query": "5, 50, 300", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "\ud83c\udf09 Bridge Pricing & Ranking", - "uid": "bridge-pricing", - "version": 0 -} \ No newline at end of file diff --git a/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/dashboard.yml b/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/dashboard.yml deleted file mode 100644 index b6fa1088..00000000 --- a/harnesses/bridge-monitor/deploy/grafana/provisioning/dashboards/dashboard.yml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: 1 - -providers: - - name: 'Bridge Latency Dashboards' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: false - options: - path: /etc/grafana/provisioning/dashboards diff --git a/harnesses/bridge-monitor/deploy/grafana/provisioning/datasources/prometheus.yml b/harnesses/bridge-monitor/deploy/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 0672b9c6..00000000 --- a/harnesses/bridge-monitor/deploy/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: https://prometheus-production-9ffe.up.railway.app - isDefault: true - editable: false diff --git a/harnesses/bridge-monitor/deploy/prometheus/Dockerfile b/harnesses/bridge-monitor/deploy/prometheus/Dockerfile deleted file mode 100644 index 21ebb852..00000000 --- a/harnesses/bridge-monitor/deploy/prometheus/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM prom/prometheus:v2.49.1 - -USER root - -# Copy config and alert rules (we're already in prometheus folder as root directory) -COPY prometheus.yml /etc/prometheus/prometheus.yml -COPY alert_rules.yml /etc/prometheus/alert_rules.yml - -EXPOSE 9090 - -# Run as root to avoid permission issues with Railway volumes -CMD ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=365d", "--web.enable-lifecycle", "--web.enable-admin-api"] diff --git a/harnesses/bridge-monitor/deploy/prometheus/alert_rules.yml b/harnesses/bridge-monitor/deploy/prometheus/alert_rules.yml deleted file mode 100644 index b1afbf9e..00000000 --- a/harnesses/bridge-monitor/deploy/prometheus/alert_rules.yml +++ /dev/null @@ -1,130 +0,0 @@ -groups: - - name: bridge_latency - interval: 30s - rules: - - alert: HighBridgeQuoteLatency - expr: histogram_quantile(0.95, rate(bridge_quote_latency_ms_bucket[5m])) > 2000 - for: 5m - labels: - severity: warning - annotations: - summary: "High bridge quote latency" - description: "{{ $labels.bridge }} quote p95 ({{ $labels.from_chain }} β†’ {{ $labels.to_chain }}) is {{ $value | humanize }}ms" - - - alert: HighBridgeExecutionLatency - expr: histogram_quantile(0.95, rate(bridge_execution_latency_ms_bucket[5m])) > 120000 - for: 5m - labels: - severity: warning - annotations: - summary: "High bridge execution latency" - description: "{{ $labels.bridge }} exec p95 ({{ $labels.from_chain }} β†’ {{ $labels.to_chain }}) is {{ $value | humanize }}ms" - - - alert: HighBridgeFees - expr: bridge_fees_percent > 5 - for: 15m - labels: - severity: warning - annotations: - summary: "High bridge fees" - description: "{{ $labels.bridge }} fees ({{ $labels.from_chain }} β†’ {{ $labels.to_chain }}) = {{ $value }}%" - - - name: bridge_reliability - interval: 30s - rules: - # Computed revert rate over a 1h window from counters (the bridge_revert_rate - # gauge was never populated β€” see PR fix). clamp_min prevents div-by-zero. - - alert: HighBridgeRevertRate - expr: | - sum by (bridge, from_chain, to_chain) (rate(bridge_reverts_total[1h])) - / - clamp_min(sum by (bridge, from_chain, to_chain) (rate(bridge_reverts_total[1h]) + rate(bridge_success_total[1h])), 0.0001) - > 0.1 - for: 5m - labels: - severity: critical - annotations: - summary: "High bridge revert rate" - description: "{{ $labels.bridge }} revert rate ({{ $labels.from_chain }} β†’ {{ $labels.to_chain }}) = {{ $value | humanizePercentage }}" - - # Three real-execution failures in a row on the same bridge β€” almost certainly a - # bridge-side outage or our integration broke. Page immediately. - - alert: BridgeConsecutiveFailures - expr: bridge_consecutive_failures >= 3 - for: 1m - labels: - severity: critical - annotations: - summary: "{{ $labels.bridge }}: {{ $value }} consecutive execution failures" - description: "{{ $labels.bridge }} has failed {{ $value }} executions in a row (region: {{ $labels.region }})" - - - alert: BridgeMonitorDown - expr: up{job="bridge-monitor"} == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "Bridge monitor is down" - description: "bridge-monitor unreachable for 2m β€” scheduler not running, no executions." - - - name: wallet_balance - interval: 30s - rules: - # Any triangle leg below $50 means the next $50 scheduled run on that chain will fail. - - alert: LowTriangleBalance - expr: wallet_balance_usd{token=~"USDC|USDT|USDT0"} < 50 - for: 5m - labels: - severity: warning - annotations: - summary: "Triangle leg low: {{ $labels.chain }}/{{ $labels.token }} = ${{ $value | printf \"%.2f\" }}" - description: "Next $50 test from {{ $labels.chain }} will fail insufficient funds. Rebalance needed." - - - alert: CriticalTriangleBalance - expr: wallet_balance_usd{token=~"USDC|USDT|USDT0"} < 10 - for: 1m - labels: - severity: critical - annotations: - summary: "Triangle leg critically low: {{ $labels.chain }}/{{ $labels.token }} = ${{ $value | printf \"%.2f\" }}" - description: "Even the daily $5 test will fail on this chain." - - # Gas: we keep ~$10 native on each chain. Alert when dropping under $3. - - alert: LowGasBalance - expr: wallet_balance_usd{token=~"ETH|SOL"} < 3 - for: 5m - labels: - severity: warning - annotations: - summary: "Low gas on {{ $labels.chain }}: {{ $labels.token }} = ${{ $value | printf \"%.2f\" }}" - description: "Top up native-token balance β€” TXs will start failing." - - # R4 meme route is one-way (TRUMP Sol β†’ BRETT Base), no inflow refill. - # Weekly $5 Γ— 3 bridges = $15 per run. Alert at $30 (~2 weekly cycles left). - - alert: LowMemeSourceBalance - expr: wallet_balance_usd{chain="Solana", token="TRUMP"} < 30 - for: 5m - labels: - severity: warning - annotations: - summary: "TRUMP balance low on Solana: ${{ $value | printf \"%.2f\" }}" - description: "R4 TRUMPβ†’BRETT will run out in ~{{ $value | humanize }} / 15 weekly cycles. Manual refill required (one-way route)." - - - alert: CriticalMemeSourceBalance - expr: wallet_balance_usd{chain="Solana", token="TRUMP"} < 15 - for: 1m - labels: - severity: critical - annotations: - summary: "TRUMP critically low: ${{ $value | printf \"%.2f\" }}" - description: "Next weekly R4 run will fail. Bridge TRUMP from another wallet to resume." - - # Balance refresh loop died: last update > 20 min ago (updater runs every 5 min). - - alert: WalletBalanceStale - expr: time() - wallet_balance_last_update_timestamp_seconds > 1200 - for: 1m - labels: - severity: warning - annotations: - summary: "Wallet balance metrics stale" - description: "Balance refresh hasn't run in {{ $value | humanizeDuration }}. Check monitor logs." diff --git a/harnesses/bridge-monitor/deploy/prometheus/prometheus.yml b/harnesses/bridge-monitor/deploy/prometheus/prometheus.yml deleted file mode 100644 index 61b5e4f7..00000000 --- a/harnesses/bridge-monitor/deploy/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - external_labels: - cluster: 'bridge-latency' - -alerting: - alertmanagers: - - static_configs: - - targets: - - alertmanager.railway.internal:9093 - -rule_files: - - 'alert_rules.yml' - -scrape_configs: - - job_name: 'bridge-monitor' - static_configs: - - targets: ['bridge-monitor.railway.internal:9090'] diff --git a/infrastructure/README.md b/infrastructure/README.md new file mode 100644 index 00000000..f70fc5fb --- /dev/null +++ b/infrastructure/README.md @@ -0,0 +1,83 @@ +# Infrastructure + +Shared services that every benchmark depends on. Hosted by the OpenChainBench Railway project; the layout here is what gets deployed. + +``` +infrastructure/ +└── prometheus/ Single shared Prometheus instance scraped by the site + β”œβ”€β”€ Dockerfile + └── prometheus.yml Scrape config β€” one job per harness +``` + +## How the system fits together + +``` +OpenChainBench Railway project +β”œβ”€β”€ prometheus ← this folder, public read-only URL +β”œβ”€β”€ aggregator-head-lag ← from harnesses/aggregator-head-lag +β”œβ”€β”€ bridge-monitor ← from harnesses/bridge-monitor +└── ← from harnesses/ + + β–² β–² + β”‚ scrape every 15 s β”‚ + β”‚ β”‚ + Each harness exposes /metrics on a known port (Railway internal DNS) + Prometheus aggregates everything and exposes a single public URL. + +Each YAML spec in benchmarks/.yml has prom_url pointing at the +public Prometheus URL. The Next.js site queries Prometheus directly. +``` + +A contributor never has to think about Prometheus. They drop their harness in `harnesses//`, append one block to `prometheus.yml` here, and the data flows. + +## Why no Grafana / Alertmanager in this folder + +Grafana and Alertmanager are useful for ops but they're not part of the public surface β€” the OpenChainBench site renders its own visualizations from PromQL, and alerting is a per-operator concern. Anyone running their own fork is welcome to add either alongside; the harnesses don't depend on them. + +If you need to debug Prometheus visually, you can: + +- Browse `https:///graph` directly (Prometheus ships its own UI). +- Spin up Grafana locally pointed at the public Prometheus URL β€” no infra changes required. + +## Scrape config β€” adding a target + +`prometheus.yml` lists one `scrape_configs` job per harness. To add a new harness: + +```yaml +- job_name: + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - .railway.internal: + labels: + benchmark: +``` + +Three things to get right: + +1. **`` matches the Railway service name.** Railway exposes each service at `.railway.internal:` β€” the job's target string must match. +2. **`` matches the port the harness binds for `/metrics`.** Conventionally 2112 (aggregator-style) or 9090 (bridge-style); whatever your harness chooses, document it in `harnesses//README.md`. +3. **`benchmark: ` label.** All metrics get this label automatically; the YAML spec uses it to scope queries to a specific harness. Keep it consistent with the spec slug. + +After merging a PR that adds a job, redeploy the `prometheus` Railway service so it picks up the new config (or hit the `/-/reload` endpoint with `--web.enable-lifecycle` enabled β€” already on by default in this Dockerfile). + +## Running this locally + +```bash +cd infrastructure/prometheus +docker build -t openbench-prom . +docker run --rm -p 9090:9090 openbench-prom +``` + +The local Prometheus will fail to scrape the `*.railway.internal` targets (those resolve only inside Railway). To exercise locally, edit `prometheus.yml` to point at `host.docker.internal:` while you run the harness on your laptop. + +## Retention + +Default retention is 365 days, configurable via the `PROM_RETENTION` Railway env var. The site queries up to 7 days back for time-series panels (the `series7d` window) β€” anything older is for archival and ad-hoc queries. + +## Why Railway over a managed Prom + +- Cost: a single ~$5/mo Railway service vs $20+/mo for managed Prom plans +- Same DNS network as the harnesses β†’ no public ingress per harness needed +- Trivial to redeploy / wipe / split if scaling demands it diff --git a/infrastructure/prometheus/Dockerfile b/infrastructure/prometheus/Dockerfile new file mode 100644 index 00000000..bb4edc29 --- /dev/null +++ b/infrastructure/prometheus/Dockerfile @@ -0,0 +1,19 @@ +FROM prom/prometheus:latest + +USER root + +COPY prometheus.yml /etc/prometheus/prometheus.yml + +# Long retention so old benchmarks remain queryable. +# Override with PROM_RETENTION at deploy time if needed. +ENV PROM_RETENTION=365d + +EXPOSE 9090 + +ENTRYPOINT ["/bin/sh", "-c", "exec /bin/prometheus \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus \ + --storage.tsdb.retention.time=${PROM_RETENTION} \ + --web.enable-lifecycle \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --web.console.templates=/usr/share/prometheus/consoles"] diff --git a/infrastructure/prometheus/prometheus.yml b/infrastructure/prometheus/prometheus.yml new file mode 100644 index 00000000..da8d5614 --- /dev/null +++ b/infrastructure/prometheus/prometheus.yml @@ -0,0 +1,46 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + project: openchainbench + +# Each scrape_configs entry corresponds to one harness service deployed +# on the OpenChainBench Railway project. The targets are Railway internal +# DNS names (`.railway.internal:`) β€” no public URLs needed. +# +# To onboard a new harness: +# 1. Create the Railway service from `harnesses//`. +# 2. Confirm the service exposes `/metrics` on a known port. +# 3. Append a `- job_name: ` block below referencing +# `.railway.internal:`. +# 4. Open a PR. Once merged, redeploy the prometheus service so it +# picks up the new target. +# +# Conventional ports per existing harness: +# aggregator-head-lag β†’ 2112 (matches `metrics.go` HTTP listener) +# bridge-monitor β†’ 9090 (matches `cmd/monitor/main.go` listener) + +scrape_configs: + - job_name: aggregator-head-lag + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - aggregator-head-lag.railway.internal:2112 + labels: + benchmark: aggregator-head-lag + + - job_name: bridge-monitor + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - bridge-monitor.railway.internal:9090 + labels: + benchmark: bridge-monitor + + # Self-scrape β€” surfaces Prometheus's own health in the same UI. + - job_name: prometheus + static_configs: + - targets: + - localhost:9090