feat: operator liveness metric

grafana/provisioning/dashboards/aligned/aggregator_batcher.json

@@ -2721,6 +2721,123 @@
  ],
  "type": "timeseries"
  },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "continuous-GrYlRd"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "noValue": "none",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Max"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "gauge"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 61
+ },
+ "id": 49,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": false
+ },
+ "pluginVersion": "10.1.10",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "floor(increase(missing_operator_count{job=\"aligned-tracker\"}[$__range]))",
+ "format": "time_series",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "interval": "1",
+ "legendFormat": "{{operator}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "# Operator Missing Tasks",
+ "transformations": [
+ {
+ "id": "reduce",
+ "options": {
+ "labelsToFields": false,
+ "reducers": [
+ "max"
+ ]
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "desc": true,
+ "field": "Max"
+ }
+ ]
+ }
+ }
+ ],
+ "type": "table"
+ },
  {
  "datasource": {
  "type": "prometheus",
@@ -3176,13 +3293,13 @@
  "list": []
  },
  "time": {
- "from": "now-5m",
+ "from": "now-30m",
  "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "System Data",
  "uid": "aggregator",
- "version": 7,
+ "version": 1,
  "weekStart": ""
 }

telemetry_api/lib/telemetry_api/operators.ex

@@ -9,6 +9,7 @@ defmodule TelemetryApi.Operators do
  alias TelemetryApi.Operators.Operator
  alias TelemetryApi.ContractManagers.OperatorStateRetriever
  alias TelemetryApi.ContractManagers.DelegationManager
+ alias TelemetryApi.PrometheusMetrics
 
  @doc """
  Returns the list of operators.
@@ -95,6 +96,18 @@ defmodule TelemetryApi.Operators do
  |> Enum.filter(fn {status, _} -> status == :ok end)
  |> Enum.map(fn {_, data} -> data end)
 
+ # Initialize new_operators metrics
+ Enum.map(new_operators, fn {_, op_data} ->
+ op_name_address = op_data.name <> " - " <> String.slice(op_data.address, 0..7)
+ PrometheusMetrics.initialize_operator_metrics(op_name_address)
+ end)
+
+ # If the server was restarted, initialize old_operators metrics
+ Enum.map(old_operators, fn {op, _} ->
+ op_name_address = op.name <> " - " <> String.slice(op.address, 0..7)
+ PrometheusMetrics.initialize_operator_metrics(op_name_address)
+ end)
+
  # Merge both lists
  operators = (new_operators ++ old_operators)
 

telemetry_api/lib/telemetry_api/periodically.ex

@@ -1,7 +1,7 @@
 defmodule TelemetryApi.Periodically do
  use GenServer
  alias TelemetryApi.Operators
- alias TelemetryApi.EthereumMetrics
+ alias TelemetryApi.PrometheusMetrics
  alias TelemetryApi.ContractManagers.RegistryCoordinatorManager
  require Logger
 
@@ -45,7 +45,7 @@ defmodule TelemetryApi.Periodically do
  def handle_info(:gas_price, _state) do
  case Ethers.current_gas_price() do
  {:ok, gas_price} ->
-EthereumMetrics.new_gas_price(gas_price)
+PrometheusMetrics.new_gas_price(gas_price)
 
  {:error, error} ->
  IO.inspect("Error fetching gas price: #{error}")

telemetry_api/lib/telemetry_api/prometheus_metrics.ex

@@ -0,0 +1,35 @@
+defmodule TelemetryApi.PrometheusMetrics do
+ use Prometheus.Metric
+
+ @gauge [name: :gas_price, help: "Ethereum Gas Price.", labels: []]
+ @counter [name: :missing_operator_count, help: "Missing Operators", labels: [:operator]]
+
+ def new_gas_price(gas_price) do
+ Gauge.set(
+ [name: :gas_price, labels: []],
+ gas_price
+ )
+ end
+
+ def missing_operator(operator) do
+ Counter.inc(
+ name: :missing_operator_count,
+ labels: [operator]
+ )
+ end
+
+ def initialize_operator_metrics(operator) do
+ value =
+ Counter.value(
+ name: :missing_operator_count,
+ labels: [operator]
+ )
+
+ if value == :undefined do
+ Counter.inc(
+ [name: :missing_operator_count, labels: [operator]],
+ 0
+ )
+ end
+ end
+end

telemetry_api/lib/telemetry_api/traces.ex

@@ -5,6 +5,7 @@ defmodule TelemetryApi.Traces do
  alias TelemetryApi.Traces.Trace
  alias TelemetryApi.Operators
  alias TelemetryApi.ContractManagers.StakeRegistry
+ alias TelemetryApi.PrometheusMetrics
 
  require OpenTelemetry.Tracer
  require OpenTelemetry.Ctx
@@ -208,7 +209,6 @@ defmodule TelemetryApi.Traces do
  :ok
  end
  end
-
 
  @doc """
  Registers the sending of a batcher task to Ethereum in the task trace.
@@ -298,7 +298,7 @@ defmodule TelemetryApi.Traces do
  :ok
  end
  end
-
+
  @doc """
  Registers a bump in the gas price when the aggregator tries to respond to a task in the task trace.
 
@@ -368,8 +368,17 @@ defmodule TelemetryApi.Traces do
  defp add_missing_operators([]), do: :ok
 
  defp add_missing_operators(missing_operators) do
+ # Concatenate name + address
+ missing_operators =
+ missing_operators
+ |> Enum.map(fn op -> op.name <> " - " <> String.slice(op.address, 0..7) end)
+
+ # Send to prometheus
+ missing_operators
+ |> Enum.map(fn o -> PrometheusMetrics.missing_operator(o) end)
+
  missing_operators =
- missing_operators |> Enum.map(fn o -> o.name end) |> Enum.join(";")
+ missing_operators |> Enum.join(";")
 
  Tracer.add_event("Missing Operators", [{:operators, missing_operators}])
  end