HYPERFLEET-1410 - feat: expose operator metrics, health probes, and ServiceMonitor

Dockerfile

@@ -2,6 +2,15 @@ ARG BASE_IMAGE=registry.access.redhat.com/ubi9-micro:latest
 
 FROM registry.access.redhat.com/ubi9/go-toolset:9.8-1788409979 AS builder
 
+# APP_VERSION/GIT_SHA are injected into the binary below via -ldflags -X, so
+# hyperfleet_operator_build_info reports the real release/commit instead of
+# falling back to "dev"/"unknown" (see internal/version and docs/metrics.md).
+# The container build has no .git directory to source them from automatically,
+# unlike `make build`/`make run`, which get them from the toolchain's own VCS
+# stamping.
+ARG APP_VERSION="0.0.0-dev"
+ARG GIT_SHA="unknown"
+
 USER root
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -18,7 +27,10 @@ COPY internal/ internal/
 
 
 RUN CGO_ENABLED=1 GOEXPERIMENT=boringcrypto \
- go build -trimpath -ldflags="-s -w" -o manager ./cmd/main.go
+ go build -trimpath -ldflags="-s -w \
+ -X github.com/openshift-hyperfleet/hyperfleet-operator/internal/version.version=${APP_VERSION} \
+ -X github.com/openshift-hyperfleet/hyperfleet-operator/internal/version.commit=${GIT_SHA}" \
+ -o manager ./cmd/main.go
 
 # Runtime stage
 FROM ${BASE_IMAGE} AS final

Makefile

@@ -169,12 +169,12 @@ GIT_DIRTY ?= $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] || echo "-m
 
 # Go build flags (FIPS compliant)
 CGO_ENABLED ?= 1
-GOEXPERIMENT ?= boringcrypto
+GOEXPERIMENT ?= boringcrypto
 GOFLAGS ?= -trimpath
-# LDFLAGS := -s -w \
-#  -X github.com/openshift-hyperfleet/hyperfleet-operator/pkg/version.Version=$(APP_VERSION) \
-#  -X github.com/openshift-hyperfleet/hyperfleet-operator/pkg/version.Commit=$(GIT_SHA) \
-#  -X 'github.com/openshift-hyperfleet/hyperfleet-operator/pkg/version.BuildTime=$(BUILD_DATE)'
+# APP_VERSION/GIT_SHA are injected into the binary via the Dockerfile's own
+# -ldflags -X (see Dockerfile and internal/version); `make build`/`make run`
+# intentionally skip ldflags and rely on the Go toolchain's automatic VCS
+# stamping from the local .git checkout instead (see internal/version).
 
 .PHONY: check-container-tool
 check-container-tool:
@@ -190,6 +190,7 @@ image: check-container-tool manifests generate fmt vet ## Build container image
  --platform $(PLATFORM) \
  --build-arg BASE_IMAGE=$(BASE_IMAGE) \
  --build-arg APP_VERSION=$(APP_VERSION) \
+ --build-arg GIT_SHA=$(GIT_SHA) \
  -t $(IMG) .
 	@echo "Image built: $(IMG)"
 	@echo "$(IMG)"

README.md

@@ -49,6 +49,19 @@ kubectl apply -k config/samples/
 
 >**NOTE**: Ensure that the samples has default values to test it out.
 
+### Observability endpoints
+
+The manager exposes the standard HyperFleet observability endpoints (defaults):
+
+- **Liveness probe:** `http://localhost:8080/healthz`
+- **Readiness probe:** `http://localhost:8080/readyz`
+- **Metrics:** `http://localhost:9090/metrics`
+
+Metrics are served as plain HTTP under the `hyperfleet_operator_*` namespace. Ports
+are configurable via `--health-probe-bind-address` and `--metrics-bind-address`.
+See [docs/metrics.md](docs/metrics.md) for the full metric catalogue, labels, and
+example PromQL queries.
+
 ### To Uninstall
 **Delete the instances (CRs) from the cluster:**
 

bundle/manifests/hyperfleet-operator-controller-manager-metrics-service_v1_service.yaml

@@ -9,10 +9,10 @@ metadata:
  name: hyperfleet-operator-controller-manager-metrics-service
 spec:
  ports:
- - name: https
- port: 8443
+ - name: metrics
+ port: 9090
  protocol: TCP
- targetPort: 8443
+ targetPort: metrics
  selector:
  app.kubernetes.io/name: hyperfleet-operator
  control-plane: controller-manager

bundle/manifests/hyperfleet-operator.clusterserviceversion.yaml

@@ -166,9 +166,10 @@ spec:
  spec:
  containers:
  - args:
- - --metrics-bind-address=:8443
  - --leader-elect
- - --health-probe-bind-address=:8081
+ - --health-probe-bind-address=:8080
+ - --metrics-bind-address=:9090
+ - --metrics-secure=false
  command:
  - /manager
  env:
@@ -180,18 +181,29 @@ spec:
  value: quay.io/redhat-services-prod/hyperfleet-tenant/hyperfleet/hyperfleet-api:latest
  image: quay.io/redhat-services-prod/hyperfleet-tenant/hyperfleet/hyperfleet-operator:latest
  livenessProbe:
+ failureThreshold: 3
  httpGet:
  path: /healthz
- port: 8081
+ port: 8080
  initialDelaySeconds: 15
  periodSeconds: 20
+ timeoutSeconds: 5
  name: manager
+ ports:
+ - containerPort: 9090
+ name: metrics
+ protocol: TCP
+ - containerPort: 8080
+ name: health
+ protocol: TCP
  readinessProbe:
+ failureThreshold: 3
  httpGet:
  path: /readyz
- port: 8081
+ port: 8080
  initialDelaySeconds: 5
  periodSeconds: 10
+ timeoutSeconds: 3
  resources:
  limits:
  cpu: 500m
@@ -209,7 +221,7 @@ spec:
  seccompProfile:
  type: RuntimeDefault
  serviceAccountName: hyperfleet-operator-controller-manager
- terminationGracePeriodSeconds: 10
+ terminationGracePeriodSeconds: 30
  permissions:
  - rules:
  - apiGroups:
@@ -251,6 +263,17 @@ spec:
  - get
  - list
  - watch
+ - apiGroups:
+ - monitoring.coreos.com
+ resources:
+ - servicemonitors
+ verbs:
+ - create
+ - get
+ - list
+ - patch
+ - update
+ - watch
  - apiGroups:
  - networking.k8s.io
  resources:

cmd/main.go

@@ -43,6 +43,8 @@ import (
 
 	hyperfleetv1alpha1 "github.com/openshift-hyperfleet/hyperfleet-operator/api/v1alpha1"
 	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/controller"
+	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/metrics"
+	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/servicemonitor"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -68,14 +70,18 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
- "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	// Defaults follow the HyperFleet health-endpoints / metrics standards, matching
+	// the API, Sentinel and Adapter components: metrics on :9090 over plain HTTP at
+	// /metrics, health/readiness on :8080. Set 0 on metrics-bind-address to disable.
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":9090", "The address the metrics endpoint binds to. "+
+ "Defaults to :9090 (HyperFleet standard). Set to 0 to disable the metrics service.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8080", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
  "Enable leader election for controller manager. "+
  "Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", true,
- "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", false,
+ "If set, the metrics endpoint is served securely via HTTPS with authn/authz. "+
+ "The HyperFleet standard scrapes plain HTTP, so this defaults to false.")
 	flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.")
 	flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.")
 	flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.")
@@ -93,6 +99,10 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Publish the build-info and up metrics into controller-runtime's registry so
+	// they are exposed on the same /metrics endpoint as the reconcile metrics.
+	metrics.Init()
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -256,6 +266,29 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	// Create the operator's own ServiceMonitor at runtime, but only when the
+	// Prometheus Operator API is present. Shipping it in the OLM bundle would fail
+	// the InstallPlan on clusters without the monitoring.coreos.com CRD and block
+	// the operator install; this bootstrapper degrades gracefully instead. See the
+	// servicemonitor package doc for the full rationale.
+	//
+	// Skipped entirely when metrics are disabled (metricsAddr == "0", the same
+	// sentinel controller-runtime's metrics server itself checks — see
+	// metricsserver.NewServer): the ServiceMonitor always points at the metrics
+	// Service's "metrics" port, so with no metrics server listening it would only
+	// give Prometheus a target that fails every scrape.
+	if metricsAddr != "0" {
+ if err := mgr.Add(&servicemonitor.Bootstrapper{
+ Config: mgr.GetConfig(),
+ Namespace: operatorNamespace,
+ }); err != nil {
+ setupLog.Error(err, "unable to add ServiceMonitor bootstrapper")
+ os.Exit(1)
+ }
+	} else {
+ setupLog.Info("metrics disabled (metrics-bind-address=0); skipping ServiceMonitor bootstrap")
+	}
+
 	if metricsCertWatcher != nil {
  setupLog.Info("Adding metrics certificate watcher to manager")
  if err := mgr.Add(metricsCertWatcher); err != nil {

config/default/kustomization.yaml

@@ -38,13 +38,9 @@ resources:
 # be able to communicate with the Webhook Server.
 #- ../network-policy
 
-# Uncomment the patches line if you enable Metrics
-patches:
-# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
-# More info: https://book.kubebuilder.io/reference/metrics
-- path: manager_metrics_patch.yaml
- target:
- kind: Deployment
+# The metrics endpoint (:9090, plain HTTP, /metrics) and health probes (:8080)
+# are configured directly on the manager Deployment (config/manager/manager.yaml)
+# per the HyperFleet standard, so no metrics args patch is needed here.
 
 # Uncomment the patches line if you enable Metrics and CertManager
 # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.

config/default/metrics_service.yaml

@@ -9,10 +9,10 @@ metadata:
  namespace: system
 spec:
  ports:
- - name: https
- port: 8443
+ - name: metrics
+ port: 9090
  protocol: TCP
- targetPort: 8443
+ targetPort: metrics
  selector:
  control-plane: controller-manager
  app.kubernetes.io/name: hyperfleet-operator

config/manager/manager.yaml

@@ -62,7 +62,11 @@ spec:
  - /manager
  args:
  - --leader-elect
- - --health-probe-bind-address=:8081
+ # HyperFleet standard: metrics on :9090 (plain HTTP, /metrics) and
+ # health/readiness on :8080, matching the other components.
+ - --health-probe-bind-address=:8080
+ - --metrics-bind-address=:9090
+ - --metrics-secure=false
  image: controller:latest
  name: manager
  env:
@@ -72,24 +76,35 @@ spec:
  valueFrom:
  fieldRef:
  fieldPath: metadata.namespace
- ports: []
+ ports:
+ - name: metrics
+ containerPort: 9090
+ protocol: TCP
+ - name: health
+ containerPort: 8080
+ protocol: TCP
  securityContext:
  allowPrivilegeEscalation: false
  capabilities:
  drop:
  - "ALL"
+ # Probe timing follows the HyperFleet health-endpoints standard.
  livenessProbe:
  httpGet:
  path: /healthz
- port: 8081
+ port: 8080
  initialDelaySeconds: 15
  periodSeconds: 20
+ timeoutSeconds: 5
+ failureThreshold: 3
  readinessProbe:
  httpGet:
  path: /readyz
- port: 8081
+ port: 8080
  initialDelaySeconds: 5
  periodSeconds: 10
+ timeoutSeconds: 3
+ failureThreshold: 3
  # TODO(user): Configure the resources accordingly based on the project requirements.
  # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
  resources:
@@ -102,4 +117,4 @@ spec:
  volumeMounts: []
  volumes: []
  serviceAccountName: controller-manager
- terminationGracePeriodSeconds: 10
+ terminationGracePeriodSeconds: 30

config/manifests/kustomization.yaml

@@ -5,6 +5,14 @@ resources:
 - ../default
 - ../samples
 - ../scorecard
+# The Prometheus ServiceMonitor is deliberately NOT included in the bundle. OLM
+# applies a bundle's arbitrary manifests but does not install the CRDs they need,
+# so bundling the ServiceMonitor would fail the InstallPlan — and block the whole
+# operator install — on clusters without the Prometheus Operator CRD. HyperFleet
+# targets generic Kubernetes, so the operator creates the ServiceMonitor itself at
+# runtime only when the monitoring.coreos.com/v1 API is present (see the
+# internal/servicemonitor package). config/prometheus remains an optional GitOps
+# overlay for users who prefer to apply it statically.
 
 # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix.
 # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager.

config/network-policy/allow-metrics-traffic.yaml

@@ -23,5 +23,5 @@ spec:
  matchLabels:
  metrics: enabled # Only from namespaces with this label
  ports:
- - port: 8443
+ - port: 9090
  protocol: TCP

config/prometheus/monitor.yaml

@@ -1,4 +1,6 @@
 # Prometheus Monitor Service (Metrics)
+# Scrapes the operator's plain-HTTP metrics endpoint on :9090, per the HyperFleet
+# metrics standard (same exposition as the API, Sentinel and Adapter components).
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -11,16 +13,14 @@ metadata:
 spec:
  endpoints:
  - path: /metrics
- port: https # Ensure this is the name of the port that exposes HTTPS metrics
- scheme: https
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- tlsConfig:
- # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
- # certificate verification, exposing the system to potential man-in-the-middle attacks.
- # For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
- # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
- # which securely references the certificate from the 'metrics-server-cert' secret.
- insecureSkipVerify: true
+ port: metrics # matches the metrics Service port name
+ # Hardcoded to the HyperFleet metrics standard's plain-HTTP default
+ # (--metrics-secure=false). If the operator is run with --metrics-secure=true,
+ # this scrape will fail: the endpoint requires HTTPS + authn/authz and this
+ # manifest is not updated automatically. Configure scheme/tlsConfig/bearerToken
+ # here yourself in that case (see config/prometheus/monitor_tls_patch.yaml).
+ scheme: http
+ interval: 30s
  selector:
  matchLabels:
  control-plane: controller-manager

config/rbac/role.yaml

@@ -81,6 +81,17 @@ rules:
  - get
  - list
  - watch
+- apiGroups:
+ - monitoring.coreos.com
+ resources:
+ - servicemonitors
+ verbs:
+ - create
+ - get
+ - list
+ - patch
+ - update
+ - watch
 - apiGroups:
  - networking.k8s.io
  resources: