Fix Traefik restart and ACME reliability

.github/workflows/agent-ci.yml

@@ -43,7 +43,7 @@ jobs:
  run: go vet ./...
 
  - name: Install staticcheck
- run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@latest
+ run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@v0.7.0
 
  - name: Staticcheck
  run: ./.bin/staticcheck ./...

.github/workflows/cli-ci.yml

@@ -43,7 +43,7 @@ jobs:
  run: go vet ./...
 
  - name: Install staticcheck
- run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@latest
+ run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@v0.7.0
 
  - name: Staticcheck
  run: ./.bin/staticcheck ./...

.github/workflows/updater-ci.yml

@@ -45,7 +45,7 @@ jobs:
  run: go vet ./...
 
  - name: Install staticcheck
- run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@latest
+ run: GOBIN=$PWD/.bin go install honnef.co/go/tools/cmd/staticcheck@v0.7.0
 
  - name: Staticcheck
  run: ./.bin/staticcheck ./...

agent/internal/traefik/reload.go

@@ -15,6 +15,7 @@ import (
 const (
 	lastReloadSuccessMetric = "traefik_config_last_reload_success"
 	pendingReloadMarkerName = ".routing-reload-pending"
+	metricsReadyTimeout = 15 * time.Second
 )
 
 var (
@@ -30,6 +31,22 @@ func LastSuccessfulReload() (time.Time, error) {
 	return readLastSuccessfulReload()
 }
 
+func waitForMetricsReady(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for {
+ if _, err := LastSuccessfulReload(); err == nil {
+ return nil
+ } else {
+ lastErr = err
+ }
+ if time.Now().After(deadline) {
+ return fmt.Errorf("traefik metrics did not become ready within %s: %w", timeout, lastErr)
+ }
+ time.Sleep(reloadPollInterval)
+	}
+}
+
 func fetchLastSuccessfulReload() (time.Time, error) {
 	response, err := metricsHTTPClient.Get(traefikMetricsURL)
 	if err != nil {

agent/internal/traefik/reload_test.go

@@ -20,6 +20,54 @@ traefik_config_last_reload_success 1.725e+09
 	}
 }
 
+func TestWaitForMetricsReadyRetriesTemporaryErrors(t *testing.T) {
+	originalReader := readLastSuccessfulReload
+	originalPollInterval := reloadPollInterval
+	t.Cleanup(func() {
+ readLastSuccessfulReload = originalReader
+ reloadPollInterval = originalPollInterval
+	})
+
+	attempts := 0
+	readLastSuccessfulReload = func() (time.Time, error) {
+ attempts++
+ if attempts < 3 {
+ return time.Time{}, os.ErrNotExist
+ }
+ return time.Now(), nil
+	}
+	reloadPollInterval = time.Millisecond
+
+	if err := waitForMetricsReady(50 * time.Millisecond); err != nil {
+ t.Fatal(err)
+	}
+	if attempts != 3 {
+ t.Fatalf("metrics read attempted %d times, want 3", attempts)
+	}
+}
+
+func TestWaitForMetricsReadyTimesOut(t *testing.T) {
+	originalReader := readLastSuccessfulReload
+	originalPollInterval := reloadPollInterval
+	t.Cleanup(func() {
+ readLastSuccessfulReload = originalReader
+ reloadPollInterval = originalPollInterval
+	})
+
+	readLastSuccessfulReload = func() (time.Time, error) {
+ return time.Time{}, os.ErrDeadlineExceeded
+	}
+	reloadPollInterval = time.Millisecond
+
+	err := waitForMetricsReady(5 * time.Millisecond)
+	if err == nil {
+ t.Fatal("metrics readiness wait unexpectedly succeeded")
+	}
+	if !strings.Contains(err.Error(), "traefik metrics did not become ready within 5ms") {
+ t.Fatalf("unexpected timeout error: %v", err)
+	}
+}
+
 func TestDynamicConfigReloadedRequiresReloadAtOrAfterNewestFile(t *testing.T) {
 	originalDir := dynamicConfigDir
 	originalReader := readLastSuccessfulReload

agent/internal/traefik/static.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"os/exec"
 	"reflect"
-	"time"
 
 	"gopkg.in/yaml.v3"
 )
@@ -16,6 +15,7 @@ const (
 	metricsEntryPointAddr = "127.0.0.1:9100"
 )
 
+// Whole-number buckets must be ints to match yaml.v3's decoded types.
 var prometheusLatencyBuckets = []interface{}{
 	0.005,
 	0.01,
@@ -26,12 +26,12 @@ var prometheusLatencyBuckets = []interface{}{
 	0.25,
 	0.5,
 	0.75,
-	1.0,
+	1,
 	2.5,
-	5.0,
-	10.0,
-	30.0,
-	60.0,
+	5,
+	10,
+	30,
+	60,
 }
 
 func validateStaticConfig(data []byte) error {
@@ -218,6 +218,5 @@ func ReloadTraefik() error {
  return fmt.Errorf("failed to restart traefik: %w", err)
 	}
 	log.Printf("[traefik] restarted traefik to apply static config changes")
-	time.Sleep(2 * time.Second)
-	return nil
+	return waitForMetricsReady(metricsReadyTimeout)
 }

agent/internal/traefik/static_test.go

@@ -1,6 +1,10 @@
 package traefik
 
-import "testing"
+import (
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
 
 func TestEnsurePrometheusMetricsConfigAddsPrivateMetricsEndpoint(t *testing.T) {
 	config := map[string]interface{}{
@@ -44,7 +48,17 @@ func TestEnsurePrometheusMetricsConfigIsStable(t *testing.T) {
 	if !ensurePrometheusMetricsConfig(config) {
  t.Fatal("expected first call to modify config")
 	}
-	if ensurePrometheusMetricsConfig(config) {
+
+	data, err := yaml.Marshal(config)
+	if err != nil {
+ t.Fatalf("failed to marshal config: %v", err)
+	}
+	var roundTripped map[string]interface{}
+	if err := yaml.Unmarshal(data, &roundTripped); err != nil {
+ t.Fatalf("failed to unmarshal config: %v", err)
+	}
+
+	if ensurePrometheusMetricsConfig(roundTripped) {
  t.Fatal("expected second call to be stable")
 	}
 }

web/lib/inngest/functions/rollout-workflow.ts

@@ -418,33 +418,57 @@ export const rolloutWorkflow = inngest.createFunction(
  });
  }
 
- const certResult = await step.run("issue-certificates", async () => {
- await db
- .update(rollouts)
- .set({ currentStage: "certificates" })
- .where(eq(rollouts.id, rolloutId));
- try {
- const result = await issueCertificatesForRevision(specification);
- if (result.issuedDomains.length > 0) {
- await ingestRolloutLog(
- rolloutId,
- serviceId,
- "certificates",
- `Certificates issued for ${result.issuedDomains.length} domain(s)`,
- );
- }
- return { success: true as const };
- } catch (error) {
- const message =
- error instanceof Error
- ? error.message
- : "Certificate provisioning failed";
- await ingestRolloutLog(rolloutId, serviceId, "certificates", message);
- return { success: false as const, reason: message };
+ let certificatesIssued = false;
+ let certificateFailureReason = "Certificate provisioning failed";
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ const certResult = await step.run(
+ `issue-certificates-${attempt}`,
+ async () => {
+ await db
+ .update(rollouts)
+ .set({ currentStage: "certificates" })
+ .where(eq(rollouts.id, rolloutId));
+ try {
+ const result = await issueCertificatesForRevision(specification);
+ if (result.issuedDomains.length > 0) {
+ await ingestRolloutLog(
+ rolloutId,
+ serviceId,
+ "certificates",
+ `Certificates issued for ${result.issuedDomains.length} domain(s)`,
+ );
+ }
+ return { success: true as const };
+ } catch (error) {
+ const message =
+ error instanceof Error
+ ? error.message
+ : "Certificate provisioning failed";
+ await ingestRolloutLog(
+ rolloutId,
+ serviceId,
+ "certificates",
+ message,
+ );
+ return { success: false as const, reason: message };
+ }
+ },
+ );
+ if (certResult.success) {
+ certificatesIssued = true;
+ break;
  }
- });
 
- if (!certResult.success) {
+ certificateFailureReason = certResult.reason;
+ if (attempt < 3) {
+ await step.sleep(
+ `wait-for-certificate-retry-${attempt}`,
+ attempt === 1 ? "10s" : "20s",
+ );
+ }
+ }
+
+ if (!certificatesIssued) {
  await step.run("handle-certificate-failure", async () => {
  await handleRolloutFailure(
  rolloutId,
@@ -455,7 +479,7 @@ export const rolloutWorkflow = inngest.createFunction(
  });
  return {
  status: "failed",
- reason: certResult.reason,
+ reason: certificateFailureReason,
  };
  }