From 9ffd563d8bd233f3544d7ef38607cd7074cdaf55 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 2 Jul 2026 14:35:36 -0400 Subject: [PATCH 01/20] Convert per-mission stellar-core Ingress to Gateway API HTTPRoute Replace the nginx Ingress (ToIngress) with an HTTPRoute (ToHttpRoute) attached to the shared traefik-gateway-private gateway. Per-pod //core|history PathPrefix matches with a ReplacePrefixMatch URLRewrite (equivalent to the old rewrite-target). - Add typed Gateway API models (GatewayApiModels) in CSLibrary; official KubernetesClient has no Gateway API types. - Create/track/delete + orphan-sweep the HTTPRoute via the custom-object API. Co-Authored-By: Claude Opus 4.8 --- src/CSLibrary/GatewayApi.cs | 161 +++++++++++++++++++++++ src/FSLibrary/StellarCoreCfg.fs | 4 + src/FSLibrary/StellarKubeSpecs.fs | 97 +++++++------- src/FSLibrary/StellarNamespaceContent.fs | 18 ++- src/FSLibrary/StellarOrphanSweep.fs | 22 +++- src/FSLibrary/StellarSupercluster.fs | 18 ++- 6 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 src/CSLibrary/GatewayApi.cs diff --git a/src/CSLibrary/GatewayApi.cs b/src/CSLibrary/GatewayApi.cs new file mode 100644 index 00000000..ef62fdab --- /dev/null +++ b/src/CSLibrary/GatewayApi.cs @@ -0,0 +1,161 @@ +// Typed models for the subset of the Gateway API (gateway.networking.k8s.io/v1) +// that Supercluster needs to route to per-pod core/history endpoints. +// +// The official KubernetesClient package ships no Gateway API models (CRDs are +// out of scope for it), so these POCOs are hand-maintained to the upstream +// gateway-api v1 schema. Only the fields Supercluster uses are modelled. +// Usable with GenericClient thanks to the [KubernetesEntity] attribute. + +using System.Collections.Generic; +using System.Text.Json.Serialization; +using k8s; +using k8s.Models; + +namespace GatewayApiModels +{ + [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRoute", PluralName = "httproutes")] + public class HTTPRoute : IKubernetesObject, ISpec + { + [JsonPropertyName("apiVersion")] + public string ApiVersion { get; set; } = "gateway.networking.k8s.io/v1"; + + [JsonPropertyName("kind")] + public string Kind { get; set; } = "HTTPRoute"; + + [JsonPropertyName("metadata")] + public V1ObjectMeta Metadata { get; set; } + + [JsonPropertyName("spec")] + public HTTPRouteSpec Spec { get; set; } + } + + [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRouteList", PluralName = "httproutes")] + public class HTTPRouteList : IKubernetesObject, IItems + { + [JsonPropertyName("apiVersion")] + public string ApiVersion { get; set; } = "gateway.networking.k8s.io/v1"; + + [JsonPropertyName("kind")] + public string Kind { get; set; } = "HTTPRouteList"; + + [JsonPropertyName("metadata")] + public V1ListMeta Metadata { get; set; } + + [JsonPropertyName("items")] + public IList Items { get; set; } + } + + public class HTTPRouteSpec + { + [JsonPropertyName("parentRefs")] + public IList ParentRefs { get; set; } + + [JsonPropertyName("hostnames")] + public IList Hostnames { get; set; } + + [JsonPropertyName("rules")] + public IList Rules { get; set; } + } + + public class ParentReference + { + [JsonPropertyName("group")] + public string Group { get; set; } + + [JsonPropertyName("kind")] + public string Kind { get; set; } + + [JsonPropertyName("namespace")] + public string Namespace { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("sectionName")] + public string SectionName { get; set; } + + [JsonPropertyName("port")] + public int? Port { get; set; } + } + + public class HTTPRouteRule + { + [JsonPropertyName("matches")] + public IList Matches { get; set; } + + [JsonPropertyName("filters")] + public IList Filters { get; set; } + + [JsonPropertyName("backendRefs")] + public IList BackendRefs { get; set; } + } + + public class HTTPRouteMatch + { + [JsonPropertyName("path")] + public HTTPPathMatch Path { get; set; } + } + + public class HTTPPathMatch + { + // "Exact" | "PathPrefix" | "RegularExpression" + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } + } + + public class HTTPRouteFilter + { + // "RequestHeaderModifier" | "URLRewrite" | ... + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("urlRewrite")] + public HTTPURLRewriteFilter UrlRewrite { get; set; } + } + + public class HTTPURLRewriteFilter + { + [JsonPropertyName("hostname")] + public string Hostname { get; set; } + + [JsonPropertyName("path")] + public HTTPPathModifier Path { get; set; } + } + + public class HTTPPathModifier + { + // "ReplaceFullPath" | "ReplacePrefixMatch" + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("replaceFullPath")] + public string ReplaceFullPath { get; set; } + + [JsonPropertyName("replacePrefixMatch")] + public string ReplacePrefixMatch { get; set; } + } + + public class HTTPBackendRef + { + [JsonPropertyName("group")] + public string Group { get; set; } + + [JsonPropertyName("kind")] + public string Kind { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("namespace")] + public string Namespace { get; set; } + + [JsonPropertyName("port")] + public int? Port { get; set; } + + [JsonPropertyName("weight")] + public int? Weight { get; set; } + } +} diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index e4c2fd4a..f800fbf0 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -19,7 +19,11 @@ open StellarDotnetSdk.Accounts // paths, labels, etc. module CfgVal = let httpPort = 11626 + let historyPort = 80 let prometheusExporterPort = 9473 + // Gateway the per-mission HTTPRoute attaches to (shared private traefik gateway). + let gatewayName = "traefik-gateway-private" + let gatewayNamespace = "traefik" let labels = Map.ofSeq [ "app", "stellar-core" ] let labelSelector = "app = stellar-core" let stellarCoreBinPath = "stellar-core" diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index df6d836d..39a625f2 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -14,6 +14,7 @@ open StellarNetworkDelays open System.Text.RegularExpressions open System.Collections.Generic open Logging +open GatewayApiModels // Containers that run stellar-core may or may-not have a final '--conf' // argument appended to their command-line. The argument is specified one of 3 @@ -885,54 +886,56 @@ type NetworkCfg with self.MapAllPeers perPodService - // Returns an Ingress object with rules that map URLs http://$ingressHost/peer-N/foo - // to the per-Pod Service within the current networkCfg named peer-N (which then, via - // DNS mapping, goes to the Pod itself). Exposing this to external traffic - // requires that you enable the nginx Ingress controller on your k8s - // cluster. - member self.ToIngress() : V1Ingress = - let coreBackend (pn: PodName) : V1IngressBackend = - let port = V1ServiceBackendPort(number = CfgVal.httpPort) - let service = V1IngressServiceBackend(pn.StringName, port = port) - V1IngressBackend(service = service) - - let historyBackend (pn: PodName) : V1IngressBackend = - let port = V1ServiceBackendPort(number = 80) - let service = V1IngressServiceBackend(pn.StringName, port = port) - V1IngressBackend(service = service) - - let corePath (coreSet: CoreSet) (i: int) : V1HTTPIngressPath = - let pn = self.PodName coreSet i - let ingressPath = V1HTTPIngressPath() - ingressPath.Backend <- coreBackend pn - ingressPath.Path <- sprintf "/%s/core(/|$)(.*)" pn.StringName - ingressPath.PathType <- "ImplementationSpecific" - ingressPath - - let historyPath (coreSet: CoreSet) (i: int) : V1HTTPIngressPath = - let pn = self.PodName coreSet i - let ingressPath = V1HTTPIngressPath() - ingressPath.Backend <- historyBackend pn - ingressPath.Path <- sprintf "/%s/history(/|$)(.*)" pn.StringName - ingressPath.PathType <- "ImplementationSpecific" - ingressPath - - let corePaths = self.MapAllPeers corePath - let historyPaths = self.MapAllPeers historyPath - - let rule = V1HTTPIngressRuleValue(paths = Array.concat [ corePaths; historyPaths ]) - - let host = self.IngressInternalHostName - let rules = [| V1IngressRule(host = host, http = rule) |] - let spec = V1IngressSpec(rules = rules) - - let annotation = - Map.ofArray [| ("kubernetes.io/ingress.class", self.missionContext.ingressClass) - ("nginx.ingress.kubernetes.io/use-regex", "true") - ("nginx.ingress.kubernetes.io/rewrite-target", "/$2") |] + // Returns an HTTPRoute (gateway.networking.k8s.io/v1) mapping + // http://$ingressHost//core|history/... to the per-Pod Service , attached to + // the shared private traefik gateway. Each pod gets a core rule (-> httpPort) and a + // history rule (-> historyPort); the //core|history prefix is stripped via a + // ReplacePrefixMatch URLRewrite (equivalent to the old nginx rewrite-target /$2). + // Replaces the former nginx Ingress. + member self.ToHttpRoute() : HTTPRoute = + let parentRef = + ParentReference( + Group = "gateway.networking.k8s.io", + Kind = "Gateway", + Namespace = CfgVal.gatewayNamespace, + Name = CfgVal.gatewayName + ) + + let rewriteFilter = + HTTPRouteFilter( + Type = "URLRewrite", + UrlRewrite = HTTPURLRewriteFilter(Path = HTTPPathModifier(Type = "ReplacePrefixMatch", ReplacePrefixMatch = "/")) + ) + + let ruleFor (pn: PodName) (suffix: string) (port: int) : HTTPRouteRule = + let m = + HTTPRouteMatch(Path = HTTPPathMatch(Type = "PathPrefix", Value = sprintf "/%s/%s" pn.StringName suffix)) + + let b = HTTPBackendRef(Name = pn.StringName, Port = System.Nullable(port)) + + HTTPRouteRule( + Matches = List([ m ]), + Filters = List([ rewriteFilter ]), + BackendRefs = List([ b ]) + ) + + let coreRule (coreSet: CoreSet) (i: int) : HTTPRouteRule = + ruleFor (self.PodName coreSet i) "core" CfgVal.httpPort + + let historyRule (coreSet: CoreSet) (i: int) : HTTPRouteRule = + ruleFor (self.PodName coreSet i) "history" CfgVal.historyPort + + let rules = Array.concat [ self.MapAllPeers coreRule; self.MapAllPeers historyRule ] + + let spec = + HTTPRouteSpec( + ParentRefs = List([ parentRef ]), + Hostnames = List([ self.IngressInternalHostName ]), + Rules = List(rules) + ) let meta = - V1ObjectMeta(name = self.IngressName, namespaceProperty = self.NamespaceProperty, annotations = annotation) + V1ObjectMeta(name = self.IngressName, namespaceProperty = self.NamespaceProperty) |> applyAnchorOwner self - V1Ingress(spec = spec, metadata = meta) + HTTPRoute(Metadata = meta, Spec = spec) diff --git a/src/FSLibrary/StellarNamespaceContent.fs b/src/FSLibrary/StellarNamespaceContent.fs index 22b62c0a..dccefca0 100644 --- a/src/FSLibrary/StellarNamespaceContent.fs +++ b/src/FSLibrary/StellarNamespaceContent.fs @@ -7,6 +7,7 @@ module StellarNamespaceContent open k8s open k8s.Models open Logging +open GatewayApiModels type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: string) = @@ -15,7 +16,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st let services : Set ref = ref Set.empty let configMaps : Set ref = ref Set.empty let statefulSets : Set ref = ref Set.empty - let ingresses : Set ref = ref Set.empty + let httpRoutes : Set ref = ref Set.empty let jobs : Set ref = ref Set.empty let daemonSets : Set ref = ref Set.empty let deployments : Set ref = ref Set.empty @@ -61,14 +62,17 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st propagationPolicy = "Foreground" )) - let delIngress (name: string) = - LogInfo "Deleting Ingress %s" name + let delHttpRoute (name: string) = + LogInfo "Deleting HTTPRoute %s" name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (apiRateLimit) ignoreError (fun _ -> - kube.DeleteNamespacedIngress( + kube.DeleteNamespacedCustomObject( + group = "gateway.networking.k8s.io", + version = "v1", namespaceParameter = namespaceProperty, + plural = "httproutes", name = name, propagationPolicy = "Foreground" )) @@ -124,7 +128,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st cleanSet delStatefulSet statefulSets cleanSet delDeployment deployments cleanSet delConfigMap configMaps - cleanSet delIngress ingresses + cleanSet delHttpRoute httpRoutes cleanSet delJob jobs cleanSet delDaemonSet daemonSets @@ -134,7 +138,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st member self.Add(statefulSet: V1StatefulSet) = addOne statefulSets statefulSet.Metadata.Name - member self.Add(ingress: V1Ingress) = addOne ingresses ingress.Metadata.Name + member self.Add(route: HTTPRoute) = addOne httpRoutes route.Metadata.Name member self.Add(job: V1Job) = addOne jobs job.Metadata.Name @@ -148,7 +152,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st member self.Del(statefulSet: V1StatefulSet) = delOne delStatefulSet statefulSets statefulSet.Metadata.Name - member self.Del(ingress: V1Ingress) = delOne delIngress ingresses ingress.Metadata.Name + member self.Del(route: HTTPRoute) = delOne delHttpRoute httpRoutes route.Metadata.Name member self.Del(job: V1Job) = delOne delJob jobs job.Metadata.Name diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index c62cf202..453b1c68 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -10,6 +10,7 @@ open k8s.Models open Logging open ScriptUtils +open GatewayApiModels // Resources that look this old must belong to a failed prior run — no healthy // mission runs anywhere near this long, and the Jenkins lock serializes CI @@ -124,12 +125,25 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( sweepKind apiRateLimit - "Ingress" + "HTTPRoute" (fun () -> - kube.ListNamespacedIngress(namespaceParameter = ns).Items - |> Seq.map (fun i -> i.Metadata)) + let gc = GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") + + gc + .ListNamespacedAsync(ns) + .GetAwaiter() + .GetResult() + .Items + |> Seq.map (fun r -> r.Metadata)) (fun n -> - kube.DeleteNamespacedIngress(namespaceParameter = ns, name = n, propagationPolicy = "Foreground") + kube.DeleteNamespacedCustomObject( + group = "gateway.networking.k8s.io", + version = "v1", + namespaceParameter = ns, + plural = "httproutes", + name = n, + propagationPolicy = "Foreground" + ) |> ignore) cutoff diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index d3b19583..13ba7467 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -17,6 +17,7 @@ open StellarStatefulSets open StellarCoreSet open StellarKubeSpecs open StellarNamespaceContent +open GatewayApiModels open System open System.Diagnostics @@ -320,11 +321,20 @@ type Kubernetes with namespaceContent.Add(service) if not (List.isEmpty statefulSets) then - let ing = nCfg.ToIngress() - LogInfo "Creating Ingress %s" ing.Metadata.Name + let route = nCfg.ToHttpRoute() + LogInfo "Creating HTTPRoute %s" route.Metadata.Name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) - let ingress = self.CreateNamespacedIngress(namespaceParameter = nsStr, body = ing) - namespaceContent.Add(ingress) + + self.CreateNamespacedCustomObject( + body = route, + group = "gateway.networking.k8s.io", + version = "v1", + namespaceParameter = nsStr, + plural = "httproutes" + ) + |> ignore + + namespaceContent.Add(route) let formation = new StellarFormation( From 2761815034055951eb6aa3e7c0faba034e0270d5 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 2 Jul 2026 14:47:37 -0400 Subject: [PATCH 02/20] Bump KubernetesClient 15.0.1 -> 17.0.14 Highest version before the 18.x model-constructor breaking change (parameterized ctors removed). Builds clean + tests pass with no code changes; 18/19 would need a repo-wide constructor migration, left as separate work. Co-Authored-By: Claude Opus 4.8 --- src/CSLibrary/CSLibrary.csproj | 2 +- src/FSLibrary/FSLibrary.fsproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSLibrary/CSLibrary.csproj b/src/CSLibrary/CSLibrary.csproj index 906f91f7..00e0cef2 100644 --- a/src/CSLibrary/CSLibrary.csproj +++ b/src/CSLibrary/CSLibrary.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/FSLibrary/FSLibrary.fsproj b/src/FSLibrary/FSLibrary.fsproj index 135d1cec..6b1422dd 100644 --- a/src/FSLibrary/FSLibrary.fsproj +++ b/src/FSLibrary/FSLibrary.fsproj @@ -86,7 +86,7 @@ - + From 69f0941188e483d2b33503560371a24bd2e3c232 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 2 Jul 2026 15:28:46 -0400 Subject: [PATCH 03/20] Per-pod services: ClusterIP + pod-name selector instead of ExternalName Traefik's Gateway provider rejects ExternalName services as HTTPRoute backends ('type ExternalName is not supported'), unlike the nginx Ingress path. Select the target pod via its statefulset.kubernetes.io/pod-name label instead. Works for both gateway and ingress; verified live on ssc-test (core /info + protocol upgrade route through the gateway). Co-Authored-By: Claude Opus 4.8 --- src/FSLibrary/StellarKubeSpecs.fs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 39a625f2..6a6d3a74 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -867,7 +867,6 @@ type NetworkCfg with member self.ToPerPodServices() : V1Service array = let perPodService (coreSet: CoreSet) i = let name = self.PodName coreSet i - let dnsName = self.PeerDnsName coreSet i let ports = [| V1ServicePort(name = "core", port = CfgVal.httpPort) @@ -879,8 +878,13 @@ type NetworkCfg with else ports - let spec = - V1ServiceSpec(``type`` = "ExternalName", ports = ports, externalName = dnsName.StringName) + // ClusterIP service selecting the single pod by its StatefulSet pod-name label. + // (Previously ExternalName -> pod DNS, which the traefik Gateway provider rejects + // as an HTTPRoute backend.) + let selector = + CfgVal.labels |> Map.add "statefulset.kubernetes.io/pod-name" name.StringName + + let spec = V1ServiceSpec(selector = selector, ports = ports) V1Service(metadata = self.NamespacedMeta name.StringName, spec = spec) From f7871cea72750844169fd74df55b919218b5df19 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 2 Jul 2026 16:03:40 -0400 Subject: [PATCH 04/20] Rename IngressName -> HttpRouteName; drop -ingress from the object name The generated object is now an HTTPRoute, not an Ingress; name it -stellar-core-http-route. --- src/FSLibrary/StellarKubeSpecs.fs | 2 +- src/FSLibrary/StellarNetworkCfg.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 6a6d3a74..37ff6190 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -939,7 +939,7 @@ type NetworkCfg with ) let meta = - V1ObjectMeta(name = self.IngressName, namespaceProperty = self.NamespaceProperty) + V1ObjectMeta(name = self.HttpRouteName, namespaceProperty = self.NamespaceProperty) |> applyAnchorOwner self HTTPRoute(Metadata = meta, Spec = spec) diff --git a/src/FSLibrary/StellarNetworkCfg.fs b/src/FSLibrary/StellarNetworkCfg.fs index 2c7edea2..cc9d0021 100644 --- a/src/FSLibrary/StellarNetworkCfg.fs +++ b/src/FSLibrary/StellarNetworkCfg.fs @@ -102,7 +102,7 @@ type NetworkCfg = member self.ServiceName : string = sprintf "%s-stellar-core" self.Nonce - member self.IngressName : string = sprintf "%s-stellar-core-ingress" self.Nonce + member self.HttpRouteName : string = sprintf "%s-stellar-core-http-route" self.Nonce member self.JobName(i: int) : string = sprintf "%s-stellar-core-job-%d" self.Nonce i From 74a7f364e195b587198fdb522eaae75c0c38a4ee Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:07:20 -0400 Subject: [PATCH 05/20] Front driver->core routing with a scalable nginx proxy; ingress->route rename Replace the per-pod ClusterIP services + fat multi-rule HTTPRoute with a single per-mission nginx proxy (Deployment + Service + ConfigMap) behind a one-rule HTTPRoute. The proxy demuxes //core|history to the pod's in-cluster DNS name, so the route needs one rule regardless of node count -- fixing the HTTPRoute 16-rule cap and the CoreDNS amplification of the old ExternalName-per-pod design. Replica count auto-scales ceil(nodes/64), capped by --http-proxy-replicas (default 10, floor 1), with a readiness wait before the driver routes through it. Rename internal ingress* identifiers -> route* (CLI flag strings kept for Jenkinsfile compatibility); drop the now-dead --ingress-class. Co-Authored-By: Claude Opus 4.8 --- src/App/Program.fs | 34 ++--- src/FSLibrary.Tests/Tests.fs | 8 +- src/FSLibrary/StellarCoreCfg.fs | 6 + src/FSLibrary/StellarCoreHTTP.fs | 8 +- src/FSLibrary/StellarKubeSpecs.fs | 172 ++++++++++++++++--------- src/FSLibrary/StellarMissionContext.fs | 8 +- src/FSLibrary/StellarNetworkCfg.fs | 19 ++- src/FSLibrary/StellarSupercluster.fs | 34 ++++- 8 files changed, 190 insertions(+), 99 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 83ade839..2687f286 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -42,10 +42,9 @@ type MissionOptions logDebugPartitions: seq, logTracePartitions: seq, namespaceProperty: string option, - ingressClass: string, - ingressInternalDomain: string, - ingressExternalHost: string option, - ingressExternalPort: int, + routeInternalDomain: string, + routeExternalHost: string option, + routeExternalPort: int, exportToPrometheus: bool, probeTimeout: int, missions: string seq, @@ -70,6 +69,7 @@ type MissionOptions avoidNodeLabels: seq, tolerateNodeTaints: seq, apiRateLimit: int, + httpProxyReplicas: int, pubnetData: string option, flatQuorum: bool option, tier1Keys: string option, @@ -155,28 +155,22 @@ type MissionOptions [] member self.NamespaceProperty = namespaceProperty - [] - member self.IngressClass = ingressClass - [] - member self.IngressInternalDomain = ingressInternalDomain + member self.RouteInternalDomain = routeInternalDomain [] - member self.IngressExternalHost = ingressExternalHost + member self.RouteExternalHost = routeExternalHost [] - member self.IngressExternalPort = ingressExternalPort + member self.RouteExternalPort = routeExternalPort [] member self.ExportToPrometheus : bool = exportToPrometheus @@ -295,6 +289,12 @@ type MissionOptions Default = 10)>] member self.ApiRateLimit = apiRateLimit + [] + member self.HttpProxyReplicas = httpProxyReplicas + [] member self.PubnetData = pubnetData @@ -775,10 +775,9 @@ let main argv = numNodes = mission.NumNodes namespaceProperty = ns logLevels = ll - ingressClass = mission.IngressClass - ingressInternalDomain = mission.IngressInternalDomain - ingressExternalHost = mission.IngressExternalHost - ingressExternalPort = mission.IngressExternalPort + routeInternalDomain = mission.RouteInternalDomain + routeExternalHost = mission.RouteExternalHost + routeExternalPort = mission.RouteExternalPort exportToPrometheus = mission.ExportToPrometheus probeTimeout = mission.ProbeTimeout coreResources = SmallTestResources @@ -788,6 +787,7 @@ let main argv = avoidNodeLabels = List.map splitLabel (List.ofSeq mission.AvoidNodeLabels) tolerateNodeTaints = List.map splitLabel (List.ofSeq mission.TolerateNodeTaints) apiRateLimit = mission.ApiRateLimit + httpProxyReplicas = mission.HttpProxyReplicas pubnetData = mission.PubnetData flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index ce280d8b..26ca1352 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -54,10 +54,9 @@ let ctx : MissionContext = numNodes = 100 namespaceProperty = "stellar-supercluster" logLevels = { LogDebugPartitions = []; LogTracePartitions = [] } - ingressClass = "ingress-private" - ingressInternalDomain = "local" - ingressExternalHost = None - ingressExternalPort = 80 + routeInternalDomain = "local" + routeExternalHost = None + routeExternalPort = 80 exportToPrometheus = false probeTimeout = 10 coreResources = SmallTestResources @@ -67,6 +66,7 @@ let ctx : MissionContext = avoidNodeLabels = [] tolerateNodeTaints = [] apiRateLimit = 10 + httpProxyReplicas = 2 pubnetData = None flatQuorum = None tier1Keys = None diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index f800fbf0..b23a1e7c 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -47,6 +47,12 @@ module CfgVal = let peerCfgFileName = "stellar-core.cfg" let peerInitCfgFileName = "stellar-core-init.cfg" let peerDelayCfgFileName = "install-delays.sh" + + + let httpProxyContainerName = "http-proxy" + let httpProxyConfigVolumeName = "proxy-tmpl" + let httpProxyConfigMountPath = "/proxy-tmpl" + let httpProxyConfigFileName = "default.conf.template" let peerNameEnvCfgFileWord : ShWord = ShWord.ShPieces [| ShBare("/cfg-") diff --git a/src/FSLibrary/StellarCoreHTTP.fs b/src/FSLibrary/StellarCoreHTTP.fs index 66d58fae..f4ceb87b 100644 --- a/src/FSLibrary/StellarCoreHTTP.fs +++ b/src/FSLibrary/StellarCoreHTTP.fs @@ -377,14 +377,14 @@ exception NodeLostSyncException of string type Peer with member self.Headers = - let host = self.networkCfg.IngressInternalHostName + let host = self.networkCfg.RouteInternalHostName [ HttpRequestHeaders.Host host ] member self.URL(path: string) : string = sprintf "http://%s:%d/%s/core/%s" - self.networkCfg.IngressExternalHostName - self.networkCfg.missionContext.ingressExternalPort + self.networkCfg.RouteExternalHostName + self.networkCfg.missionContext.routeExternalPort self.PodName.StringName path @@ -792,7 +792,7 @@ type Peer with (self.URL "tx"), httpMethod = "GET", query = [ "blob", b64 ], - headers = [ "Host", self.networkCfg.IngressInternalHostName ] + headers = [ "Host", self.networkCfg.RouteInternalHostName ] ) use reader = new System.IO.StreamReader(response.ResponseStream) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 37ff6190..64a9428a 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -855,47 +855,116 @@ type NetworkCfg with statefulSet - // Returns an array of "per-Pod" Service objects, each named according to - // the peer-N short names, and mapping (via a somewhat hacky misuse of the - // ExternalName Service type -- thanks internet!) to the _internal_ DNS - // names of each pod. - // - // This exists strictly to support the Ingress object below, that routes - // separate URL prefixes to separate Pods (which is somewhat the opposite of - // the load-balancing task Services, Pods, and Ingress systems typically - // do). - member self.ToPerPodServices() : V1Service array = - let perPodService (coreSet: CoreSet) i = - let name = self.PodName coreSet i - - let ports = - [| V1ServicePort(name = "core", port = CfgVal.httpPort) - V1ServicePort(name = "history", port = 80) |] - - let ports = - if self.missionContext.exportToPrometheus then - Array.append ports [| V1ServicePort(name = "prom-exp", port = CfgVal.prometheusExporterPort) |] - else - ports + // Metadata for a proxy object: run-scoped labels + the anchor owner ref so + // it is GC'd with the rest of the run. (Not NamespacedMeta, which would + // stamp the core "app=stellar-core" labels.) + member private self.HttpProxyMeta(name: string) : V1ObjectMeta = + V1ObjectMeta(name = name, namespaceProperty = self.NamespaceProperty, labels = self.HttpProxyLabels) + |> applyAnchorOwner self + + // ConfigMap holding the nginx server config for the proxy. The pod's actual + // CoreDNS address is not known here, so we ship a template with a + // __RESOLVER__ placeholder that the container substitutes at startup from + // its own /etc/resolv.conf (see ToHttpProxyDeployment). Requests of the + // form //core|history[/] are proxied to the pod's in-cluster DNS + // name; $is_args$args preserves the query string (the driver uses it, e.g. + // //core/tx?blob=...). + member self.ToHttpProxyConfigMap() : V1ConfigMap = + let fqdn = sprintf "%s.%s.svc.cluster.local" self.ServiceName self.NamespaceProperty + + let conf = + sprintf + """server { + listen 80 default_server; + server_name _; + resolver __RESOLVER__ ipv6=off valid=10s; + + location ~ ^/([^/]+)/core(?:/(.*))?$ { + proxy_pass http://$1.%s:%d/$2$is_args$args; + } + + location ~ ^/([^/]+)/history(?:/(.*))?$ { + proxy_pass http://$1.%s:%d/$2$is_args$args; + } +} +""" + fqdn + CfgVal.httpPort + fqdn + CfgVal.historyPort + + let data = Map.empty.Add(CfgVal.httpProxyConfigFileName, conf) + V1ConfigMap(metadata = self.HttpProxyMeta self.HttpProxyConfigMapName, data = data) + + // Scalable nginx Deployment fronting driver->pod routing. The startup shim + // reads the CoreDNS server from /etc/resolv.conf and templates it into the + // nginx config, so the same manifest works on any cluster (the image's + // nginx is too old for `resolver local=on`). + member self.ToHttpProxyDeployment() : V1Deployment = + let shim = + sprintf + "set -e; R=$(awk '/^nameserver/ {print $2; exit}' /etc/resolv.conf); sed \"s/__RESOLVER__/$R/\" %s/%s > /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'" + CfgVal.httpProxyConfigMountPath + CfgVal.httpProxyConfigFileName + + // Match the ingress-nginx private controller pods (ssc-eks) this proxy replaces. + let resources = + V1ResourceRequirements( + requests = dict [ ("cpu", ResourceQuantity("50m")); ("memory", ResourceQuantity("90Mi")) ], + limits = dict [ ("cpu", ResourceQuantity("250m")); ("memory", ResourceQuantity("768Mi")) ] + ) + + let container = + V1Container( + name = CfgVal.httpProxyContainerName, + image = self.missionContext.nginxImage, + command = [| "/bin/sh" |], + args = [| "-c"; shim |], + ports = [| V1ContainerPort(containerPort = 80, name = "http") |], + resources = resources, + volumeMounts = + [| V1VolumeMount(name = CfgVal.httpProxyConfigVolumeName, mountPath = CfgVal.httpProxyConfigMountPath) |] + ) + + let volume = + V1Volume( + name = CfgVal.httpProxyConfigVolumeName, + configMap = V1ConfigMapVolumeSource(name = self.HttpProxyConfigMapName) + ) - // ClusterIP service selecting the single pod by its StatefulSet pod-name label. - // (Previously ExternalName -> pod DNS, which the traefik Gateway provider rejects - // as an HTTPRoute backend.) - let selector = - CfgVal.labels |> Map.add "statefulset.kubernetes.io/pod-name" name.StringName + let podSpec = V1PodSpec(containers = [| container |], volumes = [| volume |]) + let podTemplate = V1PodTemplateSpec(metadata = V1ObjectMeta(labels = self.HttpProxyLabels), spec = podSpec) - let spec = V1ServiceSpec(selector = selector, ports = ports) + // The proxy only carries driver->core control HTTP (getinfo polling, + // loadgen commands, metrics) -- not the tx/overlay load, which is + // pod-to-pod. So 1 replica suffices for most missions; scale up only for + // large topologies. ceil(nodes/64), clamped to [1, cap] where cap is + // --http-proxy-replicas. Keeps the parallel-mission fan-out cheap. + let cap = max 1 self.missionContext.httpProxyReplicas + let nodesPerProxy = 64 + let replicas = self.MaxPeerCount |> fun n -> (n + nodesPerProxy - 1) / nodesPerProxy |> max 1 |> min cap - V1Service(metadata = self.NamespacedMeta name.StringName, spec = spec) + let spec = + V1DeploymentSpec( + replicas = System.Nullable(replicas), + selector = V1LabelSelector(matchLabels = self.HttpProxyLabels), + template = podTemplate + ) + + V1Deployment(metadata = self.HttpProxyMeta self.HttpProxyName, spec = spec) - self.MapAllPeers perPodService + // ClusterIP Service selecting the proxy pods; the HTTPRoute's single + // backend. + member self.ToHttpProxyService() : V1Service = + let spec = V1ServiceSpec(selector = self.HttpProxyLabels, ports = [| V1ServicePort(name = "http", port = 80) |]) + V1Service(metadata = self.HttpProxyMeta self.HttpProxyName, spec = spec) - // Returns an HTTPRoute (gateway.networking.k8s.io/v1) mapping - // http://$ingressHost//core|history/... to the per-Pod Service , attached to - // the shared private traefik gateway. Each pod gets a core rule (-> httpPort) and a - // history rule (-> historyPort); the //core|history prefix is stripped via a - // ReplacePrefixMatch URLRewrite (equivalent to the old nginx rewrite-target /$2). - // Replaces the former nginx Ingress. + // Returns an HTTPRoute (gateway.networking.k8s.io/v1) attached to the shared + // private traefik gateway that sends all of http://$routeHost/* to the + // per-run nginx HTTP proxy Service. The proxy does the //core|history + // demux internally, so this route needs exactly ONE rule regardless of pod + // count -- side-stepping the HTTPRoute 16-rule cap and the CoreDNS load of + // the former per-pod backends. Replaces the former nginx Ingress. member self.ToHttpRoute() : HTTPRoute = let parentRef = ParentReference( @@ -905,37 +974,18 @@ type NetworkCfg with Name = CfgVal.gatewayName ) - let rewriteFilter = - HTTPRouteFilter( - Type = "URLRewrite", - UrlRewrite = HTTPURLRewriteFilter(Path = HTTPPathModifier(Type = "ReplacePrefixMatch", ReplacePrefixMatch = "/")) - ) - - let ruleFor (pn: PodName) (suffix: string) (port: int) : HTTPRouteRule = - let m = - HTTPRouteMatch(Path = HTTPPathMatch(Type = "PathPrefix", Value = sprintf "/%s/%s" pn.StringName suffix)) - - let b = HTTPBackendRef(Name = pn.StringName, Port = System.Nullable(port)) - + let rule = HTTPRouteRule( - Matches = List([ m ]), - Filters = List([ rewriteFilter ]), - BackendRefs = List([ b ]) + Matches = List([ HTTPRouteMatch(Path = HTTPPathMatch(Type = "PathPrefix", Value = "/")) ]), + BackendRefs = + List([ HTTPBackendRef(Name = self.HttpProxyName, Port = System.Nullable(80)) ]) ) - let coreRule (coreSet: CoreSet) (i: int) : HTTPRouteRule = - ruleFor (self.PodName coreSet i) "core" CfgVal.httpPort - - let historyRule (coreSet: CoreSet) (i: int) : HTTPRouteRule = - ruleFor (self.PodName coreSet i) "history" CfgVal.historyPort - - let rules = Array.concat [ self.MapAllPeers coreRule; self.MapAllPeers historyRule ] - let spec = HTTPRouteSpec( ParentRefs = List([ parentRef ]), - Hostnames = List([ self.IngressInternalHostName ]), - Rules = List(rules) + Hostnames = List([ self.RouteInternalHostName ]), + Rules = List([ rule ]) ) let meta = diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index c64b4cca..b29a092a 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -55,10 +55,9 @@ type MissionContext = numNodes: int namespaceProperty: string logLevels: LogLevels - ingressClass: string - ingressInternalDomain: string - ingressExternalHost: string option - ingressExternalPort: int + routeInternalDomain: string + routeExternalHost: string option + routeExternalPort: int exportToPrometheus: bool probeTimeout: int coreResources: CoreResources @@ -68,6 +67,7 @@ type MissionContext = avoidNodeLabels: ((string * string option) list) tolerateNodeTaints: ((string * string option) list) apiRateLimit: int + httpProxyReplicas: int pubnetData: string option flatQuorum: bool option tier1Keys: string option diff --git a/src/FSLibrary/StellarNetworkCfg.fs b/src/FSLibrary/StellarNetworkCfg.fs index cc9d0021..4da8c80c 100644 --- a/src/FSLibrary/StellarNetworkCfg.fs +++ b/src/FSLibrary/StellarNetworkCfg.fs @@ -104,6 +104,17 @@ type NetworkCfg = member self.HttpRouteName : string = sprintf "%s-stellar-core-http-route" self.Nonce + // The nginx HTTP proxy (Deployment + Service) that the HTTPRoute points at. + member self.HttpProxyName : string = sprintf "%s-http-proxy" self.Nonce + + member self.HttpProxyConfigMapName : string = sprintf "%s-http-proxy-cfg" self.Nonce + + // Labels scoping the proxy Deployment/Service to this run only (the shared + // namespace can hold other runs). Distinct from the core "app=stellar-core" + // labels so the proxy Service selects only proxy pods. + member self.HttpProxyLabels : Map = + Map.ofSeq [ ("app", "stellar-core-http-proxy"); ("ssc-nonce", self.Nonce) ] + member self.JobName(i: int) : string = sprintf "%s-stellar-core-job-%d" self.Nonce i member self.PeerCfgMapName (cs: CoreSet) (i: int) : string = sprintf "%s-cfg-map" (self.PodName cs i).StringName @@ -120,11 +131,11 @@ type NetworkCfg = PeerDnsName s - member self.IngressInternalHostName : string = sprintf "%s.%s" self.Nonce self.missionContext.ingressInternalDomain + member self.RouteInternalHostName : string = sprintf "%s.%s" self.Nonce self.missionContext.routeInternalDomain - member self.IngressExternalHostName : string = - match self.missionContext.ingressExternalHost with - | None -> self.IngressInternalHostName + member self.RouteExternalHostName : string = + match self.missionContext.routeExternalHost with + | None -> self.RouteInternalHostName | Some h -> h member self.WithLive name (live: bool) = diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index 13ba7467..6d54733c 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -313,14 +313,22 @@ type Kubernetes with for statefulSet in statefulSets do namespaceContent.Add(statefulSet) - for svc in nCfg.ToPerPodServices() do - LogInfo "Creating Per-Pod Service %s" svc.Metadata.Name + if not (List.isEmpty statefulSets) then + let proxyCfg = nCfg.ToHttpProxyConfigMap() + LogInfo "Creating HTTP proxy ConfigMap %s" proxyCfg.Metadata.Name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + namespaceContent.Add(self.CreateNamespacedConfigMap(body = proxyCfg, namespaceParameter = nsStr)) - let service = self.CreateNamespacedService(namespaceParameter = nsStr, body = svc) - namespaceContent.Add(service) + let proxyDep = nCfg.ToHttpProxyDeployment() + LogInfo "Creating HTTP proxy Deployment %s (%d replicas)" proxyDep.Metadata.Name proxyDep.Spec.Replicas.Value + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + namespaceContent.Add(self.CreateNamespacedDeployment(body = proxyDep, namespaceParameter = nsStr)) + + let proxySvc = nCfg.ToHttpProxyService() + LogInfo "Creating HTTP proxy Service %s" proxySvc.Metadata.Name + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + namespaceContent.Add(self.CreateNamespacedService(body = proxySvc, namespaceParameter = nsStr)) - if not (List.isEmpty statefulSets) then let route = nCfg.ToHttpRoute() LogInfo "Creating HTTPRoute %s" route.Metadata.Name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) @@ -336,6 +344,22 @@ type Kubernetes with namespaceContent.Add(route) + // Wait for at least one proxy pod to be ready before the driver + // routes core HTTP through it, so mission startup doesn't spend + // its retry budget on 502s. + let rec waitProxyReady (n: int) = + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + let d = self.ReadNamespacedDeployment(name = proxyDep.Metadata.Name, namespaceParameter = nsStr) + let ready = d.Status.ReadyReplicas.GetValueOrDefault(0) + LogInfo "HTTP proxy %s: %d ready" proxyDep.Metadata.Name ready + + if ready < 1 then + if n >= 60 then failwithf "HTTP proxy %s not ready after 60 attempts" proxyDep.Metadata.Name + System.Threading.Thread.Sleep(2000) + waitProxyReady (n + 1) + + waitProxyReady 0 + let formation = new StellarFormation( networkCfg = nCfg, From ab49e35140e051d9eee2a4ed0e4a7d6bffa5f3dc Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:07:21 -0400 Subject: [PATCH 06/20] Convert parallel-catchup job-monitor Ingress -> HTTPRoute Co-Authored-By: Claude Opus 4.8 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 2 +- .../templates/job_monitor.yaml | 40 +++++++++++-------- .../parallel_catchup_helm/values.yaml | 5 ++- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 601e5e57..fa8f332a 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -170,7 +170,7 @@ let installProject (context: MissionContext) = | None -> () setOptions.Add(sprintf "monitor.hostname=%s" (jobMonitorHostName context)) - setOptions.Add(sprintf "monitor.path=/%s/%s/(.*)" context.namespaceProperty helmReleaseName) + setOptions.Add(sprintf "monitor.path_prefix=/%s/%s" context.namespaceProperty helmReleaseName) setOptions.Add(sprintf "monitor.logging_interval_seconds=%d" jobMonitorLoggingIntervalSecs) // Set ASAN_OPTIONS if provided diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 353abf17..10d28dcb 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -1,23 +1,29 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute metadata: - name: {{ .Release.Name }}-job-monitor-ingress - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$1 + name: {{ .Release.Name }}-job-monitor-route spec: - ingressClassName: "{{ .Values.monitor.ingress_class_name}}" + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: "{{ .Values.monitor.gateway_name }}" + namespace: "{{ .Values.monitor.gateway_namespace }}" + hostnames: + - "{{ .Values.monitor.hostname }}" rules: - - host: "{{ .Values.monitor.hostname}}" - http: - paths: - - path: "{{ .Values.monitor.path}}" - pathType: Prefix - backend: - service: - name: {{ .Release.Name }}-job-monitor - port: - number: 8080 + - matches: + - path: + type: PathPrefix + value: "{{ .Values.monitor.path_prefix }}" + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + replacePrefixMatch: / + backendRefs: + - name: {{ .Release.Name }}-job-monitor + port: 8080 --- apiVersion: v1 kind: Service diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index b47e7ab1..36f3ea3d 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -36,9 +36,10 @@ worker: "curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" monitor: - ingress_class_name: "ingress-private" + gateway_name: "traefik-gateway-private" + gateway_namespace: "traefik" hostname: "" # to be set by the mission - path: "/default/(.*)" + path_prefix: "/default" logging_interval_seconds: 300 logging_level: "INFO" # 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' resources: From 44a2c1b75f20eda14df266ae28bc647b9de70446 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:20:40 -0400 Subject: [PATCH 07/20] Address review + fix fantomas formatting - waitProxyReady: treat null Deployment.Status as 0 ready (status is populated asynchronously; don't NPE mission startup). - Orphan sweep: also reap legacy Ingresses (pre-migration orphans), alongside HTTPRoutes. - job_monitor HTTPRoute: omit spec.hostnames when monitor.hostname is empty (avoids invalid hostnames: [""]). - Clarify --ingress-{internal-domain,external-host,external-port} help text to describe the gateway route (flag names kept for Jenkinsfile compat). - Run fantomas over the changed files (fixes the failing format check). Co-Authored-By: Claude Opus 4.8 --- src/App/Program.fs | 6 ++-- src/FSLibrary/StellarCoreCfg.fs | 2 +- src/FSLibrary/StellarKubeSpecs.fs | 29 ++++++++++++++----- src/FSLibrary/StellarNetworkCfg.fs | 3 +- src/FSLibrary/StellarOrphanSweep.fs | 19 ++++++++---- src/FSLibrary/StellarSupercluster.fs | 20 ++++++++++--- .../templates/job_monitor.yaml | 2 ++ 7 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 2687f286..e09cae53 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -156,18 +156,18 @@ type MissionOptions member self.NamespaceProperty = namespaceProperty [] member self.RouteInternalDomain = routeInternalDomain [] member self.RouteExternalHost = routeExternalHost [] member self.RouteExternalPort = routeExternalPort diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index b23a1e7c..2d3b070e 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -47,7 +47,7 @@ module CfgVal = let peerCfgFileName = "stellar-core.cfg" let peerInitCfgFileName = "stellar-core-init.cfg" let peerDelayCfgFileName = "install-delays.sh" - + let httpProxyContainerName = "http-proxy" let httpProxyConfigVolumeName = "proxy-tmpl" diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 64a9428a..b94caa90 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -910,8 +910,12 @@ type NetworkCfg with // Match the ingress-nginx private controller pods (ssc-eks) this proxy replaces. let resources = V1ResourceRequirements( - requests = dict [ ("cpu", ResourceQuantity("50m")); ("memory", ResourceQuantity("90Mi")) ], - limits = dict [ ("cpu", ResourceQuantity("250m")); ("memory", ResourceQuantity("768Mi")) ] + requests = + dict [ ("cpu", ResourceQuantity("50m")) + ("memory", ResourceQuantity("90Mi")) ], + limits = + dict [ ("cpu", ResourceQuantity("250m")) + ("memory", ResourceQuantity("768Mi")) ] ) let container = @@ -923,7 +927,10 @@ type NetworkCfg with ports = [| V1ContainerPort(containerPort = 80, name = "http") |], resources = resources, volumeMounts = - [| V1VolumeMount(name = CfgVal.httpProxyConfigVolumeName, mountPath = CfgVal.httpProxyConfigMountPath) |] + [| V1VolumeMount( + name = CfgVal.httpProxyConfigVolumeName, + mountPath = CfgVal.httpProxyConfigMountPath + ) |] ) let volume = @@ -933,7 +940,9 @@ type NetworkCfg with ) let podSpec = V1PodSpec(containers = [| container |], volumes = [| volume |]) - let podTemplate = V1PodTemplateSpec(metadata = V1ObjectMeta(labels = self.HttpProxyLabels), spec = podSpec) + + let podTemplate = + V1PodTemplateSpec(metadata = V1ObjectMeta(labels = self.HttpProxyLabels), spec = podSpec) // The proxy only carries driver->core control HTTP (getinfo polling, // loadgen commands, metrics) -- not the tx/overlay load, which is @@ -942,7 +951,10 @@ type NetworkCfg with // --http-proxy-replicas. Keeps the parallel-mission fan-out cheap. let cap = max 1 self.missionContext.httpProxyReplicas let nodesPerProxy = 64 - let replicas = self.MaxPeerCount |> fun n -> (n + nodesPerProxy - 1) / nodesPerProxy |> max 1 |> min cap + + let replicas = + self.MaxPeerCount + |> fun n -> (n + nodesPerProxy - 1) / nodesPerProxy |> max 1 |> min cap let spec = V1DeploymentSpec( @@ -956,7 +968,9 @@ type NetworkCfg with // ClusterIP Service selecting the proxy pods; the HTTPRoute's single // backend. member self.ToHttpProxyService() : V1Service = - let spec = V1ServiceSpec(selector = self.HttpProxyLabels, ports = [| V1ServicePort(name = "http", port = 80) |]) + let spec = + V1ServiceSpec(selector = self.HttpProxyLabels, ports = [| V1ServicePort(name = "http", port = 80) |]) + V1Service(metadata = self.HttpProxyMeta self.HttpProxyName, spec = spec) // Returns an HTTPRoute (gateway.networking.k8s.io/v1) attached to the shared @@ -976,7 +990,8 @@ type NetworkCfg with let rule = HTTPRouteRule( - Matches = List([ HTTPRouteMatch(Path = HTTPPathMatch(Type = "PathPrefix", Value = "/")) ]), + Matches = + List([ HTTPRouteMatch(Path = HTTPPathMatch(Type = "PathPrefix", Value = "/")) ]), BackendRefs = List([ HTTPBackendRef(Name = self.HttpProxyName, Port = System.Nullable(80)) ]) ) diff --git a/src/FSLibrary/StellarNetworkCfg.fs b/src/FSLibrary/StellarNetworkCfg.fs index 4da8c80c..06ef5440 100644 --- a/src/FSLibrary/StellarNetworkCfg.fs +++ b/src/FSLibrary/StellarNetworkCfg.fs @@ -113,7 +113,8 @@ type NetworkCfg = // namespace can hold other runs). Distinct from the core "app=stellar-core" // labels so the proxy Service selects only proxy pods. member self.HttpProxyLabels : Map = - Map.ofSeq [ ("app", "stellar-core-http-proxy"); ("ssc-nonce", self.Nonce) ] + Map.ofSeq [ ("app", "stellar-core-http-proxy") + ("ssc-nonce", self.Nonce) ] member self.JobName(i: int) : string = sprintf "%s-stellar-core-job-%d" self.Nonce i diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index 453b1c68..3b9e250f 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -123,17 +123,26 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( |> ignore) cutoff + // Also sweep legacy Ingresses left by pre-migration runs (this harness no + // longer creates them, but old orphans should still be reaped). + sweepKind + apiRateLimit + "Ingress" + (fun () -> + kube.ListNamespacedIngress(namespaceParameter = ns).Items + |> Seq.map (fun i -> i.Metadata)) + (fun n -> + kube.DeleteNamespacedIngress(namespaceParameter = ns, name = n, propagationPolicy = "Foreground") + |> ignore) + cutoff + sweepKind apiRateLimit "HTTPRoute" (fun () -> let gc = GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") - gc - .ListNamespacedAsync(ns) - .GetAwaiter() - .GetResult() - .Items + gc.ListNamespacedAsync(ns).GetAwaiter().GetResult().Items |> Seq.map (fun r -> r.Metadata)) (fun n -> kube.DeleteNamespacedCustomObject( diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index 6d54733c..f5558950 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -320,7 +320,12 @@ type Kubernetes with namespaceContent.Add(self.CreateNamespacedConfigMap(body = proxyCfg, namespaceParameter = nsStr)) let proxyDep = nCfg.ToHttpProxyDeployment() - LogInfo "Creating HTTP proxy Deployment %s (%d replicas)" proxyDep.Metadata.Name proxyDep.Spec.Replicas.Value + + LogInfo + "Creating HTTP proxy Deployment %s (%d replicas)" + proxyDep.Metadata.Name + proxyDep.Spec.Replicas.Value + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) namespaceContent.Add(self.CreateNamespacedDeployment(body = proxyDep, namespaceParameter = nsStr)) @@ -349,12 +354,19 @@ type Kubernetes with // its retry budget on 502s. let rec waitProxyReady (n: int) = ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) - let d = self.ReadNamespacedDeployment(name = proxyDep.Metadata.Name, namespaceParameter = nsStr) - let ready = d.Status.ReadyReplicas.GetValueOrDefault(0) + + let d = + self.ReadNamespacedDeployment(name = proxyDep.Metadata.Name, namespaceParameter = nsStr) + // Status (and ReadyReplicas within it) is populated asynchronously and may be + // null right after creation; treat missing as 0 ready and keep polling. + let ready = if isNull d.Status then 0 else d.Status.ReadyReplicas.GetValueOrDefault(0) + LogInfo "HTTP proxy %s: %d ready" proxyDep.Metadata.Name ready if ready < 1 then - if n >= 60 then failwithf "HTTP proxy %s not ready after 60 attempts" proxyDep.Metadata.Name + if n >= 60 then + failwithf "HTTP proxy %s not ready after 60 attempts" proxyDep.Metadata.Name + System.Threading.Thread.Sleep(2000) waitProxyReady (n + 1) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 10d28dcb..e9f6050c 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -8,8 +8,10 @@ spec: kind: Gateway name: "{{ .Values.monitor.gateway_name }}" namespace: "{{ .Values.monitor.gateway_namespace }}" + {{- if .Values.monitor.hostname }} hostnames: - "{{ .Values.monitor.hostname }}" + {{- end }} rules: - matches: - path: From a357372e31f85ae540bfa9c47564c6a1d97a328e Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:25:29 -0400 Subject: [PATCH 08/20] Drop --ingress-class from CI workflow + docs (flag removed) Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-and-test.yml | 2 +- doc/k3s.md | 2 +- doc/theoretical-max-tps.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 6c5d8acd..69c8e9b9 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -36,7 +36,7 @@ jobs: - name: Test run: dotnet test --no-restore --verbosity normal - name: Run BootAndSync mission - run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --ingress-class nginx --ingress-internal-domain local --ingress-external-host localhost --uneven-sched + run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --ingress-internal-domain local --ingress-external-host localhost --uneven-sched - uses: actions/upload-artifact@v4 with: name: destination diff --git a/doc/k3s.md b/doc/k3s.md index b0e0093b..63b68737 100644 --- a/doc/k3s.md +++ b/doc/k3s.md @@ -32,5 +32,5 @@ net.ipv6.neigh.default.gc_thresh3 = 100000 - Build supercluster normally (see [getting-started.md](getting-started.md)) - - Run supercluster with these additional arguments: `--kubeconfig $KUBECONFIG --namespace default --ingress-class nginx --ingress-internal-domain local --ingress-external-host localhost --uneven-sched` + - Run supercluster with these additional arguments: `--kubeconfig $KUBECONFIG --namespace default --ingress-internal-domain local --ingress-external-host localhost --uneven-sched` diff --git a/doc/theoretical-max-tps.md b/doc/theoretical-max-tps.md index 6b03aced..a3c4e7f9 100644 --- a/doc/theoretical-max-tps.md +++ b/doc/theoretical-max-tps.md @@ -11,7 +11,7 @@ To run the test, first [set up an EKS cluster](eks.md). Accepting the default settings will produce a topology identical to what we use in our test setup. Then, run a `MaxTPSClassic` mission with the following template: ```bash -dotnet run --project src/App/App.fsproj --configuration Release -- mission MaxTPSClassic --image= --pubnet-data=/topologies/theoretical-max-tps.json --tx-rate= --max-tx-rate= --namespace default --ingress-internal-domain= --ingress-class=nginx --run-for-max-tps=classic --enable-tcp-tuning +dotnet run --project src/App/App.fsproj --configuration Release -- mission MaxTPSClassic --image= --pubnet-data=/topologies/theoretical-max-tps.json --tx-rate= --max-tx-rate= --namespace default --ingress-internal-domain= --run-for-max-tps=classic --enable-tcp-tuning ``` For more information about how to set the parameters in the above command, see [Measuring Transaction Throughput](measuring-transaction-throughput.md). From d92aa6a6dd3df39c792177136238dd957e9e9b3c Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:31:45 -0400 Subject: [PATCH 09/20] CI: provision Gateway API + traefik gateway for BootAndSync Replace the nginx-ingress setup with gateway-api CRDs + traefik as the Gateway API controller + a traefik-gateway-private Gateway (matching CfgVal), so the migrated BootAndSync smoke mission (which creates an HTTPRoute) works. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-and-test.yml | 42 ++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 69c8e9b9..752efe3c 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -17,10 +17,46 @@ jobs: uses: debianmaster/actions-k3s@master with: version: 'v1.23.17-k3s1' - - name: Setup nginx-ingress + - name: Setup Gateway API + traefik gateway run: | - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.5.1/deploy/static/provider/cloud/deploy.yaml - kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=120s + set -euo pipefail + # Gateway API standard CRDs (gateway.networking.k8s.io/v1 HTTPRoute is GA in v1.0+). + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + # Traefik as the Gateway API controller, exposing the web entrypoint on :80 via k3s servicelb. + helm repo add traefik https://traefik.github.io/charts + helm repo update + kubectl create namespace traefik + helm install traefik traefik/traefik -n traefik \ + --set providers.kubernetesGateway.enabled=true \ + --set gateway.enabled=false \ + --set service.type=LoadBalancer \ + --set ports.web.exposedPort=80 \ + --wait + # GatewayClass + the private gateway the missions attach to (matches CfgVal.gatewayName / gatewayNamespace). + kubectl apply -f - <<'YAML' + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: traefik + spec: + controllerName: traefik.io/gateway-controller + --- + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: traefik-gateway-private + namespace: traefik + spec: + gatewayClassName: traefik + listeners: + - name: web + port: 8000 + protocol: HTTP + allowedRoutes: + namespaces: + from: All + YAML + kubectl wait --namespace traefik --for=condition=Programmed gateway/traefik-gateway-private --timeout=180s - name: Setup .NET SDK 8 uses: actions/setup-dotnet@v1 with: From d17c2afc5d730e942dff8f57be45516162182106 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:34:19 -0400 Subject: [PATCH 10/20] CI: bump k3s to v1.28.5 (traefik chart needs k8s >=1.25) Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 752efe3c..541e9efe 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -16,7 +16,7 @@ jobs: - name: Setup k3s uses: debianmaster/actions-k3s@master with: - version: 'v1.23.17-k3s1' + version: 'v1.28.5-k3s1' - name: Setup Gateway API + traefik gateway run: | set -euo pipefail From 1ba2221be6884b4a48cbf479efac03533373dc15 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:37:47 -0400 Subject: [PATCH 11/20] CI: isolate our traefik from k3s bundled traefik (fullnameOverride, no IngressClass) Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-and-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 541e9efe..f39d1e3b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -26,7 +26,11 @@ jobs: helm repo add traefik https://traefik.github.io/charts helm repo update kubectl create namespace traefik - helm install traefik traefik/traefik -n traefik \ + # k3s bundles its own traefik in kube-system; isolate ours (distinct name, + # no IngressClass) so it only provides the gateway-api controller. + helm install ssc-traefik traefik/traefik -n traefik \ + --set fullnameOverride=ssc-traefik \ + --set ingressClass.enabled=false \ --set providers.kubernetesGateway.enabled=true \ --set gateway.enabled=false \ --set service.type=LoadBalancer \ From 80c895a224a80ff860fe3d4b1b44237c4a7ca867 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 13:48:39 -0400 Subject: [PATCH 12/20] Parameterize the gateway the HTTPRoute attaches to (--gateway-name/-namespace) Restore the configurability the old --ingress-class flag provided: instead of hardcoding traefik-gateway-private/traefik in CfgVal, take --gateway-name and --gateway-namespace (same defaults). Lets each cluster / CI point the route at whits own gateway. Co-Authored-By: Claude Opus 4.8 --- src/App/Program.fs | 16 ++++++++++++++++ src/FSLibrary.Tests/Tests.fs | 2 ++ src/FSLibrary/StellarCoreCfg.fs | 3 --- src/FSLibrary/StellarKubeSpecs.fs | 4 ++-- src/FSLibrary/StellarMissionContext.fs | 2 ++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index e09cae53..5d4c1c66 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -42,6 +42,8 @@ type MissionOptions logDebugPartitions: seq, logTracePartitions: seq, namespaceProperty: string option, + gatewayName: string, + gatewayNamespace: string, routeInternalDomain: string, routeExternalHost: string option, routeExternalPort: int, @@ -155,6 +157,18 @@ type MissionOptions [] member self.NamespaceProperty = namespaceProperty + [] + member self.GatewayName = gatewayName + + [] + member self.GatewayNamespace = gatewayNamespace + [ Date: Thu, 9 Jul 2026 14:18:54 -0400 Subject: [PATCH 13/20] CI: run BootAndSync on k3d + Envoy Gateway (validated locally) The migrated BootAndSync creates an HTTPRoute, so CI needs a gateway-api env. Traefik-in-k3s proved a version-compat maze (bundled-traefik conflict, chart value drift, missing TLSRoute CRD, CEL/k8s-version mismatches). Envoy Gateway ships matching CRDs + controller in one chart and programs cleanly. Recipe validated on local k3d (k3s 1.31, bundled traefik disabled, envoy-gateway v1.2.6, Gateway 'ssc-gateway'): BootAndSync brought up 3 nodes and synced, driver reaching them via the gateway -> proxy. Uses the new --gateway-name / --gateway-namespace flags. Also bump checkout@v2->v4, setup-dotnet@v1->v4. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-and-test.yml | 62 ++++++++++++++-------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f39d1e3b..6375ec48 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -12,57 +12,55 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 - - name: Setup k3s - uses: debianmaster/actions-k3s@master - with: - version: 'v1.28.5-k3s1' - - name: Setup Gateway API + traefik gateway + - uses: actions/checkout@v4 + - name: Setup k3d cluster (k3s, bundled traefik disabled) run: | set -euo pipefail - # Gateway API standard CRDs (gateway.networking.k8s.io/v1 HTTPRoute is GA in v1.0+). - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml - # Traefik as the Gateway API controller, exposing the web entrypoint on :80 via k3s servicelb. - helm repo add traefik https://traefik.github.io/charts - helm repo update - kubectl create namespace traefik - # k3s bundles its own traefik in kube-system; isolate ours (distinct name, - # no IngressClass) so it only provides the gateway-api controller. - helm install ssc-traefik traefik/traefik -n traefik \ - --set fullnameOverride=ssc-traefik \ - --set ingressClass.enabled=false \ - --set providers.kubernetesGateway.enabled=true \ - --set gateway.enabled=false \ - --set service.type=LoadBalancer \ - --set ports.web.exposedPort=80 \ + curl -s https://raw.githubusercontent.com/k3d-io/k3d/v5.9.0/install.sh | bash + # Disable k3s's bundled traefik; the gateway controller (envoy) is installed below. + # Map host :8081 -> the gateway LoadBalancer :80 so the mission driver can reach it. + k3d cluster create ssc \ + --image rancher/k3s:v1.31.5-k3s1 \ + --k3s-arg "--disable=traefik@server:*" \ + -p "8081:80@loadbalancer" \ --wait - # GatewayClass + the private gateway the missions attach to (matches CfgVal.gatewayName / gatewayNamespace). + k3d kubeconfig get ssc > "$RUNNER_TEMP/kubeconfig" + echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV" + - name: Setup Envoy Gateway + Gateway + run: | + set -euo pipefail + # Envoy Gateway ships its own matching gateway-api CRDs + controller. + helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.2.6 \ + -n envoy-gateway-system --create-namespace --wait --timeout 4m + # GatewayClass + the Gateway the per-mission HTTPRoutes attach to. + # Missions target it via --gateway-name / --gateway-namespace. + kubectl create namespace gateway kubectl apply -f - <<'YAML' apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: - name: traefik + name: eg spec: - controllerName: traefik.io/gateway-controller + controllerName: gateway.envoyproxy.io/gatewayclass-controller --- apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: - name: traefik-gateway-private - namespace: traefik + name: ssc-gateway + namespace: gateway spec: - gatewayClassName: traefik + gatewayClassName: eg listeners: - - name: web - port: 8000 + - name: http + port: 80 protocol: HTTP allowedRoutes: namespaces: from: All YAML - kubectl wait --namespace traefik --for=condition=Programmed gateway/traefik-gateway-private --timeout=180s + kubectl wait --namespace gateway --for=condition=Programmed gateway/ssc-gateway --timeout=180s - name: Setup .NET SDK 8 - uses: actions/setup-dotnet@v1 + uses: actions/setup-dotnet@v4 with: dotnet-version: '8.0.x' - name: Install dependencies @@ -76,7 +74,7 @@ jobs: - name: Test run: dotnet test --no-restore --verbosity normal - name: Run BootAndSync mission - run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --ingress-internal-domain local --ingress-external-host localhost --uneven-sched + run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --gateway-name ssc-gateway --gateway-namespace gateway --ingress-internal-domain local --ingress-external-host localhost --ingress-external-port 8081 --uneven-sched - uses: actions/upload-artifact@v4 with: name: destination From 12290dd44bc7791150f7413b6e44077605ab2318 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 14:46:44 -0400 Subject: [PATCH 14/20] Dispose the gateway GenericClient in the HTTPRoute orphan sweep Use 'use ... = new GenericClient(...)' so the client is disposed and the FS0760 IDisposable warning is cleared. Co-Authored-By: Claude Opus 4.8 --- src/FSLibrary/StellarOrphanSweep.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index 3b9e250f..b38498cc 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -140,7 +140,7 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( apiRateLimit "HTTPRoute" (fun () -> - let gc = GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") + use gc = new GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") gc.ListNamespacedAsync(ns).GetAwaiter().GetResult().Items |> Seq.map (fun r -> r.Metadata)) From a8e1a13ea7115aaa01b55ae6c1556cbbba150191 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 14:50:40 -0400 Subject: [PATCH 15/20] Refactor comments in NetworkCfg to improve clarity and remove redundancy --- src/FSLibrary/StellarKubeSpecs.fs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index ec891ca6..e352ec10 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -856,8 +856,7 @@ type NetworkCfg with // Metadata for a proxy object: run-scoped labels + the anchor owner ref so - // it is GC'd with the rest of the run. (Not NamespacedMeta, which would - // stamp the core "app=stellar-core" labels.) + // it is GC'd with the rest of the run. member private self.HttpProxyMeta(name: string) : V1ObjectMeta = V1ObjectMeta(name = name, namespaceProperty = self.NamespaceProperty, labels = self.HttpProxyLabels) |> applyAnchorOwner self @@ -898,8 +897,7 @@ type NetworkCfg with // Scalable nginx Deployment fronting driver->pod routing. The startup shim // reads the CoreDNS server from /etc/resolv.conf and templates it into the - // nginx config, so the same manifest works on any cluster (the image's - // nginx is too old for `resolver local=on`). + // nginx config. member self.ToHttpProxyDeployment() : V1Deployment = let shim = sprintf @@ -944,11 +942,9 @@ type NetworkCfg with let podTemplate = V1PodTemplateSpec(metadata = V1ObjectMeta(labels = self.HttpProxyLabels), spec = podSpec) - // The proxy only carries driver->core control HTTP (getinfo polling, - // loadgen commands, metrics) -- not the tx/overlay load, which is - // pod-to-pod. So 1 replica suffices for most missions; scale up only for + //1 replica suffices for most missions; scale up only for // large topologies. ceil(nodes/64), clamped to [1, cap] where cap is - // --http-proxy-replicas. Keeps the parallel-mission fan-out cheap. + // --http-proxy-replicas. let cap = max 1 self.missionContext.httpProxyReplicas let nodesPerProxy = 64 @@ -976,9 +972,7 @@ type NetworkCfg with // Returns an HTTPRoute (gateway.networking.k8s.io/v1) attached to the shared // private traefik gateway that sends all of http://$routeHost/* to the // per-run nginx HTTP proxy Service. The proxy does the //core|history - // demux internally, so this route needs exactly ONE rule regardless of pod - // count -- side-stepping the HTTPRoute 16-rule cap and the CoreDNS load of - // the former per-pod backends. Replaces the former nginx Ingress. + // demux internally. member self.ToHttpRoute() : HTTPRoute = let parentRef = ParentReference( From cd158d4ad8b3019f6aed93424e7f70b5d3b664e7 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 14:52:24 -0400 Subject: [PATCH 16/20] Fix orphan sweep: use 'new' without disposing the shared client The previous 'use gc = new GenericClient' disposed the shared mission kube client (GenericClient wraps it), NPEing the next API call. Use 'let ... = new': clears FS0760 via the 'new' keyword without disposing the shared client. Co-Authored-By: Claude Opus 4.8 --- src/FSLibrary/StellarOrphanSweep.fs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index b38498cc..b4c493bb 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -140,7 +140,10 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( apiRateLimit "HTTPRoute" (fun () -> - use gc = new GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") + // `new` clears the FS0760 IDisposable warning; do NOT `use`/dispose — + // GenericClient wraps the shared mission `kube` client, and disposing + // it would dispose that client out from under the rest of the run. + let gc = new GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") gc.ListNamespacedAsync(ns).GetAwaiter().GetResult().Items |> Seq.map (fun r -> r.Metadata)) From 52611e5a41dac9ec48dcc91053edffe110ad1c7e Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 15:00:22 -0400 Subject: [PATCH 17/20] Strip trailing whitespace so fantomas --check passes Co-Authored-By: Claude Opus 4.8 --- src/FSLibrary/StellarKubeSpecs.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index e352ec10..8e236e46 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -944,7 +944,7 @@ type NetworkCfg with //1 replica suffices for most missions; scale up only for // large topologies. ceil(nodes/64), clamped to [1, cap] where cap is - // --http-proxy-replicas. + // --http-proxy-replicas. let cap = max 1 self.missionContext.httpProxyReplicas let nodesPerProxy = 64 From b6d00966f392f7b7d4a69e911599a99631fc60ca Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 9 Jul 2026 15:07:07 -0400 Subject: [PATCH 18/20] Drop redundant comments in orphan sweep Co-Authored-By: Claude Opus 4.8 --- src/FSLibrary/StellarOrphanSweep.fs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index b4c493bb..ed9cf13b 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -123,8 +123,6 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( |> ignore) cutoff - // Also sweep legacy Ingresses left by pre-migration runs (this harness no - // longer creates them, but old orphans should still be reaped). sweepKind apiRateLimit "Ingress" @@ -140,9 +138,6 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( apiRateLimit "HTTPRoute" (fun () -> - // `new` clears the FS0760 IDisposable warning; do NOT `use`/dispose — - // GenericClient wraps the shared mission `kube` client, and disposing - // it would dispose that client out from under the rest of the run. let gc = new GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") gc.ListNamespacedAsync(ns).GetAwaiter().GetResult().Items From 32c31c5ef9caf5a96e9640b8908a6da37c7e6de5 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 10 Jul 2026 13:48:35 -0400 Subject: [PATCH 19/20] Address review: proxy readiness probe, route-Accepted wait, sweep guard - Add tcpSocket:80 readinessProbe to the http-proxy Deployment so ReadyReplicas reflects nginx actually listening (waitProxyReady's guarantee is now real). - waitProxyReady: after pod-ready, best-effort wait for the HTTPRoute to be Accepted by the gateway (non-fatal), closing the pod-ready -> route-programmed 404 window. Adds minimal HTTPRoute.Status model. - Orphan sweep: guard sweepKind's list() in try/catch so a missing CRD on a cluster skips that kind instead of aborting the whole sweep. - http-proxy resolver shim: fail fast if /etc/resolv.conf has no nameserver. - Reword proxy replica comment (scales with node count at creation, not a live HPA). Co-Authored-By: Claude Opus 4.8 --- src/CSLibrary/GatewayApi.cs | 18 ++++++++++++ src/FSLibrary/StellarKubeSpecs.fs | 15 +++++++--- src/FSLibrary/StellarOrphanSweep.fs | 12 +++++++- src/FSLibrary/StellarSupercluster.fs | 44 ++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/CSLibrary/GatewayApi.cs b/src/CSLibrary/GatewayApi.cs index ef62fdab..3c8767f9 100644 --- a/src/CSLibrary/GatewayApi.cs +++ b/src/CSLibrary/GatewayApi.cs @@ -27,6 +27,24 @@ public class HTTPRoute : IKubernetesObject, ISpec [JsonPropertyName("spec")] public HTTPRouteSpec Spec { get; set; } + + [JsonPropertyName("status")] + public HTTPRouteStatus Status { get; set; } + } + + // Minimal route status: each parent (Gateway) the route attached to reports + // conditions (notably "Accepted"). Used to wait for the gateway to admit the + // route before the driver sends traffic. + public class HTTPRouteStatus + { + [JsonPropertyName("parents")] + public IList Parents { get; set; } + } + + public class RouteParentStatus + { + [JsonPropertyName("conditions")] + public IList Conditions { get; set; } } [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRouteList", PluralName = "httproutes")] diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 8c148680..9b65055c 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -901,7 +901,7 @@ type NetworkCfg with member self.ToHttpProxyDeployment() : V1Deployment = let shim = sprintf - "set -e; R=$(awk '/^nameserver/ {print $2; exit}' /etc/resolv.conf); sed \"s/__RESOLVER__/$R/\" %s/%s > /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'" + "set -e; R=$(awk '/^nameserver/ {print $2; exit}' /etc/resolv.conf); [ -n \"$R\" ] || { echo 'http-proxy: no nameserver in /etc/resolv.conf' >&2; exit 1; }; sed \"s/__RESOLVER__/$R/\" %s/%s > /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'" CfgVal.httpProxyConfigMountPath CfgVal.httpProxyConfigFileName @@ -924,6 +924,12 @@ type NetworkCfg with args = [| "-c"; shim |], ports = [| V1ContainerPort(containerPort = 80, name = "http") |], resources = resources, + readinessProbe = + V1Probe( + tcpSocket = V1TCPSocketAction(port = IntstrIntOrString(value = "80")), + initialDelaySeconds = System.Nullable(1), + periodSeconds = System.Nullable(2) + ), volumeMounts = [| V1VolumeMount( name = CfgVal.httpProxyConfigVolumeName, @@ -942,9 +948,10 @@ type NetworkCfg with let podTemplate = V1PodTemplateSpec(metadata = V1ObjectMeta(labels = self.HttpProxyLabels), spec = podSpec) - //1 replica suffices for most missions; scale up only for - // large topologies. ceil(nodes/64), clamped to [1, cap] where cap is - // --http-proxy-replicas. + // Replica count scales with node count at creation time: ceil(nodes/64), + // clamped to [1, cap] where cap is --http-proxy-replicas. One replica + // suffices for most missions. Not a live HPA -- the count is fixed for + // the run, which is fine since a mission's node count never changes. let cap = max 1 self.missionContext.httpProxyReplicas let nodesPerProxy = 64 diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index ed9cf13b..dd56aaf5 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -29,7 +29,17 @@ let private sweepKind : unit = ApiRateLimit.sleepUntilNextRateLimitedApiCallTime apiRateLimit - for meta in list () do + // Guard the list() itself: on a cluster missing this kind's CRD (e.g. no + // gateway-api) it throws, and without this catch it would abort the whole + // sweep and skip the remaining kinds. + let items = + try + list () + with ex -> + LogWarn "Orphan sweep: failed to list %s (skipping kind): %s" kind ex.Message + Seq.empty + + for meta in items do if isOlderThan cutoff meta then try LogInfo diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index f5558950..4c48bfc9 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -372,6 +372,50 @@ type Kubernetes with waitProxyReady 0 + // Best-effort wait for the gateway to mark the HTTPRoute Accepted, + // closing the window between pod-ready and route-programmed (404s). + // Non-fatal: if status can't be confirmed we log and proceed, since + // mission-level retries still cover the residual race. + let routeAccepted () = + let gc = new GenericClient(self, "gateway.networking.k8s.io", "v1", "httproutes") + + gc.ListNamespacedAsync(nsStr).GetAwaiter().GetResult().Items + |> Seq.tryFind (fun r -> r.Metadata.Name = route.Metadata.Name) + |> Option.exists + (fun r -> + match r.Status with + | null -> false + | s when isNull s.Parents -> false + | s -> + s.Parents + |> Seq.exists + (fun p -> + not (isNull p.Conditions) + && p.Conditions + |> Seq.exists (fun c -> c.Type = "Accepted" && c.Status = "True"))) + + let rec waitRouteAccepted (n: int) = + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + + let accepted = + try + routeAccepted () + with ex -> + LogWarn "HTTPRoute %s status check failed (proceeding): %s" route.Metadata.Name ex.Message + true + + if accepted then + LogInfo "HTTPRoute %s Accepted by gateway" route.Metadata.Name + elif n >= 30 then + LogWarn + "HTTPRoute %s not Accepted after 30 attempts; proceeding (mission retries cover it)" + route.Metadata.Name + else + System.Threading.Thread.Sleep(2000) + waitRouteAccepted (n + 1) + + waitRouteAccepted 0 + let formation = new StellarFormation( networkCfg = nCfg, From a63bc7a10ea52b61f975a873d53d22cfb6cb12de Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 10 Jul 2026 14:18:24 -0400 Subject: [PATCH 20/20] PCv2: attach job-monitor HTTPRoute to the mission's configured Gateway The job-monitor chart values didn't propagate --gateway-name/--gateway-namespace, so its HTTPRoute always used the values.yaml defaults (traefik-gateway-private/ traefik) and would attach to the wrong Gateway on any cluster using a different one. Mirror the core route: pass monitor.gateway_name/gateway_namespace from the mission context. Verified on ssc-test: job-monitor route parentRef followed --gateway-name/--gateway-namespace. Co-Authored-By: Claude Opus 4.8 --- src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index fa8f332a..001981ea 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -172,6 +172,10 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "monitor.hostname=%s" (jobMonitorHostName context)) setOptions.Add(sprintf "monitor.path_prefix=/%s/%s" context.namespaceProperty helmReleaseName) setOptions.Add(sprintf "monitor.logging_interval_seconds=%d" jobMonitorLoggingIntervalSecs) + // Attach the job-monitor HTTPRoute to the same Gateway as the core route + // (--gateway-name/--gateway-namespace), instead of the values.yaml defaults. + setOptions.Add(sprintf "monitor.gateway_name=%s" context.gatewayName) + setOptions.Add(sprintf "monitor.gateway_namespace=%s" context.gatewayNamespace) // Set ASAN_OPTIONS if provided match context.asanOptions with