diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 6c5d8acd..6375ec48 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -12,17 +12,55 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 - - name: Setup k3s - uses: debianmaster/actions-k3s@master - with: - version: 'v1.23.17-k3s1' - - name: Setup nginx-ingress + - uses: actions/checkout@v4 + - name: Setup k3d cluster (k3s, bundled traefik disabled) + run: | + set -euo pipefail + curl -s https://raw.githubusercontent.com/k3d-io/k3d/v5.9.0/install.sh | bash + # Disable k3s's bundled traefik; the gateway controller (envoy) is installed below. + # Map host :8081 -> the gateway LoadBalancer :80 so the mission driver can reach it. + k3d cluster create ssc \ + --image rancher/k3s:v1.31.5-k3s1 \ + --k3s-arg "--disable=traefik@server:*" \ + -p "8081:80@loadbalancer" \ + --wait + k3d kubeconfig get ssc > "$RUNNER_TEMP/kubeconfig" + echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV" + - name: Setup Envoy Gateway + Gateway run: | - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.5.1/deploy/static/provider/cloud/deploy.yaml - kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=120s + set -euo pipefail + # Envoy Gateway ships its own matching gateway-api CRDs + controller. + helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.2.6 \ + -n envoy-gateway-system --create-namespace --wait --timeout 4m + # GatewayClass + the Gateway the per-mission HTTPRoutes attach to. + # Missions target it via --gateway-name / --gateway-namespace. + kubectl create namespace gateway + kubectl apply -f - <<'YAML' + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: eg + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + --- + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: ssc-gateway + namespace: gateway + spec: + gatewayClassName: eg + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: All + YAML + kubectl wait --namespace gateway --for=condition=Programmed gateway/ssc-gateway --timeout=180s - name: Setup .NET SDK 8 - uses: actions/setup-dotnet@v1 + uses: actions/setup-dotnet@v4 with: dotnet-version: '8.0.x' - name: Install dependencies @@ -36,7 +74,7 @@ jobs: - name: Test run: dotnet test --no-restore --verbosity normal - name: Run BootAndSync mission - run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --ingress-class nginx --ingress-internal-domain local --ingress-external-host localhost --uneven-sched + run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --gateway-name ssc-gateway --gateway-namespace gateway --ingress-internal-domain local --ingress-external-host localhost --ingress-external-port 8081 --uneven-sched - uses: actions/upload-artifact@v4 with: name: destination diff --git a/doc/k3s.md b/doc/k3s.md index b0e0093b..63b68737 100644 --- a/doc/k3s.md +++ b/doc/k3s.md @@ -32,5 +32,5 @@ net.ipv6.neigh.default.gc_thresh3 = 100000 - Build supercluster normally (see [getting-started.md](getting-started.md)) - - Run supercluster with these additional arguments: `--kubeconfig $KUBECONFIG --namespace default --ingress-class nginx --ingress-internal-domain local --ingress-external-host localhost --uneven-sched` + - Run supercluster with these additional arguments: `--kubeconfig $KUBECONFIG --namespace default --ingress-internal-domain local --ingress-external-host localhost --uneven-sched` diff --git a/doc/theoretical-max-tps.md b/doc/theoretical-max-tps.md index 6b03aced..a3c4e7f9 100644 --- a/doc/theoretical-max-tps.md +++ b/doc/theoretical-max-tps.md @@ -11,7 +11,7 @@ To run the test, first [set up an EKS cluster](eks.md). Accepting the default settings will produce a topology identical to what we use in our test setup. Then, run a `MaxTPSClassic` mission with the following template: ```bash -dotnet run --project src/App/App.fsproj --configuration Release -- mission MaxTPSClassic --image= --pubnet-data=/topologies/theoretical-max-tps.json --tx-rate= --max-tx-rate= --namespace default --ingress-internal-domain= --ingress-class=nginx --run-for-max-tps=classic --enable-tcp-tuning +dotnet run --project src/App/App.fsproj --configuration Release -- mission MaxTPSClassic --image= --pubnet-data=/topologies/theoretical-max-tps.json --tx-rate= --max-tx-rate= --namespace default --ingress-internal-domain= --run-for-max-tps=classic --enable-tcp-tuning ``` For more information about how to set the parameters in the above command, see [Measuring Transaction Throughput](measuring-transaction-throughput.md). diff --git a/src/App/Program.fs b/src/App/Program.fs index cf17e553..0dd5125a 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -42,10 +42,11 @@ type MissionOptions logDebugPartitions: seq, logTracePartitions: seq, namespaceProperty: string option, - ingressClass: string, - ingressInternalDomain: string, - ingressExternalHost: string option, - ingressExternalPort: int, + gatewayName: string, + gatewayNamespace: string, + routeInternalDomain: string, + routeExternalHost: string option, + routeExternalPort: int, exportToPrometheus: bool, probeTimeout: int, missions: string seq, @@ -70,6 +71,7 @@ type MissionOptions avoidNodeLabels: seq, tolerateNodeTaints: seq, apiRateLimit: int, + httpProxyReplicas: int, pubnetData: string option, flatQuorum: bool option, tier1Keys: string option, @@ -161,28 +163,34 @@ type MissionOptions [] member self.NamespaceProperty = namespaceProperty - [] - member self.IngressClass = ingressClass + Default = "traefik-gateway-private")>] + member self.GatewayName = gatewayName + + [] + member self.GatewayNamespace = gatewayNamespace [] - member self.IngressInternalDomain = ingressInternalDomain + member self.RouteInternalDomain = routeInternalDomain [] - member self.IngressExternalHost = ingressExternalHost + member self.RouteExternalHost = routeExternalHost [] - member self.IngressExternalPort = ingressExternalPort + member self.RouteExternalPort = routeExternalPort [] member self.ExportToPrometheus : bool = exportToPrometheus @@ -301,6 +309,12 @@ type MissionOptions Default = 10)>] member self.ApiRateLimit = apiRateLimit + [] + member self.HttpProxyReplicas = httpProxyReplicas + [] member self.PubnetData = pubnetData @@ -815,10 +829,11 @@ let main argv = numNodes = mission.NumNodes namespaceProperty = ns logLevels = ll - ingressClass = mission.IngressClass - ingressInternalDomain = mission.IngressInternalDomain - ingressExternalHost = mission.IngressExternalHost - ingressExternalPort = mission.IngressExternalPort + gatewayName = mission.GatewayName + gatewayNamespace = mission.GatewayNamespace + routeInternalDomain = mission.RouteInternalDomain + routeExternalHost = mission.RouteExternalHost + routeExternalPort = mission.RouteExternalPort exportToPrometheus = mission.ExportToPrometheus probeTimeout = mission.ProbeTimeout coreResources = SmallTestResources @@ -831,6 +846,7 @@ let main argv = avoidNodeLabels = List.map splitLabel (List.ofSeq mission.AvoidNodeLabels) tolerateNodeTaints = List.map splitLabel (List.ofSeq mission.TolerateNodeTaints) apiRateLimit = mission.ApiRateLimit + httpProxyReplicas = mission.HttpProxyReplicas pubnetData = mission.PubnetData flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys diff --git a/src/CSLibrary/CSLibrary.csproj b/src/CSLibrary/CSLibrary.csproj index 906f91f7..00e0cef2 100644 --- a/src/CSLibrary/CSLibrary.csproj +++ b/src/CSLibrary/CSLibrary.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/CSLibrary/GatewayApi.cs b/src/CSLibrary/GatewayApi.cs new file mode 100644 index 00000000..3c8767f9 --- /dev/null +++ b/src/CSLibrary/GatewayApi.cs @@ -0,0 +1,179 @@ +// Typed models for the subset of the Gateway API (gateway.networking.k8s.io/v1) +// that Supercluster needs to route to per-pod core/history endpoints. +// +// The official KubernetesClient package ships no Gateway API models (CRDs are +// out of scope for it), so these POCOs are hand-maintained to the upstream +// gateway-api v1 schema. Only the fields Supercluster uses are modelled. +// Usable with GenericClient thanks to the [KubernetesEntity] attribute. + +using System.Collections.Generic; +using System.Text.Json.Serialization; +using k8s; +using k8s.Models; + +namespace GatewayApiModels +{ + [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRoute", PluralName = "httproutes")] + public class HTTPRoute : IKubernetesObject, ISpec + { + [JsonPropertyName("apiVersion")] + public string ApiVersion { get; set; } = "gateway.networking.k8s.io/v1"; + + [JsonPropertyName("kind")] + public string Kind { get; set; } = "HTTPRoute"; + + [JsonPropertyName("metadata")] + public V1ObjectMeta Metadata { get; set; } + + [JsonPropertyName("spec")] + public HTTPRouteSpec Spec { get; set; } + + [JsonPropertyName("status")] + public HTTPRouteStatus Status { get; set; } + } + + // Minimal route status: each parent (Gateway) the route attached to reports + // conditions (notably "Accepted"). Used to wait for the gateway to admit the + // route before the driver sends traffic. + public class HTTPRouteStatus + { + [JsonPropertyName("parents")] + public IList Parents { get; set; } + } + + public class RouteParentStatus + { + [JsonPropertyName("conditions")] + public IList Conditions { get; set; } + } + + [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRouteList", PluralName = "httproutes")] + public class HTTPRouteList : IKubernetesObject, IItems + { + [JsonPropertyName("apiVersion")] + public string ApiVersion { get; set; } = "gateway.networking.k8s.io/v1"; + + [JsonPropertyName("kind")] + public string Kind { get; set; } = "HTTPRouteList"; + + [JsonPropertyName("metadata")] + public V1ListMeta Metadata { get; set; } + + [JsonPropertyName("items")] + public IList Items { get; set; } + } + + public class HTTPRouteSpec + { + [JsonPropertyName("parentRefs")] + public IList ParentRefs { get; set; } + + [JsonPropertyName("hostnames")] + public IList Hostnames { get; set; } + + [JsonPropertyName("rules")] + public IList Rules { get; set; } + } + + public class ParentReference + { + [JsonPropertyName("group")] + public string Group { get; set; } + + [JsonPropertyName("kind")] + public string Kind { get; set; } + + [JsonPropertyName("namespace")] + public string Namespace { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("sectionName")] + public string SectionName { get; set; } + + [JsonPropertyName("port")] + public int? Port { get; set; } + } + + public class HTTPRouteRule + { + [JsonPropertyName("matches")] + public IList Matches { get; set; } + + [JsonPropertyName("filters")] + public IList Filters { get; set; } + + [JsonPropertyName("backendRefs")] + public IList BackendRefs { get; set; } + } + + public class HTTPRouteMatch + { + [JsonPropertyName("path")] + public HTTPPathMatch Path { get; set; } + } + + public class HTTPPathMatch + { + // "Exact" | "PathPrefix" | "RegularExpression" + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } + } + + public class HTTPRouteFilter + { + // "RequestHeaderModifier" | "URLRewrite" | ... + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("urlRewrite")] + public HTTPURLRewriteFilter UrlRewrite { get; set; } + } + + public class HTTPURLRewriteFilter + { + [JsonPropertyName("hostname")] + public string Hostname { get; set; } + + [JsonPropertyName("path")] + public HTTPPathModifier Path { get; set; } + } + + public class HTTPPathModifier + { + // "ReplaceFullPath" | "ReplacePrefixMatch" + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("replaceFullPath")] + public string ReplaceFullPath { get; set; } + + [JsonPropertyName("replacePrefixMatch")] + public string ReplacePrefixMatch { get; set; } + } + + public class HTTPBackendRef + { + [JsonPropertyName("group")] + public string Group { get; set; } + + [JsonPropertyName("kind")] + public string Kind { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("namespace")] + public string Namespace { get; set; } + + [JsonPropertyName("port")] + public int? Port { get; set; } + + [JsonPropertyName("weight")] + public int? Weight { get; set; } + } +} diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 4a753333..e4b6f5c9 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -54,10 +54,11 @@ let ctx : MissionContext = numNodes = 100 namespaceProperty = "stellar-supercluster" logLevels = { LogDebugPartitions = []; LogTracePartitions = [] } - ingressClass = "ingress-private" - ingressInternalDomain = "local" - ingressExternalHost = None - ingressExternalPort = 80 + gatewayName = "traefik-gateway-private" + gatewayNamespace = "traefik" + routeInternalDomain = "local" + routeExternalHost = None + routeExternalPort = 80 exportToPrometheus = false probeTimeout = 10 coreResources = SmallTestResources @@ -68,6 +69,7 @@ let ctx : MissionContext = avoidNodeLabels = [] tolerateNodeTaints = [] apiRateLimit = 10 + httpProxyReplicas = 2 pubnetData = None flatQuorum = None tier1Keys = None diff --git a/src/FSLibrary/FSLibrary.fsproj b/src/FSLibrary/FSLibrary.fsproj index e78c6ff5..8e199c78 100644 --- a/src/FSLibrary/FSLibrary.fsproj +++ b/src/FSLibrary/FSLibrary.fsproj @@ -87,7 +87,7 @@ - + diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 601e5e57..001981ea 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -170,8 +170,12 @@ let installProject (context: MissionContext) = | None -> () setOptions.Add(sprintf "monitor.hostname=%s" (jobMonitorHostName context)) - setOptions.Add(sprintf "monitor.path=/%s/%s/(.*)" context.namespaceProperty helmReleaseName) + setOptions.Add(sprintf "monitor.path_prefix=/%s/%s" context.namespaceProperty helmReleaseName) setOptions.Add(sprintf "monitor.logging_interval_seconds=%d" jobMonitorLoggingIntervalSecs) + // Attach the job-monitor HTTPRoute to the same Gateway as the core route + // (--gateway-name/--gateway-namespace), instead of the values.yaml defaults. + setOptions.Add(sprintf "monitor.gateway_name=%s" context.gatewayName) + setOptions.Add(sprintf "monitor.gateway_namespace=%s" context.gatewayNamespace) // Set ASAN_OPTIONS if provided match context.asanOptions with diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index a6252126..190fea2b 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -19,6 +19,7 @@ open StellarDotnetSdk.Accounts // paths, labels, etc. module CfgVal = let httpPort = 11626 + let historyPort = 80 let prometheusExporterPort = 9473 let labels = Map.ofSeq [ "app", "stellar-core" ] let labelSelector = "app = stellar-core" @@ -48,6 +49,12 @@ module CfgVal = let peerInitCfgFileName = "stellar-core-init.cfg" let peerDelayCfgFileName = "install-delays.sh" + + let httpProxyContainerName = "http-proxy" + let httpProxyConfigVolumeName = "proxy-tmpl" + let httpProxyConfigMountPath = "/proxy-tmpl" + let httpProxyConfigFileName = "default.conf.template" + let peerNameEnvCfgFileWord : ShWord = ShWord.ShPieces [| ShBare("/cfg-") ShVar(ShName peerNameEnvVarName) diff --git a/src/FSLibrary/StellarCoreHTTP.fs b/src/FSLibrary/StellarCoreHTTP.fs index 66d58fae..f4ceb87b 100644 --- a/src/FSLibrary/StellarCoreHTTP.fs +++ b/src/FSLibrary/StellarCoreHTTP.fs @@ -377,14 +377,14 @@ exception NodeLostSyncException of string type Peer with member self.Headers = - let host = self.networkCfg.IngressInternalHostName + let host = self.networkCfg.RouteInternalHostName [ HttpRequestHeaders.Host host ] member self.URL(path: string) : string = sprintf "http://%s:%d/%s/core/%s" - self.networkCfg.IngressExternalHostName - self.networkCfg.missionContext.ingressExternalPort + self.networkCfg.RouteExternalHostName + self.networkCfg.missionContext.routeExternalPort self.PodName.StringName path @@ -792,7 +792,7 @@ type Peer with (self.URL "tx"), httpMethod = "GET", query = [ "blob", b64 ], - headers = [ "Host", self.networkCfg.IngressInternalHostName ] + headers = [ "Host", self.networkCfg.RouteInternalHostName ] ) use reader = new System.IO.StreamReader(response.ResponseStream) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index d31a9bf3..64c35b1f 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -14,6 +14,7 @@ open StellarNetworkDelays open System.Text.RegularExpressions open System.Collections.Generic open Logging +open GatewayApiModels // Containers that run stellar-core may or may-not have a final '--conf' // argument appended to their command-line. The argument is specified one of 3 @@ -885,85 +886,157 @@ type NetworkCfg with statefulSet - // Returns an array of "per-Pod" Service objects, each named according to - // the peer-N short names, and mapping (via a somewhat hacky misuse of the - // ExternalName Service type -- thanks internet!) to the _internal_ DNS - // names of each pod. - // - // This exists strictly to support the Ingress object below, that routes - // separate URL prefixes to separate Pods (which is somewhat the opposite of - // the load-balancing task Services, Pods, and Ingress systems typically - // do). - member self.ToPerPodServices() : V1Service array = - let perPodService (coreSet: CoreSet) i = - let name = self.PodName coreSet i - let dnsName = self.PeerDnsName coreSet i + // Metadata for a proxy object: run-scoped labels + the anchor owner ref so + // it is GC'd with the rest of the run. + member private self.HttpProxyMeta(name: string) : V1ObjectMeta = + V1ObjectMeta(name = name, namespaceProperty = self.NamespaceProperty, labels = self.HttpProxyLabels) + |> applyAnchorOwner self - let ports = - [| V1ServicePort(name = "core", port = CfgVal.httpPort) - V1ServicePort(name = "history", port = 80) |] + // ConfigMap holding the nginx server config for the proxy. The pod's actual + // CoreDNS address is not known here, so we ship a template with a + // __RESOLVER__ placeholder that the container substitutes at startup from + // its own /etc/resolv.conf (see ToHttpProxyDeployment). Requests of the + // form //core|history[/] are proxied to the pod's in-cluster DNS + // name; $is_args$args preserves the query string (the driver uses it, e.g. + // //core/tx?blob=...). + member self.ToHttpProxyConfigMap() : V1ConfigMap = + let fqdn = sprintf "%s.%s.svc.cluster.local" self.ServiceName self.NamespaceProperty + + let conf = + sprintf + """server { + listen 80 default_server; + server_name _; + resolver __RESOLVER__ ipv6=off valid=10s; + + location ~ ^/([^/]+)/core(?:/(.*))?$ { + proxy_pass http://$1.%s:%d/$2$is_args$args; + } + + location ~ ^/([^/]+)/history(?:/(.*))?$ { + proxy_pass http://$1.%s:%d/$2$is_args$args; + } +} +""" + fqdn + CfgVal.httpPort + fqdn + CfgVal.historyPort + + let data = Map.empty.Add(CfgVal.httpProxyConfigFileName, conf) + V1ConfigMap(metadata = self.HttpProxyMeta self.HttpProxyConfigMapName, data = data) + + // Scalable nginx Deployment fronting driver->pod routing. The startup shim + // reads the CoreDNS server from /etc/resolv.conf and templates it into the + // nginx config. + member self.ToHttpProxyDeployment() : V1Deployment = + let shim = + sprintf + "set -e; R=$(awk '/^nameserver/ {print $2; exit}' /etc/resolv.conf); [ -n \"$R\" ] || { echo 'http-proxy: no nameserver in /etc/resolv.conf' >&2; exit 1; }; sed \"s/__RESOLVER__/$R/\" %s/%s > /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'" + CfgVal.httpProxyConfigMountPath + CfgVal.httpProxyConfigFileName + + // Match the ingress-nginx private controller pods (ssc-eks) this proxy replaces. + let resources = + V1ResourceRequirements( + requests = + dict [ ("cpu", ResourceQuantity("50m")) + ("memory", ResourceQuantity("90Mi")) ], + limits = + dict [ ("cpu", ResourceQuantity("250m")) + ("memory", ResourceQuantity("768Mi")) ] + ) - let ports = - if self.missionContext.exportToPrometheus then - Array.append ports [| V1ServicePort(name = "prom-exp", port = CfgVal.prometheusExporterPort) |] - else - ports - - let spec = - V1ServiceSpec(``type`` = "ExternalName", ports = ports, externalName = dnsName.StringName) - - V1Service(metadata = self.NamespacedMeta name.StringName, spec = spec) - - self.MapAllPeers perPodService - - // Returns an Ingress object with rules that map URLs http://$ingressHost/peer-N/foo - // to the per-Pod Service within the current networkCfg named peer-N (which then, via - // DNS mapping, goes to the Pod itself). Exposing this to external traffic - // requires that you enable the nginx Ingress controller on your k8s - // cluster. - member self.ToIngress() : V1Ingress = - let coreBackend (pn: PodName) : V1IngressBackend = - let port = V1ServiceBackendPort(number = CfgVal.httpPort) - let service = V1IngressServiceBackend(pn.StringName, port = port) - V1IngressBackend(service = service) - - let historyBackend (pn: PodName) : V1IngressBackend = - let port = V1ServiceBackendPort(number = 80) - let service = V1IngressServiceBackend(pn.StringName, port = port) - V1IngressBackend(service = service) - - let corePath (coreSet: CoreSet) (i: int) : V1HTTPIngressPath = - let pn = self.PodName coreSet i - let ingressPath = V1HTTPIngressPath() - ingressPath.Backend <- coreBackend pn - ingressPath.Path <- sprintf "/%s/core(/|$)(.*)" pn.StringName - ingressPath.PathType <- "ImplementationSpecific" - ingressPath - - let historyPath (coreSet: CoreSet) (i: int) : V1HTTPIngressPath = - let pn = self.PodName coreSet i - let ingressPath = V1HTTPIngressPath() - ingressPath.Backend <- historyBackend pn - ingressPath.Path <- sprintf "/%s/history(/|$)(.*)" pn.StringName - ingressPath.PathType <- "ImplementationSpecific" - ingressPath - - let corePaths = self.MapAllPeers corePath - let historyPaths = self.MapAllPeers historyPath - - let rule = V1HTTPIngressRuleValue(paths = Array.concat [ corePaths; historyPaths ]) - - let host = self.IngressInternalHostName - let rules = [| V1IngressRule(host = host, http = rule) |] - let spec = V1IngressSpec(rules = rules) - - let annotation = - Map.ofArray [| ("kubernetes.io/ingress.class", self.missionContext.ingressClass) - ("nginx.ingress.kubernetes.io/use-regex", "true") - ("nginx.ingress.kubernetes.io/rewrite-target", "/$2") |] + let container = + V1Container( + name = CfgVal.httpProxyContainerName, + image = self.missionContext.nginxImage, + command = [| "/bin/sh" |], + args = [| "-c"; shim |], + ports = [| V1ContainerPort(containerPort = 80, name = "http") |], + resources = resources, + readinessProbe = + V1Probe( + tcpSocket = V1TCPSocketAction(port = IntstrIntOrString(value = "80")), + initialDelaySeconds = System.Nullable(1), + periodSeconds = System.Nullable(2) + ), + volumeMounts = + [| V1VolumeMount( + name = CfgVal.httpProxyConfigVolumeName, + mountPath = CfgVal.httpProxyConfigMountPath + ) |] + ) + + let volume = + V1Volume( + name = CfgVal.httpProxyConfigVolumeName, + configMap = V1ConfigMapVolumeSource(name = self.HttpProxyConfigMapName) + ) + + let podSpec = V1PodSpec(containers = [| container |], volumes = [| volume |]) + + let podTemplate = + V1PodTemplateSpec(metadata = V1ObjectMeta(labels = self.HttpProxyLabels), spec = podSpec) + + // Replica count scales with node count at creation time: ceil(nodes/64), + // clamped to [1, cap] where cap is --http-proxy-replicas. One replica + // suffices for most missions. Not a live HPA -- the count is fixed for + // the run, which is fine since a mission's node count never changes. + let cap = max 1 self.missionContext.httpProxyReplicas + let nodesPerProxy = 64 + + let replicas = + self.MaxPeerCount + |> fun n -> (n + nodesPerProxy - 1) / nodesPerProxy |> max 1 |> min cap + + let spec = + V1DeploymentSpec( + replicas = System.Nullable(replicas), + selector = V1LabelSelector(matchLabels = self.HttpProxyLabels), + template = podTemplate + ) + + V1Deployment(metadata = self.HttpProxyMeta self.HttpProxyName, spec = spec) + + // ClusterIP Service selecting the proxy pods; the HTTPRoute's single + // backend. + member self.ToHttpProxyService() : V1Service = + let spec = + V1ServiceSpec(selector = self.HttpProxyLabels, ports = [| V1ServicePort(name = "http", port = 80) |]) + + V1Service(metadata = self.HttpProxyMeta self.HttpProxyName, spec = spec) + + // Returns an HTTPRoute (gateway.networking.k8s.io/v1) attached to the shared + // private traefik gateway that sends all of http://$routeHost/* to the + // per-run nginx HTTP proxy Service. The proxy does the //core|history + // demux internally. + member self.ToHttpRoute() : HTTPRoute = + let parentRef = + ParentReference( + Group = "gateway.networking.k8s.io", + Kind = "Gateway", + Namespace = self.missionContext.gatewayNamespace, + Name = self.missionContext.gatewayName + ) + + let rule = + HTTPRouteRule( + Matches = + List([ HTTPRouteMatch(Path = HTTPPathMatch(Type = "PathPrefix", Value = "/")) ]), + BackendRefs = + List([ HTTPBackendRef(Name = self.HttpProxyName, Port = System.Nullable(80)) ]) + ) + + let spec = + HTTPRouteSpec( + ParentRefs = List([ parentRef ]), + Hostnames = List([ self.RouteInternalHostName ]), + Rules = List([ rule ]) + ) let meta = - V1ObjectMeta(name = self.IngressName, namespaceProperty = self.NamespaceProperty, annotations = annotation) + V1ObjectMeta(name = self.HttpRouteName, namespaceProperty = self.NamespaceProperty) |> applyAnchorOwner self - V1Ingress(spec = spec, metadata = meta) + HTTPRoute(Metadata = meta, Spec = spec) diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 01df9ae3..828048ff 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -55,10 +55,11 @@ type MissionContext = numNodes: int namespaceProperty: string logLevels: LogLevels - ingressClass: string - ingressInternalDomain: string - ingressExternalHost: string option - ingressExternalPort: int + gatewayName: string + gatewayNamespace: string + routeInternalDomain: string + routeExternalHost: string option + routeExternalPort: int exportToPrometheus: bool probeTimeout: int coreResources: CoreResources @@ -78,6 +79,7 @@ type MissionContext = avoidNodeLabels: ((string * string option) list) tolerateNodeTaints: ((string * string option) list) apiRateLimit: int + httpProxyReplicas: int pubnetData: string option flatQuorum: bool option tier1Keys: string option diff --git a/src/FSLibrary/StellarNamespaceContent.fs b/src/FSLibrary/StellarNamespaceContent.fs index 22b62c0a..dccefca0 100644 --- a/src/FSLibrary/StellarNamespaceContent.fs +++ b/src/FSLibrary/StellarNamespaceContent.fs @@ -7,6 +7,7 @@ module StellarNamespaceContent open k8s open k8s.Models open Logging +open GatewayApiModels type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: string) = @@ -15,7 +16,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st let services : Set ref = ref Set.empty let configMaps : Set ref = ref Set.empty let statefulSets : Set ref = ref Set.empty - let ingresses : Set ref = ref Set.empty + let httpRoutes : Set ref = ref Set.empty let jobs : Set ref = ref Set.empty let daemonSets : Set ref = ref Set.empty let deployments : Set ref = ref Set.empty @@ -61,14 +62,17 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st propagationPolicy = "Foreground" )) - let delIngress (name: string) = - LogInfo "Deleting Ingress %s" name + let delHttpRoute (name: string) = + LogInfo "Deleting HTTPRoute %s" name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (apiRateLimit) ignoreError (fun _ -> - kube.DeleteNamespacedIngress( + kube.DeleteNamespacedCustomObject( + group = "gateway.networking.k8s.io", + version = "v1", namespaceParameter = namespaceProperty, + plural = "httproutes", name = name, propagationPolicy = "Foreground" )) @@ -124,7 +128,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st cleanSet delStatefulSet statefulSets cleanSet delDeployment deployments cleanSet delConfigMap configMaps - cleanSet delIngress ingresses + cleanSet delHttpRoute httpRoutes cleanSet delJob jobs cleanSet delDaemonSet daemonSets @@ -134,7 +138,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st member self.Add(statefulSet: V1StatefulSet) = addOne statefulSets statefulSet.Metadata.Name - member self.Add(ingress: V1Ingress) = addOne ingresses ingress.Metadata.Name + member self.Add(route: HTTPRoute) = addOne httpRoutes route.Metadata.Name member self.Add(job: V1Job) = addOne jobs job.Metadata.Name @@ -148,7 +152,7 @@ type NamespaceContent(kube: Kubernetes, apiRateLimit: int, namespaceProperty: st member self.Del(statefulSet: V1StatefulSet) = delOne delStatefulSet statefulSets statefulSet.Metadata.Name - member self.Del(ingress: V1Ingress) = delOne delIngress ingresses ingress.Metadata.Name + member self.Del(route: HTTPRoute) = delOne delHttpRoute httpRoutes route.Metadata.Name member self.Del(job: V1Job) = delOne delJob jobs job.Metadata.Name diff --git a/src/FSLibrary/StellarNetworkCfg.fs b/src/FSLibrary/StellarNetworkCfg.fs index 2c7edea2..06ef5440 100644 --- a/src/FSLibrary/StellarNetworkCfg.fs +++ b/src/FSLibrary/StellarNetworkCfg.fs @@ -102,7 +102,19 @@ type NetworkCfg = member self.ServiceName : string = sprintf "%s-stellar-core" self.Nonce - member self.IngressName : string = sprintf "%s-stellar-core-ingress" self.Nonce + member self.HttpRouteName : string = sprintf "%s-stellar-core-http-route" self.Nonce + + // The nginx HTTP proxy (Deployment + Service) that the HTTPRoute points at. + member self.HttpProxyName : string = sprintf "%s-http-proxy" self.Nonce + + member self.HttpProxyConfigMapName : string = sprintf "%s-http-proxy-cfg" self.Nonce + + // Labels scoping the proxy Deployment/Service to this run only (the shared + // namespace can hold other runs). Distinct from the core "app=stellar-core" + // labels so the proxy Service selects only proxy pods. + member self.HttpProxyLabels : Map = + Map.ofSeq [ ("app", "stellar-core-http-proxy") + ("ssc-nonce", self.Nonce) ] member self.JobName(i: int) : string = sprintf "%s-stellar-core-job-%d" self.Nonce i @@ -120,11 +132,11 @@ type NetworkCfg = PeerDnsName s - member self.IngressInternalHostName : string = sprintf "%s.%s" self.Nonce self.missionContext.ingressInternalDomain + member self.RouteInternalHostName : string = sprintf "%s.%s" self.Nonce self.missionContext.routeInternalDomain - member self.IngressExternalHostName : string = - match self.missionContext.ingressExternalHost with - | None -> self.IngressInternalHostName + member self.RouteExternalHostName : string = + match self.missionContext.routeExternalHost with + | None -> self.RouteInternalHostName | Some h -> h member self.WithLive name (live: bool) = diff --git a/src/FSLibrary/StellarOrphanSweep.fs b/src/FSLibrary/StellarOrphanSweep.fs index c62cf202..dd56aaf5 100644 --- a/src/FSLibrary/StellarOrphanSweep.fs +++ b/src/FSLibrary/StellarOrphanSweep.fs @@ -10,6 +10,7 @@ open k8s.Models open Logging open ScriptUtils +open GatewayApiModels // Resources that look this old must belong to a failed prior run — no healthy // mission runs anywhere near this long, and the Jenkins lock serializes CI @@ -28,7 +29,17 @@ let private sweepKind : unit = ApiRateLimit.sleepUntilNextRateLimitedApiCallTime apiRateLimit - for meta in list () do + // Guard the list() itself: on a cluster missing this kind's CRD (e.g. no + // gateway-api) it throws, and without this catch it would abort the whole + // sweep and skip the remaining kinds. + let items = + try + list () + with ex -> + LogWarn "Orphan sweep: failed to list %s (skipping kind): %s" kind ex.Message + Seq.empty + + for meta in items do if isOlderThan cutoff meta then try LogInfo @@ -133,6 +144,26 @@ let private sweepWithCutoff (cutoff: DateTime) (kube: Kubernetes) (ns: string) ( |> ignore) cutoff + sweepKind + apiRateLimit + "HTTPRoute" + (fun () -> + let gc = new GenericClient(kube, "gateway.networking.k8s.io", "v1", "httproutes") + + gc.ListNamespacedAsync(ns).GetAwaiter().GetResult().Items + |> Seq.map (fun r -> r.Metadata)) + (fun n -> + kube.DeleteNamespacedCustomObject( + group = "gateway.networking.k8s.io", + version = "v1", + namespaceParameter = ns, + plural = "httproutes", + name = n, + propagationPolicy = "Foreground" + ) + |> ignore) + cutoff + sweepKind apiRateLimit "Job" diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index d3b19583..4c48bfc9 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -17,6 +17,7 @@ open StellarStatefulSets open StellarCoreSet open StellarKubeSpecs open StellarNamespaceContent +open GatewayApiModels open System open System.Diagnostics @@ -312,19 +313,108 @@ type Kubernetes with for statefulSet in statefulSets do namespaceContent.Add(statefulSet) - for svc in nCfg.ToPerPodServices() do - LogInfo "Creating Per-Pod Service %s" svc.Metadata.Name + if not (List.isEmpty statefulSets) then + let proxyCfg = nCfg.ToHttpProxyConfigMap() + LogInfo "Creating HTTP proxy ConfigMap %s" proxyCfg.Metadata.Name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + namespaceContent.Add(self.CreateNamespacedConfigMap(body = proxyCfg, namespaceParameter = nsStr)) - let service = self.CreateNamespacedService(namespaceParameter = nsStr, body = svc) - namespaceContent.Add(service) + let proxyDep = nCfg.ToHttpProxyDeployment() - if not (List.isEmpty statefulSets) then - let ing = nCfg.ToIngress() - LogInfo "Creating Ingress %s" ing.Metadata.Name + LogInfo + "Creating HTTP proxy Deployment %s (%d replicas)" + proxyDep.Metadata.Name + proxyDep.Spec.Replicas.Value + + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + namespaceContent.Add(self.CreateNamespacedDeployment(body = proxyDep, namespaceParameter = nsStr)) + + let proxySvc = nCfg.ToHttpProxyService() + LogInfo "Creating HTTP proxy Service %s" proxySvc.Metadata.Name + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + namespaceContent.Add(self.CreateNamespacedService(body = proxySvc, namespaceParameter = nsStr)) + + let route = nCfg.ToHttpRoute() + LogInfo "Creating HTTPRoute %s" route.Metadata.Name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) - let ingress = self.CreateNamespacedIngress(namespaceParameter = nsStr, body = ing) - namespaceContent.Add(ingress) + + self.CreateNamespacedCustomObject( + body = route, + group = "gateway.networking.k8s.io", + version = "v1", + namespaceParameter = nsStr, + plural = "httproutes" + ) + |> ignore + + namespaceContent.Add(route) + + // Wait for at least one proxy pod to be ready before the driver + // routes core HTTP through it, so mission startup doesn't spend + // its retry budget on 502s. + let rec waitProxyReady (n: int) = + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + + let d = + self.ReadNamespacedDeployment(name = proxyDep.Metadata.Name, namespaceParameter = nsStr) + // Status (and ReadyReplicas within it) is populated asynchronously and may be + // null right after creation; treat missing as 0 ready and keep polling. + let ready = if isNull d.Status then 0 else d.Status.ReadyReplicas.GetValueOrDefault(0) + + LogInfo "HTTP proxy %s: %d ready" proxyDep.Metadata.Name ready + + if ready < 1 then + if n >= 60 then + failwithf "HTTP proxy %s not ready after 60 attempts" proxyDep.Metadata.Name + + System.Threading.Thread.Sleep(2000) + waitProxyReady (n + 1) + + waitProxyReady 0 + + // Best-effort wait for the gateway to mark the HTTPRoute Accepted, + // closing the window between pod-ready and route-programmed (404s). + // Non-fatal: if status can't be confirmed we log and proceed, since + // mission-level retries still cover the residual race. + let routeAccepted () = + let gc = new GenericClient(self, "gateway.networking.k8s.io", "v1", "httproutes") + + gc.ListNamespacedAsync(nsStr).GetAwaiter().GetResult().Items + |> Seq.tryFind (fun r -> r.Metadata.Name = route.Metadata.Name) + |> Option.exists + (fun r -> + match r.Status with + | null -> false + | s when isNull s.Parents -> false + | s -> + s.Parents + |> Seq.exists + (fun p -> + not (isNull p.Conditions) + && p.Conditions + |> Seq.exists (fun c -> c.Type = "Accepted" && c.Status = "True"))) + + let rec waitRouteAccepted (n: int) = + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + + let accepted = + try + routeAccepted () + with ex -> + LogWarn "HTTPRoute %s status check failed (proceeding): %s" route.Metadata.Name ex.Message + true + + if accepted then + LogInfo "HTTPRoute %s Accepted by gateway" route.Metadata.Name + elif n >= 30 then + LogWarn + "HTTPRoute %s not Accepted after 30 attempts; proceeding (mission retries cover it)" + route.Metadata.Name + else + System.Threading.Thread.Sleep(2000) + waitRouteAccepted (n + 1) + + waitRouteAccepted 0 let formation = new StellarFormation( diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 353abf17..e9f6050c 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -1,23 +1,31 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute metadata: - name: {{ .Release.Name }}-job-monitor-ingress - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$1 + name: {{ .Release.Name }}-job-monitor-route spec: - ingressClassName: "{{ .Values.monitor.ingress_class_name}}" + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: "{{ .Values.monitor.gateway_name }}" + namespace: "{{ .Values.monitor.gateway_namespace }}" + {{- if .Values.monitor.hostname }} + hostnames: + - "{{ .Values.monitor.hostname }}" + {{- end }} rules: - - host: "{{ .Values.monitor.hostname}}" - http: - paths: - - path: "{{ .Values.monitor.path}}" - pathType: Prefix - backend: - service: - name: {{ .Release.Name }}-job-monitor - port: - number: 8080 + - matches: + - path: + type: PathPrefix + value: "{{ .Values.monitor.path_prefix }}" + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + replacePrefixMatch: / + backendRefs: + - name: {{ .Release.Name }}-job-monitor + port: 8080 --- apiVersion: v1 kind: Service diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index b47e7ab1..36f3ea3d 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -36,9 +36,10 @@ worker: "curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" monitor: - ingress_class_name: "ingress-private" + gateway_name: "traefik-gateway-private" + gateway_namespace: "traefik" hostname: "" # to be set by the mission - path: "/default/(.*)" + path_prefix: "/default" logging_interval_seconds: 300 logging_level: "INFO" # 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' resources: