diff --git a/.github/workflows/envelope-contract-drift.yml b/.github/workflows/envelope-contract-drift.yml new file mode 100644 index 00000000..8f18058a --- /dev/null +++ b/.github/workflows/envelope-contract-drift.yml @@ -0,0 +1,147 @@ +name: Envelope contract drift (cross-repo) + +# internal/resources/envelope_contract.json is VENDORED from +# tracebloc/client-runtime (backend#2220, RFC-BACKEND-664 §P0). client-runtime +# owns the training-envelope arithmetic +# (node_sizing.envelope_from_allocatable); this repo, the bash installer and its +# PowerShell twin are readers of it. Before that consolidation the same policy +# was typed out in all three, none derived from the others — and they disagreed: +# set.go ranked candidate nodes (cpu, memory) while the bash installer ranked +# them (memory, cpu), so on a cluster of 8c/16Gi + 4c/32Gi `resources set` and +# the installer anchored on DIFFERENT nodes. +# +# Unlike the installers, Go needs no generator: the contract is embedded verbatim +# with go:embed, so the vendored artifact is byte-identical to upstream and this +# gate is a plain diff. internal/resources/contract_test.go replays the +# contract's golden vectors through MaxRunCores/MaxRunGiB on every PR; this job +# is the other half — it catches the contract itself going stale. +# +# Pin, don't float (scripts/.client-runtime-ref), exactly as this repo already +# does for tracebloc/client and tracebloc/data-ingestors: an unrelated upstream +# commit must not redden every open CLI PR, and the weekly run catches a pin gone +# stale enough to matter. +# +# FAIL-CLOSED. client-runtime is private, so this needs a token GITHUB_TOKEN +# cannot provide; when it cannot read upstream the job FAILS rather than warning +# and exiting 0. A check that never executed must not report as a passing one — +# the activation-phase fail-open cli#536 had to remove from the backend-fixtures +# gate for exactly this reason. + +on: + schedule: + - cron: "0 6 * * 1" # weekly Monday, offset from chart-drift (05:00) + workflow_dispatch: + pull_request: + branches: [develop, main] + paths: + - "internal/resources/**" + - "scripts/.client-runtime-ref" + - ".github/workflows/envelope-contract-drift.yml" + +permissions: + contents: read + +jobs: + envelope-contract: + timeout-minutes: 10 + name: Envelope contract gate (pinned client-runtime ref) + runs-on: ubuntu-latest + steps: + - name: Checkout this CLI ref + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: cli + + - name: Resolve the pinned client-runtime ref + id: pin + # First non-comment, non-blank line of scripts/.client-runtime-ref — the + # same convention .client-ref and .data-ingestors-ref use. Shape is + # validated (SHA/branch/tag characters only, no "..") before it reaches + # the checkout action. + run: | + ref="$(grep -vE '^[[:space:]]*(#|$)' cli/scripts/.client-runtime-ref | head -1 | tr -d '[:space:]')" + if [ -z "$ref" ]; then + echo "::error file=scripts/.client-runtime-ref::no ref found — the first non-comment line must be a commit SHA" + exit 1 + fi + if ! printf '%s' "$ref" | grep -qE '^[A-Za-z0-9][A-Za-z0-9._/-]*$' || printf '%s' "$ref" | grep -q '\.\.'; then + echo "::error file=scripts/.client-runtime-ref::invalid ref shape: $ref" + exit 1 + fi + echo "ref=$ref" >> "$GITHUB_OUTPUT" + + - name: Mint a read-only installation token for client-runtime + id: token + # Least privilege per the backend#2157 sweep: named `repositories`, not + # owner-wide, and contents:read only — this job reads two files. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_TRAIN_APP_ID }} + private-key: ${{ secrets.RELEASE_TRAIN_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: client-runtime + permission-contents: read + + - name: Checkout tracebloc/client-runtime @ pinned ref + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: tracebloc/client-runtime + ref: ${{ steps.pin.outputs.ref }} + token: ${{ steps.token.outputs.token }} + path: client-runtime + persist-credentials: false + + - name: The vendored contract matches upstream, byte for byte + env: + PINNED_REF: ${{ steps.pin.outputs.ref }} + run: | + upstream="client-runtime/envelope_contract.json" + vendored="cli/internal/resources/envelope_contract.json" + if [ ! -f "$upstream" ]; then + echo "::error::$upstream is missing at $PINNED_REF — has the contract moved or been renamed?" + exit 1 + fi + if ! diff -u "$vendored" "$upstream"; then + echo "::error file=internal/resources/envelope_contract.json::the vendored envelope contract has drifted from tracebloc/client-runtime@$PINNED_REF" + echo "" + echo "To adopt the upstream change:" + echo " 1. cp /envelope_contract.json internal/resources/" + echo " 2. update the SHA in scripts/.client-runtime-ref" + echo " 3. go test ./internal/resources/... # the golden vectors WILL have moved" + echo "" + echo "If the overhead or the floors moved, that is a FLEET envelope change" + echo "(backend#2167, RFC-BACKEND-664 L0) — not a re-vendor. Say so on the PR." + exit 1 + fi + echo "vendored contract matches client-runtime@$PINNED_REF" + + - name: Upstream's own goldens are not stale against its own arithmetic + # A vendored contract can match upstream byte-for-byte while UPSTREAM's + # vectors have gone stale against upstream's code — in which case we are + # faithfully mirroring a lie. Re-derive them from client-runtime's own + # generator and require no diff. Pure-python, no cluster, no deps. + run: | + cd client-runtime + if [ ! -f scripts/gen_envelope_vectors.py ]; then + echo "::error::client-runtime@${{ steps.pin.outputs.ref }} has no scripts/gen_envelope_vectors.py — the contract's provenance cannot be verified" + exit 1 + fi + python3 scripts/gen_envelope_vectors.py + if ! git diff --exit-code -- envelope_contract.json; then + echo "::error::client-runtime@${{ steps.pin.outputs.ref }} carries goldens that its own generator does not reproduce." + echo "The pinned ref is not self-consistent — fix it upstream, then re-vendor and re-pin here." + exit 1 + fi + echo "upstream goldens reproduce from upstream code" + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: cli/go.mod + cache: true + cache-dependency-path: cli/go.sum + + - name: The CLI still agrees with the contract it vendored + run: | + cd cli + go test ./internal/resources/... -run 'Contract|Golden|DecisionA|Overhead|FloorText' -v diff --git a/internal/resources/contract.go b/internal/resources/contract.go new file mode 100644 index 00000000..65f050bf --- /dev/null +++ b/internal/resources/contract.go @@ -0,0 +1,99 @@ +package resources + +// contract.go carries the training-envelope contract, vendored from +// tracebloc/client-runtime (backend#2220, RFC-BACKEND-664 §P0). +// +// The question "how much of this machine may ONE training run have" used to be +// answered independently in four places: this package, the bash installer, its +// PowerShell twin, and a fifth 0.75-fraction policy inside client-runtime +// itself. None derived from the others, and two of them disagreed by +// construction — most sharply on the node tie-break, where this package ranked +// candidates (cpu, memory) and the bash installer ranked them (memory, cpu), so +// on a cluster of 8c/16Gi + 4c/32Gi `tracebloc resources set` and the installer +// anchored on DIFFERENT nodes and gave different answers about one machine. +// +// client-runtime now owns the arithmetic (node_sizing.envelope_from_allocatable) +// and its constants live in envelope_contract.json. Unlike the installers, Go +// needs no generator to read it: the file is embedded verbatim at compile time, +// so the vendored artifact is byte-identical to upstream and the cross-repo +// drift gate is a plain diff (.github/workflows/envelope-contract-drift.yml at +// scripts/.client-runtime-ref). +// +// What is NOT changed here is cli#143 Decision A. The number the user sets is +// still the per-run ceiling, written to RESOURCE_* verbatim; Overhead() is still +// a fit-check safety margin that is never subtracted from it. Only the duplicate +// *definition* of these four numbers is gone. + +import ( + _ "embed" + "encoding/json" + "fmt" + "sync" +) + +// contractBytes is the raw contract as vendored. Exposed for the drift test, +// which asserts the embedded bytes still parse and still carry vectors. +// +//go:embed envelope_contract.json +var contractBytes []byte + +// envelopeContract is the decoded shape. Only the fields this package needs are +// named: the anchors and rendering rules are the installers' and jobs-manager's +// business, and decoding them here would invite drift of a different kind. +type envelopeContract struct { + ContractVersion int `json:"contract_version"` + Overhead struct { + CPUMilli int64 `json:"cpu_millicores"` + MemoryBytes int64 `json:"memory_bytes"` + } `json:"overhead"` + Floor struct { + CPUMilli int64 `json:"cpu_millicores"` + MemoryBytes int64 `json:"memory_bytes"` + } `json:"floor"` + Vectors struct { + SingleNode []struct { + Label string `json:"label"` + AllocatableCPU string `json:"allocatable_cpu"` + AllocatableMemory string `json:"allocatable_memory"` + Expected *struct { + CPUMilli int64 `json:"cpu_millicores"` + MemoryBytes int64 `json:"memory_bytes"` + Viable bool `json:"viable"` + RenderGi struct { + CPU string `json:"cpu"` + Memory string `json:"memory"` + } `json:"render_gi"` + } `json:"expected"` + } `json:"single_node"` + } `json:"vectors"` +} + +// mustContract decodes and validates the embedded contract exactly once. +// +// It panics on a malformed contract, the same way regexp.MustCompile does for a +// bad literal pattern: the file is embedded at COMPILE time, so the only way it +// can be invalid is a hand-edit or a botched re-vendor, and that is a broken +// build rather than a runtime condition a user could hit. Silently falling back +// to defaults would be worse than a panic — a default here is a fifth policy, +// which is the whole thing backend#2220 removes. TestContractIsValid keeps the +// panic from ever reaching a release. +var mustContract = sync.OnceValue(func() envelopeContract { + var c envelopeContract + if err := json.Unmarshal(contractBytes, &c); err != nil { + panic(fmt.Sprintf("envelope_contract.json is not valid JSON: %v", err)) + } + if c.ContractVersion < 1 { + panic(fmt.Sprintf("envelope_contract.json has no usable contract_version: %d", c.ContractVersion)) + } + for name, v := range map[string]int64{ + "overhead.cpu_millicores": c.Overhead.CPUMilli, + "overhead.memory_bytes": c.Overhead.MemoryBytes, + "floor.cpu_millicores": c.Floor.CPUMilli, + "floor.memory_bytes": c.Floor.MemoryBytes, + } { + if v <= 0 { + panic(fmt.Sprintf("envelope_contract.json %s must be a positive int, got %d", name, v)) + } + } + return c +}) diff --git a/internal/resources/contract_test.go b/internal/resources/contract_test.go new file mode 100644 index 00000000..7891bd52 --- /dev/null +++ b/internal/resources/contract_test.go @@ -0,0 +1,191 @@ +package resources + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// backend#2220. The in-repo half of the ticket's definition of done: a mutation +// to the arithmetic in client-runtime's node_sizing.py must redden tests HERE, +// not just there. Two independent nets, and they fail for different reasons: +// +// - TestGoldenVectorsReplay pins that MaxRunCores/MaxRunGiB still agree with +// the contract's own vectors, so a re-vendored contract whose numbers moved +// cannot land silently. +// - TestDecisionAIsIntact pins cli#143's contract itself, independently of the +// contract file, so re-vendoring cannot quietly turn the overhead into +// something that gets subtracted from the user's number. + +func TestContractIsValid(t *testing.T) { + // The embedded contract is decoded by a panicking MustCompile-style helper, + // which is right for a compile-time asset but must never reach a release. + // This is the test that keeps that promise. + c := mustContract() + if c.ContractVersion < 1 { + t.Fatalf("contract_version = %d, want >= 1", c.ContractVersion) + } + if len(c.Vectors.SingleNode) == 0 { + t.Fatal("the vendored contract carries no single-node vectors — " + + "re-vendor it, or the replay below is asserting nothing") + } + if c.Overhead.CPUMilli != 1000 || c.Overhead.MemoryBytes != 3*gib { + t.Errorf("overhead moved: got %dm / %d bytes, want 1000m / %d bytes. "+ + "If that is intended it is a FLEET envelope change, not a re-vendor — "+ + "see backend#2167 and RFC-BACKEND-664 L0", + c.Overhead.CPUMilli, c.Overhead.MemoryBytes, 3*gib) + } + if c.Floor.CPUMilli != 1000 || c.Floor.MemoryBytes != 2*gib { + t.Errorf("floor moved: got %dm / %d bytes, want 1000m / %d bytes", + c.Floor.CPUMilli, c.Floor.MemoryBytes, 2*gib) + } +} + +func TestEmbeddedContractMatchesTheFileOnDisk(t *testing.T) { + // go:embed snapshots the file at compile time; this catches an edit that was + // made but not rebuilt, and — more usefully — proves the embedded bytes are + // the very bytes the cross-repo drift gate diffs. + onDisk, err := os.ReadFile("envelope_contract.json") + if err != nil { + t.Fatalf("reading envelope_contract.json: %v", err) + } + if string(onDisk) != string(contractBytes) { + t.Error("the embedded contract differs from envelope_contract.json on disk") + } +} + +// node builds a Machine the way LargestReadyNode would, from k8s quantity +// strings, so the vectors are exercised through the real parsing path. +func node(t *testing.T, cpu, mem string) Machine { + t.Helper() + c, err := resource.ParseQuantity(cpu) + if err != nil { + t.Fatalf("bad cpu %q: %v", cpu, err) + } + m, err := resource.ParseQuantity(mem) + if err != nil { + t.Fatalf("bad memory %q: %v", mem, err) + } + return Machine{CPU: c, Mem: m, GPU: map[corev1.ResourceName]resource.Quantity{}} +} + +func TestGoldenVectorsReplay(t *testing.T) { + var failures []string + for _, v := range mustContract().Vectors.SingleNode { + // Vectors whose expected is null are the unparseable cases: the contract + // says "I cannot answer", which k8s quantity parsing rejects long before + // this package sees it. Nothing for MaxRun* to agree with. + if v.Expected == nil { + continue + } + m := node(t, v.AllocatableCPU, v.AllocatableMemory) + + gotCores := MaxRunCores(m) + gotGiB := MaxRunGiB(m) + + // A machine the contract calls non-viable is one this package must clamp + // to zero — MaxRun* returning a positive number for it would be the CLI + // offering the user a ceiling the machine cannot host. + if !v.Expected.Viable { + if gotCores != 0 && gotGiB != 0 { + failures = append(failures, fmt.Sprintf( + "%s (%s/%s): contract says NOT viable, but MaxRunCores=%d MaxRunGiB=%d", + v.Label, v.AllocatableCPU, v.AllocatableMemory, gotCores, gotGiB)) + } + continue + } + + wantCores := int(v.Expected.CPUMilli / 1000) + wantGiB := int(v.Expected.MemoryBytes / gib) + if gotCores != wantCores || gotGiB != wantGiB { + failures = append(failures, fmt.Sprintf( + "%s (%s/%s): want %dc/%dGiB, got %dc/%dGiB", + v.Label, v.AllocatableCPU, v.AllocatableMemory, + wantCores, wantGiB, gotCores, gotGiB)) + } + } + if len(failures) > 0 { + t.Fatalf("MaxRunCores/MaxRunGiB no longer agree with the vendored "+ + "contract's vectors (contract v%d):\n %s\n\n"+ + "Either this package's arithmetic drifted, or the contract was "+ + "re-vendored with different numbers. Both are real changes — do not "+ + "'fix' this by editing the expectations.", + mustContract().ContractVersion, strings.Join(failures, "\n ")) + } +} + +func TestDecisionAIsIntact(t *testing.T) { + // cli#143 Decision A, pinned independently of the contract file: the number + // the user sets IS the per-run ceiling, and the overhead is a fit margin that + // is NEVER subtracted from it. backend#2220 deleted the duplicate derivation + // beside this rule; it did not touch the rule. If a future re-vendor makes + // DeriveTraining start shrinking the user's ask, this is what says so. + cpu := *resource.NewQuantity(6, resource.DecimalSI) + mem := *resource.NewQuantity(24*gib, resource.BinarySI) + + got := DeriveTraining(cpu, mem, "", resource.Quantity{}, false) + if got.CPU.Cmp(cpu) != 0 { + t.Errorf("DeriveTraining shrank the user's CPU: got %s, want %s", + got.CPU.String(), cpu.String()) + } + if got.Mem.Cmp(mem) != 0 { + t.Errorf("DeriveTraining shrank the user's memory: got %s, want %s", + got.Mem.String(), mem.String()) + } + if !got.HasCPUMem { + t.Error("DeriveTraining dropped HasCPUMem") + } + if got.HasGPU { + t.Error("DeriveTraining invented a GPU dimension") + } +} + +func TestOverheadIsStillAFitMarginOnly(t *testing.T) { + // The distinction the contract's own comment insists on: overhead is added to + // what the user asked for when checking fit, never subtracted from it. A + // 6c/24GiB ask must NOT fit a 6c/24GiB node, because the platform still needs + // its cut — that asymmetry is the whole design. + small := node(t, "6", "24Gi") + cpu := *resource.NewQuantity(6, resource.DecimalSI) + mem := *resource.NewQuantity(24*gib, resource.BinarySI) + if FitsNode(small, cpu, mem, "", resource.Quantity{}, false) { + t.Error("a full-node ask fit a node with no room for the platform overhead") + } + + big := node(t, "8", "28Gi") + if !FitsNode(big, cpu, mem, "", resource.Quantity{}, false) { + t.Error("a 6c/24Gi ask did not fit 8c/28Gi, which has room for the overhead") + } +} + +func TestFloorTextMatchesTheContractFloor(t *testing.T) { + // The user-facing strings are hand-written; the floor they describe is not. + // A re-vendor that moved the floor while these strings stayed put would make + // the CLI lie in its error messages. + c := mustContract() + if want := fmt.Sprintf("%d core", c.Floor.CPUMilli/1000); CoreFloorText() != want { + t.Errorf("CoreFloorText() = %q but the contract floor is %q", CoreFloorText(), want) + } + if want := fmt.Sprintf("%d GiB", c.Floor.MemoryBytes/gib); MemFloorText() != want { + t.Errorf("MemFloorText() = %q but the contract floor is %q", MemFloorText(), want) + } +} + +func TestContractJSONIsCanonicalFormatting(t *testing.T) { + // The cross-repo gate is a byte diff, so the vendored file must be the + // upstream bytes — not a re-serialised equivalent. Catches a well-meaning + // editor reformat that would redden the drift job for no real reason. + var probe map[string]json.RawMessage + if err := json.Unmarshal(contractBytes, &probe); err != nil { + t.Fatalf("vendored contract is not a JSON object: %v", err) + } + if !strings.HasSuffix(string(contractBytes), "}\n") { + t.Error("vendored contract should end with a closing brace and one newline, " + + "as client-runtime's generator writes it") + } +} diff --git a/internal/resources/envelope_contract.json b/internal/resources/envelope_contract.json new file mode 100644 index 00000000..4a4ae826 --- /dev/null +++ b/internal/resources/envelope_contract.json @@ -0,0 +1,507 @@ +{ + "contract_version": 1, + "issue": "tracebloc/backend#2220", + "rfc": "RFC-BACKEND-664 \u00a7P0", + "source_of_truth": "client-runtime/node_sizing.py::envelope_from_allocatable", + "regenerate_with": "python3 scripts/gen_envelope_vectors.py", + "description": [ + "The one definition of 'how much of this machine may a single training run", + "have'. Before this file the question was answered independently by the bash", + "installer, its PowerShell twin, the Go CLI, and a fourth 0.75-fraction", + "policy in node_sizing.py itself \u2014 four implementations, none derived from", + "the others, two of which disagreed by construction.", + "", + "The installer speaks bash, its twin speaks PowerShell and the CLI speaks", + "Go, so no consumer can call the Python. Each therefore vendors this file", + "and replays the golden vectors below through its own reader, in its own", + "test suite, drift-gated against this repo at a pinned ref. Mutate the", + "arithmetic without regenerating the vectors and this repo reddens at once;", + "regenerate them and every consumer's gate reddens. That is the ticket's", + "definition of done." + ], + "overhead": { + "cpu_millicores": 1000, + "memory_bytes": 3221225472, + "why": [ + "Reserved for the platform \u2014 kubelet, the k3s server, jobs-manager, the", + "requests proxy, MySQL \u2014 so a run sized to the ceiling still leaves the", + "cluster able to schedule and report it. 1 CPU / 3 GiB is what every live", + "producer already subtracts; keeping it identical is what makes this", + "ticket a no-op on installed edges rather than a fleet envelope change." + ] + }, + "floor": { + "cpu_millicores": 1000, + "memory_bytes": 2147483648, + "why": [ + "Below this a training run is not worth scheduling, so a machine that", + "cannot clear it after overhead is reported NON-VIABLE. It is not a clamp:", + "clamping up is how the 4 GiB WSL2 machine ends up asked to host an 8 GiB", + "pod that stays Pending forever." + ] + }, + "fallback_literal": { + "cpu": "2", + "memory": "8Gi", + "why": [ + "The historical fixed envelope, still hardcoded in six physical places", + "across three repos (_TRAINING_DEFAULT, the ps1's two inline copies, the", + "chart template's two, DEFAULT_JOB_RESOURCES). Recorded here so those", + "become readers too. NOTE it is LARGER than the floor: handing it to a", + "non-viable machine is the sub-8GiB bug, not a safe default." + ] + }, + "anchors": { + "largest": { + "question": "What is the largest run this machine could host?", + "rule": "the single schedulable node maximising (cpu_millicores, memory_bytes) lexicographically", + "used_by": [ + "client installer \u2014 sizing the value it writes", + "cli resources set \u2014 clamping the wizard prompt and the fit check" + ], + "why": [ + "A pod's resources all come from ONE node, so this question is", + "single-node by nature and must never sum across the cluster.", + "The tie-break is itself a consolidation: bash ranked nodes (memory,", + "cpu) and cli's nodeLarger ranked them (cpu, memory), so on a cluster of", + "8c/16Gi + 4c/32Gi the two anchored on different nodes. Nobody chose", + "that; it fell out of two independent implementations." + ] + }, + "every": { + "question": "What envelope fits on EVERY schedulable node?", + "rule": "minimum cpu_millicores and minimum memory_bytes, taken independently", + "used_by": [ + "client-runtime derive path (DERIVE_JOB_ENVELOPE, off by default)" + ], + "why": [ + "An envelope sized to the biggest node cannot schedule on the smallest.", + "Independent minima are deliberately conservative \u2014 a heterogeneous", + "cluster with a small system nodepool degrades toward the floor rather", + "than risking an unschedulable pod." + ] + } + }, + "rendering": { + "Gi": "cpu=,memory=Gi \u2014 installer + CLI surface", + "Mi": "cpu=,memory=Mi \u2014 jobs-manager pod spec", + "rounding": "always floor; a ceiling that rounds up is not a ceiling" + }, + "skipped_nodes": [ + "spec.unschedulable (cordoned)", + "allocatable cpu or memory unparseable" + ], + "vectors": { + "single_node": [ + { + "label": "field-8c-32gi", + "allocatable_cpu": "8", + "allocatable_memory": "32Gi", + "expected": { + "cpu_millicores": 7000, + "memory_bytes": 31138512896, + "viable": true, + "render_gi": { + "cpu": "7", + "memory": "29Gi" + }, + "render_mi": { + "cpu": "7", + "memory": "29696Mi" + } + } + }, + { + "label": "field-16c-64gi", + "allocatable_cpu": "16", + "allocatable_memory": "64Gi", + "expected": { + "cpu_millicores": 15000, + "memory_bytes": 65498251264, + "viable": true, + "render_gi": { + "cpu": "15", + "memory": "61Gi" + }, + "render_mi": { + "cpu": "15", + "memory": "62464Mi" + } + } + }, + { + "label": "field-4c-16gi", + "allocatable_cpu": "4", + "allocatable_memory": "16Gi", + "expected": { + "cpu_millicores": 3000, + "memory_bytes": 13958643712, + "viable": true, + "render_gi": { + "cpu": "3", + "memory": "13Gi" + }, + "render_mi": { + "cpu": "3", + "memory": "13312Mi" + } + } + }, + { + "label": "k8s-millicores-and-mi", + "allocatable_cpu": "15500m", + "allocatable_memory": "63928Mi", + "expected": { + "cpu_millicores": 14000, + "memory_bytes": 63812141056, + "viable": true, + "render_gi": { + "cpu": "14", + "memory": "59Gi" + }, + "render_mi": { + "cpu": "14", + "memory": "60856Mi" + } + } + }, + { + "label": "k8s-raw-bytes", + "allocatable_cpu": "8", + "allocatable_memory": "33285996544", + "expected": { + "cpu_millicores": 7000, + "memory_bytes": 30064771072, + "viable": true, + "render_gi": { + "cpu": "7", + "memory": "28Gi" + }, + "render_mi": { + "cpu": "7", + "memory": "28672Mi" + } + } + }, + { + "label": "laptop-2c-8gi", + "allocatable_cpu": "2", + "allocatable_memory": "8Gi", + "expected": { + "cpu_millicores": 1000, + "memory_bytes": 5368709120, + "viable": true, + "render_gi": { + "cpu": "1", + "memory": "5Gi" + }, + "render_mi": { + "cpu": "1", + "memory": "5120Mi" + } + } + }, + { + "label": "exact-floor", + "allocatable_cpu": "2", + "allocatable_memory": "5Gi", + "expected": { + "cpu_millicores": 1000, + "memory_bytes": 2147483648, + "viable": true, + "render_gi": { + "cpu": "1", + "memory": "2Gi" + }, + "render_mi": { + "cpu": "1", + "memory": "2048Mi" + } + } + }, + { + "label": "one-byte-under-mem-floor", + "allocatable_cpu": "2", + "allocatable_memory": "5368709119", + "expected": { + "cpu_millicores": 1000, + "memory_bytes": 2147483647, + "viable": false, + "render_gi": { + "cpu": "1", + "memory": "1Gi" + }, + "render_mi": { + "cpu": "1", + "memory": "2047Mi" + } + } + }, + { + "label": "one-milli-under-cpu-floor", + "allocatable_cpu": "1999m", + "allocatable_memory": "8Gi", + "expected": { + "cpu_millicores": 0, + "memory_bytes": 5368709120, + "viable": false, + "render_gi": { + "cpu": "0", + "memory": "5Gi" + }, + "render_mi": { + "cpu": "0", + "memory": "5120Mi" + } + } + }, + { + "label": "wsl2-too-small-4gi", + "allocatable_cpu": "4", + "allocatable_memory": "4Gi", + "expected": { + "cpu_millicores": 3000, + "memory_bytes": 1073741824, + "viable": false, + "render_gi": { + "cpu": "3", + "memory": "1Gi" + }, + "render_mi": { + "cpu": "3", + "memory": "1024Mi" + } + } + }, + { + "label": "absurdly-small", + "allocatable_cpu": "500m", + "allocatable_memory": "512Mi", + "expected": { + "cpu_millicores": 0, + "memory_bytes": 0, + "viable": false, + "render_gi": { + "cpu": "0", + "memory": "0Gi" + }, + "render_mi": { + "cpu": "0", + "memory": "0Mi" + } + } + }, + { + "label": "unparseable-cpu", + "allocatable_cpu": "eight", + "allocatable_memory": "8Gi", + "expected": null + }, + { + "label": "unparseable-memory", + "allocatable_cpu": "8", + "allocatable_memory": "lots", + "expected": null + } + ], + "multi_node": [ + { + "label": "heterogeneous-incomparable", + "nodes": [ + { + "cpu": "8", + "memory": "16Gi" + }, + { + "cpu": "4", + "memory": "32Gi" + } + ], + "anchored": { + "largest": { + "allocatable_cpu_millicores": 8000, + "allocatable_memory_bytes": 17179869184, + "expected": { + "cpu_millicores": 7000, + "memory_bytes": 13958643712, + "viable": true, + "render_gi": { + "cpu": "7", + "memory": "13Gi" + }, + "render_mi": { + "cpu": "7", + "memory": "13312Mi" + } + } + }, + "every": { + "allocatable_cpu_millicores": 4000, + "allocatable_memory_bytes": 17179869184, + "expected": { + "cpu_millicores": 3000, + "memory_bytes": 13958643712, + "viable": true, + "render_gi": { + "cpu": "3", + "memory": "13Gi" + }, + "render_mi": { + "cpu": "3", + "memory": "13312Mi" + } + } + } + } + }, + { + "label": "server-plus-small-agent", + "nodes": [ + { + "cpu": "16", + "memory": "64Gi" + }, + { + "cpu": "2", + "memory": "8Gi" + } + ], + "anchored": { + "largest": { + "allocatable_cpu_millicores": 16000, + "allocatable_memory_bytes": 68719476736, + "expected": { + "cpu_millicores": 15000, + "memory_bytes": 65498251264, + "viable": true, + "render_gi": { + "cpu": "15", + "memory": "61Gi" + }, + "render_mi": { + "cpu": "15", + "memory": "62464Mi" + } + } + }, + "every": { + "allocatable_cpu_millicores": 2000, + "allocatable_memory_bytes": 8589934592, + "expected": { + "cpu_millicores": 1000, + "memory_bytes": 5368709120, + "viable": true, + "render_gi": { + "cpu": "1", + "memory": "5Gi" + }, + "render_mi": { + "cpu": "1", + "memory": "5120Mi" + } + } + } + } + }, + { + "label": "identical-pair", + "nodes": [ + { + "cpu": "8", + "memory": "32Gi" + }, + { + "cpu": "8", + "memory": "32Gi" + } + ], + "anchored": { + "largest": { + "allocatable_cpu_millicores": 8000, + "allocatable_memory_bytes": 34359738368, + "expected": { + "cpu_millicores": 7000, + "memory_bytes": 31138512896, + "viable": true, + "render_gi": { + "cpu": "7", + "memory": "29Gi" + }, + "render_mi": { + "cpu": "7", + "memory": "29696Mi" + } + } + }, + "every": { + "allocatable_cpu_millicores": 8000, + "allocatable_memory_bytes": 34359738368, + "expected": { + "cpu_millicores": 7000, + "memory_bytes": 31138512896, + "viable": true, + "render_gi": { + "cpu": "7", + "memory": "29Gi" + }, + "render_mi": { + "cpu": "7", + "memory": "29696Mi" + } + } + } + } + }, + { + "label": "one-cordoned-out", + "nodes": [ + { + "cpu": "16", + "memory": "64Gi", + "unschedulable": true + }, + { + "cpu": "4", + "memory": "16Gi" + } + ], + "anchored": { + "largest": { + "allocatable_cpu_millicores": 4000, + "allocatable_memory_bytes": 17179869184, + "expected": { + "cpu_millicores": 3000, + "memory_bytes": 13958643712, + "viable": true, + "render_gi": { + "cpu": "3", + "memory": "13Gi" + }, + "render_mi": { + "cpu": "3", + "memory": "13312Mi" + } + } + }, + "every": { + "allocatable_cpu_millicores": 4000, + "allocatable_memory_bytes": 17179869184, + "expected": { + "cpu_millicores": 3000, + "memory_bytes": 13958643712, + "viable": true, + "render_gi": { + "cpu": "3", + "memory": "13Gi" + }, + "render_mi": { + "cpu": "3", + "memory": "13312Mi" + } + } + } + } + } + ] + } +} diff --git a/internal/resources/set.go b/internal/resources/set.go index 8c322309..3e9e02be 100644 --- a/internal/resources/set.go +++ b/internal/resources/set.go @@ -27,24 +27,34 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) -// overheadCPUMilli / overheadMemBytes are tracebloc's fixed platform reservation -// — the single documented constant the design calls for (~1 core / 3 GiB). It is -// used ONLY as the fit-check safety margin (see FitsNode / MaxRunCores / -// MaxRunGiB); it is never subtracted from the per-run ceiling the user sets. -const ( - overheadCPUMilli = 1000 // ~1 CPU core - overheadMemBytes = 3 * (1 << 30) // 3 GiB - minRunCPUMilli = 1000 // per-run floor: ~1 core - minRunMemBytes = 2 * (1 << 30) // per-run floor: 2 GiB - gib = int64(1) << 30 // bytes per GiB -) +// The platform reservation and the per-run floors. These used to be four +// literals typed out here, and again in client/scripts/lib/install-client-helm.sh, +// and again in client/scripts/install-k8s.ps1 — three copies of the same policy, +// none derived from the others (backend#2220). They now come from the contract +// vendored in contract.go, whose arithmetic lives in client-runtime's +// node_sizing.envelope_from_allocatable. +// +// Functions, not vars, for the same reason Overhead() is a function: a package +// var here would be mutable from anywhere in the process, and the whole point is +// that exactly one place decides these numbers. +// +// Semantics are UNCHANGED. The overhead is still used ONLY as the fit-check +// safety margin (FitsNode / MaxRunCores / MaxRunGiB) and is still never +// subtracted from the per-run ceiling the user sets — cli#143 Decision A. What +// went away is the duplicate definition, not the design. +func overheadCPUMilli() int64 { return mustContract().Overhead.CPUMilli } +func overheadMemBytes() int64 { return mustContract().Overhead.MemoryBytes } +func minRunCPUMilli() int64 { return mustContract().Floor.CPUMilli } +func minRunMemBytes() int64 { return mustContract().Floor.MemoryBytes } + +const gib = int64(1) << 30 // bytes per GiB // Overhead returns tracebloc's fixed platform reservation as quantities. Kept a // function (not exported vars) so callers can't mutate the shared value — a // resource.Quantity's Add mutates its receiver. func Overhead() (cpu, mem resource.Quantity) { - return *resource.NewMilliQuantity(overheadCPUMilli, resource.DecimalSI), - *resource.NewQuantity(overheadMemBytes, resource.BinarySI) + return *resource.NewMilliQuantity(overheadCPUMilli(), resource.DecimalSI), + *resource.NewQuantity(overheadMemBytes(), resource.BinarySI) } // DeriveTraining turns a chosen per-run ceiling into the Training spec written to @@ -129,8 +139,8 @@ func FitsNode(node Machine, cpu, mem resource.Quantity, gpuName corev1.ResourceN // the overhead on this node: floor(nodeCPU - overheadCPU). Never negative. This is // the bound the wizard clamps the "cores" prompt to, so over-asking is impossible. func MaxRunCores(node Machine) int { - milli := node.CPU.MilliValue() - overheadCPUMilli - if milli < minRunCPUMilli { + milli := node.CPU.MilliValue() - overheadCPUMilli() + if milli < minRunCPUMilli() { return 0 } return int(milli / 1000) @@ -139,8 +149,8 @@ func MaxRunCores(node Machine) int { // MaxRunGiB is the largest whole-GiB per-run memory ceiling that still leaves room // for the overhead: floor(nodeMem - overheadMem), in GiB. Never negative. func MaxRunGiB(node Machine) int { - b := node.Mem.Value() - overheadMemBytes - if b < minRunMemBytes { + b := node.Mem.Value() - overheadMemBytes() + if b < minRunMemBytes() { return 0 } return int(b / gib) @@ -162,8 +172,8 @@ func MachineGPU(m Machine) (name corev1.ResourceName, count int64, ok bool) { // BelowCoreFloor / BelowMemFloor enforce the per-run minimum (~1 core / 2 GiB): // a run smaller than this can't hold a training job and is almost always a typo. -func BelowCoreFloor(cpu resource.Quantity) bool { return cpu.MilliValue() < minRunCPUMilli } -func BelowMemFloor(mem resource.Quantity) bool { return mem.Value() < minRunMemBytes } +func BelowCoreFloor(cpu resource.Quantity) bool { return cpu.MilliValue() < minRunCPUMilli() } +func BelowMemFloor(mem resource.Quantity) bool { return mem.Value() < minRunMemBytes() } // CoreFloorText / MemFloorText are the floor values as user-facing strings, for error messages. func CoreFloorText() string { return "1 core" } diff --git a/scripts/.client-runtime-ref b/scripts/.client-runtime-ref new file mode 100644 index 00000000..d745d166 --- /dev/null +++ b/scripts/.client-runtime-ref @@ -0,0 +1,12 @@ +# Pinned tracebloc/client-runtime commit that +# internal/resources/envelope_contract.json is vendored from (backend#2220). +# +# Pin, don't float — the same rule scripts/.client-ref and +# scripts/.data-ingestors-ref follow: an unrelated client-runtime commit must +# not redden every open CLI PR. The weekly run catches a pin gone stale. +# +# To adopt an upstream contract change: +# 1. cp /envelope_contract.json internal/resources/ +# 2. update the SHA below +# 3. go test ./internal/resources/... — the golden vectors WILL have moved +6293d15f7025d06bc85886df93456e62767a8d45