diff --git a/internal/cli/resources.go b/internal/cli/resources.go new file mode 100644 index 00000000..180f6bd0 --- /dev/null +++ b/internal/cli/resources.go @@ -0,0 +1,237 @@ +package cli + +import ( + "context" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/resources" + "github.com/tracebloc/cli/internal/ui" +) + +// newResourcesCmd wires the top-level `tracebloc resources` command (cli#143): +// the one-knob view of "how much of this machine tracebloc may use." Bare +// `tracebloc resources` SHOWS the current picture (P1) — machine capacity and +// the ceiling a single training run may use — with no mutations. +// +// It is deliberately top-level, NOT under `cluster`: the user concept is about +// THIS MACHINE, not Kubernetes internals, and the design locked "top-level +// command" (issue #143, approved 2026-07-06). +// +// Raising the allowance (`set --cpu/--memory`, `set max`) and the macOS VM raise +// are approved later phases (P2/P3) — see newResourcesSetCmd and the deferral +// note in runResourcesSetDeferred. +func newResourcesCmd() *cobra.Command { + var ( + kubeconfigPath string + contextOverride string + nsOverride string + ) + + cmd := &cobra.Command{ + Use: "resources", + Short: "Show how much of this machine tracebloc may use", + Long: `Shows, in plain terms, how much of this machine tracebloc may use: + + • This machine — the CPU and memory the cluster can schedule + • tracebloc uses — the ceiling a single training run may use right now + +No Kubernetes concepts, no YAML — one number for the machine and one for +tracebloc's share of it. + +Raising the share (` + "`tracebloc resources set`" + `) is a later phase; today it's +set when this machine is first connected. Run with --verbose for the +per-node breakdown and the raw values. + +Exit codes: + 0 shown + 3 kubeconfig could not be loaded / cluster unreachable + 4 cluster reachable but no tracebloc client found here`, + // Bare `tracebloc resources` SHOWS (P1). Raising the share lives in the + // `set` subcommand, whose flags + optional `max` positional parse cleanly + // and reach an honest deferral (P2 not built) — see newResourcesSetCmd. + // NoArgs so a stray token (`resources bogus`) gets cobra's "unknown + // command", not a silently-ignored SHOW. + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + opts := cluster.KubeconfigOptions{ + Path: kubeconfigPath, + Context: contextOverride, + Namespace: nsOverride, + } + return runResourcesShow(cmd.Context(), printerFor(cmd), opts) + }, + } + + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", "", + "path to the kubeconfig file (default: $KUBECONFIG, then ~/.kube/config)") + cmd.Flags().StringVar(&contextOverride, "context", "", + "name of the kubeconfig context to use (default: kubeconfig's current-context)") + cmd.Flags().StringVarP(&nsOverride, "namespace", "n", "", + "namespace where your tracebloc client is installed (default: the context's namespace, or 'default')") + + // `set` (P2, deferred): wired now so the approved invocation shape parses and + // reaches the honest deferral instead of a cobra flag error. + cmd.AddCommand(newResourcesSetCmd()) + + return cmd +} + +// newResourcesSetCmd wires `tracebloc resources set` — the approved-but-unbuilt +// P2 that RAISES how much of this machine tracebloc may use. Two forms, both +// locked in the #143 design: +// +// tracebloc resources set --cpu 4 --memory 16Gi # explicit per-run ceiling +// tracebloc resources set max # give a run the whole machine +// +// The `--cpu`/`--memory` flags and the optional `max` positional are registered +// now for one reason: so these invocations PARSE cleanly and reach the honest +// exit-1 deferral in runResourcesSetDeferred, instead of dying on cobra's +// "unknown flag: --cpu". The shape matches the approved design so P2 slots in +// behind it once the Helm-values persistence path lands. It mutates nothing. +func newResourcesSetCmd() *cobra.Command { + setCmd := &cobra.Command{ + Use: "set [max]", + Short: "Raise how much of this machine tracebloc may use (coming soon)", + Long: `Raise the per-training-run ceiling — how much of this machine a single +training run may use. + + tracebloc resources set --cpu 4 --memory 16Gi set an explicit ceiling + tracebloc resources set max let a run use the whole machine + +This is an approved but not-yet-built phase. Today the ceiling is set when this +machine is first connected; run ` + "`tracebloc resources`" + ` to see it.`, + // Optional single positional; when present it can only be `max` (the one + // non-flag form in the design). Both guards run so `set max extra` and + // `set bogus` are rejected rather than silently accepted. + ValidArgs: []string{"max"}, + Args: cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs), + RunE: func(cmd *cobra.Command, _ []string) error { + return runResourcesSetDeferred(printerFor(cmd)) + }, + } + // Registered (unbound) purely so parsing succeeds; P2 will read them. + setCmd.Flags().String("cpu", "", "per-run CPU ceiling (e.g. 4 or 500m)") + setCmd.Flags().String("memory", "", "per-run memory ceiling (e.g. 16Gi)") + return setCmd +} + +// runResourcesShow renders the read-only allocation view. It resolves the +// cluster exactly like the data commands (active-client binding + cluster-wide +// fallback scan, exit 3 for kubeconfig / 4 for no-release), reads the machine's +// capacity from node allocatable, and reads the per-run training ceiling from +// the jobs-manager env — the same source `cluster doctor` parses, so the two +// never disagree. +func runResourcesShow(ctx context.Context, p *ui.Printer, opts cluster.KubeconfigOptions) error { + p.Banner("tracebloc", "machine resources") + + binding := bindActiveClientNamespace(&opts) + target, err := resolveClusterTarget(ctx, p, opts, binding, false) + if err != nil { + return binding.explain(err) + } + return renderResources(ctx, p, target) +} + +// renderResources is the post-resolution half of `resources show`: given an +// already-resolved cluster target, it reads capacity + the training ceiling and +// prints the view. Split out so it's exercisable with a fake clientset without +// going through the real kubeconfig-load path (same seam ingestion_run_test uses). +func renderResources(ctx context.Context, p *ui.Printer, target *clusterTarget) error { + resolved, cs, release := target.Resolved, target.Clientset, target.Release + + // Machine capacity: sum of Ready nodes' allocatable. A node-list failure is + // not fatal to the whole view — we still show the training ceiling — but it + // is called out so the machine line isn't silently zero. + var machine resources.Machine + nodeErr := error(nil) + if nodes, lerr := cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{}); lerr != nil { + nodeErr = lerr + } else { + machine = resources.MachineCapacity(nodes.Items) + } + + env := resources.JobsManagerEnv(ctx, cs, resolved.Namespace, release.ReleaseName) + train := resources.ParseTraining(env) + + p.Section("This machine") + if nodeErr != nil { + p.Field("capacity", "unavailable") + p.Hintf(" couldn't read node capacity: %v", nodeErr) + } else { + p.Field("capacity", machineLine(machine)) + } + + p.Section("tracebloc uses") + p.Field("per training run", trainingLine(train)) + + if p.Verbose() { + p.Section("Details") + p.Field("namespace", resolved.Namespace) + p.Field("client", release.ReleaseName) + if raw := firstNonEmptyEnv(env, "RESOURCE_LIMITS", "RESOURCE_REQUESTS"); raw != "" { + p.Field("resource env", raw) + } else { + p.Field("resource env", "(unset — using chart default "+resources.DefaultTraining+")") + } + if nodeErr == nil && len(machine.GPU) == 0 { + p.Field("gpu", "none detected") + } + } + + p.Newline() + p.Hintf("Raising tracebloc's share (`tracebloc resources set`) is coming; today it's set when this machine is connected.") + return nil +} + +// machineLine renders the machine-capacity value: "8 CPU · 32 GiB" (+ " · 1 GPU" +// when a device is present). +func machineLine(m resources.Machine) string { + line := resources.FormatCPU(m.CPU) + " · " + resources.FormatMem(m.Mem) + for name, qty := range m.GPU { + line += " · " + resources.FormatGPU(name, qty) + } + return line +} + +// trainingLine renders the per-run ceiling: "up to 2 CPU · 8 GiB" (+ GPU). +func trainingLine(t resources.Training) string { + line := "up to " + resources.FormatCPU(t.CPU) + " · " + resources.FormatMem(t.Mem) + if t.HasGPU { + line += " · " + resources.FormatGPU(t.GPUName, t.GPU) + } + return line +} + +// firstNonEmptyEnv returns the first present, non-empty value among keys. +func firstNonEmptyEnv(env map[string]string, keys ...string) string { + for _, k := range keys { + if v := env[k]; v != "" { + return v + } + } + return "" +} + +// runResourcesSetDeferred prints an honest "not in this build" message for the +// approved-but-unbuilt P2 (`set --cpu/--memory`, `set max`). Building it safely +// needs a persistence path the shipped groundwork doesn't yet re-expose to the +// CLI: the value must be written to Helm values (a `kubectl set env` is reverted +// by the hourly auto-upgrade CronJob), and `helm upgrade` needs the chart +// reference the installer resolves via TRACEBLOC_HELM_REPO_NAME / a dev path — +// not recoverable from `helm list` alone. Rather than shell Helm blindly at a +// customer's live training cluster, `set` is deferred to its own change. +// +// Both `set` forms funnel here (the positional-vs-flags distinction is P2's to +// consume), so the message names the command, not the specific verb. +func runResourcesSetDeferred(p *ui.Printer) error { + p.Banner("tracebloc", "machine resources") + p.Errorf("`tracebloc resources set` isn't supported in this build yet.") + p.Hintf("Today, how much of this machine tracebloc may use is set when the machine is first connected.") + p.Hintf("Run `tracebloc resources` to see the current allocation.") + // Silent (err == nil): the ✖ + hints above already explained it, so main() + // must not print a redundant "Error:" line (same contract as cluster doctor). + return &exitError{code: 1, err: nil} +} diff --git a/internal/cli/resources_test.go b/internal/cli/resources_test.go new file mode 100644 index 00000000..183a4371 --- /dev/null +++ b/internal/cli/resources_test.go @@ -0,0 +1,245 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/ui" +) + +func resNode(name, cpu, mem string, extra ...string) *corev1.Node { + alloc := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpu), + corev1.ResourceMemory: resource.MustParse(mem), + } + if len(extra) == 2 { + alloc[corev1.ResourceName(extra[0])] = resource.MustParse(extra[1]) + } + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: corev1.NodeStatus{ + Allocatable: alloc, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } +} + +func resJMDeploy(release string, env map[string]string) *appsv1.Deployment { + var vars []corev1.EnvVar + for k, v := range env { + vars = append(vars, corev1.EnvVar{Name: k, Value: v}) + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: release + "-jobs-manager", Namespace: "tracebloc"}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "jobs-manager", Env: vars}}}, + }, + }, + } +} + +func resTarget(cs *fake.Clientset) *clusterTarget { + return &clusterTarget{ + Resolved: &cluster.ResolvedConfig{Namespace: "tracebloc"}, + Clientset: cs, + Release: &cluster.ParentRelease{ReleaseName: "tb"}, + } +} + +// TestRenderResources_ShowsMachineAndTrainingCeiling: the happy path renders the +// machine capacity (summed allocatable) and the per-run training ceiling read +// from the jobs-manager env. +func TestRenderResources_ShowsMachineAndTrainingCeiling(t *testing.T) { + cs := fake.NewClientset( + resNode("n1", "8", "32Gi"), + resJMDeploy("tb", map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi", "RESOURCE_REQUESTS": "cpu=4,memory=16Gi"}), + ) + var buf bytes.Buffer + if err := renderResources(context.Background(), ui.New(&buf, ui.WithColor(false)), resTarget(cs)); err != nil { + t.Fatalf("renderResources: %v", err) + } + out := buf.String() + for _, want := range []string{"This machine", "8 CPU · 32 GiB", "tracebloc uses", "up to 4 CPU · 16 GiB"} { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + // No Kubernetes vocabulary must leak into the default view. + for _, banned := range []string{"allocatable", "RESOURCE_LIMITS", "limits", "requests"} { + if strings.Contains(out, banned) { + t.Errorf("leaked k8s term %q in default view:\n%s", banned, out) + } + } +} + +// TestRenderResources_ChartDefaultWhenEnvUnset: with no RESOURCE_* env, the +// ceiling reported is the chart default (cpu=2,memory=8Gi), not "unknown". +func TestRenderResources_ChartDefaultWhenEnvUnset(t *testing.T) { + cs := fake.NewClientset(resNode("n1", "8", "32Gi"), resJMDeploy("tb", map[string]string{})) + var buf bytes.Buffer + if err := renderResources(context.Background(), ui.New(&buf, ui.WithColor(false)), resTarget(cs)); err != nil { + t.Fatalf("renderResources: %v", err) + } + if !strings.Contains(buf.String(), "up to 2 CPU · 8 GiB") { + t.Errorf("want chart-default ceiling 2 CPU · 8 GiB:\n%s", buf.String()) + } +} + +// TestRenderResources_GPUSurfaced: a node exposing a GPU shows it on the machine +// line, and a GPU training run shows it on the ceiling line. +func TestRenderResources_GPUSurfaced(t *testing.T) { + cs := fake.NewClientset( + resNode("n1", "8", "32Gi", "nvidia.com/gpu", "1"), + resJMDeploy("tb", map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi", "GPU_LIMITS": "nvidia.com/gpu=1"}), + ) + var buf bytes.Buffer + if err := renderResources(context.Background(), ui.New(&buf, ui.WithColor(false)), resTarget(cs)); err != nil { + t.Fatalf("renderResources: %v", err) + } + if c := strings.Count(buf.String(), "1 GPU"); c < 2 { + t.Errorf("expected GPU on both machine + training lines, got %d occurrences:\n%s", c, buf.String()) + } +} + +// TestRenderResources_NodeListErrorIsNotFatal: even when node capacity can't be +// read, the training ceiling still renders and the machine line says so. +func TestRenderResources_VerboseShowsRawEnv(t *testing.T) { + cs := fake.NewClientset(resNode("n1", "8", "32Gi"), resJMDeploy("tb", map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi"})) + var buf bytes.Buffer + if err := renderResources(context.Background(), ui.New(&buf, ui.WithColor(false), ui.WithVerbose(true)), resTarget(cs)); err != nil { + t.Fatalf("renderResources: %v", err) + } + out := buf.String() + // Verbose is the ONLY place the raw env + namespace/client are allowed. + for _, want := range []string{"Details", "cpu=4,memory=16Gi", "tracebloc"} { + if !strings.Contains(out, want) { + t.Errorf("verbose view missing %q:\n%s", want, out) + } + } +} + +// TestRunResourcesShow_BadKubeconfigExit3: a broken kubeconfig fails with exit 3 +// before any cluster read (mirrors runDataList's contract). +func TestRunResourcesShow_BadKubeconfigExit3(t *testing.T) { + bad := filepath.Join(t.TempDir(), "broken.yaml") + if err := os.WriteFile(bad, []byte("}{ not valid kubeconfig"), 0o644); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + err := runResourcesShow(context.Background(), ui.New(&buf, ui.WithColor(false)), + cluster.KubeconfigOptions{Path: bad}) + var ee *exitError + if !errors.As(err, &ee) || ee.Code() != 3 { + t.Fatalf("err = %v, want *exitError code 3", err) + } +} + +// TestRunResourcesSetDeferred_HonestExit1: the deferral handler returns a +// silent (err==nil inner, so main() prints no extra "Error:" line) exit-1 with +// an honest "not supported yet" message that points back at the SHOW view — +// never a bogus success or a mangled cluster mutation. +func TestRunResourcesSetDeferred_HonestExit1(t *testing.T) { + var buf bytes.Buffer + err := runResourcesSetDeferred(ui.New(&buf, ui.WithColor(false))) + var ee *exitError + if !errors.As(err, &ee) || ee.Code() != 1 { + t.Fatalf("err = %v, want *exitError code 1", err) + } + if !IsSilentError(err) { + t.Errorf("deferred message already printed a ✖ block, so the error must be silent") + } + out := buf.String() + if !strings.Contains(out, "isn't supported in this build yet") { + t.Errorf("missing honest not-yet-supported message:\n%s", out) + } + if !strings.Contains(out, "tracebloc resources") { + t.Errorf("deferral should point back at `tracebloc resources`:\n%s", out) + } +} + +// TestResourcesSet_DesignedInvocationsReachDeferral is the regression for +// finding #1: the two locked-design invocations must PARSE (the `--cpu`/ +// `--memory` flags and the `max` positional are wired on the `set` subcommand) +// and reach the honest exit-1 deferral — NOT die on cobra's "unknown flag: +// --cpu". Driven through the real root tree so flag/arg parsing is exercised +// end-to-end, which the direct-call test above cannot cover. +func TestResourcesSet_DesignedInvocationsReachDeferral(t *testing.T) { + for _, args := range [][]string{ + {"resources", "set", "--cpu", "4", "--memory", "16Gi"}, + {"resources", "set", "max"}, + } { + root := NewRootCmd(BuildInfo{Version: "test"}) + root.SetArgs(args) + var buf bytes.Buffer + root.SetOut(&buf) + root.SetErr(&buf) + err := root.Execute() + + // The whole point: it must not be a cobra parse failure. + if err != nil && strings.Contains(err.Error(), "unknown flag") { + t.Fatalf("%v: hit cobra unknown-flag error, not the honest deferral: %v", args, err) + } + var ee *exitError + if !errors.As(err, &ee) || ee.Code() != 1 { + t.Fatalf("%v: err = %v, want honest *exitError code 1", args, err) + } + out := buf.String() + if !strings.Contains(out, "isn't supported in this build yet") { + t.Errorf("%v: missing not-yet-supported message:\n%s", args, out) + } + if !strings.Contains(out, "tracebloc resources") { + t.Errorf("%v: deferral should point at `tracebloc resources`:\n%s", args, out) + } + } +} + +// TestResourcesBare_RoutesToShow: bare `tracebloc resources` still routes to the +// SHOW path (not the `set` deferral, not an "unknown command"). Proven through +// the tree by feeding a broken --kubeconfig and asserting SHOW's exit-3 +// kubeconfig-load contract — an exit 1 here would mean it wrongly hit the +// deferral, and a nil error would mean it never ran SHOW. +func TestResourcesBare_RoutesToShow(t *testing.T) { + bad := filepath.Join(t.TempDir(), "broken.yaml") + if err := os.WriteFile(bad, []byte("}{ not valid kubeconfig"), 0o644); err != nil { + t.Fatal(err) + } + root := NewRootCmd(BuildInfo{Version: "test"}) + root.SetArgs([]string{"resources", "--kubeconfig", bad}) + var buf bytes.Buffer + root.SetOut(&buf) + root.SetErr(&buf) + err := root.Execute() + var ee *exitError + if !errors.As(err, &ee) || ee.Code() != 3 { + t.Fatalf("bare resources err = %v, want SHOW's *exitError code 3 (proves it routed to SHOW)", err) + } +} + +// TestResourcesCmd_WiredIntoTree: the command is reachable from the root tree +// and `--help` renders without error. +func TestResourcesCmd_WiredIntoTree(t *testing.T) { + root := NewRootCmd(BuildInfo{Version: "test"}) + root.SetArgs([]string{"resources", "--help"}) + var buf bytes.Buffer + root.SetOut(&buf) + root.SetErr(&buf) + if err := root.Execute(); err != nil { + t.Fatalf("resources --help: %v", err) + } + if !strings.Contains(buf.String(), "how much of this machine tracebloc may use") { + t.Errorf("help missing the one-concept summary:\n%s", buf.String()) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 54c78516..09e223f7 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -89,6 +89,8 @@ Helm, no YAML, no kubectl needed.`, root.AddCommand(newIngestCmd()) root.AddCommand(newClusterCmd()) root.AddCommand(newDataCmd()) + // cli#143: one-knob view of how much of this machine tracebloc may use. + root.AddCommand(newResourcesCmd()) // RFC-0001 (backend#830): browser sign-in + client provisioning. root.AddCommand(newLoginCmd()) root.AddCommand(newLogoutCmd()) @@ -114,6 +116,7 @@ Helm, no YAML, no kubectl needed.`, p.Infof("tracebloc data list — datasets in the cluster") p.Infof("tracebloc data delete — delete an ingested dataset") p.Infof("tracebloc cluster doctor — diagnose connection issues") + p.Infof("tracebloc resources — how much of this machine tracebloc may use") p.Infof("tracebloc delete — remove tracebloc from this machine") p.Newline() p.Hintf("Add --help to any command for the full flag list.") diff --git a/internal/resources/reader.go b/internal/resources/reader.go new file mode 100644 index 00000000..7ff18c7b --- /dev/null +++ b/internal/resources/reader.go @@ -0,0 +1,77 @@ +package resources + +import ( + "context" + "strings" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// JobsManagerEnv reads the jobs-manager Deployment's first-container literal env +// into a map (valueFrom entries have no literal value and are skipped). It is +// release-scoped: it looks up the chart's standard "-jobs-manager" +// name, falling back to a bare "jobs-manager" only when that deployment's +// app.kubernetes.io/instance label ties it to this release — never a different +// release's component (the same attribution rule doctor.findDeployment uses). +// +// Best-effort by design: an unreadable/absent deployment returns an empty map, +// and ParseTraining then reports the chart-default ceiling rather than failing — +// `resources show` is a read-only view, not a gate. +func JobsManagerEnv(ctx context.Context, cs kubernetes.Interface, ns, releaseName string) map[string]string { + env := map[string]string{} + dep := jobsManagerDeployment(ctx, cs, ns, releaseName) + if dep == nil || len(dep.Spec.Template.Spec.Containers) == 0 { + return env + } + for _, e := range dep.Spec.Template.Spec.Containers[0].Env { + if e.Value != "" { + env[e.Name] = e.Value + } + } + return env +} + +// jobsManagerDeployment resolves the release's jobs-manager Deployment, or nil. +// It mirrors doctor.findDeployment's attribution rule exactly, so a best-effort +// resources read can never be satisfied by a DIFFERENT release's component: +// +// Release known — take the chart's "-jobs-manager", or a bare +// "jobs-manager" ONLY when its app.kubernetes.io/instance +// label ties it to this release. A deployment belonging to +// another release is never accepted; missing → nil. +// Release unknown — match any "-jobs-manager"/"jobs-manager" by name, but +// only when EXACTLY ONE exists. With several (multiple +// releases, which discovery refuses to disambiguate) there is +// no safe attribution, so return nil rather than guess and +// read the wrong release's ceiling. +func jobsManagerDeployment(ctx context.Context, cs kubernetes.Interface, ns, releaseName string) *appsv1.Deployment { + const suffix = "jobs-manager" + if releaseName != "" { + if d, err := cs.AppsV1().Deployments(ns).Get(ctx, releaseName+"-"+suffix, metav1.GetOptions{}); err == nil { + return d + } + if d, err := cs.AppsV1().Deployments(ns).Get(ctx, suffix, metav1.GetOptions{}); err == nil && + d.Labels["app.kubernetes.io/instance"] == releaseName { + return d + } + return nil + } + // Release unknown: match by name suffix, accepting the deployment only when + // it's the unique carrier — otherwise there's no safe attribution. + deps, err := cs.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil + } + var match *appsv1.Deployment + for i := range deps.Items { + if n := deps.Items[i].Name; n == suffix || strings.HasSuffix(n, "-"+suffix) { + if match != nil { + return nil // ambiguous across releases — don't guess + } + match = &deps.Items[i] + } + } + return match +} diff --git a/internal/resources/resources.go b/internal/resources/resources.go new file mode 100644 index 00000000..b98bd021 --- /dev/null +++ b/internal/resources/resources.go @@ -0,0 +1,226 @@ +// Package resources backs `tracebloc resources` — the one-knob view of "how +// much of this machine tracebloc may use" (cli#143). It is deliberately +// cluster-free and pure: it reads already-fetched Kubernetes objects (nodes, +// the jobs-manager env) and turns them into the two numbers the command shows — +// the machine's capacity and the ceiling a single tracebloc training run may +// use — plus the user-language formatting for both. +// +// Grounding (verified against tracebloc/client + client-runtime, 2026-07): +// - The machine's capacity is the sum of Ready nodes' Status.Allocatable +// (the installer path is single-node, so this is normally one node). +// - A training run's ceiling is the jobs-manager env RESOURCE_LIMITS +// ("cpu=2,memory=8Gi", requests==limits for Guaranteed QoS). This is the +// exact value client-runtime's jobs_manager.py stamps on spawned jobs and +// the same value `cluster doctor`'s checkNodeFit already parses — so the two +// read it identically (di#358 lesson: a reader must mirror the writer). +// +// No Kubernetes vocabulary leaks into what this package formats: the command +// renders CPU cores and GiB, never "requests", "limits", or "allocatable". +package resources + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// DefaultTraining is the chart's default spawned-job size when the operator set +// no override — mirrors tracebloc/client's jobs-manager-deployment.yaml default +// ("cpu=2,memory=8Gi") and client-runtime's own fallback. Kept here so a `show` +// against an older chart that omits the literal env still reports the real +// effective ceiling rather than "unknown". +const DefaultTraining = "cpu=2,memory=8Gi" + +// Machine is a cluster's schedulable capacity: the sum of Ready nodes' +// allocatable CPU and memory, plus any GPU capacity keyed by its resource name +// (e.g. "nvidia.com/gpu"). The zero value is a machine with nothing Ready. +type Machine struct { + CPU resource.Quantity + Mem resource.Quantity + GPU map[corev1.ResourceName]resource.Quantity +} + +// Training is the ceiling one tracebloc training run may use, parsed from the +// jobs-manager resource env. HasCPUMem is false when neither RESOURCE_LIMITS nor +// RESOURCE_REQUESTS carried a parseable cpu+memory pair (so the command can say +// so honestly instead of printing a fabricated number). +type Training struct { + CPU resource.Quantity + Mem resource.Quantity + HasCPUMem bool + + // GPU, when requested, is the per-run GPU ceiling. Name is the k8s device + // resource name; HasGPU is false for a CPU-only run. + GPUName corev1.ResourceName + GPU resource.Quantity + HasGPU bool +} + +// MachineCapacity sums the allocatable CPU/memory (and GPU) across every Ready +// node. A pod is scheduled onto ONE node, but the machine-capacity headline is a +// whole-machine figure the user recognizes ("this machine has 8 CPU"); the +// single-node fit question is `cluster doctor`'s job (checkNodeFit), which +// deliberately never ORs capacity across nodes. On the installer's single-node +// path the sum is just that node. +func MachineCapacity(nodes []corev1.Node) Machine { + m := Machine{GPU: map[corev1.ResourceName]resource.Quantity{}} + for i := range nodes { + n := nodes[i] + if !nodeReady(n) { + continue + } + for name, qty := range n.Status.Allocatable { + switch name { + case corev1.ResourceCPU: + addInto(&m.CPU, qty) + case corev1.ResourceMemory: + addInto(&m.Mem, qty) + default: + // Surface GPUs (and any other extended device) so the machine + // line can note "· 1 GPU" — but only non-zero quantities. + if isGPUResource(name) && !qty.IsZero() { + cur := m.GPU[name] + cur.Add(qty) + m.GPU[name] = cur + } + } + } + } + return m +} + +// ParseTraining reads the per-run ceiling from a jobs-manager env map. It +// prefers RESOURCE_LIMITS (the true ceiling) and falls back to RESOURCE_REQUESTS +// (requests==limits by chart contract), then to the chart default so an older +// chart without the literal env still reports the effective size. GPU is read +// from GPU_LIMITS, then GPU_REQUESTS. +func ParseTraining(env map[string]string) Training { + spec := firstNonEmpty(env["RESOURCE_LIMITS"], env["RESOURCE_REQUESTS"]) + cpu, mem, ok := parseCPUMem(spec) + if !ok { + // Fall back to the chart default rather than reporting nothing: the + // chart injects this exact value when the operator set no override. + cpu, mem, ok = parseCPUMem(DefaultTraining) + } + t := Training{CPU: cpu, Mem: mem, HasCPUMem: ok} + + gpuName, gpuQty, gpuOK := parseGPU(firstNonEmpty(env["GPU_LIMITS"], env["GPU_REQUESTS"])) + if gpuOK { + t.GPUName, t.GPU, t.HasGPU = gpuName, gpuQty, true + } + return t +} + +// FormatCPU renders a CPU quantity as the user sees it: whole cores with no +// decimal ("4 CPU"), fractional cores to one decimal ("1.2 CPU"). Never a +// milli-suffixed Kubernetes string. +func FormatCPU(q resource.Quantity) string { + cores := float64(q.MilliValue()) / 1000.0 + return trimFloat(cores) + " CPU" +} + +// FormatMem renders a memory quantity in GiB — whole when it divides evenly +// ("16 GiB"), else one decimal ("11.5 GiB"). GiB (1024^3), matching the Gi +// suffix the chart uses, so a "16Gi" limit reads back as "16 GiB". +func FormatMem(q resource.Quantity) string { + gib := float64(q.Value()) / float64(1<<30) + return trimFloat(gib) + " GiB" +} + +// FormatGPU renders a GPU count with its short device label ("1 GPU"). The +// vendor prefix (nvidia.com/, amd.com/) is dropped — the user cares "does it +// have a GPU", not the device-plugin name. +func FormatGPU(name corev1.ResourceName, q resource.Quantity) string { + n := q.Value() + unit := "GPU" + if n != 1 { + return fmt.Sprintf("%d %ss", n, unit) + } + return fmt.Sprintf("%d %s", n, unit) +} + +// --- internal helpers ------------------------------------------------------- + +// addInto adds src into dst in place. resource.Quantity.Add is a pointer +// method; dst starts as a zero quantity (value 0), so the first add seeds it. +func addInto(dst *resource.Quantity, src resource.Quantity) { dst.Add(src) } + +// nodeReady reports whether a node's Ready condition is True. Mirrors +// doctor.nodeReady — kept local so this package stays leaf/pure and doesn't +// import the doctor command package. +func nodeReady(n corev1.Node) bool { + for _, c := range n.Status.Conditions { + if c.Type == corev1.NodeReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +// isGPUResource reports whether an extended resource name is a GPU device. Covers +// the common vendor plugins; anything else (hugepages, ephemeral-storage, …) is +// intentionally not surfaced as a GPU. +func isGPUResource(name corev1.ResourceName) bool { + s := string(name) + return strings.Contains(s, "gpu") || strings.HasPrefix(s, "nvidia.com/") || strings.HasPrefix(s, "amd.com/") +} + +// parseResourceSpec parses jobs-manager's "k1=v1,k2=v2" env into a map. Same +// shape as doctor.parseResourceSpec — the format is a stable chart contract. +func parseResourceSpec(spec string) map[string]string { + out := map[string]string{} + for _, part := range strings.Split(spec, ",") { + kv := strings.SplitN(strings.TrimSpace(part), "=", 2) + if len(kv) == 2 && strings.TrimSpace(kv[0]) != "" { + out[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + } + return out +} + +// parseCPUMem extracts cpu+memory quantities; ok is false unless both parse. +func parseCPUMem(spec string) (cpu, mem resource.Quantity, ok bool) { + m := parseResourceSpec(spec) + c, cOK := m["cpu"] + mm, mOK := m["memory"] + if !cOK || !mOK { + return resource.Quantity{}, resource.Quantity{}, false + } + cpu, errC := resource.ParseQuantity(c) + mem, errM := resource.ParseQuantity(mm) + if errC != nil || errM != nil { + return resource.Quantity{}, resource.Quantity{}, false + } + return cpu, mem, true +} + +// parseGPU extracts the GPU device name + count; requested is false when absent, +// unparseable, or zero. +func parseGPU(spec string) (name corev1.ResourceName, qty resource.Quantity, requested bool) { + for k, v := range parseResourceSpec(spec) { + q, err := resource.ParseQuantity(v) + if err == nil && !q.IsZero() { + return corev1.ResourceName(k), q, true + } + } + return "", resource.Quantity{}, false +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +// trimFloat formats a float with at most one decimal, dropping a trailing ".0" +// so whole numbers read as "16" not "16.0". +func trimFloat(f float64) string { + s := fmt.Sprintf("%.1f", f) + s = strings.TrimSuffix(s, ".0") + return s +} diff --git a/internal/resources/resources_test.go b/internal/resources/resources_test.go new file mode 100644 index 00000000..8061c342 --- /dev/null +++ b/internal/resources/resources_test.go @@ -0,0 +1,223 @@ +package resources + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func readyNode(name, cpu, mem string, extra ...string) *corev1.Node { + alloc := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpu), + corev1.ResourceMemory: resource.MustParse(mem), + } + if len(extra) == 2 { + alloc[corev1.ResourceName(extra[0])] = resource.MustParse(extra[1]) + } + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: corev1.NodeStatus{ + Allocatable: alloc, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } +} + +func TestMachineCapacity_SumsReadyNodesAndGPU(t *testing.T) { + notReady := readyNode("down", "16", "64Gi") + notReady.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}} + + nodes := []corev1.Node{ + *readyNode("a", "4", "16Gi", "nvidia.com/gpu", "1"), + *readyNode("b", "4", "16Gi"), + *notReady, // must be excluded entirely + } + m := MachineCapacity(nodes) + if got := FormatCPU(m.CPU); got != "8 CPU" { + t.Errorf("cpu sum = %q, want 8 CPU", got) + } + if got := FormatMem(m.Mem); got != "32 GiB" { + t.Errorf("mem sum = %q, want 32 GiB", got) + } + gpu, ok := m.GPU["nvidia.com/gpu"] + if !ok || gpu.Value() != 1 { + t.Errorf("gpu = %v (ok=%v), want 1", gpu, ok) + } + if len(m.GPU) != 1 { + t.Errorf("unexpected extra GPU entries: %v", m.GPU) + } +} + +func TestMachineCapacity_NoReadyNodes(t *testing.T) { + m := MachineCapacity(nil) + if !m.CPU.IsZero() || !m.Mem.IsZero() { + t.Errorf("empty machine should be zero, got cpu=%v mem=%v", m.CPU, m.Mem) + } +} + +func TestParseTraining_PrefersLimitsThenRequestsThenDefault(t *testing.T) { + t.Run("limits win over requests", func(t *testing.T) { + tr := ParseTraining(map[string]string{ + "RESOURCE_LIMITS": "cpu=4,memory=16Gi", + "RESOURCE_REQUESTS": "cpu=2,memory=8Gi", + }) + if !tr.HasCPUMem || FormatCPU(tr.CPU) != "4 CPU" || FormatMem(tr.Mem) != "16 GiB" { + t.Fatalf("=> %q %q ok=%v", FormatCPU(tr.CPU), FormatMem(tr.Mem), tr.HasCPUMem) + } + }) + t.Run("falls back to requests", func(t *testing.T) { + tr := ParseTraining(map[string]string{"RESOURCE_REQUESTS": "cpu=2,memory=8Gi"}) + if FormatCPU(tr.CPU) != "2 CPU" || FormatMem(tr.Mem) != "8 GiB" { + t.Fatalf("=> %q %q", FormatCPU(tr.CPU), FormatMem(tr.Mem)) + } + }) + t.Run("empty env falls back to chart default", func(t *testing.T) { + tr := ParseTraining(map[string]string{}) + if !tr.HasCPUMem || FormatCPU(tr.CPU) != "2 CPU" || FormatMem(tr.Mem) != "8 GiB" { + t.Fatalf("default => %q %q ok=%v", FormatCPU(tr.CPU), FormatMem(tr.Mem), tr.HasCPUMem) + } + }) + t.Run("unparseable env falls back to chart default", func(t *testing.T) { + tr := ParseTraining(map[string]string{"RESOURCE_LIMITS": "cpu=oops"}) + if !tr.HasCPUMem || FormatCPU(tr.CPU) != "2 CPU" { + t.Fatalf("=> %q ok=%v, want chart default", FormatCPU(tr.CPU), tr.HasCPUMem) + } + }) +} + +func TestParseTraining_GPU(t *testing.T) { + tr := ParseTraining(map[string]string{ + "RESOURCE_LIMITS": "cpu=4,memory=16Gi", + "GPU_LIMITS": "nvidia.com/gpu=1", + }) + if !tr.HasGPU || string(tr.GPUName) != "nvidia.com/gpu" || tr.GPU.Value() != 1 { + t.Fatalf("gpu => %v %v has=%v", tr.GPUName, tr.GPU, tr.HasGPU) + } + if tr2 := ParseTraining(map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi"}); tr2.HasGPU { + t.Errorf("cpu-only run must not report a GPU") + } +} + +func TestFormatCPU(t *testing.T) { + cases := map[string]string{ + "4": "4 CPU", + "2000m": "2 CPU", + "1200m": "1.2 CPU", + "500m": "0.5 CPU", + } + for in, want := range cases { + if got := FormatCPU(resource.MustParse(in)); got != want { + t.Errorf("FormatCPU(%q) = %q, want %q", in, got, want) + } + } +} + +func TestFormatMem(t *testing.T) { + cases := map[string]string{ + "8Gi": "8 GiB", + "16Gi": "16 GiB", + "32Gi": "32 GiB", + "1536Mi": "1.5 GiB", + "30720Mi": "30 GiB", + } + for in, want := range cases { + if got := FormatMem(resource.MustParse(in)); got != want { + t.Errorf("FormatMem(%q) = %q, want %q", in, got, want) + } + } +} + +func TestFormatGPU_Pluralization(t *testing.T) { + if got := FormatGPU("nvidia.com/gpu", resource.MustParse("1")); got != "1 GPU" { + t.Errorf("one gpu = %q, want 1 GPU", got) + } + if got := FormatGPU("nvidia.com/gpu", resource.MustParse("2")); got != "2 GPUs" { + t.Errorf("two gpus = %q, want 2 GPUs", got) + } +} + +// --- reader (fake clientset) --- + +func jmDeploy(name, instance string, env map[string]string) *appsv1.Deployment { + var vars []corev1.EnvVar + for k, v := range env { + vars = append(vars, corev1.EnvVar{Name: k, Value: v}) + } + // A valueFrom entry (no literal value) must be skipped by the reader. + vars = append(vars, corev1.EnvVar{Name: "SECRET_REF", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{Key: "k"}, + }}) + labels := map[string]string{} + if instance != "" { + labels["app.kubernetes.io/instance"] = instance + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tracebloc", Labels: labels}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "jobs-manager", Env: vars}}}, + }, + }, + } +} + +func TestJobsManagerEnv_ReleasePrefixedName(t *testing.T) { + cs := fake.NewClientset(jmDeploy("tb-jobs-manager", "tb", map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi"})) + env := JobsManagerEnv(context.Background(), cs, "tracebloc", "tb") + if env["RESOURCE_LIMITS"] != "cpu=4,memory=16Gi" { + t.Errorf("env = %v", env) + } + if _, leaked := env["SECRET_REF"]; leaked { + t.Errorf("valueFrom env must be skipped, got %v", env) + } +} + +func TestJobsManagerEnv_BareNameRequiresMatchingInstanceLabel(t *testing.T) { + // Bare "jobs-manager" belonging to a DIFFERENT release must not be read. + cs := fake.NewClientset(jmDeploy("jobs-manager", "other", map[string]string{"RESOURCE_LIMITS": "cpu=99,memory=99Gi"})) + if env := JobsManagerEnv(context.Background(), cs, "tracebloc", "tb"); len(env) != 0 { + t.Errorf("must not read another release's bare deployment, got %v", env) + } + // Bare "jobs-manager" whose instance label matches IS read. + cs2 := fake.NewClientset(jmDeploy("jobs-manager", "tb", map[string]string{"RESOURCE_LIMITS": "cpu=8,memory=32Gi"})) + if env := JobsManagerEnv(context.Background(), cs2, "tracebloc", "tb"); env["RESOURCE_LIMITS"] != "cpu=8,memory=32Gi" { + t.Errorf("matching-instance bare deployment should be read, got %v", env) + } +} + +func TestJobsManagerEnv_AbsentIsEmpty(t *testing.T) { + cs := fake.NewClientset() + if env := JobsManagerEnv(context.Background(), cs, "tracebloc", "tb"); len(env) != 0 { + t.Errorf("absent deployment should yield empty env, got %v", env) + } +} + +// TestJobsManagerEnv_ReleaseUnknownUniqueMatch: with the release undiscovered +// (releaseName==""), a single "-jobs-manager" is unambiguous and IS read — +// matching doctor.findDeployment's unknown-release branch (the OLD reader only +// Get()'d a bare "jobs-manager" and so missed a release-prefixed one). +func TestJobsManagerEnv_ReleaseUnknownUniqueMatch(t *testing.T) { + cs := fake.NewClientset(jmDeploy("tb-jobs-manager", "tb", map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi"})) + if env := JobsManagerEnv(context.Background(), cs, "tracebloc", ""); env["RESOURCE_LIMITS"] != "cpu=4,memory=16Gi" { + t.Errorf("unique suffix match should be read when release is unknown, got %v", env) + } +} + +// TestJobsManagerEnv_ReleaseUnknownAmbiguousIsEmpty: with the release +// undiscovered and TWO releases' jobs-managers present, there's no safe +// attribution — the read must return nothing rather than pick another release's +// ceiling (the attribution guard finding #2 restores). +func TestJobsManagerEnv_ReleaseUnknownAmbiguousIsEmpty(t *testing.T) { + cs := fake.NewClientset( + jmDeploy("tb-jobs-manager", "tb", map[string]string{"RESOURCE_LIMITS": "cpu=4,memory=16Gi"}), + jmDeploy("prod-jobs-manager", "prod", map[string]string{"RESOURCE_LIMITS": "cpu=99,memory=99Gi"}), + ) + if env := JobsManagerEnv(context.Background(), cs, "tracebloc", ""); len(env) != 0 { + t.Errorf("ambiguous multi-release match must yield empty env, got %v", env) + } +}