diff --git a/internal/cli/data.go b/internal/cli/data.go
index cb14b3a5..7a13a531 100644
--- a/internal/cli/data.go
+++ b/internal/cli/data.go
@@ -5,15 +5,10 @@ import (
"errors"
"fmt"
"io"
- "path/filepath"
- "strings"
"github.com/spf13/cobra"
- "gopkg.in/yaml.v3"
- "github.com/tracebloc/cli/internal/cluster"
"github.com/tracebloc/cli/internal/push"
- "github.com/tracebloc/cli/internal/schema"
)
// newDataCmd wires the `tracebloc data` subtree. The dominant
@@ -123,478 +118,30 @@ func runDataIngest(ctx context.Context, out, errOut io.Writer, a runDataIngestAr
}
}()
- // Intro header: brand + a plain-English explainer of what an ingest
- // does, so a first-time user understands it before any prompts.
- // Routed through a.Printer, so --output-json keeps it on stderr and
- // --plain/non-TTY degrade cleanly. (#31)
- // --overwrite + a reused --idempotency-key is a data-loss trap: the
- // teardown removes the existing data, then jobs-manager treats the
- // duplicate key as a REPLAY and attaches to the previous run instead of
- // ingesting anything — old data gone, new data never loaded, exit 0 from
- // the old Job's status. Refuse the combination outright.
- if a.Overwrite && a.IdempotencyKey != "" {
- return &exitError{code: 2, err: errors.New(
- "--overwrite can't be combined with --idempotency-key: a reused key makes the cluster replay the previous run instead of ingesting the new data — after --overwrite's removal that would report success while loading nothing. Drop one of the two (a fresh per-run key is the default).")}
- }
-
- a.Printer.Banner("tracebloc", "data ingest")
- a.Printer.Para(strings.TrimSpace(`
-This ingests a dataset so models can train on it. Your files never leave your
-own infrastructure — tracebloc copies them into your workspace's storage,
-checks them, and loads them into a table your training runs read from. Other
-collaborators can train against that table without ever seeing the raw files.`))
- a.Printer.Hintf("Learn more: https://docs.tracebloc.io")
-
- // 0. Guided mode: prompt for any missing core inputs before
- // validation. Flags already provided win; non-TTY / --no-input
- // leaves Prompter nil and skips straight to the flag-only path.
- if a.Interactive && a.Prompter != nil {
- if err := runInteractive(a.Printer, a.Prompter, &a, a.TaskSet); err != nil {
- if errors.Is(err, errInteractiveCancelled) {
- a.Printer.Infof("Cancelled — nothing was ingested.")
- return nil
- }
- // A typed exitError from a guided step (e.g. the path-existence
- // guard, which runInteractive runs before the family sniff)
- // already carries its own code + clean message — surface it as-is
- // rather than burying it under "interactive setup:".
- var ee *exitError
- if errors.As(err, &ee) {
- return err
- }
- return &exitError{code: 3, err: fmt.Errorf("interactive setup: %w", err)}
- }
- }
- // --intent defaults to train. Applied after the interactive block so
- // the guided flow still asks "training or test?" (it prompts on an
- // empty value); a non-interactive run that omits --intent gets train
- // without erroring (RFC-0002 §5). The wire field stays "intent".
- if a.Spec.Intent == "" {
- a.Spec.Intent = "train"
- }
- if a.LocalPath == "" {
- return &exitError{code: 3, err: errors.New(
- "local dataset path is required — pass it as an argument, or run " +
- "on a terminal without --no-input for guided prompts")}
- }
- // Expand a leading ~ ourselves. The shell expands an unquoted ~ on
- // the command line, but a path typed at the interactive prompt (or
- // a quoted/literal ~ arg) reaches us unexpanded — and filepath.Abs
- // would just prepend the CWD, yielding ".../cwd/~/...". Mirrors
- // cluster.expandPath; done here so it covers both entry points
- // before any push.Discover* call. (#37)
- a.LocalPath = expandHome(a.LocalPath)
-
- // 0b. Path existence FIRST — before any spec / schema / family
- // validation. A typo'd path should fail on the path with a plain
- // "no such file or directory", not surface later as a confusing
- // downstream error (e.g. the task gate asking which task the
- // non-existent data is for). runInteractive runs this same guard
- // before its family sniff / label preview, so the invariant holds on
- // the guided route too; this re-check covers the flag-only path and
- // is cheap (one stat). The family walk below stats again for its
- // layout-specific diagnostics; this is only about ordering the first
- // failure a customer sees. (#181)
- if err := statDatasetPath(a.LocalPath); err != nil {
- return err
- }
-
- // 1. Validate the table name BEFORE anything else. It's both
- // the MySQL identifier and the /data/shared/
/ PVC
- // subdirectory — an unsanitized traversal name (../../etc)
- // would escape that subtree once the stage Pod writes to
- // it. The embedded schema only checks minLength on `table`,
- // so this CLI-side guard is the real fix. SpecArgs.Build()
- // below calls StagedPrefix, which panics on an unsafe name —
- // so this check MUST come first.
- if err := push.ValidateTableName(a.Spec.Table); err != nil {
- return &exitError{code: 2, err: err}
- }
-
- // 2. Category gate. Runs BEFORE schema validation so an
- // unsupported category gets a clear, actionable CLI message
- // rather than the schema's terse enum / missing-property error.
- // Every schema task category is CLI-supported now; this gate stays as
- // defensive routing so a future known-but-not-yet-wired category gets a
- // clear per-category message, and a typo'd category gets the supported
- // list rather than the schema's raw enum dump.
- switch {
- case a.Spec.Category == "":
- // No task chosen. In guided mode the picker already filled this;
- // reaching here means a non-interactive run (or --no-input /
- // --output-json) that omitted --task. Give a clear, actionable
- // error instead of silently assuming images (the old default).
- return &exitError{code: 2, err: fmt.Errorf(
- "which task is this data for? pass --task — one of: %s. "+
- "(On a terminal without --no-input, tracebloc asks you to pick.)",
- push.SupportedCategoriesList())}
- case push.IsCLISupported(a.Spec.Category):
- // supported
- case push.IsKnown(a.Spec.Category):
- // A recognized category the CLI doesn't implement yet. None today — every
- // schema category is wired — but kept as defensive routing so a future
- // known-but-unsupported category gets the registry's per-category reason,
- // not a misleading "unrecognized category". Supported categories were
- // already caught above, so IsKnown here means known-but-unsupported.
- spec, _ := push.Lookup(a.Spec.Category)
- reason := ""
- if spec.UnsupportedNote != "" {
- reason = " (" + spec.UnsupportedNote + ")"
- }
- return &exitError{code: 2, err: fmt.Errorf(
- "task %q isn't supported by the CLI yet%s. Supported tasks: %s.",
- a.Spec.Category, reason, push.SupportedCategoriesList())}
- default:
- return &exitError{code: 2, err: fmt.Errorf(
- "task %q isn't a recognized task. Supported tasks: %s.",
- a.Spec.Category, push.SupportedCategoriesList())}
- }
-
- // Image-only flags. --target-size / --min-size describe image
- // resolution, so they're meaningless on a tabular / text task.
- // Reject them explicitly here: without this guard they'd be parsed
- // only inside the image branch below, so on a non-image task the
- // value — even a malformed one — was silently dropped with no error.
- if !push.IsImage(a.Spec.Category) {
- for _, f := range []struct{ name, val string }{
- {"--target-size", a.TargetSizeFlag},
- {"--min-size", a.MinSizeFlag},
- } {
- if f.val != "" {
- return &exitError{code: 2, err: fmt.Errorf(
- "%s is image tasks only; it doesn't apply to task %q",
- f.name, a.Spec.Category)}
- }
- }
- }
-
- // Task-scoped flags. Like --target-size/--min-size above, each of these is
- // read only inside the one category branch that consumes it, so passing one
- // on a task that doesn't use it silently dropped the value — and the user's
- // intent — with no error, even though the help text says each is scoped.
- // Reject a misapplied flag explicitly so it fails fast instead of being
- // ignored (the scope mirrors spec.go's build gates exactly).
- if a.SchemaFlag != "" && !push.IsTabular(a.Spec.Category) {
- return &exitError{code: 2, err: fmt.Errorf(
- "--schema is tabular/time-series tasks only; it doesn't apply to task %q", a.Spec.Category)}
- }
- if a.Spec.LabelPolicy != "" && !push.IsRegressionClass(a.Spec.Category) {
- return &exitError{code: 2, err: fmt.Errorf(
- "--label-policy is regression-class tasks only (tabular_regression, "+
- "time_series_forecasting, time_to_event_prediction); it doesn't apply to task %q",
- a.Spec.Category)}
- }
- if a.Spec.TimeColumn != "" && a.Spec.Category != "time_to_event_prediction" {
- return &exitError{code: 2, err: fmt.Errorf(
- "--time-column is time_to_event_prediction only; it doesn't apply to task %q", a.Spec.Category)}
- }
- if a.Spec.NumberOfKeypoints != 0 && a.Spec.Category != "keypoint_detection" {
- return &exitError{code: 2, err: fmt.Errorf(
- "--number-of-keypoints is keypoint_detection only; it doesn't apply to task %q", a.Spec.Category)}
- }
- // --label-column is meaningless for self-supervised text (the label is the
- // text itself); buildText drops it, so accepting it silently discarded the
- // user's value and the review echoed a column that never shipped.
- if a.Spec.LabelColumn != "" && push.SelfSupervisedText(a.Spec.Category) {
- return &exitError{code: 2, err: fmt.Errorf(
- "--label-column doesn't apply to task %q — it trains on the text itself, with no label column",
- a.Spec.Category)}
- }
-
- // 3. Walk the local directory FIRST (local "fail fast"), dispatched
- // by category family. Image categories expect labels.csv +
- // images/; tabular / time-series categories expect a single
- // data CSV. The walk also yields what the per-category
- // resolution below needs (the image list for target-size, the
- // CSV for schema inference).
- // err is the function's named return (see the --output-json defer
- // at the top), so it's not redeclared here. The walk can take a moment
- // on a large tree, so it gets a spinner — no blocking wait stays silent.
- var layout *push.LocalLayout
- walkSpin := a.Printer.Spinner("Reading your files", "")
- switch {
- case push.IsTabular(a.Spec.Category):
- layout, err = push.DiscoverTabular(a.LocalPath)
- case push.IsText(a.Spec.Category):
- layout, err = push.DiscoverText(a.Spec.Category, a.LocalPath)
- case a.Spec.Category == "object_detection":
- layout, err = push.DiscoverObjectDetection(a.LocalPath)
- case a.Spec.Category == "semantic_segmentation":
- layout, err = push.DiscoverSemanticSegmentation(a.LocalPath)
- default:
- // image_classification + keypoint_detection: labels.csv + images/.
- layout, err = push.Discover(a.LocalPath)
- }
- walkSpin.Stop()
+ // Steps 0–4 + the P3 preflight — everything local — live in
+ // resolveLocalInput (cli#283). It mutates a.Spec in place, so the defer
+ // above and the cluster steps below see the resolved spec; err feeds the
+ // named return, so the defer fires on its failures too.
+ layout, spec, specBytes, cancelled, err := resolveLocalInput(out, errOut, &a)
if err != nil {
- return &exitError{code: 3, err: err}
- }
-
- a.Printer.Step(1, 3, "Check your data")
- a.Printer.Hintf("Reading your files locally first — nothing has touched your workspace yet — so a layout or settings problem shows up right away.")
-
- // 3a. Per-category spec resolution from the local data, so the
- // synthesized spec carries the right fields before validation.
- switch {
- case push.IsTabular(a.Spec.Category):
- // P3 (cli#71): a BOM'd tabular CSV is doomed in-cluster AND would
- // corrupt InferSchema's own header read below — reject before
- // either. The rest of the content preflight runs after the spec
- // schema validation (mirroring the in-cluster order).
- if perr := push.CheckTabularBOM(layout.LabelsCSV); perr != nil {
- return &exitError{code: 3, err: perr}
- }
- if perr := push.CheckHasDataRows(layout.LabelsCSV); perr != nil {
- return &exitError{code: 3, err: perr}
- }
-
- // Column schema. An explicit --schema wins (raw flag, or the
- // optional override the interactive prompt captures into SchemaFlag).
- // Otherwise infer the types here — mirroring the ingestor's own rules
- // (di#349) — and EMIT the result explicitly (below, via a.Spec.Schema
- // → spec.schema), so the ingestor uses the CLI's answer regardless of
- // its own version. Inference runs on both a no-schema non-interactive
- // run and an interactive run where the user left the schema prompt
- // blank; the risky cases below are surfaced as warnings.
- if a.SchemaFlag != "" {
- sch, perr := push.ParseSchema(a.SchemaFlag)
- if perr != nil {
- return &exitError{code: 2, err: perr}
- }
- a.Spec.Schema = sch
- } else {
- res, ierr := push.InferSchema(layout.LabelsCSV)
- if ierr != nil {
- return &exitError{code: 3, err: fmt.Errorf("inferring schema from CSV: %w", ierr)}
- }
- a.Spec.Schema = res.Schema
- _, _ = fmt.Fprintf(out,
- "Inferred schema for %d column(s) from %s (override with --schema).\n",
- len(res.Schema), filepath.Base(layout.LabelsCSV))
- if len(res.Skipped) > 0 {
- _, _ = fmt.Fprintf(out,
- " (skipped framework-managed column(s): %s)\n", strings.Join(res.Skipped, ", "))
- }
- if len(res.Empty) > 0 {
- _, _ = fmt.Fprintf(out,
- " (warning: %d column(s) had no values in the sample and were typed VARCHAR(1): %s)\n",
- len(res.Empty), strings.Join(res.Empty, ", "))
- }
- if len(res.IDLike) > 0 {
- _, _ = fmt.Fprintf(out,
- " (warning: %d column(s) look like identifiers (all-unique integers): %s — "+
- "if any is a zero-padded code, pass --schema to type it VARCHAR)\n",
- len(res.IDLike), strings.Join(res.IDLike, ", "))
- }
- }
- case push.IsImage(a.Spec.Category):
- // keypoint_detection needs --number-of-keypoints (dataset-
- // specific, no default). Catch it here with an actionable
- // message rather than letting the ingestor fail mid-run. Split
- // UNSET from SET-BUT-INVALID (#76b): a Go 0 could mean either, so
- // key on whether the flag was passed (ChangedFlags). Unset → the
- // "requires" nudge; set to a non-positive value → name the bad
- // value so the user sees exactly what was rejected.
- if a.Spec.Category == "keypoint_detection" && a.Spec.NumberOfKeypoints <= 0 {
- if a.ChangedFlags["number-of-keypoints"] {
- return &exitError{code: 2, err: fmt.Errorf(
- "--number-of-keypoints must be a positive integer (got %d); "+
- "it's the number of keypoints per sample (e.g. 17 for COCO pose)",
- a.Spec.NumberOfKeypoints)}
- }
- return &exitError{code: 2, err: errors.New(
- "keypoint_detection requires --number-of-keypoints (e.g. " +
- "--number-of-keypoints 17); it's dataset-specific and has no default")}
- }
- // Image target resolution: the ingestor's image_classification
- // default is 512x512 and it VALIDATES (it does not resize), so
- // a mismatch hard-fails. Honour an explicit --target-size;
- // otherwise auto-detect from the first image so the common
- // "all my images are NxN" case just works.
- if a.TargetSizeFlag != "" {
- w, h, perr := push.ParseTargetSize(a.TargetSizeFlag)
- if perr != nil {
- return &exitError{code: 2, err: perr}
- }
- a.Spec.TargetSize = []int{w, h}
- } else if len(layout.Images) > 0 {
- if w, h, derr := push.DetectImageSize(layout.Images[0]); derr == nil {
- a.Spec.TargetSize = []int{w, h}
- _, _ = fmt.Fprintf(out,
- "Auto-detected image target size %dx%d from %s (override with --target-size).\n",
- w, h, filepath.Base(layout.Images[0]))
- } else {
- _, _ = fmt.Fprintf(errOut,
- "Note: couldn't auto-detect image size (%v); using the ingestor "+
- "default. Pass --target-size WxH if ingestion reports a "+
- "resolution mismatch.\n", derr)
- }
- }
- // Minimum-size floor override (#348): plumb an explicit --min-size to
- // spec.file_options.min_size. When unset, no spec field is emitted, so
- // the ingestor applies its own default (none on the deployed
- // v0.5.7/v0.6.0; 32x32 on develop post-#348) — and the local preview
- // applies NO floor either (PreflightDataset only previews the floor
- // when --min-size is set, so it never rejects an ingest the live
- // cluster accepts). The below-floor reject is previewed in
- // runLocalPreflight (ValidateImages).
- if a.MinSizeFlag != "" {
- w, h, perr := push.ParseMinSize(a.MinSizeFlag)
- if perr != nil {
- return &exitError{code: 2, err: perr}
- }
- a.Spec.MinSize = []int{w, h}
- }
- // Extension: every image must share one type, and the spec tells
- // the cluster which one to validate against (file_options.extension).
- // Without this the ingestor checked its .jpeg convention default and
- // rejected .jpg/.png datasets AFTER the full upload (cli#68).
- ext, exterr := push.DetectExtension(layout.Images)
- if exterr != nil {
- return &exitError{code: 3, err: exterr}
- }
- a.Spec.Extension = ext
- default:
- // Text family: no extra per-category resolution. The supervised text
- // tasks (text_classification, token_classification,
- // sentence_pair_classification) carry a label straight from
- // --label-column; the self-supervised ones (masked/causal language
- // modeling, seq2seq, embeddings) need neither a label nor a schema.
- // buildText emits the label for exactly the supervised set, keyed on
- // the registry's SelfSupervised flag (not a hardcoded id).
+ return err
}
-
- // 3b. Friendly missing-label pre-check (#214). Tabular / time-series tasks
- // AND semantic_segmentation carry a required label column (the ingest
- // schema's allOf requires `label` for them). With no --label-column the
- // synthesized spec's `label` is an empty string, which trips the schema's
- // label oneOf and the raw validation below dumps an opaque "got object,
- // want string" / "minLength" pair. semseg is especially prone to this —
- // its per-image label reads as vestigial beside the pixel masks, so the
- // flag is easy to forget. Intercept ONLY that specific missing case here —
- // a label present-but-not-in-the-CSV still flows to runLocalPreflight's
- // CheckLabelColumn, and every other schema error still reaches the dump —
- // and name the flag to fix instead.
- if (push.IsTabular(a.Spec.Category) || a.Spec.Category == "semantic_segmentation") && a.Spec.LabelColumn == "" {
- msg := "this task needs a label column, but --label-column wasn't set — " +
- "pass --label-column with the name of the target column in your data CSV"
- if cols := sortedKeys(a.Spec.Schema); len(cols) > 0 {
- msg += " (columns: " + strings.Join(cols, ", ") + ")"
- }
- return &exitError{code: 2, err: errors.New(msg)}
+ if cancelled {
+ return nil
}
- // 4. Synthesize the spec from flags + validate against schema.
- // Catches "bad category", "missing intent" etc. BEFORE we
- // touch the cluster. The error formatter is the same one
- // ingest validate uses, so a customer who YAML'd manually
- // first sees identical wording.
- spec := a.Spec.Build()
- specBytes, err := yaml.Marshal(spec)
+ // Steps 5–8a — cluster discovery + the destination-table guard — live in
+ // connectIngestTarget (cli#283). It may set a.Overwrite (the folded
+ // interactive replace decision); the teardown below keys on that.
+ target, existingTable, cancelled, err := connectIngestTarget(ctx, &a)
if err != nil {
- return &exitError{code: 3, err: fmt.Errorf("marshaling synthesized spec: %w", err)}
- }
- v, err := schema.NewV1Validator()
- if err != nil {
- return &exitError{code: 3, err: fmt.Errorf("loading embedded schema: %w", err)}
- }
- _, errs, parseErr := v.ValidateYAML(specBytes)
- if parseErr != nil {
- // "Parse" failing on a spec we marshaled ourselves is a
- // programming error, not a customer error — surface it
- // with the bytes so we can diagnose. Exit 3 (the
- // "internal" bucket) matches the marshal-failure branch
- // above.
- return &exitError{code: 3, err: fmt.Errorf("internal: re-parsing synthesized spec: %w\n%s", parseErr, specBytes)}
- }
- if len(errs) > 0 {
- // Use the SAME formatter `ingest validate` uses, so the
- // experience is identical whether the customer authored
- // YAML by hand or via flags. Diagnostics go to stderr
- // (matching ingest validate) so a downstream pipe of
- // stdout (e.g. piping the summary to jq once that's a
- // JSON output mode) isn't polluted by error text. Exit 2
- // is reserved for schema violations across the CLI.
- _, _ = fmt.Fprintf(errOut, "synthesized spec failed schema validation (%d issue%s):\n",
- len(errs), plural(len(errs)))
- _, _ = fmt.Fprintln(errOut, schema.FormatErrors(errs))
- return &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")}
- }
-
- // P3 content preflight (backend#828, cli#69/#71/#72/#73): preview the
- // ingestor's own validators locally — AFTER the spec schema validation,
- // mirroring the in-cluster order (jsonschema first, then validators),
- // and BEFORE any cluster work. Each check names the rule it previews;
- // parity is pinned by internal/push/parity_golden_test.go.
- if perr := runLocalPreflight(a, layout, errOut); perr != nil {
- return perr
+ return err
}
-
- printLocalSummary(a.Printer, layout, spec)
-
- // 5. Cluster discovery — same kubeconfig path as `cluster info`.
- // Errors mirror that command's exit-code contract (3 for
- // kubeconfig, 4 for missing release) so behaviour is
- // consistent across pre-flight commands.
- // Connecting to the workspace + discovering its shared storage is
- // Kubernetes plumbing (release / PVC / jobs-manager) the happy path keeps
- // quiet — it's no longer a numbered step (RFC-0002 §6), and --verbose adds
- // the release/PVC detail below. But the discovery itself is several blocking
- // apiserver round-trips (kubeconfig load, release + PVC discovery, then the
- // destination-exists check), so it still needs a visible status line — no
- // silent wait on the happy path (RFC-0002 "progress on every wait").
- // A plain line, not a spinner: discoverRelease can print its own
- // namespace-fallback note mid-call, and a spinner's \r redraw would clobber
- // it. ALL the logic below (discovery + the exit-6 destination guard) is
- // unchanged; only the presentation moved.
- a.Printer.Infof("Connecting to your workspace…")
- // 6. PVC discovery (needPVC) confirms the chart's shared-data PVC is
- // Bound before we waste time provisioning a Pod that can't mount it.
- opts := cluster.KubeconfigOptions{Path: a.Kubeconfig, Context: a.Context, Namespace: a.Namespace}
- binding := bindActiveClientNamespace(&opts)
- target, err := resolveClusterTarget(ctx, a.Printer, opts, binding, true)
- if err != nil {
- return binding.explain(err)
+ if cancelled {
+ return nil
}
resolved, cs, release, pvc := target.Resolved, target.Clientset, target.Release, target.PVC
- // release.IngestorSAName is discovered from the ingestionAuthz ConfigMap by
- // DiscoverParentRelease (#7) and flows into the stage/teardown pods + the
- // jobs-manager token mint below — no --ingestor-sa override.
-
- // 7. Under --verbose, show what we found on the cluster; the happy path
- // keeps this Kubernetes detail hidden (printClusterSummary is a no-op
- // without --verbose).
- printClusterSummary(a.Printer, release, pvc)
-
- // 8a. Destination guard (cli#70, P4-lite): re-ingesting an existing
- // table used to stage EVERYTHING and then fail the in-cluster Job
- // on the ingestor's duplicate check — a full upload burned to learn
- // the table exists. One cheap read heads that off. The check fails
- // open (dim note) — the ingestor still refuses duplicates, so a
- // broken check can't cause silent data loss.
- existingTable, checkNote := destTableExists(ctx, cs, resolved, a.Spec.Table)
- if checkNote != "" {
- a.Printer.Hintf("%s", checkNote)
- }
tableExists := existingTable != ""
- if tableExists && !a.Overwrite {
- // Folded decision (RFC-0002): in interactive mode a pre-existing table
- // is a question, not a wall. Prompt to replace it; a "no" cancels
- // cleanly (exit 0). Non-interactive (or --output-json / --no-input)
- // still hard-fails exit 6 — a script must opt in with --overwrite.
- proceed, aerr := existingTableAction(&a, existingTable)
- if aerr != nil {
- return aerr
- }
- if !proceed {
- a.Printer.Infof("Cancelled — %q was left as-is; nothing was ingested.", existingTable)
- return nil
- }
- a.Overwrite = true
- }
- if tableExists && a.Overwrite {
- a.Printer.Warnf("Table %q already exists — replacing it (table + files).", existingTable)
- }
// 8. Dry-run stop. Acknowledged success, plus a reminder of the
// live-only steps (stage + ingest) the customer just skipped.
diff --git a/internal/cli/data_ingest_cluster.go b/internal/cli/data_ingest_cluster.go
index f7718dfe..e20ac764 100644
--- a/internal/cli/data_ingest_cluster.go
+++ b/internal/cli/data_ingest_cluster.go
@@ -19,6 +19,86 @@ import (
"github.com/tracebloc/cli/internal/ui"
)
+// connectIngestTarget is the cluster half of `data ingest`'s pre-flight:
+// resolve the kubeconfig and discover the parent release + shared PVC
+// (steps 5–7), then run the destination-table guard (8a). Extracted
+// verbatim from runDataIngest (cli#283) — step order, output, and exit
+// codes unchanged.
+//
+// The folded replace decision can set a.Overwrite (interactive "replace
+// it?" answered yes); the caller's teardown step keys on that, exactly as
+// before. cancelled=true with a nil err is the user declining the replace
+// prompt: the caller exits 0, nothing ingested. existingTable is the
+// EXISTING table's exact spelling ("" when absent) — any teardown must act
+// on it, not on the flag's casing.
+func connectIngestTarget(ctx context.Context, a *runDataIngestArgs) (target *clusterTarget, existingTable string, cancelled bool, err error) {
+ // 5. Cluster discovery — same kubeconfig path as `cluster info`.
+ // Errors mirror that command's exit-code contract (3 for
+ // kubeconfig, 4 for missing release) so behaviour is
+ // consistent across pre-flight commands.
+ // Connecting to the workspace + discovering its shared storage is
+ // Kubernetes plumbing (release / PVC / jobs-manager) the happy path keeps
+ // quiet — it's no longer a numbered step (RFC-0002 §6), and --verbose adds
+ // the release/PVC detail below. But the discovery itself is several blocking
+ // apiserver round-trips (kubeconfig load, release + PVC discovery, then the
+ // destination-exists check), so it still needs a visible status line — no
+ // silent wait on the happy path (RFC-0002 "progress on every wait").
+ // A plain line, not a spinner: discoverRelease can print its own
+ // namespace-fallback note mid-call, and a spinner's \r redraw would clobber
+ // it. ALL the logic below (discovery + the exit-6 destination guard) is
+ // unchanged; only the presentation moved.
+ a.Printer.Infof("Connecting to your workspace…")
+ // 6. PVC discovery (needPVC) confirms the chart's shared-data PVC is
+ // Bound before we waste time provisioning a Pod that can't mount it.
+ opts := cluster.KubeconfigOptions{Path: a.Kubeconfig, Context: a.Context, Namespace: a.Namespace}
+ binding := bindActiveClientNamespace(&opts)
+ target, err = resolveClusterTarget(ctx, a.Printer, opts, binding, true)
+ if err != nil {
+ return nil, "", false, binding.explain(err)
+ }
+ resolved, cs, release, pvc := target.Resolved, target.Clientset, target.Release, target.PVC
+ // release.IngestorSAName is discovered from the ingestionAuthz ConfigMap by
+ // DiscoverParentRelease (#7) and flows into the stage/teardown pods + the
+ // jobs-manager token mint below — no --ingestor-sa override.
+
+ // 7. Under --verbose, show what we found on the cluster; the happy path
+ // keeps this Kubernetes detail hidden (printClusterSummary is a no-op
+ // without --verbose).
+ printClusterSummary(a.Printer, release, pvc)
+
+ // 8a. Destination guard (cli#70, P4-lite): re-ingesting an existing
+ // table used to stage EVERYTHING and then fail the in-cluster Job
+ // on the ingestor's duplicate check — a full upload burned to learn
+ // the table exists. One cheap read heads that off. The check fails
+ // open (dim note) — the ingestor still refuses duplicates, so a
+ // broken check can't cause silent data loss.
+ existingTable, checkNote := destTableExists(ctx, cs, resolved, a.Spec.Table)
+ if checkNote != "" {
+ a.Printer.Hintf("%s", checkNote)
+ }
+ tableExists := existingTable != ""
+ if tableExists && !a.Overwrite {
+ // Folded decision (RFC-0002): in interactive mode a pre-existing table
+ // is a question, not a wall. Prompt to replace it; a "no" cancels
+ // cleanly (exit 0). Non-interactive (or --output-json / --no-input)
+ // still hard-fails exit 6 — a script must opt in with --overwrite.
+ proceed, aerr := existingTableAction(a, existingTable)
+ if aerr != nil {
+ return nil, "", false, aerr
+ }
+ if !proceed {
+ a.Printer.Infof("Cancelled — %q was left as-is; nothing was ingested.", existingTable)
+ return nil, "", true, nil
+ }
+ a.Overwrite = true
+ }
+ if tableExists && a.Overwrite {
+ a.Printer.Warnf("Table %q already exists — replacing it (table + files).", existingTable)
+ }
+
+ return target, existingTable, false, nil
+}
+
// runIngestionRun is the money path's outcome tail. It mints the ingestor
// token, port-forwards to jobs-manager, POSTs the run, classifies the result
// into a status + process exit code (kept in lockstep by classifyPushOutcome),
diff --git a/internal/cli/data_ingest_local.go b/internal/cli/data_ingest_local.go
index 7a382ac3..6a9e2702 100644
--- a/internal/cli/data_ingest_local.go
+++ b/internal/cli/data_ingest_local.go
@@ -1,7 +1,9 @@
-// The local, cluster-free half of `data ingest`: path expansion + the
+// The local, cluster-free half of `data ingest`: resolveLocalInput (the
+// whole pre-cluster pipeline — prompts, validation, layout walk, spec
+// synthesis + schema validation), path expansion + the
// path-existence-first guard, the local dataset summary, and the
// preflight that previews the ingestor's validators on the local data.
-// Moved verbatim from data.go (cli#282) — behavior unchanged.
+// Moved verbatim from data.go (cli#282, cli#283) — behavior unchanged.
package cli
import (
@@ -9,10 +11,15 @@ import (
"fmt"
"io"
"os"
+ "path/filepath"
"sort"
+ "strings"
+
+ "gopkg.in/yaml.v3"
"github.com/tracebloc/cli/internal/pathutil"
"github.com/tracebloc/cli/internal/push"
+ "github.com/tracebloc/cli/internal/schema"
"github.com/tracebloc/cli/internal/ui"
)
@@ -55,6 +62,432 @@ func statDatasetPath(path string) error {
return nil
}
+// resolveLocalInput is everything `data ingest` does BEFORE touching the
+// cluster: the --overwrite/--idempotency-key guard, the intro banner, the
+// guided prompts, path expansion + the existence-first check, table-name /
+// category / misapplied-flag validation, the local layout walk, per-category
+// spec resolution, spec synthesis + schema validation, the P3 content
+// preflight, and the local summary. Extracted verbatim from runDataIngest
+// (cli#283) — step order, output, and exit codes unchanged.
+//
+// It mutates a in place (a.LocalPath's ~-expansion; a.Spec's intent default
+// and resolved schema / target size / min size / extension), exactly as the
+// inline code mutated its copy — so runDataIngest's --output-json error
+// defer (which stays next to the named return it reads) and the cluster
+// steps that follow see the resolved spec. cancelled=true with a nil err is
+// the guided flow's clean Ctrl-C: the caller exits 0, nothing ingested.
+func resolveLocalInput(out, errOut io.Writer, a *runDataIngestArgs) (layout *push.LocalLayout, spec map[string]any, specBytes []byte, cancelled bool, err error) {
+ // Intro header: brand + a plain-English explainer of what an ingest
+ // does, so a first-time user understands it before any prompts.
+ // Routed through a.Printer, so --output-json keeps it on stderr and
+ // --plain/non-TTY degrade cleanly. (#31)
+ // --overwrite + a reused --idempotency-key is a data-loss trap: the
+ // teardown removes the existing data, then jobs-manager treats the
+ // duplicate key as a REPLAY and attaches to the previous run instead of
+ // ingesting anything — old data gone, new data never loaded, exit 0 from
+ // the old Job's status. Refuse the combination outright.
+ if a.Overwrite && a.IdempotencyKey != "" {
+ return nil, nil, nil, false, &exitError{code: 2, err: errors.New(
+ "--overwrite can't be combined with --idempotency-key: a reused key makes the cluster replay the previous run instead of ingesting the new data — after --overwrite's removal that would report success while loading nothing. Drop one of the two (a fresh per-run key is the default).")}
+ }
+
+ a.Printer.Banner("tracebloc", "data ingest")
+ a.Printer.Para(strings.TrimSpace(`
+This ingests a dataset so models can train on it. Your files never leave your
+own infrastructure — tracebloc copies them into your workspace's storage,
+checks them, and loads them into a table your training runs read from. Other
+collaborators can train against that table without ever seeing the raw files.`))
+ a.Printer.Hintf("Learn more: https://docs.tracebloc.io")
+
+ // 0. Guided mode: prompt for any missing core inputs before
+ // validation. Flags already provided win; non-TTY / --no-input
+ // leaves Prompter nil and skips straight to the flag-only path.
+ if a.Interactive && a.Prompter != nil {
+ if err := runInteractive(a.Printer, a.Prompter, a, a.TaskSet); err != nil {
+ if errors.Is(err, errInteractiveCancelled) {
+ a.Printer.Infof("Cancelled — nothing was ingested.")
+ return nil, nil, nil, true, nil
+ }
+ // A typed exitError from a guided step (e.g. the path-existence
+ // guard, which runInteractive runs before the family sniff)
+ // already carries its own code + clean message — surface it as-is
+ // rather than burying it under "interactive setup:".
+ var ee *exitError
+ if errors.As(err, &ee) {
+ return nil, nil, nil, false, err
+ }
+ return nil, nil, nil, false, &exitError{code: 3, err: fmt.Errorf("interactive setup: %w", err)}
+ }
+ }
+ // --intent defaults to train. Applied after the interactive block so
+ // the guided flow still asks "training or test?" (it prompts on an
+ // empty value); a non-interactive run that omits --intent gets train
+ // without erroring (RFC-0002 §5). The wire field stays "intent".
+ if a.Spec.Intent == "" {
+ a.Spec.Intent = "train"
+ }
+ if a.LocalPath == "" {
+ return nil, nil, nil, false, &exitError{code: 3, err: errors.New(
+ "local dataset path is required — pass it as an argument, or run " +
+ "on a terminal without --no-input for guided prompts")}
+ }
+ // Expand a leading ~ ourselves. The shell expands an unquoted ~ on
+ // the command line, but a path typed at the interactive prompt (or
+ // a quoted/literal ~ arg) reaches us unexpanded — and filepath.Abs
+ // would just prepend the CWD, yielding ".../cwd/~/...". Mirrors
+ // cluster.expandPath; done here so it covers both entry points
+ // before any push.Discover* call. (#37)
+ a.LocalPath = expandHome(a.LocalPath)
+
+ // 0b. Path existence FIRST — before any spec / schema / family
+ // validation. A typo'd path should fail on the path with a plain
+ // "no such file or directory", not surface later as a confusing
+ // downstream error (e.g. the task gate asking which task the
+ // non-existent data is for). runInteractive runs this same guard
+ // before its family sniff / label preview, so the invariant holds on
+ // the guided route too; this re-check covers the flag-only path and
+ // is cheap (one stat). The family walk below stats again for its
+ // layout-specific diagnostics; this is only about ordering the first
+ // failure a customer sees. (#181)
+ if err := statDatasetPath(a.LocalPath); err != nil {
+ return nil, nil, nil, false, err
+ }
+
+ // 1. Validate the table name BEFORE anything else. It's both
+ // the MySQL identifier and the /data/shared// PVC
+ // subdirectory — an unsanitized traversal name (../../etc)
+ // would escape that subtree once the stage Pod writes to
+ // it. The embedded schema only checks minLength on `table`,
+ // so this CLI-side guard is the real fix. SpecArgs.Build()
+ // below calls StagedPrefix, which panics on an unsafe name —
+ // so this check MUST come first.
+ if err := push.ValidateTableName(a.Spec.Table); err != nil {
+ return nil, nil, nil, false, &exitError{code: 2, err: err}
+ }
+
+ // 2. Category gate. Runs BEFORE schema validation so an
+ // unsupported category gets a clear, actionable CLI message
+ // rather than the schema's terse enum / missing-property error.
+ // Every schema task category is CLI-supported now; this gate stays as
+ // defensive routing so a future known-but-not-yet-wired category gets a
+ // clear per-category message, and a typo'd category gets the supported
+ // list rather than the schema's raw enum dump.
+ switch {
+ case a.Spec.Category == "":
+ // No task chosen. In guided mode the picker already filled this;
+ // reaching here means a non-interactive run (or --no-input /
+ // --output-json) that omitted --task. Give a clear, actionable
+ // error instead of silently assuming images (the old default).
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "which task is this data for? pass --task — one of: %s. "+
+ "(On a terminal without --no-input, tracebloc asks you to pick.)",
+ push.SupportedCategoriesList())}
+ case push.IsCLISupported(a.Spec.Category):
+ // supported
+ case push.IsKnown(a.Spec.Category):
+ // A recognized category the CLI doesn't implement yet. None today — every
+ // schema category is wired — but kept as defensive routing so a future
+ // known-but-unsupported category gets the registry's per-category reason,
+ // not a misleading "unrecognized category". Supported categories were
+ // already caught above, so IsKnown here means known-but-unsupported.
+ spec, _ := push.Lookup(a.Spec.Category)
+ reason := ""
+ if spec.UnsupportedNote != "" {
+ reason = " (" + spec.UnsupportedNote + ")"
+ }
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "task %q isn't supported by the CLI yet%s. Supported tasks: %s.",
+ a.Spec.Category, reason, push.SupportedCategoriesList())}
+ default:
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "task %q isn't a recognized task. Supported tasks: %s.",
+ a.Spec.Category, push.SupportedCategoriesList())}
+ }
+
+ // Image-only flags. --target-size / --min-size describe image
+ // resolution, so they're meaningless on a tabular / text task.
+ // Reject them explicitly here: without this guard they'd be parsed
+ // only inside the image branch below, so on a non-image task the
+ // value — even a malformed one — was silently dropped with no error.
+ if !push.IsImage(a.Spec.Category) {
+ for _, f := range []struct{ name, val string }{
+ {"--target-size", a.TargetSizeFlag},
+ {"--min-size", a.MinSizeFlag},
+ } {
+ if f.val != "" {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "%s is image tasks only; it doesn't apply to task %q",
+ f.name, a.Spec.Category)}
+ }
+ }
+ }
+
+ // Task-scoped flags. Like --target-size/--min-size above, each of these is
+ // read only inside the one category branch that consumes it, so passing one
+ // on a task that doesn't use it silently dropped the value — and the user's
+ // intent — with no error, even though the help text says each is scoped.
+ // Reject a misapplied flag explicitly so it fails fast instead of being
+ // ignored (the scope mirrors spec.go's build gates exactly).
+ if a.SchemaFlag != "" && !push.IsTabular(a.Spec.Category) {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "--schema is tabular/time-series tasks only; it doesn't apply to task %q", a.Spec.Category)}
+ }
+ if a.Spec.LabelPolicy != "" && !push.IsRegressionClass(a.Spec.Category) {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "--label-policy is regression-class tasks only (tabular_regression, "+
+ "time_series_forecasting, time_to_event_prediction); it doesn't apply to task %q",
+ a.Spec.Category)}
+ }
+ if a.Spec.TimeColumn != "" && a.Spec.Category != "time_to_event_prediction" {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "--time-column is time_to_event_prediction only; it doesn't apply to task %q", a.Spec.Category)}
+ }
+ if a.Spec.NumberOfKeypoints != 0 && a.Spec.Category != "keypoint_detection" {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "--number-of-keypoints is keypoint_detection only; it doesn't apply to task %q", a.Spec.Category)}
+ }
+ // --label-column is meaningless for self-supervised text (the label is the
+ // text itself); buildText drops it, so accepting it silently discarded the
+ // user's value and the review echoed a column that never shipped.
+ if a.Spec.LabelColumn != "" && push.SelfSupervisedText(a.Spec.Category) {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "--label-column doesn't apply to task %q — it trains on the text itself, with no label column",
+ a.Spec.Category)}
+ }
+
+ // 3. Walk the local directory FIRST (local "fail fast"), dispatched
+ // by category family. Image categories expect labels.csv +
+ // images/; tabular / time-series categories expect a single
+ // data CSV. The walk also yields what the per-category
+ // resolution below needs (the image list for target-size, the
+ // CSV for schema inference).
+ // err and layout are this function's named returns, so neither is
+ // redeclared here. The walk can take a moment on a large tree, so it
+ // gets a spinner — no blocking wait stays silent.
+ walkSpin := a.Printer.Spinner("Reading your files", "")
+ switch {
+ case push.IsTabular(a.Spec.Category):
+ layout, err = push.DiscoverTabular(a.LocalPath)
+ case push.IsText(a.Spec.Category):
+ layout, err = push.DiscoverText(a.Spec.Category, a.LocalPath)
+ case a.Spec.Category == "object_detection":
+ layout, err = push.DiscoverObjectDetection(a.LocalPath)
+ case a.Spec.Category == "semantic_segmentation":
+ layout, err = push.DiscoverSemanticSegmentation(a.LocalPath)
+ default:
+ // image_classification + keypoint_detection: labels.csv + images/.
+ layout, err = push.Discover(a.LocalPath)
+ }
+ walkSpin.Stop()
+ if err != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: err}
+ }
+
+ a.Printer.Step(1, 3, "Check your data")
+ a.Printer.Hintf("Reading your files locally first — nothing has touched your workspace yet — so a layout or settings problem shows up right away.")
+
+ // 3a. Per-category spec resolution from the local data, so the
+ // synthesized spec carries the right fields before validation.
+ switch {
+ case push.IsTabular(a.Spec.Category):
+ // P3 (cli#71): a BOM'd tabular CSV is doomed in-cluster AND would
+ // corrupt InferSchema's own header read below — reject before
+ // either. The rest of the content preflight runs after the spec
+ // schema validation (mirroring the in-cluster order).
+ if perr := push.CheckTabularBOM(layout.LabelsCSV); perr != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: perr}
+ }
+ if perr := push.CheckHasDataRows(layout.LabelsCSV); perr != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: perr}
+ }
+
+ // Column schema. An explicit --schema wins (raw flag, or the
+ // optional override the interactive prompt captures into SchemaFlag).
+ // Otherwise infer the types here — mirroring the ingestor's own rules
+ // (di#349) — and EMIT the result explicitly (below, via a.Spec.Schema
+ // → spec.schema), so the ingestor uses the CLI's answer regardless of
+ // its own version. Inference runs on both a no-schema non-interactive
+ // run and an interactive run where the user left the schema prompt
+ // blank; the risky cases below are surfaced as warnings.
+ if a.SchemaFlag != "" {
+ sch, perr := push.ParseSchema(a.SchemaFlag)
+ if perr != nil {
+ return nil, nil, nil, false, &exitError{code: 2, err: perr}
+ }
+ a.Spec.Schema = sch
+ } else {
+ res, ierr := push.InferSchema(layout.LabelsCSV)
+ if ierr != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: fmt.Errorf("inferring schema from CSV: %w", ierr)}
+ }
+ a.Spec.Schema = res.Schema
+ _, _ = fmt.Fprintf(out,
+ "Inferred schema for %d column(s) from %s (override with --schema).\n",
+ len(res.Schema), filepath.Base(layout.LabelsCSV))
+ if len(res.Skipped) > 0 {
+ _, _ = fmt.Fprintf(out,
+ " (skipped framework-managed column(s): %s)\n", strings.Join(res.Skipped, ", "))
+ }
+ if len(res.Empty) > 0 {
+ _, _ = fmt.Fprintf(out,
+ " (warning: %d column(s) had no values in the sample and were typed VARCHAR(1): %s)\n",
+ len(res.Empty), strings.Join(res.Empty, ", "))
+ }
+ if len(res.IDLike) > 0 {
+ _, _ = fmt.Fprintf(out,
+ " (warning: %d column(s) look like identifiers (all-unique integers): %s — "+
+ "if any is a zero-padded code, pass --schema to type it VARCHAR)\n",
+ len(res.IDLike), strings.Join(res.IDLike, ", "))
+ }
+ }
+ case push.IsImage(a.Spec.Category):
+ // keypoint_detection needs --number-of-keypoints (dataset-
+ // specific, no default). Catch it here with an actionable
+ // message rather than letting the ingestor fail mid-run. Split
+ // UNSET from SET-BUT-INVALID (#76b): a Go 0 could mean either, so
+ // key on whether the flag was passed (ChangedFlags). Unset → the
+ // "requires" nudge; set to a non-positive value → name the bad
+ // value so the user sees exactly what was rejected.
+ if a.Spec.Category == "keypoint_detection" && a.Spec.NumberOfKeypoints <= 0 {
+ if a.ChangedFlags["number-of-keypoints"] {
+ return nil, nil, nil, false, &exitError{code: 2, err: fmt.Errorf(
+ "--number-of-keypoints must be a positive integer (got %d); "+
+ "it's the number of keypoints per sample (e.g. 17 for COCO pose)",
+ a.Spec.NumberOfKeypoints)}
+ }
+ return nil, nil, nil, false, &exitError{code: 2, err: errors.New(
+ "keypoint_detection requires --number-of-keypoints (e.g. " +
+ "--number-of-keypoints 17); it's dataset-specific and has no default")}
+ }
+ // Image target resolution: the ingestor's image_classification
+ // default is 512x512 and it VALIDATES (it does not resize), so
+ // a mismatch hard-fails. Honour an explicit --target-size;
+ // otherwise auto-detect from the first image so the common
+ // "all my images are NxN" case just works.
+ if a.TargetSizeFlag != "" {
+ w, h, perr := push.ParseTargetSize(a.TargetSizeFlag)
+ if perr != nil {
+ return nil, nil, nil, false, &exitError{code: 2, err: perr}
+ }
+ a.Spec.TargetSize = []int{w, h}
+ } else if len(layout.Images) > 0 {
+ if w, h, derr := push.DetectImageSize(layout.Images[0]); derr == nil {
+ a.Spec.TargetSize = []int{w, h}
+ _, _ = fmt.Fprintf(out,
+ "Auto-detected image target size %dx%d from %s (override with --target-size).\n",
+ w, h, filepath.Base(layout.Images[0]))
+ } else {
+ _, _ = fmt.Fprintf(errOut,
+ "Note: couldn't auto-detect image size (%v); using the ingestor "+
+ "default. Pass --target-size WxH if ingestion reports a "+
+ "resolution mismatch.\n", derr)
+ }
+ }
+ // Minimum-size floor override (#348): plumb an explicit --min-size to
+ // spec.file_options.min_size. When unset, no spec field is emitted, so
+ // the ingestor applies its own default (none on the deployed
+ // v0.5.7/v0.6.0; 32x32 on develop post-#348) — and the local preview
+ // applies NO floor either (PreflightDataset only previews the floor
+ // when --min-size is set, so it never rejects an ingest the live
+ // cluster accepts). The below-floor reject is previewed in
+ // runLocalPreflight (ValidateImages).
+ if a.MinSizeFlag != "" {
+ w, h, perr := push.ParseMinSize(a.MinSizeFlag)
+ if perr != nil {
+ return nil, nil, nil, false, &exitError{code: 2, err: perr}
+ }
+ a.Spec.MinSize = []int{w, h}
+ }
+ // Extension: every image must share one type, and the spec tells
+ // the cluster which one to validate against (file_options.extension).
+ // Without this the ingestor checked its .jpeg convention default and
+ // rejected .jpg/.png datasets AFTER the full upload (cli#68).
+ ext, exterr := push.DetectExtension(layout.Images)
+ if exterr != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: exterr}
+ }
+ a.Spec.Extension = ext
+ default:
+ // Text family: no extra per-category resolution. The supervised text
+ // tasks (text_classification, token_classification,
+ // sentence_pair_classification) carry a label straight from
+ // --label-column; the self-supervised ones (masked/causal language
+ // modeling, seq2seq, embeddings) need neither a label nor a schema.
+ // buildText emits the label for exactly the supervised set, keyed on
+ // the registry's SelfSupervised flag (not a hardcoded id).
+ }
+
+ // 3b. Friendly missing-label pre-check (#214). Tabular / time-series tasks
+ // AND semantic_segmentation carry a required label column (the ingest
+ // schema's allOf requires `label` for them). With no --label-column the
+ // synthesized spec's `label` is an empty string, which trips the schema's
+ // label oneOf and the raw validation below dumps an opaque "got object,
+ // want string" / "minLength" pair. semseg is especially prone to this —
+ // its per-image label reads as vestigial beside the pixel masks, so the
+ // flag is easy to forget. Intercept ONLY that specific missing case here —
+ // a label present-but-not-in-the-CSV still flows to runLocalPreflight's
+ // CheckLabelColumn, and every other schema error still reaches the dump —
+ // and name the flag to fix instead.
+ if (push.IsTabular(a.Spec.Category) || a.Spec.Category == "semantic_segmentation") && a.Spec.LabelColumn == "" {
+ msg := "this task needs a label column, but --label-column wasn't set — " +
+ "pass --label-column with the name of the target column in your data CSV"
+ if cols := sortedKeys(a.Spec.Schema); len(cols) > 0 {
+ msg += " (columns: " + strings.Join(cols, ", ") + ")"
+ }
+ return nil, nil, nil, false, &exitError{code: 2, err: errors.New(msg)}
+ }
+
+ // 4. Synthesize the spec from flags + validate against schema.
+ // Catches "bad category", "missing intent" etc. BEFORE we
+ // touch the cluster. The error formatter is the same one
+ // ingest validate uses, so a customer who YAML'd manually
+ // first sees identical wording.
+ spec = a.Spec.Build()
+ specBytes, err = yaml.Marshal(spec)
+ if err != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: fmt.Errorf("marshaling synthesized spec: %w", err)}
+ }
+ v, err := schema.NewV1Validator()
+ if err != nil {
+ return nil, nil, nil, false, &exitError{code: 3, err: fmt.Errorf("loading embedded schema: %w", err)}
+ }
+ _, errs, parseErr := v.ValidateYAML(specBytes)
+ if parseErr != nil {
+ // "Parse" failing on a spec we marshaled ourselves is a
+ // programming error, not a customer error — surface it
+ // with the bytes so we can diagnose. Exit 3 (the
+ // "internal" bucket) matches the marshal-failure branch
+ // above.
+ return nil, nil, nil, false, &exitError{code: 3, err: fmt.Errorf("internal: re-parsing synthesized spec: %w\n%s", parseErr, specBytes)}
+ }
+ if len(errs) > 0 {
+ // Use the SAME formatter `ingest validate` uses, so the
+ // experience is identical whether the customer authored
+ // YAML by hand or via flags. Diagnostics go to stderr
+ // (matching ingest validate) so a downstream pipe of
+ // stdout (e.g. piping the summary to jq once that's a
+ // JSON output mode) isn't polluted by error text. Exit 2
+ // is reserved for schema violations across the CLI.
+ _, _ = fmt.Fprintf(errOut, "synthesized spec failed schema validation (%d issue%s):\n",
+ len(errs), plural(len(errs)))
+ _, _ = fmt.Fprintln(errOut, schema.FormatErrors(errs))
+ return nil, nil, nil, false, &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")}
+ }
+
+ // P3 content preflight (backend#828, cli#69/#71/#72/#73): preview the
+ // ingestor's own validators locally — AFTER the spec schema validation,
+ // mirroring the in-cluster order (jsonschema first, then validators),
+ // and BEFORE any cluster work. Each check names the rule it previews;
+ // parity is pinned by internal/push/parity_golden_test.go.
+ if perr := runLocalPreflight(*a, layout, errOut); perr != nil {
+ return nil, nil, nil, false, perr
+ }
+
+ printLocalSummary(a.Printer, layout, spec)
+
+ return layout, spec, specBytes, false, nil
+}
+
// printLocalSummary shows what the CLI found on disk plus the ingest
// settings it assembled — the detail under step 1 ("Check your data").
// Mirrors `cluster info`'s section/Field layout.