feat(data ingest): wire the 5 text tasks (#182)

internal/cli/data.go

@@ -559,12 +559,12 @@ collaborators can train against that table without ever seeing the raw files.`))
 	case push.IsCLISupported(a.Spec.Category):
  // supported
 	case push.IsKnown(a.Spec.Category):
- // A recognized category data ingest doesn't implement yet — image
- // (semantic_segmentation) or text (causal_language_modeling, seq2seq,
- // …). Routed here (not the default branch) so the
- // user gets the registry's per-category pending-support reason, not a
- // misleading "unrecognized category". Supported categories were already
- // caught above, so IsKnown here means known-but-unsupported.
+ // A recognized category data ingest doesn't implement yet — today just
+ // semantic_segmentation (awaiting the ingestor's mask_id link column +
+ // training sign-off, backend#816). Routed here (not the default branch)
+ // so the user gets the registry's per-category pending-support reason,
+ // not a misleading "unrecognized category". Supported categories were
+ // already caught above, so IsKnown here means known-but-unsupported.
  spec, _ := push.Lookup(a.Spec.Category)
  return &exitError{code: 2, err: fmt.Errorf(
  "task %q isn't supported by the CLI yet (%s). Supported tasks: %s.",
@@ -724,9 +724,13 @@ collaborators can train against that table without ever seeing the raw files.`))
  }
  a.Spec.Extension = ext
 	default:
- // Text family: no extra per-category resolution. The label (for
- // text_classification) comes straight from --label-column;
- // masked_language_modeling needs neither a label nor a schema.
+ // Text family: no extra per-category resolution. The supervised text
+ // tasks (text_classification, token_classification,
+ // sentence_pair_classification) carry a label straight from
+ // --label-column; the self-supervised ones (masked/causal language
+ // modeling, seq2seq, embeddings) need neither a label nor a schema.
+ // buildText emits the label for exactly the supervised set, keyed on
+ // the registry's SelfSupervised flag (not a hardcoded id).
 	}
 
 	// 4. Synthesize the spec from flags + validate against schema.

internal/cli/data_test.go

@@ -94,7 +94,7 @@ func execDataIngest(t *testing.T, args []string) (exitCode int, stdout, stderr s
 func TestDataIngest_UnsupportedCategory_ExitsTwo(t *testing.T) {
 	root := imgcLayout(t)
 	for _, badCategory := range []string{
- "semantic_segmentation", // known but blocked on the ingestor (data-ingestors#136)
+ "semantic_segmentation", // known but pending (awaiting mask_id + training sign-off, backend#816)
  "instance_segmentation", // dead — removed from the registry (#1005), now unrecognized
  "definitely-not-a-category", // nonsense; gate catches this too
 	} {
@@ -114,19 +114,19 @@ func TestDataIngest_UnsupportedCategory_ExitsTwo(t *testing.T) {
 }
 
 // TestDataIngest_KnownUnsupportedCategory_PendingNote pins the Bugbot fix
-// (v0.4.0 RC): a registry-known but CLI-unsupported NON-image category
-// (causal_language_modeling) must get the registry's pending-support note, not
-// the misleading "isn't a recognized task category" message. execDataIngest
-// discards the error and SilenceErrors swallows it, so run the command here and
-// inspect the returned error directly.
+// (v0.4.0 RC): a registry-known but CLI-unsupported category
+// (semantic_segmentation — the sole remaining one after phase 4) must get the
+// registry's pending-support note, not the misleading "isn't a recognized task
+// category" message. execDataIngest discards the error and SilenceErrors
+// swallows it, so run the command here and inspect the returned error directly.
 func TestDataIngest_KnownUnsupportedCategory_PendingNote(t *testing.T) {
 	root := imgcLayout(t)
 	rootCmd := NewRootCmd(BuildInfo{Version: "test"})
 	rootCmd.SetOut(&bytes.Buffer{})
 	rootCmd.SetErr(&bytes.Buffer{})
 	rootCmd.SetArgs([]string{"data", "ingest",
  "--kubeconfig=/tmp/tracebloc-cli-test-nonexistent-" + t.Name(),
- root, "--name=t1", "--task=causal_language_modeling",
+ root, "--name=t1", "--task=semantic_segmentation",
  "--intent=train", "--label-column=label"})
 	err := rootCmd.Execute()
 	if err == nil {

internal/cli/interactive_test.go

@@ -240,13 +240,13 @@ func TestRunInteractive_ExplicitTaskSkipsSniff(t *testing.T) {
 }
 
 // TestPickTask_FamilyScoped: the picker offers only the given family's
-// tasks, wires the friendly display names + the locked glosses, and lists
-// the not-yet-supported ones (greyed, with a reason) — never the other
-// families' tasks.
+// tasks, wires the friendly display names + the locked glosses — never the
+// other families' tasks. After RFC-0002 phase 4 every text task is wired, so
+// the text picker has no "Not yet in the CLI" section at all.
 func TestPickTask_FamilyScoped(t *testing.T) {
-	// Text family: fill-mask (gloss) is available; seq2seq
-	// (translation / summarization, gloss) + token_classification are
-	// pending; image/tabular tasks must not appear.
+	// Text family: all tasks are available now — fill-mask (gloss),
+	// classification, the two structured-pair tasks, and the two seq tasks;
+	// image/tabular tasks must not appear.
 	f := &fakePrompter{answers: map[string]string{"Which task?": "Text classification"}}
 	var buf bytes.Buffer
 	p := ui.New(&buf, ui.WithColor(false))
@@ -260,17 +260,22 @@ func TestPickTask_FamilyScoped(t *testing.T) {
 	out := buf.String()
 	for _, want := range []string{
  "Tasks for text data",
- "fill-mask", // MLM gloss (available)
- "Text classification", // label
- "Not yet in the CLI:", // pending header
- "translation / summarization", // seq2seq gloss (pending)
- "token_classification", // pending id
- "schema-recognized", // an UnsupportedNote fragment
+ "Available now:",
+ "fill-mask", // MLM gloss (available)
+ "Text classification", // label
+ "translation / summarization", // seq2seq gloss (now available)
+ "token_classification", // now available
+ "sentence_pair_classification", // now available
+ "Embeddings", // now available
 	} {
  if !strings.Contains(out, want) {
  t.Errorf("picker output missing %q:\n%s", want, out)
  }
 	}
+	// Every text task is wired now — no pending section.
+	if strings.Contains(out, "Not yet in the CLI:") {
+ t.Errorf("text picker should have no pending section now:\n%s", out)
+	}
 	// Other families must not leak in.
 	for _, unwanted := range []string{"Image classification", "Tabular classification", "Survival analysis"} {
  if strings.Contains(out, unwanted) {
@@ -279,6 +284,30 @@ func TestPickTask_FamilyScoped(t *testing.T) {
 	}
 }
 
+// TestPickTask_ImagePending: semantic_segmentation is the sole remaining
+// CLI-pending task, so the image picker still renders a greyed "Not yet in the
+// CLI" section with its backend#816 reason.
+func TestPickTask_ImagePending(t *testing.T) {
+	f := &fakePrompter{answers: map[string]string{"Which task?": "Image classification"}}
+	var buf bytes.Buffer
+	p := ui.New(&buf, ui.WithColor(false))
+	if _, err := pickTask(p, f, push.FamilyImage); err != nil {
+ t.Fatalf("pickTask: %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{
+ "Available now:",
+ "Image classification",
+ "Not yet in the CLI:",
+ "semantic_segmentation",
+ "backend#816", // the UnsupportedNote reason
+	} {
+ if !strings.Contains(out, want) {
+ t.Errorf("image picker missing %q:\n%s", want, out)
+ }
+	}
+}
+
 // TestPickTask_TabularGloss: the tabular picker shows the survival-analysis
 // gloss for time_to_event_prediction and can select it back to its id.
 func TestPickTask_TabularGloss(t *testing.T) {

internal/push/category.go

@@ -1,6 +1,9 @@
 package push
 
-import "strings"
+import (
+	"fmt"
+	"strings"
+)
 
 // CategorySpec is the single source of truth for one task category's
 // CLI-relevant rules. It mirrors data-ingestors'
@@ -36,12 +39,26 @@ type CategorySpec struct {
 	// never ships to the central backend by default.
 	RegressionClass bool
 	// SelfSupervised marks text categories that train without an explicit
-	// label column — the target is derived from the text itself (MLM masks
-	// tokens; CLM predicts the next token), so the interactive flow skips
-	// the "which column is the label?" question. A registry fact rather
-	// than a hardcoded id list so a new self-supervised task can't be added
-	// without deciding this (SelfSupervisedText reads it).
+	// label column — no `label` travels in labels.csv. For MLM/CLM the target
+	// is derived from the text itself (mask a token; predict the next one); for
+	// seq2seq and embeddings it comes from the record's own paired fields
+	// (source→target, anchor/positive/negative). Either way there's no label
+	// column, so the interactive flow skips the "which column is the label?"
+	// question. A registry fact rather than a hardcoded id list so a new
+	// self-supervised task can't be added without deciding this
+	// (SelfSupervisedText reads it). Mirrors the ingestor registry's
+	// is_self_supervised (data-ingestors modalities/registry.py).
 	SelfSupervised bool
+	// IsClassification marks categories the ingestor treats as classification
+	// (registry ModalitySpec.is_classification) — the ones whose validator
+	// chain gets a LabelDiversityValidator, so the dataset needs >= 2 distinct
+	// labels. Mirrors the ingestor exactly: the image family +
+	// text_classification + sentence_pair_classification + tabular_classification
+	// are true; token_classification is NOT (its labels are BIO tag sequences,
+	// checked by BIOLabelValidator, not class labels), nor are the regression /
+	// self-supervised tasks. The label-diversity preflight reads it so it can't
+	// drift from the ingestor's wiring.
+	IsClassification bool
 	// CLISupported reports whether `dataset push` implements the category
 	// today. semantic_segmentation is known (the schema defines it) but
 	// not yet pushable.
@@ -76,42 +93,41 @@ const (
 // nor carry an extra the ingestor won't accept (the instance_segmentation
 // half-ingest class — data-ingestors #240/#99, #1005).
 var categoryRegistry = []CategorySpec{
-	{ID: "image_classification", Family: FamilyImage, Label: "Image classification", CLISupported: true,
+	{ID: "image_classification", Family: FamilyImage, Label: "Image classification", CLISupported: true, IsClassification: true,
  Blurb: "sort images into classes"},
-	{ID: "object_detection", Family: FamilyImage, Label: "Object detection", CLISupported: true,
+	{ID: "object_detection", Family: FamilyImage, Label: "Object detection", CLISupported: true, IsClassification: true,
  Blurb: "draw boxes around objects in an image"},
-	{ID: "keypoint_detection", Family: FamilyImage, Label: "Keypoint detection", CLISupported: true,
+	{ID: "keypoint_detection", Family: FamilyImage, Label: "Keypoint detection", CLISupported: true, IsClassification: true,
  Blurb: "locate landmark points on an image (e.g. pose)"},
-	{ID: "text_classification", Family: FamilyText, Label: "Text classification", CLISupported: true,
+	{ID: "text_classification", Family: FamilyText, Label: "Text classification", CLISupported: true, IsClassification: true,
  Blurb: "sort text snippets into classes"},
 	{ID: "masked_language_modeling", Family: FamilyText, Label: "Masked language modeling", Gloss: "fill-mask", CLISupported: true, SelfSupervised: true,
  Blurb: "predict masked-out words — no labels needed"},
-	{ID: "tabular_classification", Family: FamilyTabular, Label: "Tabular classification", CLISupported: true,
+	{ID: "tabular_classification", Family: FamilyTabular, Label: "Tabular classification", CLISupported: true, IsClassification: true,
  Blurb: "predict a class from table columns"},
 	{ID: "tabular_regression", Family: FamilyTabular, Label: "Tabular regression", RegressionClass: true, CLISupported: true,
  Blurb: "predict a number from table columns"},
 	{ID: "time_series_forecasting", Family: FamilyTabular, Label: "Time-series forecasting", RegressionClass: true, CLISupported: true,
  Blurb: "predict future values from past ones"},
 	{ID: "time_to_event_prediction", Family: FamilyTabular, Label: "Time-to-event prediction", Gloss: "Survival analysis", RegressionClass: true, CLISupported: true,
  Blurb: "predict how long until an event happens"},
-	{ID: "semantic_segmentation", Family: FamilyImage, Label: "Semantic segmentation", CLISupported: false,
+	{ID: "causal_language_modeling", Family: FamilyText, Label: "Causal language modeling", CLISupported: true, SelfSupervised: true,
+ Blurb: "predict the next word in a sequence"},
+	{ID: "seq2seq", Family: FamilyText, Label: "Sequence-to-sequence", Gloss: "translation / summarization", CLISupported: true, SelfSupervised: true,
+ Blurb: "map an input sequence to an output one"},
+	{ID: "token_classification", Family: FamilyText, Label: "Token classification", CLISupported: true,
+ Blurb: "label each word in a sequence"},
+	{ID: "sentence_pair_classification", Family: FamilyText, Label: "Sentence-pair classification", CLISupported: true, IsClassification: true,
+ Blurb: "label how two texts relate"},
+	{ID: "embeddings", Family: FamilyText, Label: "Embeddings", CLISupported: true, SelfSupervised: true,
+ Blurb: "learn vector representations from text pairs"},
+	// semantic_segmentation stays CLI-pending: di#136 (mask sidecar) shipped,
+	// but the ingestor doesn't yet populate the mask_id link column the
+	// contract requires, and the training-side sign-off is tracked in
+	// backend#816. Wire it once those land (RFC-0002 phase 4 follow-up).
+	{ID: "semantic_segmentation", Family: FamilyImage, Label: "Semantic segmentation", CLISupported: false, IsClassification: true,
  Blurb: "label every pixel in an image",
- UnsupportedNote: "blocked on the ingestor's mask-sidecar support (data-ingestors#136)"},
-	{ID: "causal_language_modeling", Family: FamilyText, Label: "Causal language modeling", CLISupported: false, SelfSupervised: true,
- Blurb: "predict the next word in a sequence",
- UnsupportedNote: "schema-recognized (data-ingestors#805); `tracebloc ingest` discover/build for its raw-.txt / prompt\\tcompletion `texts` layout is pending"},
-	{ID: "seq2seq", Family: FamilyText, Label: "Sequence-to-sequence", Gloss: "translation / summarization", CLISupported: false,
- Blurb: "map an input sequence to an output one",
- UnsupportedNote: "schema-recognized; `tracebloc ingest` discover/build for its raw-.txt / source\\ttarget `texts` layout is pending"},
-	{ID: "token_classification", Family: FamilyText, Label: "Token classification", CLISupported: false,
- Blurb: "label each word in a sequence",
- UnsupportedNote: "schema-recognized; the CLI doesn't stage its per-token-label `texts` layout yet"},
-	{ID: "sentence_pair_classification", Family: FamilyText, Label: "Sentence-pair classification", CLISupported: false,
- Blurb: "label how two texts relate",
- UnsupportedNote: "schema-recognized; `tracebloc ingest` discover/build for its raw-.txt / text_a\\ttext_b `texts` layout is pending"},
-	{ID: "embeddings", Family: FamilyText, Label: "Embeddings", CLISupported: false,
- Blurb: "learn vector representations from text pairs",
- UnsupportedNote: "schema-recognized; `tracebloc ingest` discover/build for its raw-.txt / anchor\\tpositive[\\tnegative] `texts` layout is pending"},
+ UnsupportedNote: "schema-recognized; awaiting the ingestor's mask_id link column + training sign-off (backend#816)"},
 }
 
 // categoryByID indexes the registry for O(1) lookup, built once.
@@ -206,12 +222,13 @@ func FamilyNouns() []string {
 }
 
 // SelfSupervisedText reports whether a text category trains without an
-// explicit label column — the target is derived from the text itself, so
-// the CLI skips the "which column is the label?" question. MLM masks
-// tokens; CLM predicts the next token; neither reads a labels column. The
-// answer is the registry's SelfSupervised flag, so a new self-supervised
-// task is handled the moment it's added to the registry — not when someone
-// remembers to edit this function.
+// explicit label column, so the CLI skips the "which column is the label?"
+// question. MLM/CLM derive the target from the text itself (mask a token;
+// predict the next one); seq2seq and embeddings derive it from the record's
+// own paired fields (source→target, anchor/positive/negative) — none reads a
+// labels column. The answer is the registry's SelfSupervised flag, so a new
+// self-supervised task is handled the moment it's added to the registry — not
+// when someone remembers to edit this function.
 func SelfSupervisedText(category string) bool {
 	c, ok := categoryByID[category]
 	return ok && c.SelfSupervised
@@ -249,6 +266,14 @@ func IsText(category string) bool {
 // therefore needs label.policy (object label form).
 func IsRegressionClass(category string) bool { return categoryByID[category].RegressionClass }
 
+// IsClassification reports whether the ingestor treats category as a
+// classification task (registry ModalitySpec.is_classification) — i.e. its
+// validator chain includes LabelDiversityValidator. The label-diversity
+// preflight gates on this so the CLI mirrors the ingestor's wiring rather than
+// hardcoding a category id (which is exactly how the text-family preflight
+// drifted when it only knew text_classification).
+func IsClassification(category string) bool { return categoryByID[category].IsClassification }
+
 // SupportedCategoryIDs returns the ids `dataset push` supports, in display
 // order. Used to build the --task help, the interactive picker, and
 // the accept-gate's "Supported:" lists from one place.
@@ -275,13 +300,27 @@ func AllCategoryIDs() []string {
 // and gate error messages.
 func SupportedCategoriesList() string { return strings.Join(SupportedCategoryIDs(), ", ") }
 
-// TextSidecarDir returns the sidecar directory name a text category
-// expects: "sequences" for masked_language_modeling, "texts" for
-// text_classification. (Used both as the local subdir to stage and the
-// spec field to emit.)
+// TextSidecarDir returns the sidecar directory name a text category expects
+// ("sequences" for masked_language_modeling, "texts" for every other text
+// task). Used both as the local subdir to stage and the spec field to emit.
+//
+// The value is READ from the vendored layout contract's primary_subdir — the
+// ingestor owns this fact (data-ingestors registry ModalitySpec.file_subdir),
+// so the CLI mirrors it rather than keeping a Go fork of the same rule
+// (RFC-0002 Principle 6).
+//
+// A text category with no primary_subdir in the contract is a vendoring/drift
+// bug, not a runtime condition: every text task pins one and
+// TestTextSidecarDirMirrorsContract enforces the registry and contract agree.
+// Silently falling back to "texts" would stage files into a directory the
+// ingestor never reads, so fail loud instead — the only way here is a broken
+// vendored contract, which scripts/sync-schema.sh --check catches in CI.
 func TextSidecarDir(category string) string {
-	if category == "masked_language_modeling" {
- return "sequences"
+	if layout, ok := LayoutFor(category); ok && layout.PrimarySubdir != nil {
+ return *layout.PrimarySubdir
 	}
-	return "texts"
+	panic(fmt.Sprintf(
+ "text category %q has no primary_subdir in the vendored layout contract — "+
+ "the Go registry has drifted from layout.v1.json; re-run scripts/sync-schema.sh",
+ category))
 }