diff --git a/internal/push/category.go b/internal/push/category.go index 9ea14c3d..b48640dc 100644 --- a/internal/push/category.go +++ b/internal/push/category.go @@ -68,6 +68,8 @@ var categoryRegistry = []CategorySpec{ UnsupportedNote: "blocked on the ingestor's mask-sidecar support (data-ingestors#136)"}, {ID: "instance_segmentation", Family: FamilyImage, Label: "Instance segmentation", CLISupported: false, UnsupportedNote: "not implemented"}, + {ID: "causal_language_modeling", Family: FamilyText, Label: "Causal language modeling", CLISupported: false, + UnsupportedNote: "schema-recognized (data-ingestors#805); `tracebloc ingest` discover/build for its raw-.txt / prompt\\tcompletion `texts` layout is pending"}, } // categoryByID indexes the registry for O(1) lookup, built once. diff --git a/internal/push/category_registry_test.go b/internal/push/category_registry_test.go index 1bb40ea3..997fc2ee 100644 --- a/internal/push/category_registry_test.go +++ b/internal/push/category_registry_test.go @@ -13,7 +13,7 @@ func TestRegistryKnownCategories(t *testing.T) { want := []string{ "image_classification", "object_detection", "keypoint_detection", "semantic_segmentation", "instance_segmentation", - "text_classification", "masked_language_modeling", + "text_classification", "masked_language_modeling", "causal_language_modeling", "tabular_classification", "tabular_regression", "time_series_forecasting", "time_to_event_prediction", } @@ -40,9 +40,9 @@ func TestSupportedCategories(t *testing.T) { t.Errorf("SupportedCategoryIDs returned %q but IsCLISupported is false", id) } } - // semantic_/instance_segmentation are known but not yet pushable, and - // must explain why. - for _, id := range []string{"semantic_segmentation", "instance_segmentation"} { + // segmentation + causal_language_modeling are known but not yet pushable, + // and must explain why. + for _, id := range []string{"semantic_segmentation", "instance_segmentation", "causal_language_modeling"} { if !IsKnown(id) { t.Errorf("%s should be known", id) } diff --git a/internal/schema/ingest.v1.json b/internal/schema/ingest.v1.json index 7f4f4a04..b5332b99 100644 --- a/internal/schema/ingest.v1.json +++ b/internal/schema/ingest.v1.json @@ -36,7 +36,8 @@ "tabular_regression", "time_series_forecasting", "time_to_event_prediction", - "masked_language_modeling" + "masked_language_modeling", + "causal_language_modeling" ], "description": "Task category. Drives convention defaults: validators, data_format, default columns, default file extensions, default validator set." }, @@ -86,7 +87,7 @@ "texts": { "type": "string", "minLength": 1, - "description": "Directory holding text files referenced by the labels CSV. Required for text_classification and token_classification." + "description": "Directory holding text files referenced by the labels CSV. Required for text_classification, token_classification, and causal_language_modeling (raw .txt: plain text, or a tab-separated prompt\\tcompletion pair)." }, "sequences": { @@ -336,6 +337,14 @@ }, "then": { "required": ["sequences"] } }, + { + "description": "causal_language_modeling requires `texts` (raw .txt samples). It is self-supervised, so unlike text/token classification it does NOT require `label`.", + "if": { + "properties": { "category": { "const": "causal_language_modeling" } }, + "required": ["category"] + }, + "then": { "required": ["texts"] } + }, { "description": "tabular and time-series categories require `schema`.", "if": { @@ -411,7 +420,8 @@ "properties": { "category": { "enum": [ - "masked_language_modeling" + "masked_language_modeling", + "causal_language_modeling" ] } },