Skip to content

chore(schema): re-vendor ingest.v1.json from data-ingestors (fix drift check)#103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
saadqbal merged 1 commit into from
Jun 24, 2026
Merged

chore(schema): re-vendor ingest.v1.json from data-ingestors (fix drift check) #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions internal/push/category.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -68,6 +68,8 @@ var categoryRegistry = []CategorySpec{
UnsupportedNote: "blocked on the ingestor's mask-sidecar support (data-ingestors#136)"},
{ID: "instance_segmentation", Family: FamilyImage, Label: "Instance segmentation", CLISupported: false,
UnsupportedNote: "not implemented"},
{ID: "causal_language_modeling", Family: FamilyText, Label: "Causal language modeling", CLISupported: false,
UnsupportedNote: "schema-recognized (data-ingestors#805); `tracebloc ingest` discover/build for its raw-.txt / prompt\\tcompletion `texts` layout is pending"},
}

// categoryByID indexes the registry for O(1) lookup, built once.
Expand Down
8 changes: 4 additions & 4 deletions internal/push/category_registry_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,7 +13,7 @@ func TestRegistryKnownCategories(t *testing.T) {
want := []string{
"image_classification", "object_detection", "keypoint_detection",
"semantic_segmentation", "instance_segmentation",
"text_classification", "masked_language_modeling",
"text_classification", "masked_language_modeling", "causal_language_modeling",
"tabular_classification", "tabular_regression",
"time_series_forecasting", "time_to_event_prediction",
}
Expand All@@ -40,9 +40,9 @@ func TestSupportedCategories(t *testing.T) {
t.Errorf("SupportedCategoryIDs returned %q but IsCLISupported is false", id)
}
}
// semantic_/instance_segmentation are known but not yet pushable, and
// must explain why.
for _, id := range []string{"semantic_segmentation", "instance_segmentation"} {
// segmentation + causal_language_modeling are known but not yet pushable,
// and must explain why.
for _, id := range []string{"semantic_segmentation", "instance_segmentation", "causal_language_modeling"} {
if !IsKnown(id) {
t.Errorf("%s should be known", id)
}
Expand Down
16 changes: 13 additions & 3 deletions internal/schema/ingest.v1.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,7 +36,8 @@
"tabular_regression",
"time_series_forecasting",
"time_to_event_prediction",
"masked_language_modeling"
"masked_language_modeling",
"causal_language_modeling"
],
"description": "Task category. Drives convention defaults: validators, data_format, default columns, default file extensions, default validator set."
},
Expand DownExpand Up@@ -86,7 +87,7 @@
"texts": {
"type": "string",
"minLength": 1,
"description": "Directory holding text files referenced by the labels CSV. Required for text_classificationand token_classification."
"description": "Directory holding text files referenced by the labels CSV. Required for text_classification, token_classification, and causal_language_modeling (raw .txt: plain text, or a tab-separated prompt\\tcompletion pair)."
},

"sequences": {
Expand DownExpand Up@@ -336,6 +337,14 @@
},
"then": { "required": ["sequences"] }
},
{
"description": "causal_language_modeling requires `texts` (raw .txt samples). It is self-supervised, so unlike text/token classification it does NOT require `label`.",
"if": {
"properties": { "category": { "const": "causal_language_modeling" } },
"required": ["category"]
},
"then": { "required": ["texts"] }
},
{
"description": "tabular and time-series categories require `schema`.",
"if": {
Expand DownExpand Up@@ -411,7 +420,8 @@
"properties": {
"category": {
"enum": [
"masked_language_modeling"
"masked_language_modeling",
"causal_language_modeling"
]
}
},
Expand Down
Loading