diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 3e4675e..711f13c 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -44,3 +44,18 @@ representation line are closed negative; v1 is final). Published reference: Homo-GE2PE homograph 76.89% — v1 is above the published SOTA on this benchmark. + +## heb-diac-small-1.0 — Hebrew student distillation (2026-08-20) + +Logit KD from the s43 teacher (rababa_hebrew_byt5_s43/run-001/best): +KL + CE on hebrew-v4, ByT5-small init. Harness: greedy decode, Nakdimon +IMF test split (1,864 long sentences), same harness for both models +(`src/gpu/modal_distill.py::evaluate`). + +| Model | DER | CER | +|---|---|---| +| Teacher (s43, ByT5-base) | 24.79% | 22.14% | +| **Student (ByT5-small, gate)** | **30.37%** | 24.47% | + +Shrink cost +5.58pp — inside the ~5.6pp budget pre-accepted for this +pair (rababa docs/DISTILL-SOURCE-PROMPT.md section 2). diff --git a/models/heb-diac-small/heb-diac-small-1.0.README.md b/models/heb-diac-small/heb-diac-small-1.0.README.md new file mode 100644 index 0000000..2c2056a --- /dev/null +++ b/models/heb-diac-small/heb-diac-small-1.0.README.md @@ -0,0 +1,15 @@ +# heb-diac-small-1.0 + +Hebrew diacritization (adds nikud), client tier. ByT5-small student +(300M) distilled from the s43 teacher via logit KD — same vocab +(byte-level), so the teacher's soft distributions transfer directly. + +Gate: student 30.37% DER vs teacher 24.79% (greedy, Nakdimon IMF test +split, 1,864 long sentences) = +5.58pp, inside the ~5.6pp shrink budget +pre-accepted for this pair (docs/DISTILL-SOURCE-PROMPT.md section 2). + +```python +from interscript_ml import Model +model = Model.load("heb-diac-small-1.0") +model.translate("שלום") +``` diff --git a/models/heb-diac-small/heb-diac-small-1.0.metadata.yaml b/models/heb-diac-small/heb-diac-small-1.0.metadata.yaml new file mode 100644 index 0000000..7db3cd2 --- /dev/null +++ b/models/heb-diac-small/heb-diac-small-1.0.metadata.yaml @@ -0,0 +1,29 @@ +format: imf-v1 +id: heb-diac-small-1.0 +task: diacritization +source_script: Hebr +target: Hebr +tokenizer: bytes +opset: 14 +decoder: kv +precision: fp32 +license: BSD-3-Clause +trained_from: >- + logit KD from the s43 teacher (rababa_hebrew_byt5_s43/run-001/best, + RELEASE-FROZEN): KL(teacher_soft || student_soft) + CE on the + hebrew-v4 corpus, ByT5-small init google/byt5-small; checkpoint + rababa-checkpoints:/rababa_hebrew_distill_small/run-001/best +metrics: + - name: der_teacher_greedy + value: 24.79 + protocol: >- + greedy decode; Nakdimon IMF test split, 1,864 long sentences; + s43 teacher on the same harness as the student + source: interscript/ml-models docs/RESULTS.md#heb-diac-small-1.0 + - name: der_student_greedy + value: 30.37 + protocol: >- + greedy decode; same harness; +5.58pp shrink cost, inside the + ~5.6pp budget pre-accepted for this pair + (docs/DISTILL-SOURCE-PROMPT.md section 2) + source: interscript/ml-models docs/RESULTS.md#heb-diac-small-1.0 diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml index d6173ea..1a1c7aa 100644 --- a/models/metrics-sources.yaml +++ b/models/metrics-sources.yaml @@ -58,3 +58,12 @@ fas-g2p-1.0: tables: - {row: "CER (v1 test split, greedy, editdistance)", as: cer} - {row: "SentenceBench homograph (ezafe-normalized)", as: sb_homograph_ezafe_norm} +heb-diac-small-1.0: + repo: interscript/interscript-ml + ref: main + path: docs/RESULTS.md + anchor: heb-diac-small-1.0 + protocol: "greedy decode; Nakdimon IMF test split, 1,864 long sentences" + tables: + - {row: "Teacher (s43, ByT5-base)", column: DER, as: der_teacher_greedy} + - {row: "Student (ByT5-small, gate)", column: DER, as: der_student_greedy} diff --git a/src/gpu/modal_export.py b/src/gpu/modal_export.py index e5b1ea6..684cccb 100644 --- a/src/gpu/modal_export.py +++ b/src/gpu/modal_export.py @@ -95,6 +95,15 @@ "test_data": "urdu-diacrit/test.jsonl", "probe": "اردو", }, + "heb-diac-small": { + "volume": "/volumes/rababa-checkpoints", + "checkpoint": "rababa_hebrew_distill_small/run-001/best", + "metadata": "models/heb-diac-small/heb-diac-small-1.0.metadata.yaml", + "readme": "models/heb-diac-small/heb-diac-small-1.0.README.md", + "test_volume": "/datasets/rababa", + "test_data": "nakdimon/test-imf.jsonl", + "probe": "שלום", + }, "tha-g2p-base": { "volume": "/volumes/secryst-checkpoints", "checkpoint": "secryst_thai_g2p_distill_small/run-004/best",