heathsanchez · heathsanchez · Aug 20, 2026 · Aug 20, 2026 · heathsanchez · heathsanchez
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())
diff --git a/.github/workflows/trace-ace-v125-nested-calibration.yml b/.github/workflows/trace-ace-v125-nested-calibration.yml
@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+ pull_request:
+ branches: [agent/v111-runner]
+ paths:
+ - 'competitions/trace_the_ace/v125_nested_calibration.py'
+ - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+ workflow_dispatch:
+
+jobs:
+ calibration:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+ - name: Restore frozen transcripts
+ uses: actions/cache/restore@v4
+ with:
+ path: transcripts.zip
+ key: trace-ace-transcripts-v1-603547640
+ fail-on-cache-miss: true
+ - name: Validate and extract data
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "$(stat -c%s transcripts.zip)" = "603547640"
+ unzip -tq transcripts.zip >/dev/null
+ test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+ gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+ mkdir -p data/meta data/transcripts
+ unzip -q metadata.zip -d data/meta
+ unzip -q transcripts.zip -d data/transcripts
+ echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+ echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+ FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+ echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+ - name: Run V125
+ run: |
+ cd competitions/trace_the_ace
+ python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+ - name: Show decision
+ if: always()
+ run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: trace-ace-v125-nested-calibration
+ path: v125_nested_calibration.json
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+ m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+ p75=m.predict_proba(X75[va])[:,1]
+ r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+ pr=r.predict_proba(Xr[va])[:,1]
+ vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+ seen=np.array([d.get(x,0)>0 for x in key[va]])
+ return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+ return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+ outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+ pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+ for k,(tr,va) in enumerate(outer):
+ inner_groups=groups[tr]
+ inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+ pi=np.zeros(len(tr))
+ for itr,iva in inn:
+ pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+ cal=fit_cal(pi,y[tr])
+ rng=np.random.default_rng(SEED+125+k)
+ sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+ raw=endpoint(X75,Xr,y,key,tr,va)
+ q=cal.predict_proba(logit(raw)[:,None])[:,1]
+ qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+ pb[va]=raw;pc[va]=q;ps[va]=qs
+ folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+ 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+ 'intercept':float(cal.intercept_[0])})
+ base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+ return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+ 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('features columns',list(f.columns),flush=True)
+ ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+ f=f.iloc[ix].reset_index(drop=True)
+ y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+ obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+ t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+ if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+ X75=build_v75(f,cache);Xr=build_control(rt,rz)
+ ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+ def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+ verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+ out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+ 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+ Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())