Test nested V97 calibration residual

.github/workflows/trace-ace-v125-nested-calibration.yml

@@ -0,0 +1,56 @@
+name: Trace Ace V125 Nested Calibration
+on:
+  pull_request:
+    branches: [agent/v111-runner]
+    paths:
+      - 'competitions/trace_the_ace/v125_nested_calibration.py'
+      - '.github/workflows/trace-ace-v125-nested-calibration.yml'
+  workflow_dispatch:
+
+jobs:
+  calibration:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+      - name: Restore frozen transcripts
+        uses: actions/cache/restore@v4
+        with:
+          path: transcripts.zip
+          key: trace-ace-transcripts-v1-603547640
+          fail-on-cache-miss: true
+      - name: Validate and extract data
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "$(stat -c%s transcripts.zip)" = "603547640"
+          unzip -tq transcripts.zip >/dev/null
+          test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d"
+          gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+          mkdir -p data/meta data/transcripts
+          unzip -q metadata.zip -d data/meta
+          unzip -q transcripts.zip -d data/transcripts
+          echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+          echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+          FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+          echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+      - name: Run V125
+        run: |
+          cd competitions/trace_the_ace
+          python v125_nested_calibration.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v125_nested_calibration.json
+      - name: Show decision
+        if: always()
+        run: test -f v125_nested_calibration.json && cat v125_nested_calibration.json || true
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: trace-ace-v125-nested-calibration
+          path: v125_nested_calibration.json
+          retention-days: 14
+          if-no-files-found: warn

competitions/trace_the_ace/v125_nested_calibration.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""V125: nested calibration residual over frozen V97.
+
+Question: is V97 leaving lawful log-loss improvement in probability calibration,
+without adding new information or exploiting a particular validation geometry?
+
+Frozen protocol:
+- deterministic 2500-row response-id sample;
+- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported);
+- 4-fold outer objective-grouped and session-grouped OOF;
+- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold;
+- intervention = one global Platt map sigmoid(a + b*logit(p97));
+- control = same map fit after deterministic shuffle of inner-OOF probabilities;
+- no hyperparameter sweep.
+
+Promote only if calibration gains >= .001 log loss in BOTH geometries and beats
+the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law.
+"""
+from __future__ import annotations
+import argparse, hashlib, json
+from pathlib import Path
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+
+from v71_mastery_events import load_transcript
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+
+def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS)))
+def logit(p):
+    p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+
+def endpoint(X75,Xr,y,key,tr,va):
+    m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+    p75=m.predict_proba(X75[va])[:,1]
+    r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+    pr=r.predict_proba(Xr[va])[:,1]
+    vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts))
+    seen=np.array([d.get(x,0)>0 for x in key[va]])
+    return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS)
+
+def fit_cal(p,y):
+    return LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y)
+
+def geometry(name,groups,X75,Xr,y,key):
+    outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups))
+    pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[]
+    for k,(tr,va) in enumerate(outer):
+        inner_groups=groups[tr]
+        inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups))
+        pi=np.zeros(len(tr))
+        for itr,iva in inn:
+            pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva])
+        cal=fit_cal(pi,y[tr])
+        rng=np.random.default_rng(SEED+125+k)
+        sh=fit_cal(pi[rng.permutation(len(pi))],y[tr])
+        raw=endpoint(X75,Xr,y,key,tr,va)
+        q=cal.predict_proba(logit(raw)[:,None])[:,1]
+        qs=sh.predict_proba(logit(raw)[:,None])[:,1]
+        pb[va]=raw;pc[va]=q;ps[va]=qs
+        folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q),
+                      'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]),
+                      'intercept':float(cal.intercept_[0])})
+    base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps)
+    return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal,
+            'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds}
+
+def run(a):
+    f=load_training(a.features,a.labels).reset_index(drop=True)
+    print('features columns',list(f.columns),flush=True)
+    ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows]
+    f=f.iloc[ix].reset_index(drop=True)
+    y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy()
+    obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+    sess=f.session_id.astype(str).to_numpy()
+    cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+    rt=[];rz=[]
+    for i,r in f.iterrows():
+        t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+        if (i+1)%500==0: print('prepared rows',i+1,flush=True)
+    X75=build_v75(f,cache);Xr=build_control(rt,rz)
+    ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key)
+    def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001
+    verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION'
+    out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True},
+         'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}}
+    Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args())