V136 full-data V135 verification

.github/workflows/trace-ace-v136-full-v135-verification.yml

@@ -0,0 +1,70 @@
+name: Trace Ace V136 Full V135 Verification
+on:
+  pull_request:
+    branches: [agent/v135-nested-supported-stack]
+    paths:
+      - 'competitions/trace_the_ace/v136_full_v135_verification.py'
+      - '.github/workflows/trace-ace-v136-full-v135-verification.yml'
+  workflow_dispatch:
+
+jobs:
+  verify:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+      - name: Restore frozen transcript archive from GitHub cache only
+        id: transcript-cache
+        uses: actions/cache@v4
+        with:
+          path: transcripts.zip
+          key: trace-ace-transcripts-v1-603547640
+      - name: Require exact transcript cache
+        run: |
+          set -euo pipefail
+          test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true'
+          test "$(stat -c%s transcripts.zip)" = '603547640'
+          echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d  transcripts.zip' | sha256sum -c -
+      - name: Download frozen small metadata archive only
+        run: |
+          set -euo pipefail
+          gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+          mkdir -p data/transcripts data/meta
+          unzip -q transcripts.zip -d data/transcripts
+          unzip -q metadata.zip -d data/meta
+      - name: Resolve and audit schemas
+        run: |
+          set -euo pipefail
+          FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)
+          LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)
+          FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+          TRANSCRIPTS=$(dirname "$FIRST")
+          test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS"
+          python - <<'PY' "$FEATURES" "$LABELS"
+          import pandas as pd,sys
+          f=pd.read_csv(sys.argv[1]); y=pd.read_csv(sys.argv[2])
+          print('FEATURE_COLUMNS',list(f.columns)); print('LABEL_COLUMNS',list(y.columns)); print('SHAPES',f.shape,y.shape)
+          assert len(f)==35072 and len(y)==35072
+          PY
+          echo "FEATURES=$FEATURES" >> "$GITHUB_ENV"
+          echo "LABELS=$LABELS" >> "$GITHUB_ENV"
+          echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV"
+      - name: Run frozen V136 full verification
+        run: |
+          cd competitions/trace_the_ace
+          python v136_full_v135_verification.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v136_full_v135_verification.json
+      - name: Show decision
+        if: always()
+        run: cat v136_full_v135_verification.json || true
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: trace-ace-v136-full-v135-verification
+          path: v136_full_v135_verification.json
+          retention-days: 14

competitions/trace_the_ace/v136_full_v135_verification.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""V136: full-data untouched verification of promoted V135.
+
+This is verification, not discovery. V135's operator and hyperparameters are frozen:
+- V75 + RELATED + smoothed exact-objective difficulty prior (alpha=10)
+- logistic stack C=.10
+- stack fit only from inner-OOF support-seen rows
+- exact V97 fallback whenever target objective is unsupported by outer training
+- 4 outer / 3 inner folds, no sweep
+
+V136 rebuilds V75 and RELATED from the full competition training corpus and evaluates
+that exact operator on all training rows. Primary gate is session-grouped transfer;
+objective-grouped is a structural non-regression audit and must remain exactly V97.
+"""
+from __future__ import annotations
+import argparse,json,time
+from pathlib import Path
+import numpy as np
+from v75_canonical_trajectory import load_training
+from v71_mastery_events import load_transcript
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control,build_control
+from v135_nested_supported_stack import nested_geom,ALPHA,C
+
+def main(a):
+    t0=time.time()
+    f=load_training(a.features,a.labels).reset_index(drop=True)
+    print('FULL_ROWS',len(f),'COLUMNS',list(f.columns),flush=True)
+    y=f.target.to_numpy(int)
+    obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+    support=f.learning_objective.astype(str).to_numpy()
+    sessions=f.session_id.astype(str).to_numpy()
+    cache={}
+    us=np.unique(sessions)
+    for j,sid in enumerate(us,1):
+        cache[str(sid)]=load_transcript(a.transcripts/f'{sid}.csv')
+        if j%2500==0: print('TRANSCRIPTS',j,'/',len(us),'elapsed',round(time.time()-t0,1),flush=True)
+    print('BUILD_V75',flush=True)
+    X75=build_v75(f,cache)
+    print('V75_SHAPE',X75.shape,'nnz',X75.nnz,'elapsed',round(time.time()-t0,1),flush=True)
+    rt=[];rz=[]
+    for i,r in f.iterrows():
+        text,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related')
+        rt.append(text);rz.append(z)
+        if (i+1)%5000==0: print('RELATED_ROWS',i+1,'elapsed',round(time.time()-t0,1),flush=True)
+    Xr=build_control(rt,rz)
+    print('RELATED_SHAPE',Xr.shape,'nnz',Xr.nnz,'elapsed',round(time.time()-t0,1),flush=True)
+
+    # Primary untouched verification world: unseen sessions, mostly support-seen objectives.
+    S=nested_geom('session_grouped_full',sessions,X75,Xr,y,support)
+    print('SESSION_RESULT',json.dumps(S,indent=2),flush=True)
+    # Structural safety audit: objective-held-out rows must use exact V97 fallback.
+    O=nested_geom('objective_grouped_full',obj,X75,Xr,y,support)
+    print('OBJECTIVE_RESULT',json.dumps(O,indent=2),flush=True)
+
+    sg=S['full_stack']['gain']; og=O['full_stack']['gain']; inc=S['prior_incremental_gain']
+    all_session_folds=all(x['full_stack_ll'] < x['v97_ll'] for x in S['folds'])
+    obj_exact=abs(og)<=1e-12 and all(abs(x['full_stack_ll']-x['v97_ll'])<=1e-12 for x in O['folds'])
+    verdict=('PROMOTE_V135_TO_RUNTIME' if sg>=.005 and inc>=.003 and all_session_folds and obj_exact
+             else 'RETAIN_V135_PARTIAL' if sg>=.002 and obj_exact
+             else 'SUPPRESS_V135_FULL_TRANSFER')
+    out={
+      'protocol':'V136_FULL_V135_VERIFICATION',
+      'rows':int(len(f)),'sessions':int(len(np.unique(sessions))),'objectives':int(len(np.unique(obj))),
+      'frozen_operator':{'stack_C':C,'prior_alpha':ALPHA,'outer_folds':4,'inner_folds':3,'unsupported_fallback':'exact_v97'},
+      'precommit':{'primary_session_gain':.005,'prior_incremental_gain':.003,'all_session_folds_improve':True,'objective_exact_nonregression':True},
+      'session_grouped':S,'objective_grouped':O,
+      'decision':{'session_gain':float(sg),'prior_incremental_gain':float(inc),'all_session_folds_improve':bool(all_session_folds),'objective_exact_nonregression':bool(obj_exact),'verdict':verdict},
+      'elapsed_seconds':float(time.time()-t0)
+    }
+    Path(a.out).write_text(json.dumps(out,indent=2));print('FINAL',json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',default='v136_full_v135_verification.json');main(p.parse_args())