Run final frozen V121 evaluator

.github/workflows/trace-ace-v121-eval-final.yml

@@ -0,0 +1,62 @@
+name: Trace Ace V121 Eval Final
+
+on:
+  pull_request:
+    branches: [agent/v121-cache-batch2]
+    paths:
+      - '.github/workflows/trace-ace-v121-eval-final.yml'
+      - 'competitions/trace_the_ace/v121_eval_only_fixed.py'
+  workflow_dispatch:
+
+env:
+  PREPARED_RUN_ID: '32400309220'
+  EMBEDDING_RUN_ID: '32402681183'
+
+jobs:
+  evaluate:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install evaluation dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Verify frozen manifest
+        run: |
+          test -f v121_prepared/manifest.json
+          grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json
+      - name: Download all 128 frozen embedding shards from original run
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p v121_embedding_shards
+          for i in $(seq 0 127); do
+            gh run download "$EMBEDDING_RUN_ID" -n "v121-embeddings-$i" -D "v121_embedding_shards/$i"
+          done
+          test "$(find v121_embedding_shards -name 'v121_embeddings_shard_*.npz' | wc -l)" -eq 128
+      - name: Evaluate unchanged frozen V121 precommit
+        run: |
+          cd competitions/trace_the_ace
+          python v121_eval_only_fixed.py --dir ../../v121_prepared \
+            --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json
+      - name: Show decision
+        run: cat v121_pretrained_semantic_residual.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-v121-pretrained-semantic-residual-final
+          path: v121_pretrained_semantic_residual.json
+          retention-days: 14

competitions/trace_the_ace/v121_eval_only_fixed.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Infrastructure-only evaluator for the already-frozen V121 artifacts.
+
+The only repair relative to v121_staged_transport.evaluate is loading the frozen
+string arrays with allow_pickle=True. Scientific features, folds, controls,
+models, thresholds, and interpretation are unchanged.
+"""
+import argparse, json
+from pathlib import Path
+import numpy as np
+from scipy.sparse import load_npz
+from v121_pretrained_semantic_residual import MODEL_NAME, eval_geometry, within_objective_shuffle
+from v121_staged_transport import load_embeddings
+
+
+def main(a):
+    d=Path(a.dir)
+    X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz')
+    z=np.load(d/'arrays.npz', allow_pickle=True)
+    y=z['y']; objectives=z['objectives']; support=z['support']; sessions=z['sessions']
+    E_obj,E_sem=load_embeddings(Path(a.embeddings))
+    if len(y)!=E_obj.shape[0] or len(y)!=E_sem.shape[0]:
+        raise RuntimeError('evaluation embedding row mismatch')
+    E_shuf=within_objective_shuffle(E_sem, objectives)
+    manifest=json.loads((d/'manifest.json').read_text())
+    results={
+      'protocol':'V121_PRETRAINED_SEMANTIC_RESIDUAL','model':MODEL_NAME,
+      'rows':int(len(y)),'objectives':int(len(np.unique(objectives))),
+      'sessions':int(len(np.unique(sessions))),
+      'transport_manifest':manifest,'transport':{'embedding_shards_merged':True,'serialization_repair':'allow_pickle_for_frozen_string_arrays'},
+      'precommit':{'semantic_gain_each_geometry':.003,'semantic_minus_shuffle_each_geometry':.002,'hard_collision_gain_each_geometry':'>0','no_hyperparameter_sweep':True},
+    }
+    results['objective_grouped']=eval_geometry('objective_grouped',objectives,X75,Xr,y,support,objectives,E_obj,E_sem,E_shuf)
+    results['session_grouped']=eval_geometry('session_grouped',sessions,X75,Xr,y,support,objectives,E_obj,E_sem,E_shuf)
+    def passes(r):
+        return r['semantic']['gain']>=.003 and r['semantic_minus_shuffle_gain']>=.002 and r['hard_collision'].get('semantic_gain',-1.)>0
+    ok_obj=passes(results['objective_grouped']); ok_sess=passes(results['session_grouped'])
+    if ok_obj and ok_sess:
+        verdict='PHASE_CHANGE_CANDIDATE'; nxt='Promote pretrained semantic residual to larger frozen validation and public-probe packaging.'
+    else:
+        verdict='NO_ROBUST_SEMANTIC_PHASE_CHANGE'; nxt='Treat remaining oracle gap as largely unidentifiable from supplied transcript/objective observables; pivot to validation geometry / assessment-process inference rather than more text feature search.'
+    results['decision']={'objective_grouped_pass':bool(ok_obj),'session_grouped_pass':bool(ok_sess),'verdict':verdict,'next':nxt}
+    Path(a.out).write_text(json.dumps(results,indent=2)); print(json.dumps(results,indent=2),flush=True)
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser(); p.add_argument('--dir',required=True); p.add_argument('--embeddings',required=True); p.add_argument('--out',required=True); main(p.parse_args())