From 8f32f5b182ee512368d6d441265dc5617c4c48b3 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:57:42 +1200 Subject: [PATCH 1/2] Add full-data V135 verification --- .../v136_full_v135_verification.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 competitions/trace_the_ace/v136_full_v135_verification.py diff --git a/competitions/trace_the_ace/v136_full_v135_verification.py b/competitions/trace_the_ace/v136_full_v135_verification.py new file mode 100644 index 00000000..01f6abde --- /dev/null +++ b/competitions/trace_the_ace/v136_full_v135_verification.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""V136: full-data untouched verification of promoted V135. + +This is verification, not discovery. V135's operator and hyperparameters are frozen: +- V75 + RELATED + smoothed exact-objective difficulty prior (alpha=10) +- logistic stack C=.10 +- stack fit only from inner-OOF support-seen rows +- exact V97 fallback whenever target objective is unsupported by outer training +- 4 outer / 3 inner folds, no sweep + +V136 rebuilds V75 and RELATED from the full competition training corpus and evaluates +that exact operator on all training rows. Primary gate is session-grouped transfer; +objective-grouped is a structural non-regression audit and must remain exactly V97. +""" +from __future__ import annotations +import argparse,json,time +from pathlib import Path +import numpy as np +from v75_canonical_trajectory import load_training +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v135_nested_supported_stack import nested_geom,ALPHA,C + +def main(a): + t0=time.time() + f=load_training(a.features,a.labels).reset_index(drop=True) + print('FULL_ROWS',len(f),'COLUMNS',list(f.columns),flush=True) + y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support=f.learning_objective.astype(str).to_numpy() + sessions=f.session_id.astype(str).to_numpy() + cache={} + us=np.unique(sessions) + for j,sid in enumerate(us,1): + cache[str(sid)]=load_transcript(a.transcripts/f'{sid}.csv') + if j%2500==0: print('TRANSCRIPTS',j,'/',len(us),'elapsed',round(time.time()-t0,1),flush=True) + print('BUILD_V75',flush=True) + X75=build_v75(f,cache) + print('V75_SHAPE',X75.shape,'nnz',X75.nnz,'elapsed',round(time.time()-t0,1),flush=True) + rt=[];rz=[] + for i,r in f.iterrows(): + text,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related') + rt.append(text);rz.append(z) + if (i+1)%5000==0: print('RELATED_ROWS',i+1,'elapsed',round(time.time()-t0,1),flush=True) + Xr=build_control(rt,rz) + print('RELATED_SHAPE',Xr.shape,'nnz',Xr.nnz,'elapsed',round(time.time()-t0,1),flush=True) + + # Primary untouched verification world: unseen sessions, mostly support-seen objectives. + S=nested_geom('session_grouped_full',sessions,X75,Xr,y,support) + print('SESSION_RESULT',json.dumps(S,indent=2),flush=True) + # Structural safety audit: objective-held-out rows must use exact V97 fallback. + O=nested_geom('objective_grouped_full',obj,X75,Xr,y,support) + print('OBJECTIVE_RESULT',json.dumps(O,indent=2),flush=True) + + sg=S['full_stack']['gain']; og=O['full_stack']['gain']; inc=S['prior_incremental_gain'] + all_session_folds=all(x['full_stack_ll'] < x['v97_ll'] for x in S['folds']) + obj_exact=abs(og)<=1e-12 and all(abs(x['full_stack_ll']-x['v97_ll'])<=1e-12 for x in O['folds']) + verdict=('PROMOTE_V135_TO_RUNTIME' if sg>=.005 and inc>=.003 and all_session_folds and obj_exact + else 'RETAIN_V135_PARTIAL' if sg>=.002 and obj_exact + else 'SUPPRESS_V135_FULL_TRANSFER') + out={ + 'protocol':'V136_FULL_V135_VERIFICATION', + 'rows':int(len(f)),'sessions':int(len(np.unique(sessions))),'objectives':int(len(np.unique(obj))), + 'frozen_operator':{'stack_C':C,'prior_alpha':ALPHA,'outer_folds':4,'inner_folds':3,'unsupported_fallback':'exact_v97'}, + 'precommit':{'primary_session_gain':.005,'prior_incremental_gain':.003,'all_session_folds_improve':True,'objective_exact_nonregression':True}, + 'session_grouped':S,'objective_grouped':O, + 'decision':{'session_gain':float(sg),'prior_incremental_gain':float(inc),'all_session_folds_improve':bool(all_session_folds),'objective_exact_nonregression':bool(obj_exact),'verdict':verdict}, + 'elapsed_seconds':float(time.time()-t0) + } + Path(a.out).write_text(json.dumps(out,indent=2));print('FINAL',json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',default='v136_full_v135_verification.json');main(p.parse_args()) From 0b6bea5caf4096b376e7abdb651002558b191b63 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:57:58 +1200 Subject: [PATCH 2/2] Add full-data V135 verification workflow --- .../trace-ace-v136-full-v135-verification.yml | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/trace-ace-v136-full-v135-verification.yml diff --git a/.github/workflows/trace-ace-v136-full-v135-verification.yml b/.github/workflows/trace-ace-v136-full-v135-verification.yml new file mode 100644 index 00000000..77e79517 --- /dev/null +++ b/.github/workflows/trace-ace-v136-full-v135-verification.yml @@ -0,0 +1,70 @@ +name: Trace Ace V136 Full V135 Verification +on: + pull_request: + branches: [agent/v135-nested-supported-stack] + paths: + - 'competitions/trace_the_ace/v136_full_v135_verification.py' + - '.github/workflows/trace-ace-v136-full-v135-verification.yml' + workflow_dispatch: + +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact transcript cache + run: | + set -euo pipefail + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download frozen small metadata archive only + run: | + set -euo pipefail + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve and audit schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS" + python - <<'PY' "$FEATURES" "$LABELS" + import pandas as pd,sys + f=pd.read_csv(sys.argv[1]); y=pd.read_csv(sys.argv[2]) + print('FEATURE_COLUMNS',list(f.columns)); print('LABEL_COLUMNS',list(y.columns)); print('SHAPES',f.shape,y.shape) + assert len(f)==35072 and len(y)==35072 + PY + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run frozen V136 full verification + run: | + cd competitions/trace_the_ace + python v136_full_v135_verification.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v136_full_v135_verification.json + - name: Show decision + if: always() + run: cat v136_full_v135_verification.json || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v136-full-v135-verification + path: v136_full_v135_verification.json + retention-days: 14