V141 compile V135 runtime assets

.github/workflows/trace-ace-v141-v135-runtime-assets.yml

@@ -0,0 +1,53 @@
+name: Trace Ace V141 V135 Runtime Assets
+# Frozen science; synchronize PR after default-branch runner registration.
+on:
+  push:
+    branches: [agent/v141-v135-runtime-assets]
+    paths:
+      - 'competitions/trace_the_ace/train_v135_runtime_assets.py'
+      - '.github/workflows/trace-ace-v141-v135-runtime-assets.yml'
+  pull_request:
+    branches: [agent/v140-crossfit-calibration]
+    paths:
+      - 'competitions/trace_the_ace/train_v135_runtime_assets.py'
+      - '.github/workflows/trace-ace-v141-v135-runtime-assets.yml'
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: {python-version: '3.12', cache: pip}
+      - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+      - name: Restore transcript cache
+        id: tc
+        uses: actions/cache@v4
+        with:
+          path: transcripts.zip
+          key: trace-ace-transcripts-v1-603547640
+      - name: Verify cache
+        run: |
+          set -euo pipefail
+          test '${{ steps.tc.outputs.cache-hit }}' = 'true'
+          test "$(stat -c%s transcripts.zip)" = '603547640'
+          echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d  transcripts.zip' | sha256sum -c -
+      - name: Restore metadata
+        run: |
+          gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+          mkdir -p data/transcripts data/meta
+          unzip -q transcripts.zip -d data/transcripts
+          unzip -q metadata.zip -d data/meta
+          echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+          echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+          FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit); echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+      - name: Build frozen V135 assets
+        run: |
+          cd competitions/trace_the_ace
+          python train_v135_runtime_assets.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v135_runtime_assets
+      - uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-v141-v135-runtime-assets
+          path: v135_runtime_assets
+          retention-days: 14

competitions/trace_the_ace/train_v135_runtime_assets.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import argparse,json
+import numpy as np
+from scipy.sparse import csr_matrix,hstack
+from sklearn.feature_extraction.text import HashingVectorizer
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import GroupKFold
+from v75_canonical_trajectory import load_training,trajectory_views,SEED
+from v71_mastery_events import load_transcript
+from v94_related_control import segmented_control
+from v135_nested_supported_stack import prior_apply,feats,fit_stack,ALPHA,EPS
+
+BASE_C=.25
+
+def fit_base_model(X,y): return LogisticRegression(C=BASE_C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y)
+def pred(m,X): return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS)
+
+def main(a):
+    f=load_training(a.features,a.labels).reset_index(drop=True); y=f.target.to_numpy(int)
+    sessions=f.session_id.astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy()
+    cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in np.unique(sessions)}
+    views=[]; nums=[]; rt=[]; rz=[]
+    for i,r in f.iterrows():
+        v,n,_=trajectory_views(cache[str(r.session_id)],str(r.learning_objective)); views.append(v); nums.append(n)
+        t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+        if (i+1)%5000==0: print('ROWS',i+1,flush=True)
+    hv75=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True)
+    parts=[hv75.transform([f'[OBJECTIVE] {x}' for x in f.learning_objective])]
+    for k in views[0].keys(): parts.append(hv75.transform([f'[{k.upper()}] '+v[k] for v in views]))
+    N=np.vstack(nums).astype(float); nmean=N.mean(0); nstd=N.std(0)+1e-6; parts.append(csr_matrix((N-nmean)/nstd)); X75=hstack(parts,format='csr')
+    hvr=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True)
+    R=np.vstack(rz).astype(float); rmean=R.mean(0); rstd=R.std(0)+1e-6; Xr=hstack([hvr.transform(rt),csr_matrix((R-rmean)/rstd)],format='csr')
+    print('MATRICES',X75.shape,Xr.shape,flush=True)
+    # Session-OOF component field for leakage-safe final stack training.
+    p75=np.zeros(len(y)); pr=np.zeros(len(y)); pp=np.zeros(len(y)); cc=np.zeros(len(y)); seen=np.zeros(len(y),bool)
+    for k,(tr,va) in enumerate(GroupKFold(4).split(np.zeros(len(y)),y,sessions),1):
+        m75=fit_base_model(X75[tr],y[tr]); mr=fit_base_model(Xr[tr],y[tr]); p75[va]=pred(m75,X75[va]); pr[va]=pred(mr,Xr[va])
+        q,c,s=prior_apply(y,tr,va,support);pp[va]=q;cc[va]=c;seen[va]=s;print('OOF',k,flush=True)
+    stack=fit_stack(feats(p75[seen],pr[seen],pp[seen],cc[seen],True),y[seen])
+    final75=fit_base_model(X75,y); finalr=fit_base_model(Xr,y)
+    global_mean=float(y.mean()); sums={};counts={}
+    for k,v in zip(support,y): sums[k]=sums.get(k,0.0)+float(v);counts[k]=counts.get(k,0)+1
+    out=Path(a.out);out.mkdir(parents=True,exist_ok=True)
+    np.savez_compressed(out/'v135_assets.npz',v75_coef=final75.coef_.ravel(),v75_intercept=final75.intercept_,v75_num_mean=nmean,v75_num_std=nstd,
+        related_coef=finalr.coef_.ravel(),related_intercept=finalr.intercept_,related_num_mean=rmean,related_num_std=rstd,
+        stack_coef=stack.coef_.ravel(),stack_intercept=stack.intercept_)
+    man={'protocol':'V141_V135_RUNTIME_ASSETS','rows':len(y),'base_C':BASE_C,'stack_C':0.10,'prior_alpha':ALPHA,'global_mean':global_mean,
+         'objective_counts':counts,'objective_sums':sums,'stack_training':'4-fold session-grouped OOF supported rows only','seed':SEED}
+    (out/'manifest.json').write_text(json.dumps(man,indent=2))
+    # Save a small exact component fixture for parity checks.
+    idx=np.arange(min(512,len(y)))
+    np.savez_compressed(out/'parity_fixture.npz',idx=idx,p75=p75[idx],related=pr[idx],prior=pp[idx],count=cc[idx],seen=seen[idx],y=y[idx])
+    print(json.dumps({'rows':len(y),'supported_oof':int(seen.sum()),'assets':str(out)},indent=2),flush=True)
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',type=Path,required=True);main(p.parse_args())