Run isolated V123 official reference scalar test

.github/workflows/trace-ace-v123-official-reference-scalars.yml

@@ -0,0 +1,90 @@
+name: Trace Ace V123 Official Reference Scalars
+on:
+  pull_request:
+    branches: [agent/trace-ace-mastery-events]
+    paths:
+      - 'competitions/trace_the_ace/v123_official_reference_scalars.py'
+      - '.github/workflows/trace-ace-v123-official-reference-scalars.yml'
+  workflow_dispatch:
+
+# All transcript-heavy Trace Ace workflows should use this same group/key.
+# It prevents a PR fan-out from hammering the 604 MB Drive object in parallel.
+# Rerun marker: shared default-branch cache seed installed 2026-08-19.
+concurrency:
+  group: trace-ace-frozen-transcripts-v1
+  cancel-in-progress: false
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+
+      - name: Restore frozen transcript archive from GitHub cache
+        id: transcript-cache
+        uses: actions/cache@v4
+        with:
+          path: transcripts.zip
+          key: trace-ace-transcripts-v1-603547640
+
+      - name: Download frozen transcript archive on cache miss
+        if: steps.transcript-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Original frozen object first; isolated fresh copy second.
+          # Never launch these in parallel: Drive applies a per-file/content quota.
+          for id in \
+            1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI \
+            1vkgFdcw-rnlueCgGsKKbUUXql8GyGmET
+          do
+            rm -f transcripts.zip
+            echo "Trying frozen transcript source $id"
+            if gdown "$id" -O transcripts.zip; then
+              bytes=$(stat -c%s transcripts.zip)
+              echo "Downloaded $bytes bytes"
+              if [ "$bytes" = "603547640" ]; then
+                echo "Frozen transcript archive size verified"
+                break
+              fi
+              echo "Unexpected transcript archive size: $bytes" >&2
+            fi
+            rm -f transcripts.zip
+          done
+          test -f transcripts.zip
+          test "$(stat -c%s transcripts.zip)" = "603547640"
+
+      - name: Download metadata and extract frozen data
+        shell: bash
+        run: |
+          set -euo pipefail
+          gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip
+          mkdir -p data/meta data/transcripts
+          unzip -q metadata.zip -d data/meta
+          unzip -q transcripts.zip -d data/transcripts
+          echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV"
+          echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV"
+          FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit)
+          test -n "$FIRST"
+          echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV"
+
+      - name: Preflight
+        run: python -m py_compile competitions/trace_the_ace/v123_official_reference_scalars.py
+      - name: Run V123
+        run: python competitions/trace_the_ace/v123_official_reference_scalars.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --rows 2500 --out v123_official_reference_scalars.json
+      - name: Show decision
+        if: always()
+        run: cat v123_official_reference_scalars.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: trace-ace-v123-official-reference-scalars
+          path: v123_official_reference_scalars.json
+          retention-days: 14

competitions/trace_the_ace/results/v124_oracle_contamination_audit.json

@@ -0,0 +1,34 @@
+{
+  "experiment": "V124_ORACLE_CONTAMINATION_AUDIT",
+  "status": "COMPLETED_FROM_FROZEN_METADATA_AND_COMMITTED_V91_V102_CODE",
+  "rows": 35072,
+  "sessions": 22821,
+  "objectives": 398,
+  "global_correct_rate": 0.7024692062043796,
+  "session_structure": {
+    "singleton_session_rows": 14457,
+    "singleton_session_row_rate": 0.4122091697080292,
+    "multiresponse_session_rows": 20615,
+    "multiresponse_session_row_rate": 0.5877908302919708,
+    "duplicate_session_objective_rows": 0,
+    "same_session_pair_count": 17797,
+    "same_session_pair_label_agreement": 0.7009046468505928
+  },
+  "sibling_label_diagnostic": {
+    "description": "For each row, use labels of other responses in its session only; singleton sessions fall back to global rate. Beta-style shrinkage strength fixed here at 5 for diagnostic scale, not deployment.",
+    "global_baseline_ll": 0.6087576094209947,
+    "sibling_shrunk_ll": 0.5978126232975202,
+    "gain": 0.010944986123474498,
+    "multiresponse_only_ll": 0.5443138144369557,
+    "multiresponse_only_gain_vs_global": 0.01862054588030526
+  },
+  "code_audit": {
+    "v91": "row endpoint oracle chooses endpoint from realized y; valid only as an outcome-informed complementarity ceiling",
+    "v102": "oracle_group minimizes loss against y[ix] for the same scored group; target row's own label participates in session selection; session_objective groups are all singleton in this dataset"
+  },
+  "decision": {
+    "verdict": "DOWNGRADE_V91_V102_ORACLES_TO_DIAGNOSTIC_CEILINGS",
+    "law": "Do not interpret V91/V102 oracle headroom as evidence that an equivalent sample-local observable router exists. Treat it only as expert error complementarity unless a label-free separator is independently verified.",
+    "next": "Prioritize independently motivated observable features (e.g. official productive-math-talk scalars) or genuinely stronger semantic encoders; do not chase oracle gap directly."
+  }
+}

competitions/trace_the_ace/v123_official_reference_scalars.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""V123 — inject the three official DrivenData productive-math-talk scalars into V97.
+
+Residual: V75/V97 use L2-normalized hashed text plus event-state numerics, so they do not
+explicitly retain absolute student word volume or the reference solution's two number-use ratios.
+
+Frozen fast protocol:
+- deterministic 2500-row sample (same stable hash style as V112)
+- baseline: fold-local V97
+- intervention: logistic residual [logit(V97), n_student_words,
+  numeric_turns_per_word, digit_chars_per_word]
+- two untouched geometries: objective-grouped and session-grouped 4-fold OOF
+- control: deterministic permutation of the three scalar rows before residual fitting
+- no hyperparameter search; C=.15
+- retain as a law only if gain >= .0015 in BOTH geometries and each exceeds its shuffle
+  control by >= .001. >= .003 in both is a phase-change candidate.
+
+Scientific protocol frozen before execution; this line only retriggers the isolated PR workflow.
+"""
+from __future__ import annotations
+import argparse, hashlib, json, re
+from pathlib import Path
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+from sklearn.model_selection import GroupKFold
+from sklearn.preprocessing import StandardScaler
+
+from v71_mastery_events import load_transcript, normalize_roles
+from v75_canonical_trajectory import load_training, SEED
+from v85_evidence_state import build_v75
+from v94_related_control import segmented_control, build_control
+
+EPS=1e-5
+WORD_RE=re.compile(r"[a-z0-9]+(?:'[a-z]+)?",re.I)
+DIGIT_RE=re.compile(r"\d")
+
+def stable(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)
+def logit(p):
+ p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p))
+def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1]))
+
+def reference_scalars(df: pd.DataFrame):
+ d=normalize_roles(df).reset_index(drop=True)
+ roles=d.role_repaired.astype(str).str.lower()
+ student=d.loc[roles.eq('student'),'content'].fillna('').astype(str)
+ text=' '.join(student.tolist())
+ n_words=len(WORD_RE.findall(text))
+ if n_words<=0: return np.array([0.,0.,0.],float)
+ return np.array([
+   float(n_words),
+   float(student.str.contains(r'\d',regex=True).sum()/n_words),
+   float(len(DIGIT_RE.findall(text))/n_words),
+ ],float)
+
+def base_predictions(X75,Xr,y,support,groups):
+ q=np.zeros(len(y),float)
+ for tr,va in GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(len(y)),y,groups):
+   m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr])
+   mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr])
+   p0=m0.predict_proba(X75[va])[:,1]; pr=mr.predict_proba(Xr[va])[:,1]
+   counts=pd.Series(support[tr]).value_counts()
+   unseen=np.array([counts.get(x,0)==0 for x in support[va]])
+   q[va]=np.where(unseen,.65*p0+.35*pr,p0)
+ return np.clip(q,EPS,1-EPS)
+
+def residual_oof(base,Z,y,groups):
+ q=np.zeros(len(y),float)
+ splits=list(GroupKFold(min(4,len(np.unique(groups)))).split(Z,y,groups))
+ for tr,va in splits:
+   sc=StandardScaler().fit(Z[tr])
+   xt=np.c_[logit(base[tr]),sc.transform(Z[tr])]
+   xv=np.c_[logit(base[va]),sc.transform(Z[va])]
+   m=LogisticRegression(C=.15,max_iter=250,solver='liblinear',random_state=SEED).fit(xt,y[tr])
+   q[va]=m.predict_proba(xv)[:,1]
+ return np.clip(q,EPS,1-EPS)
+
+def run(a):
+ f=load_training(a.features,a.labels).reset_index(drop=True)
+ print('joined columns',list(f.columns),flush=True)
+ oid=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ order=np.argsort(np.array([stable(x) for x in f.response_id.astype(str)]))[:a.rows]
+ f=f.iloc[order].reset_index(drop=True)
+ y=f.target.to_numpy(int); oid=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy()
+ sess=f.session_id.astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy()
+ cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}
+ session_Z={s:reference_scalars(cache[s]) for s in np.unique(sess)}
+ Z=np.vstack([session_Z[s] for s in sess])
+ print('scalar means',Z.mean(0).tolist(),'scalar stds',Z.std(0).tolist(),flush=True)
+ rt=[];rz=[]
+ for i,r in f.iterrows():
+   t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z)
+   if (i+1)%500==0: print('prepared',i+1,flush=True)
+ X75=build_v75(f,cache); Xr=build_control(rt,rz)
+ rng=np.random.default_rng(20260818); perm=rng.permutation(len(Z)); Zs=Z[perm]
+ out={'rows':int(len(f)),'sessions':int(len(np.unique(sess))),'objectives':int(len(np.unique(oid))),
+      'features':['n_student_words','numeric_turns_per_word','digit_chars_per_word'],'geometries':{}}
+ for name,g in [('objective_cold',oid),('session_cold',sess)]:
+   b=base_predictions(X75,Xr,y,support,g); q=residual_oof(b,Z,y,g); qs=residual_oof(b,Zs,y,g)
+   base_ll=ll(y,b); qll=ll(y,q); sll=ll(y,qs)
+   out['geometries'][name]={'v97_ll':base_ll,'reference_scalar_ll':qll,'gain':base_ll-qll,
+                            'shuffle_ll':sll,'shuffle_gain':base_ll-sll,'ablation_margin':sll-qll}
+   print(name,out['geometries'][name],flush=True)
+ go=out['geometries']['objective_cold']['gain']; gs=out['geometries']['session_cold']['gain']
+ mo=out['geometries']['objective_cold']['ablation_margin']; ms=out['geometries']['session_cold']['ablation_margin']
+ if go>=.003 and gs>=.003 and mo>=.001 and ms>=.001: verdict='PHASE_CHANGE_SCALARS'
+ elif go>=.0015 and gs>=.0015 and mo>=.001 and ms>=.001: verdict='RETAIN_REFERENCE_SCALAR_LAW'
+ else: verdict='SUPPRESS_REFERENCE_SCALAR_INJECTION'
+ out['decision']={'verdict':verdict,'rule':'retain >=.0015 gain both geometries and >=.001 ablation margin both; phase-change >=.003 both'}
+ Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True)
+
+if __name__=='__main__':
+ p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v123_official_reference_scalars.json');run(p.parse_args())