diff --git a/.github/workflows/trace-ace-v123-official-reference-scalars.yml b/.github/workflows/trace-ace-v123-official-reference-scalars.yml new file mode 100644 index 00000000..24850e69 --- /dev/null +++ b/.github/workflows/trace-ace-v123-official-reference-scalars.yml @@ -0,0 +1,90 @@ +name: Trace Ace V123 Official Reference Scalars +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v123_official_reference_scalars.py' + - '.github/workflows/trace-ace-v123-official-reference-scalars.yml' + workflow_dispatch: + +# All transcript-heavy Trace Ace workflows should use this same group/key. +# It prevents a PR fan-out from hammering the 604 MB Drive object in parallel. +# Rerun marker: shared default-branch cache seed installed 2026-08-19. +concurrency: + group: trace-ace-frozen-transcripts-v1 + cancel-in-progress: false + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Restore frozen transcript archive from GitHub cache + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + + - name: Download frozen transcript archive on cache miss + if: steps.transcript-cache.outputs.cache-hit != 'true' + shell: bash + run: | + set -euo pipefail + # Original frozen object first; isolated fresh copy second. + # Never launch these in parallel: Drive applies a per-file/content quota. + for id in \ + 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI \ + 1vkgFdcw-rnlueCgGsKKbUUXql8GyGmET + do + rm -f transcripts.zip + echo "Trying frozen transcript source $id" + if gdown "$id" -O transcripts.zip; then + bytes=$(stat -c%s transcripts.zip) + echo "Downloaded $bytes bytes" + if [ "$bytes" = "603547640" ]; then + echo "Frozen transcript archive size verified" + break + fi + echo "Unexpected transcript archive size: $bytes" >&2 + fi + rm -f transcripts.zip + done + test -f transcripts.zip + test "$(stat -c%s transcripts.zip)" = "603547640" + + - name: Download metadata and extract frozen data + shell: bash + run: | + set -euo pipefail + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta data/transcripts + unzip -q metadata.zip -d data/meta + unzip -q transcripts.zip -d data/transcripts + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + test -n "$FIRST" + echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV" + + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v123_official_reference_scalars.py + - name: Run V123 + run: python competitions/trace_the_ace/v123_official_reference_scalars.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --rows 2500 --out v123_official_reference_scalars.json + - name: Show decision + if: always() + run: cat v123_official_reference_scalars.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v123-official-reference-scalars + path: v123_official_reference_scalars.json + retention-days: 14 diff --git a/competitions/trace_the_ace/results/v124_oracle_contamination_audit.json b/competitions/trace_the_ace/results/v124_oracle_contamination_audit.json new file mode 100644 index 00000000..92b0886a --- /dev/null +++ b/competitions/trace_the_ace/results/v124_oracle_contamination_audit.json @@ -0,0 +1,34 @@ +{ + "experiment": "V124_ORACLE_CONTAMINATION_AUDIT", + "status": "COMPLETED_FROM_FROZEN_METADATA_AND_COMMITTED_V91_V102_CODE", + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "global_correct_rate": 0.7024692062043796, + "session_structure": { + "singleton_session_rows": 14457, + "singleton_session_row_rate": 0.4122091697080292, + "multiresponse_session_rows": 20615, + "multiresponse_session_row_rate": 0.5877908302919708, + "duplicate_session_objective_rows": 0, + "same_session_pair_count": 17797, + "same_session_pair_label_agreement": 0.7009046468505928 + }, + "sibling_label_diagnostic": { + "description": "For each row, use labels of other responses in its session only; singleton sessions fall back to global rate. Beta-style shrinkage strength fixed here at 5 for diagnostic scale, not deployment.", + "global_baseline_ll": 0.6087576094209947, + "sibling_shrunk_ll": 0.5978126232975202, + "gain": 0.010944986123474498, + "multiresponse_only_ll": 0.5443138144369557, + "multiresponse_only_gain_vs_global": 0.01862054588030526 + }, + "code_audit": { + "v91": "row endpoint oracle chooses endpoint from realized y; valid only as an outcome-informed complementarity ceiling", + "v102": "oracle_group minimizes loss against y[ix] for the same scored group; target row's own label participates in session selection; session_objective groups are all singleton in this dataset" + }, + "decision": { + "verdict": "DOWNGRADE_V91_V102_ORACLES_TO_DIAGNOSTIC_CEILINGS", + "law": "Do not interpret V91/V102 oracle headroom as evidence that an equivalent sample-local observable router exists. Treat it only as expert error complementarity unless a label-free separator is independently verified.", + "next": "Prioritize independently motivated observable features (e.g. official productive-math-talk scalars) or genuinely stronger semantic encoders; do not chase oracle gap directly." + } +} diff --git a/competitions/trace_the_ace/v123_official_reference_scalars.py b/competitions/trace_the_ace/v123_official_reference_scalars.py new file mode 100644 index 00000000..8af7a636 --- /dev/null +++ b/competitions/trace_the_ace/v123_official_reference_scalars.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""V123 — inject the three official DrivenData productive-math-talk scalars into V97. + +Residual: V75/V97 use L2-normalized hashed text plus event-state numerics, so they do not +explicitly retain absolute student word volume or the reference solution's two number-use ratios. + +Frozen fast protocol: +- deterministic 2500-row sample (same stable hash style as V112) +- baseline: fold-local V97 +- intervention: logistic residual [logit(V97), n_student_words, + numeric_turns_per_word, digit_chars_per_word] +- two untouched geometries: objective-grouped and session-grouped 4-fold OOF +- control: deterministic permutation of the three scalar rows before residual fitting +- no hyperparameter search; C=.15 +- retain as a law only if gain >= .0015 in BOTH geometries and each exceeds its shuffle + control by >= .001. >= .003 in both is a phase-change candidate. + +Scientific protocol frozen before execution; this line only retriggers the isolated PR workflow. +""" +from __future__ import annotations +import argparse, hashlib, json, re +from pathlib import Path +import numpy as np +import pandas as pd +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler + +from v71_mastery_events import load_transcript, normalize_roles +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +WORD_RE=re.compile(r"[a-z0-9]+(?:'[a-z]+)?",re.I) +DIGIT_RE=re.compile(r"\d") + +def stable(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def logit(p): + p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p)) +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) + +def reference_scalars(df: pd.DataFrame): + d=normalize_roles(df).reset_index(drop=True) + roles=d.role_repaired.astype(str).str.lower() + student=d.loc[roles.eq('student'),'content'].fillna('').astype(str) + text=' '.join(student.tolist()) + n_words=len(WORD_RE.findall(text)) + if n_words<=0: return np.array([0.,0.,0.],float) + return np.array([ + float(n_words), + float(student.str.contains(r'\d',regex=True).sum()/n_words), + float(len(DIGIT_RE.findall(text))/n_words), + ],float) + +def base_predictions(X75,Xr,y,support,groups): + q=np.zeros(len(y),float) + for tr,va in GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(len(y)),y,groups): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + p0=m0.predict_proba(X75[va])[:,1]; pr=mr.predict_proba(Xr[va])[:,1] + counts=pd.Series(support[tr]).value_counts() + unseen=np.array([counts.get(x,0)==0 for x in support[va]]) + q[va]=np.where(unseen,.65*p0+.35*pr,p0) + return np.clip(q,EPS,1-EPS) + +def residual_oof(base,Z,y,groups): + q=np.zeros(len(y),float) + splits=list(GroupKFold(min(4,len(np.unique(groups)))).split(Z,y,groups)) + for tr,va in splits: + sc=StandardScaler().fit(Z[tr]) + xt=np.c_[logit(base[tr]),sc.transform(Z[tr])] + xv=np.c_[logit(base[va]),sc.transform(Z[va])] + m=LogisticRegression(C=.15,max_iter=250,solver='liblinear',random_state=SEED).fit(xt,y[tr]) + q[va]=m.predict_proba(xv)[:,1] + return np.clip(q,EPS,1-EPS) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('joined columns',list(f.columns),flush=True) + oid=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + order=np.argsort(np.array([stable(x) for x in f.response_id.astype(str)]))[:a.rows] + f=f.iloc[order].reset_index(drop=True) + y=f.target.to_numpy(int); oid=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + session_Z={s:reference_scalars(cache[s]) for s in np.unique(sess)} + Z=np.vstack([session_Z[s] for s in sess]) + print('scalar means',Z.mean(0).tolist(),'scalar stds',Z.std(0).tolist(),flush=True) + rt=[];rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z) + if (i+1)%500==0: print('prepared',i+1,flush=True) + X75=build_v75(f,cache); Xr=build_control(rt,rz) + rng=np.random.default_rng(20260818); perm=rng.permutation(len(Z)); Zs=Z[perm] + out={'rows':int(len(f)),'sessions':int(len(np.unique(sess))),'objectives':int(len(np.unique(oid))), + 'features':['n_student_words','numeric_turns_per_word','digit_chars_per_word'],'geometries':{}} + for name,g in [('objective_cold',oid),('session_cold',sess)]: + b=base_predictions(X75,Xr,y,support,g); q=residual_oof(b,Z,y,g); qs=residual_oof(b,Zs,y,g) + base_ll=ll(y,b); qll=ll(y,q); sll=ll(y,qs) + out['geometries'][name]={'v97_ll':base_ll,'reference_scalar_ll':qll,'gain':base_ll-qll, + 'shuffle_ll':sll,'shuffle_gain':base_ll-sll,'ablation_margin':sll-qll} + print(name,out['geometries'][name],flush=True) + go=out['geometries']['objective_cold']['gain']; gs=out['geometries']['session_cold']['gain'] + mo=out['geometries']['objective_cold']['ablation_margin']; ms=out['geometries']['session_cold']['ablation_margin'] + if go>=.003 and gs>=.003 and mo>=.001 and ms>=.001: verdict='PHASE_CHANGE_SCALARS' + elif go>=.0015 and gs>=.0015 and mo>=.001 and ms>=.001: verdict='RETAIN_REFERENCE_SCALAR_LAW' + else: verdict='SUPPRESS_REFERENCE_SCALAR_INJECTION' + out['decision']={'verdict':verdict,'rule':'retain >=.0015 gain both geometries and >=.001 ablation margin both; phase-change >=.003 both'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v123_official_reference_scalars.json');run(p.parse_args())