diff --git a/.github/workflows/trace-ace-v129-tutor-uptake.yml b/.github/workflows/trace-ace-v129-tutor-uptake.yml new file mode 100644 index 00000000..bbcf3e8c --- /dev/null +++ b/.github/workflows/trace-ace-v129-tutor-uptake.yml @@ -0,0 +1,58 @@ +name: Trace Ace V129 Tutor Uptake + +on: + pull_request: + branches: [agent/v121-cache-batch2] + paths: + - '.github/workflows/trace-ace-v129-tutor-uptake.yml' + - 'competitions/trace_the_ace/v129_tutor_uptake.py' + workflow_dispatch: + +env: + PREPARED_RUN_ID: '32400309220' + +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 12 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas fastembed==0.8.0 + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact cache hit + run: | + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V129 + run: python competitions/trace_the_ace/v129_tutor_uptake.py --archive transcripts.zip --dir v121_prepared --out v129_tutor_uptake.json + - name: Show decision + run: cat v129_tutor_uptake.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v129-tutor-uptake + path: v129_tutor_uptake.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v132-nested-applicability-gate.yml b/.github/workflows/trace-ace-v132-nested-applicability-gate.yml new file mode 100644 index 00000000..ef1b85c6 --- /dev/null +++ b/.github/workflows/trace-ace-v132-nested-applicability-gate.yml @@ -0,0 +1,57 @@ +name: Trace Ace V132 Nested Applicability Gate +on: + pull_request: + branches: [agent/v121-cache-batch2] + paths: + - 'competitions/trace_the_ace/v129_tutor_uptake.py' + - 'competitions/trace_the_ace/v132_nested_applicability_gate.py' + - '.github/workflows/trace-ace-v132-nested-applicability-gate.yml' + workflow_dispatch: + +env: + PREPARED_RUN_ID: 32400309220 + +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas fastembed==0.8.0 + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact cache hit + run: | + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ github.token }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V132 + run: python competitions/trace_the_ace/v132_nested_applicability_gate.py --archive transcripts.zip --dir v121_prepared --out v132_nested_applicability_gate.json + - name: Show decision + if: always() + run: cat v132_nested_applicability_gate.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v132-nested-applicability-gate + path: v132_nested_applicability_gate.json + retention-days: 14 diff --git a/competitions/trace_the_ace/v129_tutor_uptake.py b/competitions/trace_the_ace/v129_tutor_uptake.py new file mode 100644 index 00000000..7421b662 --- /dev/null +++ b/competitions/trace_the_ace/v129_tutor_uptake.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V129 TUTOR UPTAKE residual over frozen V97. + +Primary separator: whether the tutor's next utterance lexically/structurally takes +up the student's immediately preceding language. V75 raw hashing preserves words, +but not this cross-turn relation explicitly. + +Control: deterministically rotate tutor replies among student->tutor pairs inside +each session, preserving the exact student/tutor text multisets and pair count while +destroying local response alignment. No labels enter feature construction. No sweep. +""" +from __future__ import annotations +import argparse,csv,io,json,re,zipfile,hashlib +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from v110_residual_collider_state_discovery import ll,logit +from v121_pretrained_semantic_residual import p97_oof,collision_mask +from v75_canonical_trajectory import SEED + +TOK=re.compile(r"[a-z0-9]+(?:\.[0-9]+)?",re.I) +NEG=re.compile(r"\b(?:no|not quite|incorrect|wrong|careful|try again|almost|remember|instead|actually)\b",re.I) +POS=re.compile(r"\b(?:yes|yeah|correct|right|exactly|perfect|good|great|well done|that's it|thats it|you got it|spot on)\b",re.I) +QUESTION=re.compile(r"\?|\b(?:what|which|how|why|can you|could you|tell me|work out|calculate|solve|find)\b",re.I) +STOP={'the','a','an','and','or','to','of','in','on','for','with','is','are','be','as','by','from','this','that','these','those','you','your','we','it','its'} +EPS=1e-5 + +def toks(s): return {x for x in TOK.findall(str(s).lower()) if len(x)>1 and x not in STOP} +def jac(a,b): + A,B=toks(a),toks(b) + return len(A&B)/len(A|B) if A and B else 0.0 + +def pairs(rows): + out=[] + for i,r in enumerate(rows[:-1]): + if str(r.get('role','')).lower()!='student': continue + s=str(r.get('content','')) + j=None + for k in range(i+1,min(len(rows),i+4)): + if str(rows[k].get('role','')).lower()=='tutor': j=k; break + if str(rows[k].get('role','')).lower()=='student': break + if j is not None: out.append((s,str(rows[j].get('content','')))) + return out + +def feats(P,shift=0): + if not P: return np.zeros(10,float) + S=[p[0] for p in P]; T=[p[1] for p in P]; n=len(P) + if shift and n>1: T=T[shift%n:]+T[:shift%n] + o=np.array([jac(s,t) for s,t in zip(S,T)],float) + neg=np.array([bool(NEG.search(t)) for t in T]); pos=np.array([bool(POS.search(t)) for t in T]); q=np.array([bool(QUESTION.search(t)) for t in T]) + substantive=np.array([len(toks(s))>=2 or bool(re.search(r'\d|[=+\-/*×÷%]',s)) for s in S]) + def mean(mask): return float(o[mask].mean()) if mask.any() else 0.0 + return np.array([o.mean(),np.quantile(o,.75),np.quantile(o,.9),o.max(),mean(neg),mean(pos),mean(q),mean(substantive),float((o>0).mean()),np.log1p(n)],float) + +def residual_oof(P,X,y,splits): + q=np.zeros(len(y),float) + for tr,va in splits: + mu=X[tr].mean(0); sd=X[tr].std(0)+1e-6 + A=np.c_[logit(P[tr]),(X[tr]-mu)/sd]; B=np.c_[logit(P[va]),(X[va]-mu)/sd] + m=LogisticRegression(C=.05,max_iter=300,solver='liblinear',random_state=SEED).fit(A,y[tr]) + q[va]=m.predict_proba(B)[:,1] + return np.clip(q,EPS,1-EPS) + +def evalg(name,groups,X75,Xr,y,support,obj,Xreal,Xctrl): + P,splits=p97_oof(X75,Xr,y,groups,support); Q=residual_oof(P,Xreal,y,splits); C=residual_oof(P,Xctrl,y,splits) + b=ll(y,P); r=ll(y,Q); c=ll(y,C); mask=collision_mask(P,y,obj,.01) + out={'geometry':name,'baseline_v97_ll':b,'uptake':{'ll':r,'gain':b-r},'rotated_reply_control':{'ll':c,'gain':b-c},'uptake_minus_control_gain':c-r,'hard_collision':{'rows':int(mask.sum())}} + if mask.any(): + bb=ll(y[mask],P[mask]); rr=ll(y[mask],Q[mask]); cc=ll(y[mask],C[mask]) + out['hard_collision'].update({'baseline_ll':bb,'uptake_ll':rr,'uptake_gain':bb-rr,'control_ll':cc,'uptake_minus_control_gain':cc-rr}) + return out + +def main(a): + d=Path(a.dir); z=np.load(d/'arrays.npz',allow_pickle=True); y=z['y']; obj=z['objectives']; support=z['support']; sessions=z['sessions'] + X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz'); cache={} + with zipfile.ZipFile(a.archive) as za: + names=set(za.namelist()) + for sid in np.unique(sessions): + name=f'{sid}.csv' + if name not in names: raise RuntimeError('missing '+name) + with za.open(name) as f: rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline=''))) + P=pairs(rows); shift=1+(int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(P)-1)) if len(P)>1 else 0 + cache[str(sid)]=(feats(P,0),feats(P,shift)) + R=np.vstack([cache[str(s)][0] for s in sessions]); C=np.vstack([cache[str(s)][1] for s in sessions]) + res={'protocol':'V129_TUTOR_UPTAKE','rows':int(len(y)),'primary':'student -> next tutor lexical uptake','control':'deterministic within-session rotation of tutor replies','precommit':{'promote_gain_each_geometry':.0015,'phase_change_gain_each_geometry':.003,'real_minus_control_each_geometry':.001,'hard_collision_gain_each_geometry':'>0','no_parameter_sweep':True}} + res['objective_grouped']=evalg('objective_grouped',obj,X75,Xr,y,support,obj,R,C); res['session_grouped']=evalg('session_grouped',sessions,X75,Xr,y,support,obj,R,C) + def ok(x,t): return x['uptake']['gain']>=t and x['uptake_minus_control_gain']>=.001 and x['hard_collision'].get('uptake_gain',-1)>0 + po=ok(res['objective_grouped'],.0015); ps=ok(res['session_grouped'],.0015); ph=ok(res['objective_grouped'],.003) and ok(res['session_grouped'],.003) + verdict='PHASE_CHANGE_CANDIDATE' if ph else 'PROMOTE_TUTOR_UPTAKE_LAW' if po and ps else 'SUPPRESS_TUTOR_UPTAKE' + res['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':verdict} + Path(a.out).write_text(json.dumps(res,indent=2)); print(json.dumps(res,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--archive',required=True); p.add_argument('--dir',required=True); p.add_argument('--out',required=True); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v132_nested_applicability_gate.py b/competitions/trace_the_ace/v132_nested_applicability_gate.py new file mode 100644 index 00000000..16eb83bd --- /dev/null +++ b/competitions/trace_the_ace/v132_nested_applicability_gate.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""V132: leakage-safe learned applicability gate for V129 tutor uptake. + +This is derived from the Residual Constraint Graph, not a feature sweep. +V129 shows control-separated uptake signal only in a label-defined ambiguity regime; +V131 shows prediction density alone is too broad. V132 asks whether the missing +activation predicate is itself learnable from runtime-visible state. + +For each OUTER fold: +1. Produce INNER-OOF V97 predictions on the outer-training rows. +2. Produce INNER-OOF uptake and rotated-control corrections on those rows. +3. Define correction-benefit targets only on outer-training rows from per-row logloss. +4. Fit one frozen logistic applicability gate from runtime-visible features. +5. Fit the correction model on all outer-training INNER-OOF base predictions. +6. Apply both correction and gate to untouched outer validation rows. + +No validation labels enter feature construction, correction fitting, or gating. +No threshold/C/feature sweep. The identical procedure is run for the rotated-reply +control so an apparent benefit from generic second-stage selection is not enough. +""" +from __future__ import annotations +import argparse,csv,hashlib,io,json,zipfile +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold + +from v110_residual_collider_state_discovery import ll,logit,p97_predict +from v129_tutor_uptake import feats,pairs +from v75_canonical_trajectory import SEED + +EPS=1e-5 +GATE_C=.10 +CORR_C=.05 + + +def row_loss(y,p): + p=np.clip(np.asarray(p,float),EPS,1-EPS); y=np.asarray(y,int) + return -(y*np.log(p)+(1-y)*np.log(1-p)) + + +def crowd_features(P,obj): + """Runtime-visible geometry, no labels.""" + P=np.asarray(P,float); obj=np.asarray(obj,str) + nearest=np.ones(len(P),float); count=np.ones(len(P),float) + for o in np.unique(obj): + z=np.where(obj==o)[0]; count[z]=len(z) + if len(z)>1: + d=np.abs(P[z,None]-P[None,z]); np.fill_diagonal(d,np.inf) + nearest[z]=np.min(d,axis=1) + return np.c_[np.log1p(count),nearest,(nearest<=.01).astype(float)] + + +def gate_features(P,Q,R,obj): + P=np.asarray(P,float); Q=np.asarray(Q,float); R=np.asarray(R,float) + d=Q-P + # Every field is available at inference time once V97 and the frozen uptake + # correction have produced their probabilities. + return np.c_[P,Q,d,np.abs(d),np.abs(P-.5),logit(P),crowd_features(P,obj),R] + + +def residual_fit(P,R,y): + mu=R.mean(0); sd=R.std(0)+1e-6 + X=np.c_[logit(P),(R-mu)/sd] + m=LogisticRegression(C=CORR_C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y) + return mu,sd,m + + +def residual_apply(model,P,R): + mu,sd,m=model + X=np.c_[logit(P),(R-mu)/sd] + return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS) + + +def inner_oof(X75,Xr,y,groups,support,R): + nsp=min(3,len(np.unique(groups))) + splits=list(GroupKFold(nsp).split(np.zeros(len(y)),y,groups)) + P=np.zeros(len(y)); Q=np.zeros(len(y)) + # First get base OOF for all rows. + for tr,va in splits: + P[va],_=p97_predict(X75,Xr,y,tr,va,support) + # Then fit each residual correction on OOF base probabilities of its training + # rows and validate on the same untouched inner validation block. + for tr,va in splits: + mod=residual_fit(P[tr],R[tr],y[tr]) + Q[va]=residual_apply(mod,P[va],R[va]) + return np.clip(P,EPS,1-EPS),np.clip(Q,EPS,1-EPS) + + +def fit_gate(P,Q,R,obj,y): + X=gate_features(P,Q,R,obj) + target=(row_loss(y,Q)=.5 + out=np.asarray(P,float).copy(); out[take]=Q[take] + return np.clip(out,EPS,1-EPS),take + + +def eval_geometry(name,groups,X75,Xr,y,support,obj,R,C): + groups=np.asarray(groups); nsp=min(4,len(np.unique(groups))) + outer=list(GroupKFold(nsp).split(np.zeros(len(y)),y,groups)) + PB=np.zeros(len(y)); GR=np.zeros(len(y)); GC=np.zeros(len(y)); + takeR=np.zeros(len(y),bool); takeC=np.zeros(len(y),bool); foldrows=[] + for k,(tr,va) in enumerate(outer,1): + # Clean outer validation base prediction. + pva,_=p97_predict(X75,Xr,y,tr,va,support) + # Training state for gate/correction is itself OOF. + pin,qin=inner_oof(X75[tr],Xr[tr],y[tr],groups[tr],support[tr],R[tr]) + _pin_c,qin_c=inner_oof(X75[tr],Xr[tr],y[tr],groups[tr],support[tr],C[tr]) + # pin and _pin_c are deterministically identical; do not use labels from va. + gateR=fit_gate(pin,qin,R[tr],obj[tr],y[tr]); gateC=fit_gate(pin,qin_c,C[tr],obj[tr],y[tr]) + corrR=residual_fit(pin,R[tr],y[tr]); corrC=residual_fit(pin,C[tr],y[tr]) + qva=residual_apply(corrR,pva,R[va]); qva_c=residual_apply(corrC,pva,C[va]) + gr,tR=apply_gate(gateR,pva,qva,R[va],obj[va]); gc,tC=apply_gate(gateC,pva,qva_c,C[va],obj[va]) + PB[va]=pva; GR[va]=gr; GC[va]=gc; takeR[va]=tR; takeC[va]=tC + foldrows.append({'fold':k,'rows':int(len(va)),'base_ll':ll(y[va],pva),'gate_ll':ll(y[va],gr),'control_gate_ll':ll(y[va],gc),'train_benefit_rate':gateR['rate'],'take_rate':float(tR.mean())}) + b=ll(y,PB); r=ll(y,GR); c=ll(y,GC) + return { + 'geometry':name,'baseline_v97_ll':b, + 'nested_gate':{'ll':r,'gain':b-r,'take_rate':float(takeR.mean())}, + 'nested_rotated_control_gate':{'ll':c,'gain':b-c,'take_rate':float(takeC.mean())}, + 'real_minus_control_gain':c-r, + 'folds':foldrows, + } + + +def main(a): + d=Path(a.dir); z=np.load(d/'arrays.npz',allow_pickle=True) + y=z['y']; obj=z['objectives']; support=z['support']; sessions=z['sessions'] + X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz'); cache={} + with zipfile.ZipFile(a.archive) as za: + names=set(za.namelist()) + for sid in np.unique(sessions): + name=f'{sid}.csv' + if name not in names: raise RuntimeError('missing '+name) + with za.open(name) as f: rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline=''))) + P=pairs(rows); shift=1+(int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(P)-1)) if len(P)>1 else 0 + cache[str(sid)]=(feats(P,0),feats(P,shift)) + R=np.vstack([cache[str(s)][0] for s in sessions]); C=np.vstack([cache[str(s)][1] for s in sessions]) + out={'protocol':'V132_NESTED_APPLICABILITY_GATE','rows':int(len(y)), + 'hypothesis':'V129 contains local relational information but requires a learned runtime-visible ambiguity/applicability predicate', + 'precommit':{'outer_folds':4,'inner_folds':3,'gate_C':GATE_C,'correction_C':CORR_C,'gate_threshold':.5,'no_sweep':True, + 'promote_gain_each_geometry':.0005,'real_minus_control_each_geometry':.0005}} + out['objective_grouped']=eval_geometry('objective_grouped',obj,X75,Xr,y,support,obj,R,C) + out['session_grouped']=eval_geometry('session_grouped',sessions,X75,Xr,y,support,obj,R,C) + def ok(r): return r['nested_gate']['gain']>=.0005 and r['real_minus_control_gain']>=.0005 + po,ps=ok(out['objective_grouped']),ok(out['session_grouped']) + out['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':'PROMOTE_LEARNED_APPLICABILITY_GATE' if po and ps else 'SUPPRESS_LEARNED_UPTAKE_APPLICABILITY_FAMILY'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--archive',required=True); p.add_argument('--dir',required=True); p.add_argument('--out',required=True); main(p.parse_args())