V133 verified mathematical evidence residual

.github/workflows/trace-ace-v133-verified-math-evidence.yml

@@ -0,0 +1,56 @@
+name: Trace Ace V133 Verified Math Evidence
+on:
+  pull_request:
+    branches: [agent/v132-nested-applicability-gate]
+    paths:
+      - 'competitions/trace_the_ace/v133_verified_math_evidence.py'
+      - '.github/workflows/trace-ace-v133-verified-math-evidence.yml'
+  workflow_dispatch:
+
+env:
+  PREPARED_RUN_ID: 32400309220
+
+jobs:
+  evaluate:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install dependencies
+        run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas
+      - name: Restore frozen transcript archive from GitHub cache only
+        id: transcript-cache
+        uses: actions/cache@v4
+        with:
+          path: transcripts.zip
+          key: trace-ace-transcripts-v1-603547640
+      - name: Require exact cache hit
+        run: |
+          test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true'
+          test "$(stat -c%s transcripts.zip)" = '603547640'
+          echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d  transcripts.zip' | sha256sum -c -
+      - name: Download exact frozen prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ github.token }}
+      - name: Verify frozen sample
+        run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json
+      - name: Run frozen V133
+        run: python competitions/trace_the_ace/v133_verified_math_evidence.py --archive transcripts.zip --dir v121_prepared --out v133_verified_math_evidence.json
+      - name: Show decision
+        if: always()
+        run: cat v133_verified_math_evidence.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: trace-ace-v133-verified-math-evidence
+          path: v133_verified_math_evidence.json
+          retention-days: 14

competitions/trace_the_ace/v133_verified_math_evidence.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""V133: verifier-derived mathematical evidence over frozen V97.
+
+Constraint-derived hypothesis: current residual requires a new sample-local observable,
+not another semantic/routing feature. V75 records tutor feedback but never independently
+checks whether an explicit arithmetic student answer is mathematically correct.
+
+Primary separator: exact arithmetic question->student-answer verification.
+Control: deterministically rotate student numeric answers among the same session's
+verifiable questions, preserving question/answer marginals but destroying the relation.
+No threshold/feature sweep. Each outer fold fits the residual only from inner-OOF V97
+predictions on outer-training rows, then evaluates untouched outer validation rows.
+"""
+from __future__ import annotations
+import argparse,csv,hashlib,io,json,re,zipfile
+from pathlib import Path
+import numpy as np
+from scipy.sparse import load_npz
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import GroupKFold
+from v110_residual_collider_state_discovery import ll,logit,p97_predict
+from v75_canonical_trajectory import SEED
+
+EPS=1e-5
+C=.05
+NUM=r"(-?\d+(?:\.\d+)?(?:\s*/\s*-?\d+(?:\.\d+)?)?)"
+PATTERNS=[
+ (re.compile(rf"{NUM}\s*(?:\+|plus)\s*{NUM}",re.I),lambda a,b:a+b),
+ (re.compile(rf"{NUM}\s*(?:-|minus)\s*{NUM}",re.I),lambda a,b:a-b),
+ (re.compile(rf"{NUM}\s*(?:x|×|\*|times|multiplied\s+by)\s*{NUM}",re.I),lambda a,b:a*b),
+ (re.compile(rf"{NUM}\s*(?:/|÷|divided\s+by)\s*{NUM}",re.I),lambda a,b:a/b if abs(b)>1e-12 else np.nan),
+]
+ANS_RE=re.compile(NUM)
+QUESTION_RE=re.compile(r"\?|\b(?:what|calculate|work out|solve|how much|how many)\b",re.I)
+
+def number(s):
+    s=str(s).replace(' ','')
+    try:
+        if '/' in s:
+            a,b=s.split('/',1); b=float(b); return float(a)/b if abs(b)>1e-12 else np.nan
+        return float(s)
+    except Exception:return np.nan
+
+def expected(q):
+    q=str(q).replace(',','')
+    for p,op in PATTERNS:
+        m=p.search(q)
+        if m:
+            a,b=number(m.group(1)),number(m.group(2))
+            if np.isfinite(a) and np.isfinite(b):
+                try:return float(op(a,b))
+                except Exception:return np.nan
+    return np.nan
+
+def answer_value(a):
+    m=ANS_RE.search(str(a).replace(',',''))
+    return number(m.group(1)) if m else np.nan
+
+def events(rows):
+    out=[]
+    for i,r in enumerate(rows):
+        if str(r.get('role','')).lower()!='tutor':continue
+        q=str(r.get('content',''))
+        if not QUESTION_RE.search(q):continue
+        e=expected(q)
+        if not np.isfinite(e):continue
+        ai=None
+        for j in range(i+1,min(len(rows),i+6)):
+            role=str(rows[j].get('role','')).lower(); txt=str(rows[j].get('content',''))
+            if role=='student' and txt.strip(): ai=j; break
+            if role=='tutor' and QUESTION_RE.search(txt) and j>i+1: break
+        if ai is None:continue
+        av=answer_value(rows[ai].get('content',''))
+        if np.isfinite(av):out.append((e,av,ai/max(1,len(rows)-1)))
+    return out
+
+def vec(E,shift=0):
+    n=len(E)
+    if not n:return np.zeros(8,float)
+    ans=np.asarray([x[1] for x in E],float)
+    if shift and n>1: ans=np.roll(ans,shift)
+    exp=np.asarray([x[0] for x in E],float); rec=np.asarray([x[2] for x in E],float)
+    ok=np.isclose(ans,exp,rtol=1e-6,atol=1e-6).astype(float); bad=1-ok
+    w=np.exp(2*(rec-1)); w/=w.sum()+1e-12
+    return np.asarray([np.log1p(n),ok.mean(),bad.mean(),ok[-1]-bad[-1],float((w*ok).sum()),float((w*bad).sum()),float(rec[ok>0].max()) if np.any(ok>0) else 0.,float(rec[bad>0].max()) if np.any(bad>0) else 0.],float)
+
+def inner_base(X75,Xr,y,groups,support):
+    nsp=min(3,len(np.unique(groups))); P=np.zeros(len(y))
+    for tr,va in GroupKFold(nsp).split(np.zeros(len(y)),y,groups): P[va],_=p97_predict(X75,Xr,y,tr,va,support)
+    return np.clip(P,EPS,1-EPS)
+def fit_res(P,R,y):
+    mu=R.mean(0); sd=R.std(0)+1e-6
+    X=np.c_[logit(P),(R-mu)/sd]
+    m=LogisticRegression(C=C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y)
+    return mu,sd,m
+def apply_res(mod,P,R):
+    mu,sd,m=mod; X=np.c_[logit(P),(R-mu)/sd]
+    return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS)
+def eval_geom(name,groups,X75,Xr,y,support,R,A,covered):
+    groups=np.asarray(groups); PB=np.zeros(len(y)); QR=np.zeros(len(y)); QA=np.zeros(len(y)); rows=[]
+    for k,(tr,va) in enumerate(GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(len(y)),y,groups),1):
+        pva,_=p97_predict(X75,Xr,y,tr,va,support); pin=inner_base(X75[tr],Xr[tr],y[tr],groups[tr],support[tr])
+        qr=apply_res(fit_res(pin,R[tr],y[tr]),pva,R[va]); qa=apply_res(fit_res(pin,A[tr],y[tr]),pva,A[va])
+        PB[va]=pva;QR[va]=qr;QA[va]=qa
+        rows.append({'fold':k,'rows':int(len(va)),'base_ll':ll(y[va],pva),'verified_ll':ll(y[va],qr),'control_ll':ll(y[va],qa),'covered':int(covered[va].sum())})
+    b,r,a=ll(y,PB),ll(y,QR),ll(y,QA); cov=np.asarray(covered,bool)
+    return {'geometry':name,'baseline_v97_ll':b,'verified':{'ll':r,'gain':b-r},'rotated_control':{'ll':a,'gain':b-a},'real_minus_control_gain':a-r,'coverage_fraction':float(cov.mean()),'covered_rows':int(cov.sum()),'covered_only':({'baseline_ll':ll(y[cov],PB[cov]),'verified_ll':ll(y[cov],QR[cov]),'gain':ll(y[cov],PB[cov])-ll(y[cov],QR[cov])} if cov.any() else None),'folds':rows}
+def main(a):
+    d=Path(a.dir); z=np.load(d/'arrays.npz',allow_pickle=True); y=z['y']; obj=z['objectives']; support=z['support']; sessions=z['sessions']
+    X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz'); cache={}
+    with zipfile.ZipFile(a.archive) as za:
+        names=set(za.namelist())
+        for sid in np.unique(sessions):
+            name=f'{sid}.csv'
+            if name not in names:raise RuntimeError('missing '+name)
+            with za.open(name) as f: rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline='')))
+            E=events(rows); shift=(1+int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(E)-1)) if len(E)>1 else 0
+            cache[str(sid)]=(vec(E,0),vec(E,shift),len(E))
+    R=np.vstack([cache[str(s)][0] for s in sessions]); A=np.vstack([cache[str(s)][1] for s in sessions]); covered=np.asarray([cache[str(s)][2]>0 for s in sessions])
+    out={'protocol':'V133_VERIFIED_MATH_EVIDENCE','rows':int(len(y)),'hypothesis':'independent arithmetic verification is a missing sample-local observable over V97','precommit':{'residual_C':C,'outer_folds':4,'inner_folds':3,'no_sweep':True,'promote_gain_each_geometry':.001,'real_minus_control_each_geometry':.0005,'phase_change_gain_each_geometry':.003}}
+    out['objective_grouped']=eval_geom('objective_grouped',obj,X75,Xr,y,support,R,A,covered)
+    out['session_grouped']=eval_geom('session_grouped',sessions,X75,Xr,y,support,R,A,covered)
+    def ok(x,t=.001):return x['verified']['gain']>=t and x['real_minus_control_gain']>=.0005
+    po,ps=ok(out['objective_grouped']),ok(out['session_grouped']); phase=ok(out['objective_grouped'],.003) and ok(out['session_grouped'],.003)
+    out['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':'PHASE_CHANGE_VERIFIED_MATH_EVIDENCE' if phase else 'PROMOTE_VERIFIED_MATH_EVIDENCE' if po and ps else 'SUPPRESS_VERIFIED_MATH_EVIDENCE'}
+    Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True)
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--archive',required=True);p.add_argument('--dir',required=True);p.add_argument('--out',required=True);main(p.parse_args())