diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml new file mode 100644 index 00000000..4b00ff00 --- /dev/null +++ b/.github/workflows/trace-ace-mastery.yml @@ -0,0 +1,152 @@ +name: Trace the Ace mastery experiment + +on: + workflow_dispatch: + inputs: + run_full: + description: "Run full experiment using public Drive bundles" + required: false + default: false + type: boolean + limit: + description: "Optional row limit (0 = all rows)" + required: false + default: "0" + type: string + push: + branches: + - agent/trace-ace-mastery-events + paths: + - "competitions/trace_the_ace/**" + - ".github/workflows/trace-ace-mastery.yml" + +jobs: + self-test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install experiment dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Run mastery extractor self-test + run: python competitions/trace_the_ace/v71_mastery_events.py --self-test + - name: Run supervision audit self-test + run: python competitions/trace_the_ace/v72_supervision_audit.py --self-test + - name: Run contrastive mastery self-test + run: python competitions/trace_the_ace/v73_contrastive_mastery.py --self-test + - name: Run semantic objective prior self-test + run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test + - name: Run canonical trajectory self-test + run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test + - name: Run unseen validation self-test + run: python competitions/trace_the_ace/v76_unseen_validation.py --self-test + - name: Run incremental mastery stack self-test + run: python competitions/trace_the_ace/v77_incremental_mastery_stack.py --self-test + - name: Run official-runtime validation harness self-test + run: python competitions/trace_the_ace/runtime_validate.py self-test + + full-experiment: + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }} + needs: self-test + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install experiment dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download public transcript archive + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/transcripts + python - <<'PY' + import os, gdown + file_id = os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'] + out = '/tmp/trace_ace/transcripts_download' + path = gdown.download(id=file_id, output=out, quiet=False) + if not path: + raise SystemExit('Google Drive transcript download failed') + PY + unzip -q /tmp/trace_ace/transcripts_download -d /tmp/trace_ace/transcripts + rm -f /tmp/trace_ace/transcripts_download + - name: Download public feature/label bundle + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta + python - <<'PY' + import os, gdown + file_id = os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'] + out = '/tmp/trace_ace/meta.zip' + path = gdown.download(id=file_id, output=out, quiet=False) + if not path: + raise SystemExit('Google Drive metadata download failed; confirm Anyone with the link can view') + PY + unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta + rm -f /tmp/trace_ace/meta.zip + - name: Locate inputs by schema and run experiments + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, shlex + from pathlib import Path + roots = [Path('/tmp/trace_ace/meta'), Path('/tmp/trace_ace/transcripts')] + features = labels = transcript_dir = None + for root in roots: + for path in root.rglob('*.csv'): + try: + with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f: + header = next(csv.reader(f)) + except Exception: + continue + cols = set(header) + if features is None and {'response_id','session_id','learning_objective'}.issubset(cols): + features = path; print('FEATURE HEADER', header) + if labels is None and 'response_id' in cols and ('is_correct' in cols or 'correct' in cols): + labels = path; print('LABEL HEADER', header) + if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(cols): + transcript_dir = path.parent; print('TRANSCRIPT HEADER', header) + if features and labels and transcript_dir: break + if features and labels and transcript_dir: break + if not (features and labels and transcript_dir): + raise SystemExit('Could not identify all Trace the Ace inputs by schema') + with open('/tmp/trace_ace/paths.env','w') as f: + f.write('FEATURES=' + shlex.quote(str(features)) + '\n') + f.write('LABELS=' + shlex.quote(str(labels)) + '\n') + f.write('TRANSCRIPTS=' + shlex.quote(str(transcript_dir)) + '\n') + PY + source /tmp/trace_ace/paths.env + LIMIT="${{ inputs.limit }}"; LIMIT="${LIMIT:-0}" + EXTRA=(); if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi + python competitions/trace_the_ace/v71_mastery_events.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v71_mastery_results.json "${EXTRA[@]}" + python competitions/trace_the_ace/v72_supervision_audit.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v72_supervision_audit.json "${EXTRA[@]}" + python competitions/trace_the_ace/v73_contrastive_mastery.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v73_contrastive_mastery.json "${EXTRA[@]}" + python competitions/trace_the_ace/v74_semantic_objective_prior.py --features "$FEATURES" --labels "$LABELS" --out v74_semantic_objective_prior.json "${EXTRA[@]}" + python competitions/trace_the_ace/v75_canonical_trajectory.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v75_canonical_trajectory.json "${EXTRA[@]}" + python competitions/trace_the_ace/v76_unseen_validation.py --features "$FEATURES" --labels "$LABELS" --out-protocol v76_validation_protocol.csv --out-summary v76_validation_summary.json + python competitions/trace_the_ace/v77_incremental_mastery_stack.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v77_incremental_mastery_stack.json "${EXTRA[@]}" + - name: Upload aggregate results only + uses: actions/upload-artifact@v4 + with: + name: trace-ace-aggregate-results + path: | + v71_mastery_results.json + v72_supervision_audit.json + v73_contrastive_mastery.json + v74_semantic_objective_prior.json + v75_canonical_trajectory.json + v76_validation_summary.json + v77_incremental_mastery_stack.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-runtime.yml b/.github/workflows/trace-ace-runtime.yml new file mode 100644 index 00000000..dd55e6ed --- /dev/null +++ b/.github/workflows/trace-ace-runtime.yml @@ -0,0 +1,139 @@ +name: Trace the Ace V75 official runtime + +on: + workflow_dispatch: + push: + branches: + - agent/trace-ace-mastery-events + paths: + - "competitions/trace_the_ace/runtime_v75/**" + - "competitions/trace_the_ace/train_v75_runtime_assets.py" + - "competitions/trace_the_ace/v71_mastery_events.py" + - "competitions/trace_the_ace/v75_canonical_trajectory.py" + - "competitions/trace_the_ace/runtime_validate.py" + - ".github/workflows/trace-ace-runtime.yml" + +jobs: + build-and-test: + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[runtime-build]') }} + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + GITHUB_ACTIONS_NO_TTY: "true" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: astral-sh/setup-uv@v6 + - uses: extractions/setup-just@v2 + - name: Install training dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Download training inputs + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/transcripts /tmp/trace_ace/meta + python - <<'PY' + import os, gdown + pairs = [ + ('TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID','/tmp/trace_ace/transcripts.zip'), + ('TRACE_ACE_METADATA_DRIVE_FILE_ID','/tmp/trace_ace/meta.zip'), + ] + for env, out in pairs: + p = gdown.download(id=os.environ[env], output=out, quiet=False) + if not p: + raise SystemExit(f'download failed: {env}') + PY + unzip -q /tmp/trace_ace/transcripts.zip -d /tmp/trace_ace/transcripts + unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta + rm -f /tmp/trace_ace/*.zip + + - name: Resolve schemas + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, shlex + from pathlib import Path + roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')] + features=labels=transcripts=None + for root in roots: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if transcripts is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): transcripts=p.parent; print('TRANSCRIPT HEADER',h) + if not all((features,labels,transcripts)): raise SystemExit('failed schema resolution') + with open('/tmp/trace_ace/paths.env','w') as f: + f.write('FEATURES='+shlex.quote(str(features))+'\n') + f.write('LABELS='+shlex.quote(str(labels))+'\n') + f.write('TRANSCRIPTS='+shlex.quote(str(transcripts))+'\n') + PY + + - name: Train promoted V75 all-views assets on all training rows + shell: bash + run: | + set -euo pipefail + source /tmp/trace_ace/paths.env + python competitions/trace_the_ace/train_v75_runtime_assets.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --out-dir /tmp/v75_assets + + - name: Assemble official submission_src + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/drivendataorg/tutoring-outcomes-runtime.git /tmp/runtime + rm -rf /tmp/runtime/submission_src/* + cp competitions/trace_the_ace/runtime_v75/main.py /tmp/runtime/submission_src/main.py + cp competitions/trace_the_ace/v71_mastery_events.py /tmp/runtime/submission_src/v71_mastery_events.py + cp competitions/trace_the_ace/v75_canonical_trajectory.py /tmp/runtime/submission_src/v75_canonical_trajectory.py + mkdir -p /tmp/runtime/submission_src/assets + cp /tmp/v75_assets/v75_runtime_assets.npz /tmp/runtime/submission_src/assets/ + cp /tmp/v75_assets/manifest.json /tmp/runtime/submission_src/assets/ + find /tmp/runtime/submission_src -maxdepth 2 -type f -printf '%P %s bytes\n' + + - name: Pack and check with official runtime + working-directory: /tmp/runtime + run: | + just pack-submission + just check-submission + + - name: Pull official runtime image + working-directory: /tmp/runtime + run: just pull + + - name: Test submission offline in official container + working-directory: /tmp/runtime + env: + BLOCK_INTERNET: "true" + GITHUB_ACTIONS_NO_TTY: "true" + SUBMISSION_IMAGE: tutoringoutcomeschallengeprodacr.azurecr.io/tutoring-outcomes-runtime:gpu-latest + run: just test-submission + + - name: Validate generated output contract + shell: bash + run: | + set -euo pipefail + python competitions/trace_the_ace/runtime_validate.py output \ + --format /tmp/runtime/data-demo/submission_format.csv \ + --predictions /tmp/runtime/submission/submission.csv + + - name: Upload candidate and evidence + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v75-official-runtime-candidate + retention-days: 14 + path: | + /tmp/runtime/submission/submission.zip + /tmp/runtime/submission/submission.csv + /tmp/runtime/submission/log.txt + /tmp/v75_assets/manifest.json diff --git a/.github/workflows/trace-ace-v100-nested-stacker.yml b/.github/workflows/trace-ace-v100-nested-stacker.yml new file mode 100644 index 00000000..e7ed503b --- /dev/null +++ b/.github/workflows/trace-ace-v100-nested-stacker.yml @@ -0,0 +1,53 @@ +name: Trace the Ace V100 nested calibrated stacker +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v100_nested_stacker.py' + - '.github/workflows/trace-ace-v100-nested-stacker.yml' +jobs: + stacker: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + shell: bash + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V100 nested stacker + run: | + cd competitions/trace_the_ace + python v100_nested_stacker.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v100_nested_stacker.json + - name: Show decision + if: always() + run: cat v100_nested_stacker.json || true + - name: Upload V100 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v100-nested-stacker + path: v100_nested_stacker.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v101-objective-router.yml b/.github/workflows/trace-ace-v101-objective-router.yml new file mode 100644 index 00000000..72e9b751 --- /dev/null +++ b/.github/workflows/trace-ace-v101-objective-router.yml @@ -0,0 +1,55 @@ +name: Trace the Ace V101 objective-level router +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v101_objective_level_router.py' + - '.github/workflows/trace-ace-v101-objective-router.yml' + +jobs: + router: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V101 objective-level router + run: | + cd competitions/trace_the_ace + python v101_objective_level_router.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v101_objective_level_router.json + - name: Show decision + if: always() + run: cat v101_objective_level_router.json + - name: Upload V101 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v101-objective-router + path: v101_objective_level_router.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v102-applicability-resolution.yml b/.github/workflows/trace-ace-v102-applicability-resolution.yml new file mode 100644 index 00000000..646c057a --- /dev/null +++ b/.github/workflows/trace-ace-v102-applicability-resolution.yml @@ -0,0 +1,51 @@ +name: Trace the Ace V102 applicability resolution audit +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v102_applicability_resolution.py' + - '.github/workflows/trace-ace-v102-applicability-resolution.yml' +jobs: + audit: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V102 applicability resolution audit + run: | + cd competitions/trace_the_ace + python v102_applicability_resolution.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v102_applicability_resolution.json + - name: Show decision + if: always() + run: cat v102_applicability_resolution.json + - name: Upload V102 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v102-applicability-resolution + path: v102_applicability_resolution.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v103-session-router.yml b/.github/workflows/trace-ace-v103-session-router.yml new file mode 100644 index 00000000..53ed30dc --- /dev/null +++ b/.github/workflows/trace-ace-v103-session-router.yml @@ -0,0 +1,55 @@ +name: Trace the Ace V103 session-level router +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v103_session_level_router.py' + - '.github/workflows/trace-ace-v103-session-router.yml' + +jobs: + router: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V103 session-level router + run: | + cd competitions/trace_the_ace + python v103_session_level_router.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v103_session_level_router.json + - name: Show decision + if: always() + run: cat v103_session_level_router.json + - name: Upload V103 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v103-session-router + path: v103_session_level_router.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v104-single-session-separator.yml b/.github/workflows/trace-ace-v104-single-session-separator.yml new file mode 100644 index 00000000..21a7f85e --- /dev/null +++ b/.github/workflows/trace-ace-v104-single-session-separator.yml @@ -0,0 +1,55 @@ +name: Trace the Ace V104 single session separator +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v104_single_session_separator.py' + - '.github/workflows/trace-ace-v104-single-session-separator.yml' + +jobs: + separator: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V104 single session separator + run: | + cd competitions/trace_the_ace + python v104_single_session_separator.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v104_single_session_separator.json + - name: Show decision + if: always() + run: cat v104_single_session_separator.json + - name: Upload V104 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v104-single-session-separator + path: v104_single_session_separator.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v105-prior-state-composition.yml b/.github/workflows/trace-ace-v105-prior-state-composition.yml new file mode 100644 index 00000000..866e70d8 --- /dev/null +++ b/.github/workflows/trace-ace-v105-prior-state-composition.yml @@ -0,0 +1,57 @@ +name: Trace the Ace V105 prior-state composition +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v105_prior_state_composition.py' + - '.github/workflows/trace-ace-v105-prior-state-composition.yml' + +jobs: + composition: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Syntax check V105 + run: python -m py_compile competitions/trace_the_ace/v105_prior_state_composition.py + - name: Run V105 prior-state composition + run: | + cd competitions/trace_the_ace + python v105_prior_state_composition.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v105_prior_state_composition.json + - name: Show decision + if: always() + run: cat v105_prior_state_composition.json + - name: Upload V105 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v105-prior-state-composition + path: v105_prior_state_composition.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v106-fixed-prior-state.yml b/.github/workflows/trace-ace-v106-fixed-prior-state.yml new file mode 100644 index 00000000..3bc2a9ff --- /dev/null +++ b/.github/workflows/trace-ace-v106-fixed-prior-state.yml @@ -0,0 +1,57 @@ +name: Trace the Ace V106 fixed prior-state confirmation +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v106_fixed_prior_state_law.py' + - '.github/workflows/trace-ace-v106-fixed-prior-state.yml' + +jobs: + confirmation: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Syntax check V106 + run: python -m py_compile competitions/trace_the_ace/v106_fixed_prior_state_law.py + - name: Run V106 fixed prior-state confirmation + run: | + cd competitions/trace_the_ace + python v106_fixed_prior_state_law.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v106_fixed_prior_state_law.json + - name: Show decision + if: always() + run: cat v106_fixed_prior_state_law.json + - name: Upload V106 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v106-fixed-prior-state + path: v106_fixed_prior_state_law.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v107-support-conditioned-prior.yml b/.github/workflows/trace-ace-v107-support-conditioned-prior.yml new file mode 100644 index 00000000..10b64ec3 --- /dev/null +++ b/.github/workflows/trace-ace-v107-support-conditioned-prior.yml @@ -0,0 +1,57 @@ +name: Trace the Ace V107 support-conditioned prior +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v107_support_conditioned_prior.py' + - '.github/workflows/trace-ace-v107-support-conditioned-prior.yml' + +jobs: + confirmation: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Syntax check V107 + run: python -m py_compile competitions/trace_the_ace/v107_support_conditioned_prior.py + - name: Run V107 support-conditioned prior + run: | + cd competitions/trace_the_ace + python v107_support_conditioned_prior.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v107_support_conditioned_prior.json + - name: Show decision + if: always() + run: cat v107_support_conditioned_prior.json + - name: Upload V107 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v107-support-conditioned-prior + path: v107_support_conditioned_prior.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v108-structural-interactions.yml b/.github/workflows/trace-ace-v108-structural-interactions.yml new file mode 100644 index 00000000..a18f8b12 --- /dev/null +++ b/.github/workflows/trace-ace-v108-structural-interactions.yml @@ -0,0 +1,29 @@ +name: Trace the Ace V108 structural interactions +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: ['competitions/trace_the_ace/v108_objective_transcript_interactions.py','.github/workflows/trace-ace-v108-structural-interactions.yml'] +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12', cache: pip} + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta; unzip -q transcripts.zip -d data/transcripts; unzip -q metadata.zip -d data/meta + echo "FEATURES=$(find data/meta -name 'train_features*.csv' -print -quit)" >> $GITHUB_ENV + echo "LABELS=$(find data/meta -name 'train_labels*.csv' -print -quit)" >> $GITHUB_ENV + echo "TRANSCRIPTS=$(dirname $(find data/transcripts -name '*.csv' -print -quit))" >> $GITHUB_ENV + - run: python -m py_compile competitions/trace_the_ace/v108_objective_transcript_interactions.py + - name: Run V108 + run: cd competitions/trace_the_ace && python v108_objective_transcript_interactions.py --features ../../$FEATURES --labels ../../$LABELS --transcripts ../../$TRANSCRIPTS --out ../../v108.json + - if: always() + uses: actions/upload-artifact@v4 + with: {name: trace-ace-v108-structural-interactions, path: v108.json, retention-days: 14} diff --git a/.github/workflows/trace-ace-v109-supervised-semantics.yml b/.github/workflows/trace-ace-v109-supervised-semantics.yml new file mode 100644 index 00000000..fba6cf6b --- /dev/null +++ b/.github/workflows/trace-ace-v109-supervised-semantics.yml @@ -0,0 +1,29 @@ +name: Trace the Ace V109 supervised semantics +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: ['competitions/trace_the_ace/v109_supervised_objective_semantics.py','.github/workflows/trace-ace-v109-supervised-semantics.yml'] +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12', cache: pip} + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta; unzip -q transcripts.zip -d data/transcripts; unzip -q metadata.zip -d data/meta + echo "FEATURES=$(find data/meta -name 'train_features*.csv' -print -quit)" >> $GITHUB_ENV + echo "LABELS=$(find data/meta -name 'train_labels*.csv' -print -quit)" >> $GITHUB_ENV + echo "TRANSCRIPTS=$(dirname $(find data/transcripts -name '*.csv' -print -quit))" >> $GITHUB_ENV + - run: python -m py_compile competitions/trace_the_ace/v109_supervised_objective_semantics.py + - name: Run V109 + run: cd competitions/trace_the_ace && python v109_supervised_objective_semantics.py --features ../../$FEATURES --labels ../../$LABELS --transcripts ../../$TRANSCRIPTS --out ../../v109.json + - if: always() + uses: actions/upload-artifact@v4 + with: {name: trace-ace-v109-supervised-semantics, path: v109.json, retention-days: 14} diff --git a/.github/workflows/trace-ace-v110-residual-collider-state.yml b/.github/workflows/trace-ace-v110-residual-collider-state.yml new file mode 100644 index 00000000..9e5ab760 --- /dev/null +++ b/.github/workflows/trace-ace-v110-residual-collider-state.yml @@ -0,0 +1,58 @@ +name: Trace the Ace V110 residual collider state discovery +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v110_residual_collider_state_discovery.py' + - '.github/workflows/trace-ace-v110-residual-collider-state.yml' + pull_request: + branches: [main] + paths: + - 'competitions/trace_the_ace/v110_residual_collider_state_discovery.py' + - '.github/workflows/trace-ace-v110-residual-collider-state.yml' + workflow_dispatch: +jobs: + discovery: + runs-on: ubuntu-24.04 + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Syntax check + run: python -m py_compile competitions/trace_the_ace/v110_residual_collider_state_discovery.py + - name: Run V110 phase-change search + run: | + cd competitions/trace_the_ace + python v110_residual_collider_state_discovery.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v110_residual_collider_state_discovery.json + - name: Show decision + if: always() + run: cat v110_residual_collider_state_discovery.json || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v110-residual-collider-state + path: v110_residual_collider_state_discovery.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v111-fast-residual-screen.yml b/.github/workflows/trace-ace-v111-fast-residual-screen.yml new file mode 100644 index 00000000..e08e5426 --- /dev/null +++ b/.github/workflows/trace-ace-v111-fast-residual-screen.yml @@ -0,0 +1,43 @@ +name: Trace Ace V112 Fast Raw Observable Screen +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v112_fast_raw_observable_screen.py' + - '.github/workflows/trace-ace-v111-fast-residual-screen.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12', cache: pip} + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V112 shared fast pass + run: | + cd competitions/trace_the_ace + python v112_fast_raw_observable_screen.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v112_fast_raw_observable_screen.json + - run: cat v112_fast_raw_observable_screen.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v112-fast-raw-observable-screen + path: v112_fast_raw_observable_screen.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v112-fast-raw-observable.yml b/.github/workflows/trace-ace-v112-fast-raw-observable.yml new file mode 100644 index 00000000..f68bef01 --- /dev/null +++ b/.github/workflows/trace-ace-v112-fast-raw-observable.yml @@ -0,0 +1,46 @@ +name: Trace Ace V112 Fast Raw Observable Screen +on: + pull_request: + branches: [main] + paths: + - 'competitions/trace_the_ace/v112_fast_raw_observable_screen.py' + - '.github/workflows/trace-ace-v112-fast-raw-observable.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V112 shared fast pass + run: | + cd competitions/trace_the_ace + python v112_fast_raw_observable_screen.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v112_fast_raw_observable_screen.json + - name: Show decision + run: cat v112_fast_raw_observable_screen.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v112-fast-raw-observable-screen + path: v112_fast_raw_observable_screen.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v113-applicability-regime-fast.yml b/.github/workflows/trace-ace-v113-applicability-regime-fast.yml new file mode 100644 index 00000000..271cd53a --- /dev/null +++ b/.github/workflows/trace-ace-v113-applicability-regime-fast.yml @@ -0,0 +1,53 @@ +name: Trace Ace V113 Applicability Regime Fast +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v113_applicability_regime_fast.py' + - '.github/workflows/trace-ace-v113-applicability-regime-fast.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + TEST=$(find data/meta -type f \( -name 'test_features*.csv' -o -name 'test*.csv' \) -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TEST=$TEST" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v113_applicability_regime_fast.py + - name: Run V113 shared fast pass + run: | + cd competitions/trace_the_ace + if [ -n "$TEST" ]; then EXTRA="--test-features ../../$TEST"; else EXTRA=""; fi + python v113_applicability_regime_fast.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 $EXTRA --out ../../v113_applicability_regime_fast.json + - name: Show decision + run: cat v113_applicability_regime_fast.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v113-applicability-regime-fast + path: v113_applicability_regime_fast.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v114-representation-applicability.yml b/.github/workflows/trace-ace-v114-representation-applicability.yml new file mode 100644 index 00000000..0db76e56 --- /dev/null +++ b/.github/workflows/trace-ace-v114-representation-applicability.yml @@ -0,0 +1,50 @@ +name: Trace Ace V114 Representation Applicability +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v114_representation_applicability.py' + - '.github/workflows/trace-ace-v114-representation-applicability.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py + - name: Run V114 + run: | + cd competitions/trace_the_ace + python v114_representation_applicability.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v114_representation_applicability.json + - name: Show decision + run: cat v114_representation_applicability.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v114-representation-applicability + path: v114_representation_applicability.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v115-collision-resolution.yml b/.github/workflows/trace-ace-v115-collision-resolution.yml new file mode 100644 index 00000000..f0ebe432 --- /dev/null +++ b/.github/workflows/trace-ace-v115-collision-resolution.yml @@ -0,0 +1,51 @@ +name: Trace Ace V115 Collision Resolution +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v115_collision_resolution_knn.py' + - 'competitions/trace_the_ace/v114_representation_applicability.py' + - '.github/workflows/trace-ace-v115-collision-resolution.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py v115_collision_resolution_knn.py + - name: Run V115 + run: | + cd competitions/trace_the_ace + python v115_collision_resolution_knn.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v115_collision_resolution_knn.json + - name: Show decision + run: cat v115_collision_resolution_knn.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v115-collision-resolution-knn + path: v115_collision_resolution_knn.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v115-reality-audit.yml b/.github/workflows/trace-ace-v115-reality-audit.yml new file mode 100644 index 00000000..3a0b82c5 --- /dev/null +++ b/.github/workflows/trace-ace-v115-reality-audit.yml @@ -0,0 +1,50 @@ +name: Trace Ace V115 Reality Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v115_reality_audit.py' + - '.github/workflows/trace-ace-v115-reality-audit.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight compile + run: | + cd competitions/trace_the_ace + python -m py_compile v115_reality_audit.py + - name: Run V115 reality audit + run: | + cd competitions/trace_the_ace + python v115_reality_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v115_reality_audit.json + - name: Show decision + run: cat v115_reality_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v115-reality-audit + path: v115_reality_audit.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml b/.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml new file mode 100644 index 00000000..c3af2cd0 --- /dev/null +++ b/.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml @@ -0,0 +1,45 @@ +name: Trace Ace V115b Pure V74 Reality Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v115b_pure_v74_reality_audit.py' + - '.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download metadata + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + - name: Preflight compile + run: | + cd competitions/trace_the_ace + python -m py_compile v115b_pure_v74_reality_audit.py + - name: Run V115b + run: | + cd competitions/trace_the_ace + python v115b_pure_v74_reality_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --out ../../v115b_pure_v74_reality_audit.json + - name: Show decision + run: cat v115b_pure_v74_reality_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v115b-pure-v74-reality-audit + path: v115b_pure_v74_reality_audit.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v116-row-alignment-alias.yml b/.github/workflows/trace-ace-v116-row-alignment-alias.yml new file mode 100644 index 00000000..36ded640 --- /dev/null +++ b/.github/workflows/trace-ace-v116-row-alignment-alias.yml @@ -0,0 +1,51 @@ +name: Trace Ace V116 Row Alignment Alias Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v116_row_alignment_alias_audit.py' + - 'competitions/trace_the_ace/v114_representation_applicability.py' + - '.github/workflows/trace-ace-v116-row-alignment-alias.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py v116_row_alignment_alias_audit.py + - name: Run V116 + run: | + cd competitions/trace_the_ace + python v116_row_alignment_alias_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v116_row_alignment_alias_audit.json + - name: Show decision + run: cat v116_row_alignment_alias_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v116-row-alignment-alias-audit + path: v116_row_alignment_alias_audit.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v117-oracle-information.yml b/.github/workflows/trace-ace-v117-oracle-information.yml new file mode 100644 index 00000000..094f9ac1 --- /dev/null +++ b/.github/workflows/trace-ace-v117-oracle-information.yml @@ -0,0 +1,50 @@ +name: Trace Ace V117 Oracle Information Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v117_oracle_information_audit.py' + - '.github/workflows/trace-ace-v117-oracle-information.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py v117_oracle_information_audit.py + - name: Run V117 + run: | + cd competitions/trace_the_ace + python v117_oracle_information_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v117_oracle_information_audit.json + - name: Show decision + run: cat v117_oracle_information_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v117-oracle-information-audit + path: v117_oracle_information_audit.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v118-hidden-test-geometry.yml b/.github/workflows/trace-ace-v118-hidden-test-geometry.yml new file mode 100644 index 00000000..fa41d7d0 --- /dev/null +++ b/.github/workflows/trace-ace-v118-hidden-test-geometry.yml @@ -0,0 +1,29 @@ +name: Trace Ace V118 Hidden Test Geometry Probe +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [agent/trace-ace-mastery-events] + workflow_dispatch: +jobs: + probe: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check numpy scipy + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v118_hidden_test_geometry_probe.py + - name: Run V118 + run: | + cd competitions/trace_the_ace + python v118_hidden_test_geometry_probe.py + - name: Show decision + run: cat competitions/trace_the_ace/v118_hidden_test_geometry_probe.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v118-hidden-test-geometry + path: competitions/trace_the_ace/v118_hidden_test_geometry_probe.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v119-public-anchor-geometry.yml b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml new file mode 100644 index 00000000..07ae6bd2 --- /dev/null +++ b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml @@ -0,0 +1,40 @@ +name: Trace Ace V119 Public Anchor Geometry +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v119_public_anchor_geometry.py' + - '.github/workflows/trace-ace-v119-public-anchor-geometry.yml' +jobs: + geometry: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + gdown 1UWuHKIJ86yINhQOBGx2fFJ890hiEa6Md -O transcripts.zip + mkdir -p data/meta data/transcripts + unzip -q metadata.zip -d data/meta + unzip -q transcripts.zip -d data/transcripts + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV" + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v119_public_anchor_geometry.py + - name: Run V119 + run: python competitions/trace_the_ace/v119_public_anchor_geometry.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --sessions 4000 --out competitions/trace_the_ace/v119_public_anchor_geometry.json + - name: Show decision + run: cat competitions/trace_the_ace/v119_public_anchor_geometry.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v119-public-anchor-geometry + path: competitions/trace_the_ace/v119_public_anchor_geometry.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v120-objective-identity.yml b/.github/workflows/trace-ace-v120-objective-identity.yml new file mode 100644 index 00000000..f085e9e4 --- /dev/null +++ b/.github/workflows/trace-ace-v120-objective-identity.yml @@ -0,0 +1,76 @@ +name: Trace Ace V120 Objective Identity Audit + +on: + push: + branches: [agent/v111-runner] + paths: + - 'competitions/trace_the_ace/v120_objective_identity_audit.py' + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' + - '.github/workflows/trace-ace-v120-objective-identity.yml' + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v120_objective_identity_audit.py' + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' + - '.github/workflows/trace-ace-v120-objective-identity.yml' + workflow_dispatch: + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download metadata only + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + TRAIN=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + TEST=$(find data/meta -type f \( -name 'test_features*.csv' -o -name 'submission_features*.csv' \) -print -quit) + echo "TRAIN=$TRAIN" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TEST=$TEST" >> "$GITHUB_ENV" + echo "TRAIN=$TRAIN" + echo "LABELS=$LABELS" + echo "TEST=$TEST" + - name: Inspect headers and run V120 + run: | + python - <<'PY' + import os, pandas as pd + tr=os.environ['TRAIN'] + print('train columns', list(pd.read_csv(tr,nrows=0).columns)) + te=os.environ.get('TEST','') + if te: + print('test columns', list(pd.read_csv(te,nrows=0).columns)) + PY + if [ -n "$TEST" ]; then + python competitions/trace_the_ace/v120_objective_identity_audit.py --train-features "$TRAIN" --test-features "$TEST" --out v120_objective_identity_audit.json + else + python competitions/trace_the_ace/v120_objective_identity_audit.py --train-features "$TRAIN" --out v120_objective_identity_audit.json + fi + - name: Run V122 independent metadata audit + run: | + python -m py_compile competitions/trace_the_ace/v122_id_morphology_regime_audit.py + python competitions/trace_the_ace/v122_id_morphology_regime_audit.py --features "$TRAIN" --labels "$LABELS" --out v122_id_morphology_regime_audit.json + - name: Show decisions + run: | + cat v120_objective_identity_audit.json + cat v122_id_morphology_regime_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v120-objective-identity-audit + path: v120_objective_identity_audit.json + retention-days: 14 + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v122-id-morphology-regime-audit + path: v122_id_morphology_regime_audit.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v121-128shard.yml b/.github/workflows/trace-ace-v121-128shard.yml new file mode 100644 index 00000000..527dadd6 --- /dev/null +++ b/.github/workflows/trace-ace-v121-128shard.yml @@ -0,0 +1,102 @@ +name: Trace Ace V121 Frozen 128 Shard + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-128shard.yml' + workflow_dispatch: + +concurrency: + group: trace-ace-v121-frozen-128shard + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + +jobs: + embed: + strategy: + fail-fast: false + max-parallel: 16 + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install exact embedding dependencies + run: python -m pip install --disable-pip-version-check numpy fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen manifest + run: | + test -f v121_prepared/manifest.json + grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Embed exact frozen shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_embed_batch1_transport.py --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 128 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate: + needs: embed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install evaluation dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Download all exact embedding shards + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards + merge-multiple: false + - name: Evaluate unchanged frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v121-embed-only.yml b/.github/workflows/trace-ace-v121-embed-only.yml new file mode 100644 index 00000000..84cc9f75 --- /dev/null +++ b/.github/workflows/trace-ace-v121-embed-only.yml @@ -0,0 +1,104 @@ +name: Trace Ace V121 Frozen Embed Only + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-embed-only.yml' + workflow_dispatch: + +# Reuses the already-successful exact frozen V121 prepared artifact from run 32400309220. +# Scientific model/text/sample/folds/gates are unchanged. +concurrency: + group: trace-ace-v121-embed-only-16-p4 + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + +jobs: + embed: + strategy: + fail-fast: false + max-parallel: 4 + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen preparation manifest exists + run: | + test -f v121_prepared/manifest.json + cat v121_prepared/manifest.json + - name: Embed exact frozen V121 texts shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py embed --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 16 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate: + needs: embed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Download all exact embedding shards + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards + merge-multiple: false + - name: Evaluate frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v121-eval-only.yml b/.github/workflows/trace-ace-v121-eval-only.yml new file mode 100644 index 00000000..4c7d0e74 --- /dev/null +++ b/.github/workflows/trace-ace-v121-eval-only.yml @@ -0,0 +1,65 @@ +name: Trace Ace V121 Eval Only + +on: + push: + branches: [agent/v121-cache-batch2] + paths: + - '.github/workflows/trace-ace-v121-eval-only.yml' + workflow_dispatch: + +concurrency: + group: trace-ace-v121-eval-only + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + EMBEDDING_RUN_ID: '32402681183' + +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install evaluation dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen manifest + run: | + test -f v121_prepared/manifest.json + grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Download all 128 frozen embedding shards from original run + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p v121_embedding_shards + for i in $(seq 0 127); do + gh run download "$EMBEDDING_RUN_ID" -n "v121-embeddings-$i" -D "v121_embedding_shards/$i" + done + test "$(find v121_embedding_shards -name 'v121_embeddings_shard_*.npz' | wc -l)" -eq 128 + - name: Evaluate unchanged frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v121-local-bootstrap.yml b/.github/workflows/trace-ace-v121-local-bootstrap.yml new file mode 100644 index 00000000..f18b81de --- /dev/null +++ b/.github/workflows/trace-ace-v121-local-bootstrap.yml @@ -0,0 +1,49 @@ +name: Trace Ace V121 Local Bootstrap + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-local-bootstrap.yml' + workflow_dispatch: + +jobs: + bootstrap: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Download Python 3.13 wheelhouse + run: | + mkdir -p wheelhouse + python -m pip download --only-binary=:all: --dest wheelhouse fastembed==0.8.0 + - name: Install FastEmbed and fetch exact Jina model + run: | + python -m pip install --no-index --find-links wheelhouse fastembed==0.8.0 + python - <<'PY' + from fastembed import TextEmbedding + m = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en') + print('model_ready', type(m).__name__) + PY + mkdir -p model_cache + for p in "$HOME/.cache/fastembed" "$HOME/.cache/huggingface"; do + if [ -d "$p" ]; then cp -a "$p" model_cache/; fi + done + find model_cache -maxdepth 4 -type f -printf '%p %s\n' | head -100 + - uses: actions/upload-artifact@v4 + with: + name: v121-python313-wheelhouse + path: wheelhouse/ + retention-days: 2 + compression-level: 0 + - uses: actions/upload-artifact@v4 + with: + name: v121-jina-model-cache + path: model_cache/ + retention-days: 2 + compression-level: 0 diff --git a/.github/workflows/trace-ace-v121-model-export.yml b/.github/workflows/trace-ace-v121-model-export.yml new file mode 100644 index 00000000..b8c0e63c --- /dev/null +++ b/.github/workflows/trace-ace-v121-model-export.yml @@ -0,0 +1,41 @@ +name: Trace Ace V121 Model Export + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-model-export.yml' + workflow_dispatch: + +jobs: + export: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Install exact FastEmbed + run: python -m pip install --disable-pip-version-check fastembed==0.8.0 + - name: Fetch and locate exact Jina model + run: | + python - <<'PY' + from fastembed import TextEmbedding + import tempfile, pathlib, json + cache = pathlib.Path('/tmp/v121_fastembed_cache') + m = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en', cache_dir=str(cache)) + print('MODEL_DICT', json.dumps({k:str(v) for k,v in m.__dict__.items()}, default=str, indent=2)) + print('CACHE', cache) + PY + echo 'MODEL FILES:' + find /tmp/v121_fastembed_cache -type f -printf '%p %s\n' | sort + du -sh /tmp/v121_fastembed_cache + - uses: actions/upload-artifact@v4 + with: + name: v121-jina-fastembed-cache + path: /tmp/v121_fastembed_cache/ + retention-days: 2 + compression-level: 0 diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml new file mode 100644 index 00000000..3e43b290 --- /dev/null +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -0,0 +1,139 @@ +name: Trace Ace V121 Pretrained Semantic Residual + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - 'competitions/trace_the_ace/v121_pretrained_semantic_residual.py' + - 'competitions/trace_the_ace/v121_staged_transport.py' + - '.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml' + workflow_dispatch: + +# Infrastructure-only serialization for the frozen 16-shard V121 transport. +concurrency: + group: trace-ace-v121-frozen-16shard-p4 + cancel-in-progress: false + +env: + TRANSCRIPT_KEY: trace-ace-transcripts-v1-603547640 + TRANSCRIPT_SHA: e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d + +jobs: + prepare: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown fastembed==0.8.0 + - name: Restore frozen transcript archive + uses: actions/cache/restore@v4 + with: + path: transcripts.zip + key: ${{ env.TRANSCRIPT_KEY }} + fail-on-cache-miss: true + - name: Validate frozen transcript archive + shell: bash + run: | + set -euo pipefail + test "$(stat -c%s transcripts.zip)" = "603547640" + unzip -tq transcripts.zip >/dev/null + test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "$TRANSCRIPT_SHA" + - name: Download frozen metadata and extract + shell: bash + run: | + set -euo pipefail + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta data/transcripts + unzip -q metadata.zip -d data/meta + unzip -q transcripts.zip -d data/transcripts + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV" + - name: Prepare exact frozen V121 inputs + run: | + python -m py_compile competitions/trace_the_ace/v121_pretrained_semantic_residual.py competitions/trace_the_ace/v121_staged_transport.py + cd competitions/trace_the_ace + python v121_staged_transport.py prepare \ + --features "../../$FEATURES" --labels "../../$LABELS" \ + --transcripts "../../$TRANSCRIPTS" --rows 2500 --dir ../../v121_prepared + - uses: actions/upload-artifact@v4 + with: + name: v121-prepared + path: v121_prepared/ + retention-days: 2 + compression-level: 6 + + embed: + needs: prepare + strategy: + fail-fast: false + max-parallel: 4 + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + - name: Embed exact frozen V121 texts shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py embed --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 16 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate: + needs: embed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + - name: Download all exact embedding shards + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards + merge-multiple: false + - name: Evaluate frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v121-tail-recovery.yml b/.github/workflows/trace-ace-v121-tail-recovery.yml new file mode 100644 index 00000000..fab73e9b --- /dev/null +++ b/.github/workflows/trace-ace-v121-tail-recovery.yml @@ -0,0 +1,111 @@ +name: Trace Ace V121 Tail Recovery + +on: + push: + branches: [agent/v121-cache-batch2] + paths: + - '.github/workflows/trace-ace-v121-tail-recovery.yml' + workflow_dispatch: + +concurrency: + group: trace-ace-v121-tail-recovery-push + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + ORIGINAL_RUN_ID: '32402681183' + +jobs: + embed_tail: + strategy: + fail-fast: false + max-parallel: 4 + matrix: + shard: [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install exact embedding dependencies + run: python -m pip install --disable-pip-version-check numpy fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen manifest + run: | + test -f v121_prepared/manifest.json + grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Embed exact frozen shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_embed_batch1_transport.py --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 128 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate_union: + needs: embed_tail + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install evaluation dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Download successful original shards 0 through 64 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p v121_embedding_shards/original + for i in $(seq 0 64); do + gh run download "$ORIGINAL_RUN_ID" -n "v121-embeddings-$i" -D "v121_embedding_shards/original/$i" + done + - name: Download recovered tail shards 65 through 127 + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards/recovered + merge-multiple: false + - name: Evaluate unchanged frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v122-id-morphology.yml b/.github/workflows/trace-ace-v122-id-morphology.yml new file mode 100644 index 00000000..44cf7104 --- /dev/null +++ b/.github/workflows/trace-ace-v122-id-morphology.yml @@ -0,0 +1,45 @@ +name: Trace Ace V122 ID Morphology Audit + +on: + push: + branches: [agent/v111-runner] + paths: + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' + - '.github/workflows/trace-ace-v122-id-morphology.yml' + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' + - '.github/workflows/trace-ace-v122-id-morphology.yml' + workflow_dispatch: + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download metadata only + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v122_id_morphology_regime_audit.py + - name: Run V122 + run: python competitions/trace_the_ace/v122_id_morphology_regime_audit.py --features "$FEATURES" --labels "$LABELS" --out v122_id_morphology_regime_audit.json + - name: Show decision + run: cat v122_id_morphology_regime_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v122-id-morphology-regime-audit + path: v122_id_morphology_regime_audit.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v129-tutor-uptake.yml b/.github/workflows/trace-ace-v129-tutor-uptake.yml new file mode 100644 index 00000000..bbcf3e8c --- /dev/null +++ b/.github/workflows/trace-ace-v129-tutor-uptake.yml @@ -0,0 +1,58 @@ +name: Trace Ace V129 Tutor Uptake + +on: + pull_request: + branches: [agent/v121-cache-batch2] + paths: + - '.github/workflows/trace-ace-v129-tutor-uptake.yml' + - 'competitions/trace_the_ace/v129_tutor_uptake.py' + workflow_dispatch: + +env: + PREPARED_RUN_ID: '32400309220' + +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 12 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas fastembed==0.8.0 + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact cache hit + run: | + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V129 + run: python competitions/trace_the_ace/v129_tutor_uptake.py --archive transcripts.zip --dir v121_prepared --out v129_tutor_uptake.json + - name: Show decision + run: cat v129_tutor_uptake.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v129-tutor-uptake + path: v129_tutor_uptake.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v132-nested-applicability-gate.yml b/.github/workflows/trace-ace-v132-nested-applicability-gate.yml new file mode 100644 index 00000000..ef1b85c6 --- /dev/null +++ b/.github/workflows/trace-ace-v132-nested-applicability-gate.yml @@ -0,0 +1,57 @@ +name: Trace Ace V132 Nested Applicability Gate +on: + pull_request: + branches: [agent/v121-cache-batch2] + paths: + - 'competitions/trace_the_ace/v129_tutor_uptake.py' + - 'competitions/trace_the_ace/v132_nested_applicability_gate.py' + - '.github/workflows/trace-ace-v132-nested-applicability-gate.yml' + workflow_dispatch: + +env: + PREPARED_RUN_ID: 32400309220 + +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas fastembed==0.8.0 + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact cache hit + run: | + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ github.token }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V132 + run: python competitions/trace_the_ace/v132_nested_applicability_gate.py --archive transcripts.zip --dir v121_prepared --out v132_nested_applicability_gate.json + - name: Show decision + if: always() + run: cat v132_nested_applicability_gate.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v132-nested-applicability-gate + path: v132_nested_applicability_gate.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v133-verified-math-evidence.yml b/.github/workflows/trace-ace-v133-verified-math-evidence.yml new file mode 100644 index 00000000..9160dfcf --- /dev/null +++ b/.github/workflows/trace-ace-v133-verified-math-evidence.yml @@ -0,0 +1,56 @@ +name: Trace Ace V133 Verified Math Evidence +on: + pull_request: + branches: [agent/v132-nested-applicability-gate] + paths: + - 'competitions/trace_the_ace/v133_verified_math_evidence.py' + - '.github/workflows/trace-ace-v133-verified-math-evidence.yml' + workflow_dispatch: + +env: + PREPARED_RUN_ID: 32400309220 + +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact cache hit + run: | + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ github.token }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V133 + run: python competitions/trace_the_ace/v133_verified_math_evidence.py --archive transcripts.zip --dir v121_prepared --out v133_verified_math_evidence.json + - name: Show decision + if: always() + run: cat v133_verified_math_evidence.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v133-verified-math-evidence + path: v133_verified_math_evidence.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v134-verifier-feedback-contradiction.yml b/.github/workflows/trace-ace-v134-verifier-feedback-contradiction.yml new file mode 100644 index 00000000..710f35f4 --- /dev/null +++ b/.github/workflows/trace-ace-v134-verifier-feedback-contradiction.yml @@ -0,0 +1,55 @@ +name: Trace Ace V134 Verifier Feedback Contradiction +on: + pull_request: + branches: [agent/v133-verified-math-evidence] + paths: + - 'competitions/trace_the_ace/v134_verifier_feedback_contradiction.py' + - '.github/workflows/trace-ace-v134-verifier-feedback-contradiction.yml' + workflow_dispatch: + +env: + PREPARED_RUN_ID: 32400309220 +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact cache hit + run: | + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ github.token }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V134 + run: python competitions/trace_the_ace/v134_verifier_feedback_contradiction.py --archive transcripts.zip --dir v121_prepared --out v134_verifier_feedback_contradiction.json + - name: Show decision + if: always() + run: cat v134_verifier_feedback_contradiction.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v134-verifier-feedback-contradiction + path: v134_verifier_feedback_contradiction.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v135-nested-supported-stack.yml b/.github/workflows/trace-ace-v135-nested-supported-stack.yml new file mode 100644 index 00000000..4f8268b1 --- /dev/null +++ b/.github/workflows/trace-ace-v135-nested-supported-stack.yml @@ -0,0 +1,44 @@ +name: Trace Ace V135 Nested Supported Stack +on: + pull_request: + branches: [agent/v134-verifier-feedback-contradiction] + paths: + - 'competitions/trace_the_ace/v135_nested_supported_stack.py' + - '.github/workflows/trace-ace-v135-nested-supported-stack.yml' + workflow_dispatch: + +env: + PREPARED_RUN_ID: 32400309220 +jobs: + evaluate: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy scipy scikit-learn pandas + - name: Download exact frozen prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ github.token }} + - name: Verify frozen sample + run: grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Run frozen V135 + run: python competitions/trace_the_ace/v135_nested_supported_stack.py --dir v121_prepared --out v135_nested_supported_stack.json + - name: Show decision + if: always() + run: cat v135_nested_supported_stack.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v135-nested-supported-stack + path: v135_nested_supported_stack.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v136-full-v135-verification.yml b/.github/workflows/trace-ace-v136-full-v135-verification.yml new file mode 100644 index 00000000..77e79517 --- /dev/null +++ b/.github/workflows/trace-ace-v136-full-v135-verification.yml @@ -0,0 +1,70 @@ +name: Trace Ace V136 Full V135 Verification +on: + pull_request: + branches: [agent/v135-nested-supported-stack] + paths: + - 'competitions/trace_the_ace/v136_full_v135_verification.py' + - '.github/workflows/trace-ace-v136-full-v135-verification.yml' + workflow_dispatch: + +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Restore frozen transcript archive from GitHub cache only + id: transcript-cache + uses: actions/cache@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + - name: Require exact transcript cache + run: | + set -euo pipefail + test '${{ steps.transcript-cache.outputs.cache-hit }}' = 'true' + test "$(stat -c%s transcripts.zip)" = '603547640' + echo 'e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d transcripts.zip' | sha256sum -c - + - name: Download frozen small metadata archive only + run: | + set -euo pipefail + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve and audit schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS" + python - <<'PY' "$FEATURES" "$LABELS" + import pandas as pd,sys + f=pd.read_csv(sys.argv[1]); y=pd.read_csv(sys.argv[2]) + print('FEATURE_COLUMNS',list(f.columns)); print('LABEL_COLUMNS',list(y.columns)); print('SHAPES',f.shape,y.shape) + assert len(f)==35072 and len(y)==35072 + PY + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run frozen V136 full verification + run: | + cd competitions/trace_the_ace + python v136_full_v135_verification.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v136_full_v135_verification.json + - name: Show decision + if: always() + run: cat v136_full_v135_verification.json || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v136-full-v135-verification + path: v136_full_v135_verification.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v74-runtime-package.yml b/.github/workflows/trace-ace-v74-runtime-package.yml new file mode 100644 index 00000000..ecdd367e --- /dev/null +++ b/.github/workflows/trace-ace-v74-runtime-package.yml @@ -0,0 +1,50 @@ +name: Trace Ace V74 Runtime Package +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/runtime_v74/**' + - 'competitions/trace_the_ace/train_v74_runtime_assets.py' + - '.github/workflows/trace-ace-v74-runtime-package.yml' +jobs: + package: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn joblib gdown + - name: Download metadata + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + - name: Preflight + run: | + python -m py_compile competitions/trace_the_ace/runtime_v74/main.py competitions/trace_the_ace/runtime_v74/v74_runtime_core.py competitions/trace_the_ace/train_v74_runtime_assets.py + - name: Train frozen assets and parity test + run: | + python competitions/trace_the_ace/train_v74_runtime_assets.py --features "$FEATURES" --labels "$LABELS" --assets competitions/trace_the_ace/runtime_v74/assets + - name: Package + run: | + rm -rf v74_submission && mkdir -p v74_submission/assets + cp competitions/trace_the_ace/runtime_v74/main.py v74_submission/main.py + cp competitions/trace_the_ace/runtime_v74/v74_runtime_core.py v74_submission/v74_runtime_core.py + cp competitions/trace_the_ace/runtime_v74/assets/* v74_submission/assets/ + (cd v74_submission && zip -q -r ../trace-ace-v74-pure-runtime.zip .) + unzip -l trace-ace-v74-pure-runtime.zip + cat competitions/trace_the_ace/runtime_v74/assets/manifest.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v74-pure-runtime + path: | + trace-ace-v74-pure-runtime.zip + competitions/trace_the_ace/runtime_v74/assets/manifest.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v75-parity.yml b/.github/workflows/trace-ace-v75-parity.yml new file mode 100644 index 00000000..185998e5 --- /dev/null +++ b/.github/workflows/trace-ace-v75-parity.yml @@ -0,0 +1,195 @@ +name: Trace the Ace V75 parity and independence + +on: + workflow_dispatch: + push: + branches: + - agent/trace-ace-mastery-events + paths: + - ".github/workflows/trace-ace-v75-parity.yml" + +jobs: + parity-independence: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Download immutable promoted V75 candidate artifact + uses: actions/download-artifact@v4 + with: + name: trace-ace-v75-official-runtime-candidate + path: /tmp/candidate_artifact + github-token: ${{ github.token }} + repository: heathSanchez/mathgraph + run-id: 31863804281 + + - name: Extract candidate submission and frozen assets + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/candidate + unzip -q /tmp/candidate_artifact/runtime/submission/submission.zip -d /tmp/candidate + test -f /tmp/candidate/main.py + test -f /tmp/candidate/assets/v75_runtime_assets.npz + sha256sum /tmp/candidate_artifact/runtime/submission/submission.zip | tee /tmp/submission_sha256.txt + + - name: Download held-out fixture source data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os, gdown + p = gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'], output='/tmp/meta.zip', quiet=False) + if not p: raise SystemExit('metadata download failed') + p = gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'], output='/tmp/transcripts.zip', quiet=False) + if not p: raise SystemExit('transcript download failed') + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + + - name: Resolve schemas and build four competition-shaped batches + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, json, shutil + from pathlib import Path + import pandas as pd + + roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')] + features=None; transcript_dir=None + for root in roots: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): + features=p; print('FEATURE HEADER',h) + if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): + transcript_dir=p.parent; print('TRANSCRIPT HEADER',h) + if not features or not transcript_dir: raise SystemExit('schema discovery failed') + f=pd.read_csv(features).reset_index(drop=True) + target=f.iloc[[0]].copy() + target_id=str(target.iloc[0].response_id) + # A: target alone; B: target + unrelated; C: same as B reordered; D: target + different unrelated batch. + A=target + B=pd.concat([target,f.iloc[1:32]],ignore_index=True).drop_duplicates('response_id') + C=B.sample(frac=1,random_state=20260815).reset_index(drop=True) + D=pd.concat([target,f.iloc[100:132]],ignore_index=True).drop_duplicates('response_id') + batches={'A':A,'B':B,'C':C,'D':D} + for name,df in batches.items(): + root=Path('/tmp/batches')/name + tdir=root/'test_transcripts'; tdir.mkdir(parents=True,exist_ok=True) + df.to_csv(root/'test_features.csv',index=False) + pd.DataFrame({'response_id':df.response_id.astype(str),'probability':0.5}).to_csv(root/'submission_format.csv',index=False) + for sid in df.session_id.astype(str).unique(): + src=transcript_dir/f'{sid}.csv' + if not src.exists(): raise SystemExit(f'missing transcript {src}') + shutil.copy2(src,tdir/src.name) + Path('/tmp/fixture.json').write_text(json.dumps({'target_response_id':target_id,'features':str(features),'transcript_dir':str(transcript_dir)},indent=2)) + print('TARGET',target_id) + print({k:len(v) for k,v in batches.items()}) + PY + + - name: Generate independent research-reference probabilities + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import json, sys + from pathlib import Path + import numpy as np, pandas as pd + from scipy.sparse import csr_matrix, hstack + from sklearn.feature_extraction.text import HashingVectorizer + + sys.path.insert(0, str(Path('competitions/trace_the_ace').resolve())) + from v71_mastery_events import load_transcript + from v75_canonical_trajectory import trajectory_views + + a=np.load('/tmp/candidate/assets/v75_runtime_assets.npz') + coef=a['coef'].astype(np.float64); intercept=float(a['intercept'].ravel()[0]) + mean=a['num_mean'].astype(np.float64); std=a['num_std'].astype(np.float64) + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + def sigmoid(x): return 1/(1+np.exp(-x)) + for name in ['A','B','C','D']: + root=Path('/tmp/batches')/name; df=pd.read_csv(root/'test_features.csv') + cache={}; views=[]; nums=[] + for r in df.itertuples(index=False): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(root/'test_transcripts'/f'{sid}.csv') + v,n,_=trajectory_views(cache[sid],str(r.learning_objective)); views.append(v); nums.append(n) + z=(np.vstack(nums).astype(np.float64)-mean)/std + parts=[ + hv.transform(['[OBJECTIVE] '+str(x) for x in df.learning_objective]), + hv.transform(['[RAW] '+v['raw'] for v in views]), + hv.transform(['[STUDENT] '+v['student'] for v in views]), + hv.transform(['[LOCAL] '+v['local'] for v in views]), + hv.transform(['[STATE] '+v['canonical'] for v in views]), + hv.transform(['[TERMINAL] '+v['terminal'] for v in views]), + csr_matrix(z), + ] + X=hstack(parts,format='csr') + p=np.clip(sigmoid(np.asarray(X@coef).ravel()+intercept),1e-5,1-1e-5) + pd.DataFrame({'response_id':df.response_id.astype(str),'probability':p}).to_csv(f'/tmp/reference_{name}.csv',index=False) + PY + + - name: Run immutable runtime candidate on all four batches + shell: bash + run: | + set -euo pipefail + sudo rm -rf /code_execution + sudo mkdir -p /code_execution + sudo chmod 0777 /code_execution + for NAME in A B C D; do + rm -rf /code_execution/data /code_execution/run + cp -a "/tmp/batches/$NAME" /code_execution/data + cp -a /tmp/candidate /code_execution/run + (cd /code_execution/run && python main.py) + cp /code_execution/run/submission.csv "/tmp/runtime_${NAME}.csv" + done + + - name: Gate 2 research/runtime parity + shell: bash + run: | + set -euo pipefail + for NAME in A B C D; do + python competitions/trace_the_ace/runtime_validate.py parity --reference "/tmp/reference_${NAME}.csv" --runtime "/tmp/runtime_${NAME}.csv" --tol 1e-8 + done + + - name: Gate 7 sample-independence metamorphic audit + shell: bash + run: | + set -euo pipefail + TARGET=$(python -c "import json; print(json.load(open('/tmp/fixture.json'))['target_response_id'])") + python competitions/trace_the_ace/runtime_validate.py independence --response-id "$TARGET" --predictions /tmp/runtime_A.csv /tmp/runtime_B.csv /tmp/runtime_C.csv /tmp/runtime_D.csv --tol 1e-8 + + - name: Validate every runtime output contract + shell: bash + run: | + set -euo pipefail + for NAME in A B C D; do + python competitions/trace_the_ace/runtime_validate.py output --format "/tmp/batches/${NAME}/submission_format.csv" --predictions "/tmp/runtime_${NAME}.csv" + done + + - name: Upload parity evidence + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v75-parity-independence-evidence + path: | + /tmp/submission_sha256.txt + /tmp/fixture.json + /tmp/reference_*.csv + /tmp/runtime_*.csv + retention-days: 14 diff --git a/.github/workflows/trace-ace-v78-seismic.yml b/.github/workflows/trace-ace-v78-seismic.yml new file mode 100644 index 00000000..a5c00ade --- /dev/null +++ b/.github/workflows/trace-ace-v78-seismic.yml @@ -0,0 +1,71 @@ +name: Trace the Ace V78 seismic semantic + +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - "competitions/trace_the_ace/v78_seismic_semantic.py" + - ".github/workflows/trace-ace-v78-seismic.yml" + +jobs: + seismic: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown sentence-transformers torch + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + if not gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False): raise SystemExit('metadata download failed') + if not gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False): raise SystemExit('transcript download failed') + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas + shell: bash + run: | + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=td=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if td is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): td=p.parent; print('TRANSCRIPT HEADER',h) + if not all([features,labels,td]): raise SystemExit('schema discovery failed') + with open('/tmp/paths.env','w') as f: + f.write('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANSCRIPTS='+shlex.quote(str(td))+'\n') + PY + - name: Run V78 MiniLM seismic test + shell: bash + env: + TOKENIZERS_PARALLELISM: "false" + run: | + set -euo pipefail + source /tmp/paths.env + python competitions/trace_the_ace/v78_seismic_semantic.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v78_seismic_semantic.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v78-seismic-result + path: v78_seismic_semantic.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v79-retrieval.yml b/.github/workflows/trace-ace-v79-retrieval.yml new file mode 100644 index 00000000..5771f47d --- /dev/null +++ b/.github/workflows/trace-ace-v79-retrieval.yml @@ -0,0 +1,69 @@ +name: Trace the Ace V79 retrieval gain + +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - "competitions/trace_the_ace/v79_retrieval_gain.py" + - ".github/workflows/trace-ace-v79-retrieval.yml" + +jobs: + retrieval-gain: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown sentence-transformers torch + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False) + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas + shell: bash + run: | + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=tdir=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if tdir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): tdir=p.parent; print('TRANSCRIPT HEADER',h) + if not(features and labels and tdir): raise SystemExit('schema discovery failed') + Path('/tmp/paths.env').write_text('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANSCRIPTS='+shlex.quote(str(tdir))+'\n') + PY + - name: Run V79 objective retrieval and learning-gain test + shell: bash + run: | + set -euo pipefail + source /tmp/paths.env + python competitions/trace_the_ace/v79_retrieval_gain.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --model sentence-transformers/all-MiniLM-L6-v2 --topk 6 --out v79_retrieval_gain.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v79-retrieval-gain + path: v79_retrieval_gain.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v80-bge.yml b/.github/workflows/trace-ace-v80-bge.yml new file mode 100644 index 00000000..444dc236 --- /dev/null +++ b/.github/workflows/trace-ace-v80-bge.yml @@ -0,0 +1,68 @@ +name: Trace the Ace V80 BGE retrieval teacher + +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - ".github/workflows/trace-ace-v80-bge.yml" + +jobs: + bge-retrieval: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown sentence-transformers torch + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False) + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas + shell: bash + run: | + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=tdir=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if tdir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): tdir=p.parent; print('TRANSCRIPT HEADER',h) + if not(features and labels and tdir): raise SystemExit('schema discovery failed') + Path('/tmp/paths.env').write_text('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANSCRIPTS='+shlex.quote(str(tdir))+'\n') + PY + - name: Run V80 BGE-large objective retrieval teacher + shell: bash + run: | + set -euo pipefail + source /tmp/paths.env + python competitions/trace_the_ace/v79_retrieval_gain.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --model BAAI/bge-large-en-v1.5 --batch 32 --topk 6 --out v80_bge_retrieval_gain.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v80-bge-retrieval-gain + path: v80_bge_retrieval_gain.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v81-phase.yml b/.github/workflows/trace-ace-v81-phase.yml new file mode 100644 index 00000000..e5244312 --- /dev/null +++ b/.github/workflows/trace-ace-v81-phase.yml @@ -0,0 +1,63 @@ +name: Trace the Ace V81 target segment phase +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - competitions/trace_the_ace/v81_target_segment_phase.py + - .github/workflows/trace-ace-v81-phase.yml +jobs: + v81: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False) + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas and run V81 + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=trans=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if trans is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): trans=p.parent; print('TRANSCRIPT HEADER',h) + if not all([features,labels,trans]): raise SystemExit('schema discovery failed') + Path('/tmp/paths.env').write_text('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANS='+shlex.quote(str(trans))+'\n') + PY + source /tmp/paths.env + python competitions/trace_the_ace/v81_target_segment_phase.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANS" --out v81_target_segment_phase.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v81-target-segment-phase + path: v81_target_segment_phase.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v82-modernbert.yml b/.github/workflows/trace-ace-v82-modernbert.yml new file mode 100644 index 00000000..d5ab4337 --- /dev/null +++ b/.github/workflows/trace-ace-v82-modernbert.yml @@ -0,0 +1,76 @@ +name: Trace the Ace V82 supervised ModernBERT + +on: + workflow_dispatch: + push: + branches: + - agent/trace-ace-mastery-events + paths: + - "competitions/trace_the_ace/v82_modernbert_supervised.py" + - ".github/workflows/trace-ace-v82-modernbert.yml" + +jobs: + v82: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + TOKENIZERS_PARALLELISM: "false" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown torch transformers accelerate + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/transcripts /tmp/trace_ace/meta + python - <<'PY' + import os, gdown + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'], output='/tmp/trace_ace/transcripts.zip', quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'], output='/tmp/trace_ace/meta.zip', quiet=False) + PY + unzip -q /tmp/trace_ace/transcripts.zip -d /tmp/trace_ace/transcripts + unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta + - name: Resolve schemas and run V82 + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, shlex + from pathlib import Path + roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')] + f=l=t=None + for root in roots: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as h: cols=next(csv.reader(h)) + except Exception: continue + s=set(cols) + if f is None and {'response_id','session_id','learning_objective'}.issubset(s): f=p + if l is None and 'response_id' in s and ('is_correct' in s or 'correct' in s): l=p + if t is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(s): t=p.parent + if not (f and l and t): raise SystemExit('Could not identify inputs') + print('FEATURE HEADER', list(csv.reader(open(f,encoding='utf-8-sig')))[0]) + print('LABEL HEADER', list(csv.reader(open(l,encoding='utf-8-sig')))[0]) + with open('/tmp/trace_ace/paths.env','w') as h: + h.write('FEATURES='+shlex.quote(str(f))+'\nLABELS='+shlex.quote(str(l))+'\nTRANSCRIPTS='+shlex.quote(str(t))+'\n') + PY + source /tmp/trace_ace/paths.env + python competitions/trace_the_ace/v82_modernbert_supervised.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --out v82_modernbert_supervised.json \ + --model answerdotai/ModernBERT-large \ + --folds 1 --epochs 1 --top-blocks 2 --max-len 768 --batch 2 --eval-batch 4 + - name: Upload aggregate result + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v82-modernbert-supervised + path: v82_modernbert_supervised.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v83-talkmove.yml b/.github/workflows/trace-ace-v83-talkmove.yml new file mode 100644 index 00000000..acfe0b49 --- /dev/null +++ b/.github/workflows/trace-ace-v83-talkmove.yml @@ -0,0 +1,35 @@ +name: Trace the Ace V83 TalkMove supervised +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v83_talkmove_supervised.py' + - '.github/workflows/trace-ace-v83-talkmove.yml' +jobs: + v83: + runs-on: ubuntu-latest + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12'} + - name: Install dependencies + run: pip install -q pandas numpy scipy scikit-learn torch transformers sentencepiece gdown + - name: Download data + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + unzip -q metadata.zip -d data + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + mkdir -p transcripts && unzip -q transcripts.zip -d transcripts + - name: Resolve schemas and run V83 + working-directory: competitions/trace_the_ace + run: | + F=$(find ../../data -type f -iname '*features*.csv' | head -1) + L=$(find ../../data -type f -iname '*labels*.csv' | head -1) + T=$(find ../../transcripts -type f -name '*.csv' -printf '%h\n' | head -1) + python v83_talkmove_supervised.py --features "$F" --labels "$L" --transcripts "$T" --out ../../v83_talkmove_supervised.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v83-talkmove-result + path: v83_talkmove_supervised.json diff --git a/.github/workflows/trace-ace-v84-student-evidence.yml b/.github/workflows/trace-ace-v84-student-evidence.yml new file mode 100644 index 00000000..4f7a1685 --- /dev/null +++ b/.github/workflows/trace-ace-v84-student-evidence.yml @@ -0,0 +1,35 @@ +name: Trace the Ace V84 student evidence +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v84_student_evidence.py' + - '.github/workflows/trace-ace-v84-student-evidence.yml' +jobs: + v84: + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12'} + - name: Install dependencies + run: pip install -q pandas numpy scipy scikit-learn gdown + - name: Download data + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + unzip -q metadata.zip -d data + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + mkdir -p transcripts && unzip -q transcripts.zip -d transcripts + - name: Resolve schemas and run V84 + working-directory: competitions/trace_the_ace + run: | + F=$(find ../../data -type f -iname '*features*.csv' | head -1) + L=$(find ../../data -type f -iname '*labels*.csv' | head -1) + T=$(find ../../transcripts -type f -name '*.csv' -printf '%h\n' | head -1) + python v84_student_evidence.py --features "$F" --labels "$L" --transcripts "$T" --out ../../v84_student_evidence.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v84-student-evidence-result + path: v84_student_evidence.json diff --git a/.github/workflows/trace-ace-v85-evidence-state.yml b/.github/workflows/trace-ace-v85-evidence-state.yml new file mode 100644 index 00000000..517915da --- /dev/null +++ b/.github/workflows/trace-ace-v85-evidence-state.yml @@ -0,0 +1,41 @@ +name: Trace the Ace V85 EvidenceEvent state +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v85_evidence_state.py' + - '.github/workflows/trace-ace-v85-evidence-state.yml' + workflow_dispatch: +jobs: + v85: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + find data -maxdepth 3 -type f | head + - name: Resolve schemas and run V85 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + echo "$FEATURES $LABELS $TRANSCRIPTS" + cd competitions/trace_the_ace + python v85_evidence_state.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v85_evidence_state.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v85-evidence-state + path: v85_evidence_state.json diff --git a/.github/workflows/trace-ace-v86-knowledge-state.yml b/.github/workflows/trace-ace-v86-knowledge-state.yml new file mode 100644 index 00000000..616d16dd --- /dev/null +++ b/.github/workflows/trace-ace-v86-knowledge-state.yml @@ -0,0 +1,41 @@ +name: Trace the Ace V86 knowledge-state dynamics +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v86_knowledge_state_dynamics.py' + - '.github/workflows/trace-ace-v86-knowledge-state.yml' + workflow_dispatch: +jobs: + v86: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas and run V86 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "$FEATURES $LABELS $TRANSCRIPTS" + cd competitions/trace_the_ace + python v86_knowledge_state_dynamics.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v86_knowledge_state_dynamics.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v86-knowledge-state-dynamics + path: v86_knowledge_state_dynamics.json diff --git a/.github/workflows/trace-ace-v87-composition.yml b/.github/workflows/trace-ace-v87-composition.yml new file mode 100644 index 00000000..e029ad00 --- /dev/null +++ b/.github/workflows/trace-ace-v87-composition.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V87 RGRS composition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v87_rgrs_composition.py' + - '.github/workflows/trace-ace-v87-composition.yml' + workflow_dispatch: +jobs: + v87: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas and run V87 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v87_rgrs_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v87_rgrs_composition.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v87-rgrs-composition + path: v87_rgrs_composition.json diff --git a/.github/workflows/trace-ace-v88-crossfit-evidence.yml b/.github/workflows/trace-ace-v88-crossfit-evidence.yml new file mode 100644 index 00000000..159a6db1 --- /dev/null +++ b/.github/workflows/trace-ace-v88-crossfit-evidence.yml @@ -0,0 +1,40 @@ +name: Trace the Ace V88 crossfit evidence composition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v88_crossfit_evidence_composition.py' + - '.github/workflows/trace-ace-v88-crossfit-evidence.yml' + workflow_dispatch: +jobs: + v88: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V88 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v88_crossfit_evidence_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v88_crossfit_evidence_composition.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v88-crossfit-evidence + path: v88_crossfit_evidence_composition.json +# retrigger-registration diff --git a/.github/workflows/trace-ace-v89-relative-ability.yml b/.github/workflows/trace-ace-v89-relative-ability.yml new file mode 100644 index 00000000..4c557e32 --- /dev/null +++ b/.github/workflows/trace-ace-v89-relative-ability.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V89 relative ability +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v89_relative_ability_composition.py' + - '.github/workflows/trace-ace-v89-relative-ability.yml' + workflow_dispatch: +jobs: + v89: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V89 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v89_relative_ability_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v89_relative_ability.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v89-relative-ability + path: v89_relative_ability.json diff --git a/.github/workflows/trace-ace-v90-regime-gated.yml b/.github/workflows/trace-ace-v90-regime-gated.yml new file mode 100644 index 00000000..e6740b9b --- /dev/null +++ b/.github/workflows/trace-ace-v90-regime-gated.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V90 regime gated composition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v90_regime_gated_composition.py' + - '.github/workflows/trace-ace-v90-regime-gated.yml' + workflow_dispatch: +jobs: + v90: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V90 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v90_regime_gated_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v90_regime_gated.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v90-regime-gated + path: v90_regime_gated.json diff --git a/.github/workflows/trace-ace-v91-disagreement-frontier.yml b/.github/workflows/trace-ace-v91-disagreement-frontier.yml new file mode 100644 index 00000000..9fd9e5f1 --- /dev/null +++ b/.github/workflows/trace-ace-v91-disagreement-frontier.yml @@ -0,0 +1,38 @@ +name: Trace the Ace V91 disagreement frontier +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v91_disagreement_frontier.py' + - '.github/workflows/trace-ace-v91-disagreement-frontier.yml' + workflow_dispatch: +jobs: + v91: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V91 + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v91_disagreement_frontier.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v91_disagreement_frontier.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v91-disagreement-frontier + path: v91_disagreement_frontier.json diff --git a/.github/workflows/trace-ace-v92-latent-state.yml b/.github/workflows/trace-ace-v92-latent-state.yml new file mode 100644 index 00000000..593ecec6 --- /dev/null +++ b/.github/workflows/trace-ace-v92-latent-state.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V92 latent-state decomposition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v92_latent_state_decomposition.py' + - '.github/workflows/trace-ace-v92-latent-state.yml' + workflow_dispatch: +jobs: + v92: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V92 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v92_latent_state_decomposition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v92_latent_state_decomposition.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v92-latent-state + path: v92_latent_state_decomposition.json diff --git a/.github/workflows/trace-ace-v93-shift-robust.yml b/.github/workflows/trace-ace-v93-shift-robust.yml new file mode 100644 index 00000000..2fa8b1a8 --- /dev/null +++ b/.github/workflows/trace-ace-v93-shift-robust.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V93 shift robust validation +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v93_shift_robust_validation.py' + - '.github/workflows/trace-ace-v93-shift-robust.yml' + workflow_dispatch: +jobs: + v93: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V93 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v93_shift_robust_validation.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v93_shift_robust.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v93-shift-robust + path: v93_shift_robust.json diff --git a/.github/workflows/trace-ace-v94-related-control.yml b/.github/workflows/trace-ace-v94-related-control.yml new file mode 100644 index 00000000..8b3e49c5 --- /dev/null +++ b/.github/workflows/trace-ace-v94-related-control.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V94 related control +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v94_related_control.py' + - '.github/workflows/trace-ace-v94-related-control.yml' + workflow_dispatch: +jobs: + v94: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V94 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v94_related_control.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v94_related_control.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v94-related-control + path: v94_related_control.json diff --git a/.github/workflows/trace-ace-v95-objective-support.yml b/.github/workflows/trace-ace-v95-objective-support.yml new file mode 100644 index 00000000..9fe6e05d --- /dev/null +++ b/.github/workflows/trace-ace-v95-objective-support.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V95 objective support +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v95_objective_support_activation.py' + - '.github/workflows/trace-ace-v95-objective-support.yml' + workflow_dispatch: +jobs: + v95: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V95 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v95_objective_support_activation.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v95_objective_support_activation.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v95-objective-support + path: v95_objective_support_activation.json diff --git a/.github/workflows/trace-ace-v96-effective-support.yml b/.github/workflows/trace-ace-v96-effective-support.yml new file mode 100644 index 00000000..ac63209d --- /dev/null +++ b/.github/workflows/trace-ace-v96-effective-support.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V96 effective support +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v96_effective_support_separator.py' + - '.github/workflows/trace-ace-v96-effective-support.yml' + workflow_dispatch: +jobs: + v96: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V96 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v96_effective_support_separator.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v96_effective_support.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v96-effective-support + path: v96_effective_support.json diff --git a/.github/workflows/trace-ace-v97-submission-sprint.yml b/.github/workflows/trace-ace-v97-submission-sprint.yml new file mode 100644 index 00000000..0bb80870 --- /dev/null +++ b/.github/workflows/trace-ace-v97-submission-sprint.yml @@ -0,0 +1,143 @@ +name: Trace the Ace V97 submission sprint +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v97_support_gate.py' + - 'competitions/trace_the_ace/train_v97_runtime_assets.py' + - 'competitions/trace_the_ace/runtime_v97/**' + - '.github/workflows/trace-ace-v97-submission-sprint.yml' + +jobs: + validate-and-build: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - uses: astral-sh/setup-uv@v6 + - uses: extractions/setup-just@v2 + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + + - name: Resolve schemas + shell: bash + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + test -n "$FEATURES" && test -n "$LABELS" && test -n "$FIRST" + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + export FEATURES LABELS TRANSCRIPTS + python - <<'PY' + import pandas as pd, os + print('features columns', list(pd.read_csv(os.environ['FEATURES'], nrows=0).columns)) + print('labels columns', list(pd.read_csv(os.environ['LABELS'], nrows=0).columns)) + PY + + - name: Run frozen V97 four-world gate + run: | + set -euo pipefail + cd competitions/trace_the_ace + python v97_support_gate.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v97_support_gate.json + + - name: Decide submission promotion + id: gate + shell: bash + run: | + python - <<'PY' + import json, os + r=json.load(open('v97_support_gate.json')) + verdict=r['decision']['verdict'] + print(json.dumps(r['decision'], indent=2)) + with open(os.environ['GITHUB_OUTPUT'],'a') as f: + f.write('promote=' + ('true' if verdict=='BUILD_SUBMISSION_NOW' else 'false') + '\n') + PY + + - name: Train frozen V97 runtime assets + if: steps.gate.outputs.promote == 'true' + run: | + set -euo pipefail + cd competitions/trace_the_ace + python train_v97_runtime_assets.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out-dir ../../v97_assets + + - name: Assemble official runtime + if: steps.gate.outputs.promote == 'true' + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/drivendataorg/tutoring-outcomes-runtime.git /tmp/runtime + rm -rf /tmp/runtime/submission_src/* + cp competitions/trace_the_ace/runtime_v97/main.py /tmp/runtime/submission_src/main.py + for F in v71_mastery_events.py v75_canonical_trajectory.py v81_target_segment_phase.py v85_evidence_state.py v89_relative_ability_composition.py v93_shift_robust_validation.py v94_related_control.py; do + cp "competitions/trace_the_ace/$F" "/tmp/runtime/submission_src/$F" + done + mkdir -p /tmp/runtime/submission_src/assets + cp v97_assets/v97_assets.npz v97_assets/manifest.json /tmp/runtime/submission_src/assets/ + find /tmp/runtime/submission_src -maxdepth 2 -type f -printf '%P %s bytes\n' + + - name: Pack and check official submission + if: steps.gate.outputs.promote == 'true' + working-directory: /tmp/runtime + run: | + just pack-submission + just check-submission + just pull + + - name: Test official submission offline + if: steps.gate.outputs.promote == 'true' + working-directory: /tmp/runtime + env: + BLOCK_INTERNET: 'true' + GITHUB_ACTIONS_NO_TTY: 'true' + SUBMISSION_IMAGE: tutoringoutcomeschallengeprodacr.azurecr.io/tutoring-outcomes-runtime:gpu-latest + run: just test-submission + + - name: Validate output contract + if: steps.gate.outputs.promote == 'true' + run: | + python competitions/trace_the_ace/runtime_validate.py output \ + --format /tmp/runtime/data-demo/submission_format.csv \ + --predictions /tmp/runtime/submission/submission.csv + + - name: Upload gate evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v97-gate-evidence + path: v97_support_gate.json + retention-days: 14 + + - name: Upload submission candidate + if: steps.gate.outputs.promote == 'true' + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v97-official-runtime-candidate + path: | + /tmp/runtime/submission/submission.zip + /tmp/runtime/submission/submission.csv + /tmp/runtime/submission/log.txt + v97_assets/manifest.json + v97_support_gate.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v98-finishing-sweep.yml b/.github/workflows/trace-ace-v98-finishing-sweep.yml new file mode 100644 index 00000000..91854f17 --- /dev/null +++ b/.github/workflows/trace-ace-v98-finishing-sweep.yml @@ -0,0 +1,58 @@ +name: Trace the Ace V98 finishing sweep +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v98_finishing_sweep.py' + - '.github/workflows/trace-ace-v98-finishing-sweep.yml' + +jobs: + sweep: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + export FEATURES LABELS + python - <<'PY' + import os,pandas as pd + print('features columns',list(pd.read_csv(os.environ['FEATURES'],nrows=0).columns)) + print('labels columns',list(pd.read_csv(os.environ['LABELS'],nrows=0).columns)) + PY + - name: Run V98 finishing sweep + run: | + set -euo pipefail + cd competitions/trace_the_ace + python v98_finishing_sweep.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v98_finishing_sweep.json + - name: Show decision + run: python -c "import json; r=json.load(open('v98_finishing_sweep.json')); print(json.dumps(r['decision'],indent=2)); print(json.dumps(r['ranking'][:5],indent=2))" + - name: Upload V98 evidence + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v98-finishing-sweep + path: v98_finishing_sweep.json + retention-days: 14 diff --git a/.github/workflows/trace-ace-v99-oof-expert-gate.yml b/.github/workflows/trace-ace-v99-oof-expert-gate.yml new file mode 100644 index 00000000..da6cdabd --- /dev/null +++ b/.github/workflows/trace-ace-v99-oof-expert-gate.yml @@ -0,0 +1,64 @@ +name: Trace the Ace V99 nested expert gate +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v99_oof_expert_gate.py' + - '.github/workflows/trace-ace-v99-oof-expert-gate.yml' + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + shell: bash + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + test -n "$FEATURES" && test -n "$LABELS" && test -n "$FIRST" + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + export FEATURES LABELS TRANSCRIPTS + python - <<'PY' + import pandas as pd, os + print('features columns', list(pd.read_csv(os.environ['FEATURES'], nrows=0).columns)) + print('labels columns', list(pd.read_csv(os.environ['LABELS'], nrows=0).columns)) + PY + - name: Run V99 nested applicability gate + run: | + set -euo pipefail + cd competitions/trace_the_ace + python v99_oof_expert_gate.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v99_oof_expert_gate.json + - name: Show decision + if: always() + run: cat v99_oof_expert_gate.json || true + - name: Upload V99 evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v99-oof-expert-gate + path: v99_oof_expert_gate.json + retention-days: 14 diff --git a/competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md b/competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md new file mode 100644 index 00000000..4b125f31 --- /dev/null +++ b/competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md @@ -0,0 +1,229 @@ +# Trace the Ace — Official Runtime Promotion Protocol + +Primary objective: lowest expected unseen/private log loss. Public leaderboard movement is not a promotion criterion by itself. + +## Scope + +Apply this protocol to the winning V71–V77 candidate after frozen OOF evaluation. Do not alter model logic while translating it into runtime form. Any material model change creates a new candidate and must return to OOF evaluation. + +## Gate 0 — Freeze the candidate + +Record in a machine-readable manifest: + +- source Git commit SHA; +- candidate/arm name; +- all hyperparameters; +- frozen fold-definition version/hash; +- overall and per-fold session-cold log loss; +- objective-cold, semantic-family-cold, rare-objective, worst-fold and calibration metrics; +- hashes of all fitted assets; +- expected prediction implementation/version. + +No leaderboard-derived calibration or smoke-test tuning is permitted in this frozen candidate. + +## Gate 1 — `submission_src` contract + +Develop the submission in the official runtime repository's `submission_src/` directory. The packed archive must contain `main.py` at ZIP root. + +Recommended source layout: + +```text +submission_src/ +├── main.py +├── assets/ +│ ├── manifest.json +│ ├── objective_model.* +│ ├── objective_statistics.* +│ ├── vectorizer.* +│ ├── residual_model.* +│ └── calibration.* +└── trace_ace/ + ├── canonicalize.py + ├── mastery.py + ├── features.py + └── predict.py +``` + +`main.py` must: + +1. read `/code_execution/data/test_features.csv`; +2. read required transcript CSVs from `/code_execution/data/test_transcripts/`; +3. construct exactly the frozen representation; +4. load frozen fitted assets only; +5. generate one probability per response independently of unrelated test cases; +6. apply only frozen calibration; +7. use `/code_execution/data/submission_format.csv` as the output contract; +8. write `/code_execution/submission.csv` with exactly `response_id,probability`. + +Forbidden during inference: + +- fitting/updating model weights or fitted feature parameters from test data; +- pseudo-labeling; +- corpus-wide test statistics used as features; +- network calls; +- package installation; +- manual test annotations; +- cross-test-case information that changes a sample prediction. + +## Gate 2 — Research/runtime parity + +Create a frozen held-out fixture and run both the research implementation and `submission_src` implementation. + +For deterministic components require: + +```text +max_abs_probability_difference < 1e-8 +``` + +If a GPU component makes bitwise equality impossible, predeclare a justified tolerance and require no meaningful log-loss change. + +Also require the runtime implementation's held-out log loss to reproduce the frozen candidate within numerical tolerance. Failure means STOP: fix translation, do not submit. + +## Gate 3 — Official container + +Use the official `drivendataorg/tutoring-outcomes-runtime` repository and run, in order: + +```bash +just pull +just pack-submission +just check-submission +just test-submission +``` + +All commands must exit successfully. Preserve `submission/log.txt` and the generated `submission/submission.csv` as validation evidence. + +## Gate 4 — Competition-shaped held-out test + +Create a local, non-Git-tracked data directory from held-out training sessions: + +```text +data/ +├── submission_format.csv +├── test_features.csv +└── test_transcripts/ + └── .csv +``` + +Run: + +```bash +DATA_DIR=/absolute/path/to/data just test-submission +``` + +Require prediction parity with the frozen held-out research implementation and expected held-out log loss. + +Competition data must not be committed to the public repository. + +## Gate 5 — Offline/cold-container audit + +Run with the official default network isolation. Do not enable internet access. + +Audit logs for: + +- attempted downloads; +- Hugging Face Hub/network calls; +- package installation attempts; +- missing assets; +- hidden filesystem assumptions; +- warnings that alter model behavior. + +Require successful cold-container inference with all required assets either packaged or officially preloaded. + +## Gate 6 — Output integrity + +Programmatically require: + +```text +columns == ["response_id", "probability"] +row_count == submission_format row_count +response_ids exactly match submission_format +no duplicate response_ids +no NaN +no +/-inf +0 <= probability <= 1 +``` + +Also inspect probability min/max and extreme-confidence counts. Unexpected tails are an investigation trigger because the competition metric is log loss. + +## Gate 7 — Sample-independence metamorphic audit + +Choose held-out response `x`. Predict it under: + +- A: `x` alone; +- B: `x` plus unrelated held-out responses; +- C: same batch reordered; +- D: `x` plus a different unrelated batch. + +Require: + +```text +p_A(x) == p_B(x) == p_C(x) == p_D(x) +``` + +within the predeclared numerical tolerance. + +This is a hard rule-compliance gate. + +## Gate 8 — Runtime/resource budget + +Record elapsed wall time, peak RAM, peak GPU VRAM if applicable, archive size and output size. + +Hard competition constraints are six hours for full inference and 20 minutes for smoke. Internal promotion target: + +```text +projected full-test runtime < 4 hours +``` + +Prefer substantially more headroom. Any candidate close to a hard resource limit is not promoted without a documented reason. + +## Gate 9 — Platform smoke test + +Only after Gates 0–8 pass, upload the exact packed archive as a smoke test. + +Use smoke only for runtime validation: + +- process starts; +- assets resolve; +- platform paths are correct; +- output is produced; +- runtime is comfortably under 20 minutes; +- logs comply with competition restrictions. + +Do not tune model/calibration from smoke score. + +## Gate 10 — Full-submission promotion + +A candidate may consume one of the limited full submissions only if ALL conditions hold: + +1. session-cold OOF log loss improves on the current frozen champion; +2. no unacceptable objective-cold regression; +3. no unacceptable semantic-family-cold regression; +4. rare-objective behavior is acceptable; +5. worst-fold risk is acceptable; +6. calibration is sound; +7. runtime implementation reproduces frozen predictions; +8. official offline container passes; +9. sample-independence audit passes; +10. platform smoke test passes; +11. projected full runtime has safe headroom; +12. expected private log loss is competitive with the winning target, not merely an incremental public-board improvement. + +Current strategic target: do not spend a full submission on a candidate unless robust evidence makes private log loss around or below 0.595 plausible, unless a later evidence-based threshold supersedes this one. + +## State machine + +```text +OOF WIN + -> FREEZE + -> RUNTIME PARITY + -> OFFICIAL CONTAINER + -> HELD-OUT CONTAINER + -> OFFLINE AUDIT + -> OUTPUT INTEGRITY + -> SAMPLE INDEPENDENCE + -> RESOURCE GATE + -> SMOKE + -> FULL SUBMIT +``` + +Any failed gate returns the candidate to the appropriate earlier stage. Never bypass a failed gate because of a favorable public leaderboard score. diff --git a/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md b/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md new file mode 100644 index 00000000..192c37db --- /dev/null +++ b/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md @@ -0,0 +1,220 @@ +# Trace the Ace — Unseen Log-Loss Plan + +## Primary objective + +The optimization target is **minimum log loss on genuinely unseen/private evaluation data**. Public leaderboard movement, AUC, model novelty, and write-up appeal are secondary. A change is retained only when it improves robust out-of-sample probability quality or provides a clearly orthogonal signal that improves a validated ensemble. + +Formally, prefer models that minimize expected unseen log loss and avoid catastrophic regime failures: + +`E[-y log p - (1-y) log(1-p)]` + +Promotion requires evidence across multiple plausible test regimes, not merely a better mean on one split. + +## Validation hierarchy + +Every material experiment should report at least: + +1. **Session-cold grouped OOF** — no session leakage. +2. **Objective-cold grouped OOF** — exact skills held out. +3. **Hard/rare-objective stress** — long-tail objectives receive explicit scrutiny. +4. **Fold dispersion / worst-fold loss** — a large mean gain that creates a catastrophic regime is not automatically promoted. +5. **Calibration diagnostics** — log loss is the target; overconfident mistakes matter more than ranking gains. + +When historical public scores are available, use them only to audit whether a validation regime is predictive of transfer. Do not derive inference-time constants from leaderboard feedback. + +## Measured anchor — V74 objective difficulty + +Full 35,072-row training evaluation on 2026-08-15 established V74 as a mandatory independent prior: + +- session-grouped global-prior log loss: **0.608771** +- session-grouped V74 hierarchical log loss: **0.552734** +- delta: **-0.056037** +- session-fold range: **0.548964 to 0.554711** +- objective-cold global-prior log loss: **0.611715** +- objective-cold semantic V74 log loss: **0.601736** +- objective-cold delta: **-0.009980** + +V74 uses no transcripts. The session-cold gain shows objective difficulty is a very large component of the target; the objective-cold gain shows semantic transfer between objective descriptions is real but materially weaker and regime-dependent. V74 is therefore an **anchor/prior**, not a complete solution. Every transcript branch should now be measured primarily by residual log-loss improvement beyond leakage-safe V74 OOF predictions. + +Aggregate result: `results/v74_full_training_oof_2026-08-15.json`. + +## Priority order + +### P0 — Preserve strong baselines and validation integrity + +- Keep the best historical lexical/model predictions as an independent view. +- Keep V74 as a mandatory objective-difficulty prior. +- Reconstruct exact OOF predictions whenever possible. +- Do not accumulate features by version number. +- Reject interventions that improve one regime while materially degrading plausible unseen regimes unless they add independently validated ensemble value. + +### P1 — High-signal transcript preprocessing / measurement + +Before larger-model work, convert raw dialogue into cleaner evidence of student knowledge while preserving educationally meaningful variation. + +1. **Conservative speaker-role repair** + - retain original role, repaired role, and repair confidence; + - never globally flip speakers from weak evidence. + +2. **Interaction episode segmentation** + - tutor question -> student answer -> feedback/correction/hint -> retry; + - preserve chronological links between retries and feedback. + +3. **Student-vs-tutor evidence separation** + - tutor exposition is context; + - student production is mastery evidence; + - tutor-confirmed student correctness is weak supervision. + +4. **Low-information turn down-weighting** + - greetings, connection checks, scheduling, generic acknowledgements and boilerplate receive low mastery weight rather than blind deletion. + +5. **Objective-conditioned relevance** + - rank episodes by semantic relevance to each learning objective; + - retain prerequisite/follow-up context when it helps objective transfer. + +6. **Assistance / independence canonicalization** + - distinguish independent correct, correct after prompt, correct after hint, copied/repeated answer, self-correction, unresolved error, repeated error, agreement-only, tutor exposition. + +7. **Chronology and terminal-state emphasis** + - represent transitions such as ERROR -> HINT -> INDEPENDENT_CORRECT; + - terminal independent evidence should be available explicitly, not lost in bagged text. + +8. **Math surface normalization with raw-text preservation** + - normalize Unicode operators, spacing, simple fraction/decimal variants where safe; + - keep original wording as an additional view. + +9. **Multiple retained views** + - raw transcript; + - student-only transcript; + - objective-local transcript; + - canonical episode sequence; + - terminal mastery window. + +The rule is: **remove nuisance variation, not educational variation**. + +### P2 — V71 mastery-event branch + +Extract objective-conditioned micro-assessments and aggregate trajectory features. Evaluate whether these events improve unseen log loss **beyond V74 OOF**, not merely beyond a global prior. + +### P3 — V72 hidden-supervision audit + +Quantify: + +- multi-objective session frequency; +- same-session mixed outcomes; +- opposite-label contrastive pairs; +- micro-assessment density; +- rare-objective structure. + +Use this only to determine whether the richer training formulations have enough support. + +### P4 — V73 same-session contrastive mastery + +Exploit pairs from the same transcript with different objective outcomes. Same-session differencing suppresses generic session ability and forces the model toward objective-specific mastery evidence. + +Primary question: does the contrastive residual improve held-out row log loss when added to V74 plus the strongest transcript evidence? + +### P5 — V74 semantic/hierarchical objective difficulty — PROMOTED ANCHOR + +Model objective difficulty explicitly and shrink rare objectives toward semantically related objectives. Exact objective identity should not be required for useful predictions. + +Measured full-data results promote this branch as an independent probability prior. Future work should preserve its OOF predictions and train transcript models on its residuals or combine through leakage-safe stacking. + +### P6 — V75 canonical student-state trajectory + +Current transcript priority before larger pretrained models. + +Canonical event alphabet should include at minimum: + +- INDEPENDENT_CORRECT +- CORRECT_AFTER_PROMPT +- CORRECT_AFTER_HINT +- SELF_CORRECTION +- TUTOR_CORRECTION +- UNRESOLVED_ERROR +- REPEATED_ERROR +- AGREEMENT_ONLY +- TUTOR_EXPOSITION +- TRANSFER_SUCCESS when reliably detectable + +For each `(session, objective)`, output both the ordered event sequence and compact numeric summaries: terminal state, number of hints, recurrence, recency, independence, correction distance, and objective relevance. + +Compare raw/localized lexical views against canonical-state views under identical folds, always reporting incremental log loss beyond V74. + +### P7 — Semantic objective-conditioned retrieval + +Only after P1-P6 are measured. Use a compliant pretrained encoder to retrieve the most objective-relevant episodes from long transcripts. Larger models are justified only if they improve unseen log loss over cheaper lexical/event retrieval. + +### P8 — Latent student-state model + +Combine distinct factors rather than forcing one text classifier to infer all of them implicitly: + +`logit P(correct) = objective_difficulty + session_state + objective_mastery + contrastive_residual + calibrated_residual_views` + +Session state must be inferable from the individual test sample at inference time; no cross-test aggregation is allowed. + +### P9 — Heterogeneous ensemble and calibration + +Retain only genuinely different information channels, e.g.: + +- robust lexical baseline; +- V74 hierarchical objective prior; +- mastery trajectory; +- contrastive residual; +- semantic retrieval/encoder signal. + +Fit ensemble weights strictly OOF. Optimize log loss directly. Test temperature/logit scaling, isotonic or other calibration only when fitted without leakage and when improvement is stable across validation regimes. + +### P10 — Submission discipline + +Full submissions are scarce and should answer causal transfer questions, not tune small hyperparameters. + +A candidate is submission-worthy only when: + +- its unseen-oriented validation is materially better; +- no major plausible regime collapses; +- calibration improves or remains safe; +- runtime and code-execution constraints are satisfied; +- the inference path processes each test sample independently as required by the rules. + +## Promotion rule + +Default decision hierarchy: + +1. Lower aggregate session-cold log loss **relative to V74 plus the strongest retained base**. +2. Lower or non-inferior hard/rare/objective-cold loss. +3. Lower worst-fold / tail risk. +4. Better calibration, especially fewer high-confidence errors. +5. Orthogonal residual value in a strictly OOF ensemble. +6. Only then consider runtime, elegance, interpretability, or write-up value. + +A model that looks clever but worsens expected unseen log loss is rejected. + +## Current working decomposition + +The leading hypothesis is: + +`P(next correct | transcript, objective)` + +should be decomposed into: + +- `D_o`: semantic objective difficulty — **measured and promoted via V74**; +- `A_s`: broad session/student competence state; +- `M_so`: objective-specific mastery evidence; +- `C_so`: same-session contrastive residual; +- `T_so`: trajectory / independence / recency. + +The transcript is therefore treated as a measurement instrument for latent student state, not merely as a document to classify. + +## Immediate next experiment + +**V75/V71 residual-on-V74 is now the next decisive experiment.** Use the transcript archive plus the official feature/label CSVs to generate identical session folds and compare: + +1. V74 OOF anchor; +2. V74 + V71 numeric mastery features; +3. V74 + raw/localized lexical transcript view; +4. V74 + V75 canonical trajectory; +5. V74 + lexical + V75; +6. add V73 contrastive residual. + +Use identical frozen folds and save OOF predictions for stacking. Promote only on unseen-oriented log-loss evidence. diff --git a/competitions/trace_the_ace/RGRS_LEDGER.md b/competitions/trace_the_ace/RGRS_LEDGER.md new file mode 100644 index 00000000..f8b8c2c8 --- /dev/null +++ b/competitions/trace_the_ace/RGRS_LEDGER.md @@ -0,0 +1,151 @@ +# Trace the Ace — Residual-Guided Representation Search Ledger + +This ledger applies Residual-Guided Representation Search (RGRS) to the Trace the Ace programme. The purpose is to prevent blind search inside a representation after repeated evidence says the missing information is representational. + +## Frozen current baseline + +- Public champion retained architecture: V75 canonical trajectory, public log loss 0.6047. +- Primary unseen proxy: exact objective-cold validation. +- V75 objective-cold aggregate: ~0.59235. +- V81 best blend objective-cold: ~0.59097. +- Promotion target: several-thousandths objective-cold improvement with no regression in the official runtime/independence gates before spending a full public submission. + +## Residual records + +### TTA-ρ-001 — Generic semantics does not recover mastery + +`rho = (R6 Representation, semantic prediction layer, V78 MiniLM standalone ~0.7145 and only ~0.0004 gain when blended with V75, objective-cold, high)` + +Material interventions inside the old text-prediction representation: +- frozen MiniLM semantic view; +- retrieval/learning-gain semantic view (V79). + +Both are weak standalone and only add a small orthogonal residual correction. + +**Decision:** do not classify the main gap as R1 search. More generic embedding capacity is not justified by the residual. + +### TTA-ρ-002 — Hard target scoping loses useful information + +`rho = (R5 Applicability, lesson-segment scoping, target-only V81 ~0.5991 worse than whole-session ~0.5924 while blended whole+target+phase reaches ~0.5910, objective-cold, high)` + +Observed separator: +- 91.6% of examples are multi-segment; +- target region averages ~32.1% of session; +- deleting non-target context hurts; +- exposing target context alongside whole context helps. + +**Decision:** target scoping is conditionally useful. Search for the predicate/weight governing when evidence belongs to the assessed objective; do not globally discard the rest of the session. + +### TTA-ρ-003 — Tutor feedback is not equivalent to mastery evidence + +`rho = (R6 Representation, evidence object, transcript audit contains label-0 sessions ending in strong tutor praise and label-1 sessions with later unrelated failures, audited examples, high)` + +The current text representation conflates: +- student-generated competence evidence; +- tutor evaluation language; +- assistance supplied before an answer; +- later unrelated lesson performance. + +The language cannot cleanly state the distinction needed to explain the residual. + +**Representation candidate:** replace transcript-as-example with an objective-conditioned evidence graph/state sequence. + +### TTA-ρ-004 — Large supervised encoder on CPU is an infrastructure failure + +`rho = (R10 Infrastructure, V82 ModernBERT-large execution, hours-long CPU runner without deciding result, GitHub Actions CPU environment, high)` + +**Decision:** draw no semantic conclusion about supervised transformers from V82 runtime. Use a feasible domain encoder (V83) or GPU for the large-model hypothesis. + +## Current primary residual + +The strongest current diagnosis is: + +`R6 Representation + R5 Applicability` + +The missing object is not "better transcript semantics". It is approximately: + +`EvidenceEvent = (objective_match, phase, question, student_answer, assistance_before_answer, correction_state, independence, transfer, position)` + +with an objective-conditioned state trajectory: + +`K_pre -> K_guided -> K_independent -> K_application` + +and an applicability predicate deciding which events should influence the assessed objective. + +## Smallest deciding representation test + +### Hypothesis H85 + +An explicit objective-conditioned student-evidence representation will outperform an otherwise matched representation that includes tutor-evaluation wording as first-class predictive text. + +### One intervention + +Create an `EvidenceEvent` view from the existing transcript while holding vectorizer/model/folds/hyperparameters fixed. + +### Frozen arms + +- **A0 — V75/V81-style text evidence:** existing whole + target + canonical views. +- **A1 — Student-evidence IR:** objective-matched question -> student-answer episodes, phase tags, assistance tags, canonical state, whole-context summary; tutor praise removed from raw predictive text. +- **A2 — Causal ablation:** same as A1 but remove assistance/independence tags while preserving all event text and ordering. + +### Opposing discriminators + +1. Cases where tutor praise is high but independent student evidence is weak: A1 should improve over A0. +2. Cases where independent evidence is strong but later unrelated struggle exists: A1 should preserve/promote probability relative to target-only deletion. +3. Cases with no meaningful target evidence: A1 should not manufacture confidence; A0 behavior/prior should be preserved. + +### Primary metric + +Exact objective-cold log loss, frozen folds. + +### Precommitted interpretation + +- A1 beats A0 by >= 0.003 and A2 materially weakens the gain -> **clean mechanism win; escalate representation**. +- A1 beats A0 by < 0.001 -> **insufficient; preserve negative law**. +- A1 helps only a subset -> **R5 Applicability; learn/certify event activation predicate**. +- A1 wins but A2 is equal -> **causal attribution fails; do not admit assistance/independence tags**. + +## Current live experiments + +### V83 — TalkMove-BERT supervised + +Purpose: test whether domain-matched tutoring-language supervision supplies a useful semantic layer once V81 structure is exposed. + +RGRS classification before result: **R1/R6 separator test**, not an admitted representation change. + +If V83 is weak, generic/model-capacity search is further demoted and the programme should prioritize explicit evidence-state IR. + +### V84 — Student evidence ablation + +Purpose: preliminary test of the TTA-ρ-003 hypothesis by suppressing raw tutor-praise language and retaining student/objective evidence. + +RGRS classification: **R6 candidate separator**. + +## Admission gate for a new Trace-the-Ace representation + +A proposed representation is retained only when all hold: + +1. **Semantic/data-contract gate** — legal competition features only; no test adaptation/leakage; official runtime contract passes. +2. **Causal gate** — removing the proposed representational feature materially weakens the gain. +3. **Predictive resource gate** — frozen primary metric improves; no post-hoc metric switching. +4. **Reproducibility gate** — same commit/folds/config reproduces and official inference remains sample-independent. + +State machine: + +`PROPOSED -> SEPARATED -> VERIFIED -> ADMITTED` + +otherwise: + +`REJECTED` or `OBSTRUCTED`. + +## Governing law for this competition + +> Never add model capacity merely because the score is imperfect when repeated residuals show that the model is being asked to predict mastery from the wrong object. + +The current highest-value search direction is therefore: + +`raw transcript -> objective-conditioned evidence events -> assistance/independence-aware knowledge state -> calibrated predictor` + +not: + +`raw transcript -> larger generic embedding -> classifier`. diff --git a/competitions/trace_the_ace/V111_RUN_NOW.md b/competitions/trace_the_ace/V111_RUN_NOW.md new file mode 100644 index 00000000..e101740a --- /dev/null +++ b/competitions/trace_the_ace/V111_RUN_NOW.md @@ -0,0 +1,5 @@ +# V111 isolated runner trigger + +Run the frozen minutes-scale residual screen independently. + +Trigger: 2 diff --git a/competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json b/competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json new file mode 100644 index 00000000..c522e7e3 --- /dev/null +++ b/competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json @@ -0,0 +1,31 @@ +{ + "experiment": "V122_ID_MORPHOLOGY_REGIME_AUDIT", + "status": "COMPLETED_LOCAL_FROM_AUTHENTICATED_FROZEN_METADATA", + "frozen_metadata_file_id": "1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz", + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "families": { + "SESSION_ID": { + "session_cold": {"ll": 0.61001884715157, "baseline_ll": 0.6087712429519364, "gain": -0.0012476041996335452}, + "objective_cold": {"ll": 0.6108649000534013, "baseline_ll": 0.6117153814673667, "gain": 0.000850481413965376} + }, + "OBJECTIVE_ID": { + "session_cold": {"ll": 0.5538378377601135, "baseline_ll": 0.6087712429519364, "gain": 0.05493340519182288}, + "objective_cold": {"ll": 0.6144847625399319, "baseline_ll": 0.6117153814673667, "gain": -0.0027693810725651913} + }, + "SESSION_X_OBJECTIVE": { + "session_cold": {"ll": 0.5594418013741027, "baseline_ll": 0.6087712429519364, "gain": 0.0493294415778337}, + "objective_cold": {"ll": 0.614578017753569, "baseline_ll": 0.6117153814673667, "gain": -0.0028626362862023136} + } + }, + "shuffle": { + "SESSION_ID": {"session_cold_gain": -0.0017297072836235383, "objective_cold_gain": -0.0014020282276541174}, + "OBJECTIVE_ID": {"session_cold_gain": -0.0015091921768788374, "objective_cold_gain": -0.0002698104527871781}, + "SESSION_X_OBJECTIVE": {"session_cold_gain": -0.001574849153598623, "objective_cold_gain": -0.0006396369760830467} + }, + "decision": { + "verdict": "ID_MORPHOLOGY_NOT_DECISION_CHANGING", + "interpretation": "Exact objective recurrence recreates the V74 session-cold prior; identifier morphology does not generalize across held-out objectives, and session-id morphology is null. Do not pursue IDs as a provider/applicability channel." + } +} diff --git a/competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json b/competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json new file mode 100644 index 00000000..d969a7fa --- /dev/null +++ b/competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json @@ -0,0 +1,42 @@ +{ + "diagnostics": { + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "positive_rate": 0.7024692062043796 + }, + "session": { + "global_logloss": 0.6087712429519364, + "hierarchical_logloss": 0.5527343454820751, + "semantic_only_logloss": 0.5828972867379437, + "hierarchical_auc": 0.7057912533493755, + "delta_vs_global": -0.05603689746986129, + "folds": [ + {"fold": 1, "rows": 7015, "global_logloss": 0.609471457418674, "hierarchical_logloss": 0.5527513784708268, "semantic_only_logloss": 0.5815525167928709}, + {"fold": 2, "rows": 7015, "global_logloss": 0.6062975979938539, "hierarchical_logloss": 0.5541348721713956, "semantic_only_logloss": 0.5825157859864996}, + {"fold": 3, "rows": 7014, "global_logloss": 0.6071844652285676, "hierarchical_logloss": 0.5489638944862815, "semantic_only_logloss": 0.5803165236898602}, + {"fold": 4, "rows": 7014, "global_logloss": 0.6138402315997151, "hierarchical_logloss": 0.5547113077644491, "semantic_only_logloss": 0.5866520609625011}, + {"fold": 5, "rows": 7014, "global_logloss": 0.6070627153604012, "hierarchical_logloss": 0.5531100724131053, "semantic_only_logloss": 0.5834497923758504} + ] + }, + "objective": { + "global_logloss": 0.6117153814673667, + "hierarchical_logloss": 0.6017357708418917, + "semantic_only_logloss": 0.6017357708418917, + "hierarchical_auc": 0.5736305336524183, + "delta_vs_global": -0.009979610625475033, + "folds": [ + {"fold": 1, "rows": 7015, "global_logloss": 0.6333952466528439, "hierarchical_logloss": 0.6412045179518634, "semantic_only_logloss": 0.6412045179518634}, + {"fold": 2, "rows": 7015, "global_logloss": 0.5698093295839226, "hierarchical_logloss": 0.5518002193250016, "semantic_only_logloss": 0.5518002193250016}, + {"fold": 3, "rows": 7014, "global_logloss": 0.5604892890991403, "hierarchical_logloss": 0.5491725706211095, "semantic_only_logloss": 0.5491725706211095}, + {"fold": 4, "rows": 7014, "global_logloss": 0.6626437649772484, "hierarchical_logloss": 0.6375851308655951, "semantic_only_logloss": 0.6375851308655951}, + {"fold": 5, "rows": 7014, "global_logloss": 0.6322421607115449, "hierarchical_logloss": 0.6289179077191152, "semantic_only_logloss": 0.6289179077191152} + ] + }, + "provenance": { + "features": "train_features_TMQTWsB.csv", + "labels": "train_labels_44ujmj2.csv", + "transcripts_used": false, + "note": "Aggregate metrics only; no competition data committed." + } +} diff --git a/competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json b/competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json new file mode 100644 index 00000000..2cf7b3f2 --- /dev/null +++ b/competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json @@ -0,0 +1,37 @@ +{ + "search_space": { + "k": [2, 4, 8, 16, 32], + "smooth": [2, 5, 10, 20, 40, 80], + "trust_denom": [2, 5, 10, 20, 40] + }, + "previous_default": { + "k": 8, + "smooth": 20, + "trust_denom": 10, + "session_logloss": 0.5527343454820751, + "objective_logloss": 0.6017357708418917, + "session_worst_fold": 0.5547113077644491, + "objective_worst_fold": 0.6412045179518634 + }, + "promoted": { + "k": 16, + "smooth": 2, + "trust_denom": 10, + "session_logloss": 0.551526796719344, + "objective_logloss": 0.5995980929074717, + "session_worst_fold": 0.553280629957868, + "objective_worst_fold": 0.638792193453735, + "session_delta_vs_previous": -0.0012075487627311, + "objective_delta_vs_previous": -0.00213767793442 + }, + "selection_rule": "Promote only configurations improving session-grouped log loss while remaining non-inferior or better on objective-cold loss and tail risk.", + "provenance": { + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "features": "train_features_TMQTWsB.csv", + "labels": "train_labels_44ujmj2.csv", + "transcripts_used": false, + "note": "Aggregate metrics only; no competition data committed." + } +} diff --git a/competitions/trace_the_ace/runtime_v74/main.py b/competitions/trace_the_ace/runtime_v74/main.py new file mode 100644 index 00000000..04260ebe --- /dev/null +++ b/competitions/trace_the_ace/runtime_v74/main.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +from pathlib import Path +import joblib,pandas as pd +HERE=Path(__file__).resolve().parent +DATA=Path('/code_execution/data') +from v74_runtime_core import predict + +def main(): + f=pd.read_csv(DATA/'test_features.csv') + fmt=pd.read_csv(DATA/'submission_format.csv') + model=joblib.load(HERE/'assets/v74_model.joblib') + p=predict(model,f.learning_objective.astype(str).tolist()) + gen=pd.DataFrame({'response_id':f.response_id.astype(str),'probability':p}) + out=fmt[['response_id']].astype({'response_id':str}).merge(gen,on='response_id',how='left',validate='one_to_one') + if out.probability.isna().any(): raise RuntimeError('missing predictions') + if not ((out.probability>=0)&(out.probability<=1)).all(): raise RuntimeError('invalid probability') + out.to_csv(DATA.parent/'submission.csv',index=False) +if __name__=='__main__': main() diff --git a/competitions/trace_the_ace/runtime_v74/v74_runtime_core.py b/competitions/trace_the_ace/runtime_v74/v74_runtime_core.py new file mode 100644 index 00000000..f487e151 --- /dev/null +++ b/competitions/trace_the_ace/runtime_v74/v74_runtime_core.py @@ -0,0 +1,22 @@ +from __future__ import annotations +import numpy as np +from sklearn.metrics.pairwise import cosine_similarity + +def predict(model, objectives): + obj=[str(x) for x in objectives] + B=model['vectorizer'].transform(obj) + sims=cosine_similarity(B,model['A']) + kk=min(int(model['k']),sims.shape[1]) + idx=np.argpartition(-sims,kth=kk-1,axis=1)[:,:kk] + rows=np.arange(len(obj))[:,None] + w=sims[rows,idx] + npv=model['posterior'][idx] + sem=(w*npv).sum(axis=1)/(w.sum(axis=1)+1e-9) + sem=np.where(w.sum(axis=1)>1e-8,sem,float(model['global_p'])) + mp=model['mapped']; ct=model['counts'] + mapped=np.array([mp.get(x,np.nan) for x in obj],float) + missing=np.isnan(mapped); mapped[missing]=sem[missing] + counts=np.array([ct.get(x,0.0) for x in obj],float) + trust=counts/(counts+10.0) + p=trust*mapped+(1-trust)*sem + return np.clip(p,1e-5,1-1e-5) diff --git a/competitions/trace_the_ace/runtime_v75/main.py b/competitions/trace_the_ace/runtime_v75/main.py new file mode 100644 index 00000000..5a29fee3 --- /dev/null +++ b/competitions/trace_the_ace/runtime_v75/main.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Official-runtime inference entrypoint for promoted V75 all-views model.""" +from __future__ import annotations + +from pathlib import Path +import sys + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer + +HERE = Path(__file__).resolve().parent +ASSETS = HERE / "assets" +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from v71_mastery_events import load_transcript # noqa: E402 +from v75_canonical_trajectory import trajectory_views # noqa: E402 + +DATA = Path("/code_execution/data") +N_HASH = 2**18 +BATCH = 256 + + +def sigmoid(x): + x = np.asarray(x, dtype=np.float64) + out = np.empty_like(x) + pos = x >= 0 + out[pos] = 1.0 / (1.0 + np.exp(-x[pos])) + e = np.exp(x[~pos]) + out[~pos] = e / (1.0 + e) + return out + + +def build_batch(rows, transcript_cache, hv, num_mean, num_std): + views, nums = [], [] + for row in rows.itertuples(index=False): + sid = str(row.session_id) + if sid not in transcript_cache: + transcript_cache[sid] = load_transcript(DATA / "test_transcripts" / f"{sid}.csv") + v, n, _ = trajectory_views(transcript_cache[sid], str(row.learning_objective)) + views.append(v) + nums.append(n) + numeric = np.vstack(nums).astype(np.float64) + z = (numeric - num_mean) / num_std + objective = hv.transform(["[OBJECTIVE] " + str(x) for x in rows.learning_objective]) + raw = hv.transform(["[RAW] " + v["raw"] for v in views]) + student = hv.transform(["[STUDENT] " + v["student"] for v in views]) + local = hv.transform(["[LOCAL] " + v["local"] for v in views]) + canonical = hv.transform(["[STATE] " + v["canonical"] for v in views]) + terminal = hv.transform(["[TERMINAL] " + v["terminal"] for v in views]) + return hstack([objective, raw, student, local, canonical, terminal, csr_matrix(z)], format="csr") + + +def main(): + features = pd.read_csv(DATA / "test_features.csv") + fmt = pd.read_csv(DATA / "submission_format.csv") + required = {"response_id", "session_id", "learning_objective"} + if not required.issubset(features.columns): + raise RuntimeError("unexpected test_features schema") + + a = np.load(ASSETS / "v75_runtime_assets.npz") + coef = a["coef"].astype(np.float64) + intercept = float(a["intercept"].ravel()[0]) + num_mean = a["num_mean"].astype(np.float64) + num_std = a["num_std"].astype(np.float64) + + hv = HashingVectorizer( + n_features=N_HASH, + alternate_sign=False, + norm="l2", + ngram_range=(1, 2), + lowercase=True, + ) + transcript_cache = {} + pred = np.empty(len(features), dtype=np.float64) + for start in range(0, len(features), BATCH): + stop = min(len(features), start + BATCH) + X = build_batch(features.iloc[start:stop], transcript_cache, hv, num_mean, num_std) + if X.shape[1] != coef.shape[0]: + raise RuntimeError("runtime feature dimension does not match frozen model") + logits = np.asarray(X @ coef).ravel() + intercept + pred[start:stop] = sigmoid(logits) + + generated = pd.DataFrame({"response_id": features.response_id.astype(str), "probability": np.clip(pred, 1e-5, 1 - 1e-5)}) + out = fmt[["response_id"]].astype({"response_id": str}).merge(generated, on="response_id", how="left", validate="one_to_one") + if out.probability.isna().any() or len(out) != len(fmt): + raise RuntimeError("could not generate exactly one prediction per submission row") + out[["response_id", "probability"]].to_csv(HERE / "submission.csv", index=False) + +if __name__ == "__main__": + main() diff --git a/competitions/trace_the_ace/runtime_v97/main.py b/competitions/trace_the_ace/runtime_v97/main.py new file mode 100644 index 00000000..4732de76 --- /dev/null +++ b/competitions/trace_the_ace/runtime_v97/main.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Official runtime for V97 fixed exact-support gate.""" +from pathlib import Path +import json, sys +import numpy as np, pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +HERE=Path(__file__).resolve().parent; DATA=Path('/code_execution/data'); sys.path.insert(0,str(HERE)) +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import trajectory_views +from v94_related_control import segmented_control + + +def sigmoid(x): return 1/(1+np.exp(-np.clip(np.asarray(x,float),-40,40))) +def main(): + f=pd.read_csv(DATA/'test_features.csv'); fmt=pd.read_csv(DATA/'submission_format.csv') + a=np.load(HERE/'assets/v97_assets.npz'); man=json.loads((HERE/'assets/manifest.json').read_text()) + cache={}; views=[]; vnums=[]; rt=[];rz=[] + for r in f.itertuples(index=False): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(DATA/'test_transcripts'/f'{sid}.csv') + obj=str(r.learning_objective); v,n,_=trajectory_views(cache[sid],obj); views.append(v);vnums.append(n) + t,z=segmented_control(cache[sid],obj,'related');rt.append(t);rz.append(z) + hv75=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + Z=(np.vstack(vnums)-a['v75_num_mean'])/a['v75_num_std'] + X0=hstack([hv75.transform(['[OBJECTIVE] '+str(x) for x in f.learning_objective]), + hv75.transform(['[RAW] '+v['raw'] for v in views]),hv75.transform(['[STUDENT] '+v['student'] for v in views]), + hv75.transform(['[LOCAL] '+v['local'] for v in views]),hv75.transform(['[STATE] '+v['canonical'] for v in views]), + hv75.transform(['[TERMINAL] '+v['terminal'] for v in views]),csr_matrix(Z)],format='csr') + p0=sigmoid(np.asarray(X0@a['v75_coef']).ravel()+float(a['v75_intercept'][0])) + hvr=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + R=(np.vstack(rz)-a['related_num_mean'])/a['related_num_std']; Xr=hstack([hvr.transform(rt),csr_matrix(R)],format='csr') + pr=sigmoid(np.asarray(Xr@a['related_coef']).ravel()+float(a['related_intercept'][0])) + keys=f.learning_objective.astype(str); counts=man['objective_counts'] + w=np.array([man['unseen_weight'] if int(counts.get(str(k),0))==0 else 0.0 for k in keys],float) + p=np.clip((1-w)*p0+w*pr,1e-5,1-1e-5) + gen=pd.DataFrame({'response_id':f.response_id.astype(str),'probability':p}) + out=fmt[['response_id']].astype({'response_id':str}).merge(gen,on='response_id',how='left',validate='one_to_one') + if out.probability.isna().any(): raise RuntimeError('missing predictions') + out.to_csv(HERE/'submission.csv',index=False) +if __name__=='__main__': main() diff --git a/competitions/trace_the_ace/runtime_validate.py b/competitions/trace_the_ace/runtime_validate.py new file mode 100644 index 00000000..3f41d902 --- /dev/null +++ b/competitions/trace_the_ace/runtime_validate.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Rule/runtime validation helpers for a frozen Trace the Ace submission. + +This script is intentionally model-agnostic. It validates the generated +submission contract, compares frozen research/runtime predictions, and checks +sample-independence fixtures produced by the runtime candidate. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import numpy as np +import pandas as pd + + +def read_headers(path: Path) -> list[str]: + return list(pd.read_csv(path, nrows=0).columns) + + +def validate_output(fmt_path: Path, pred_path: Path) -> dict: + print("submission_format columns:", read_headers(fmt_path)) + print("submission columns:", read_headers(pred_path)) + fmt = pd.read_csv(fmt_path) + pred = pd.read_csv(pred_path) + required = ["response_id", "probability"] + if list(pred.columns) != required: + raise SystemExit(f"FAIL columns: expected {required}, got {list(pred.columns)}") + if len(pred) != len(fmt): + raise SystemExit(f"FAIL row count: {len(pred)} != {len(fmt)}") + if pred.response_id.duplicated().any(): + raise SystemExit("FAIL duplicate response_id") + if pred.response_id.astype(str).tolist() != fmt.response_id.astype(str).tolist(): + raise SystemExit("FAIL response IDs/order differ from submission_format") + p = pred.probability.to_numpy(float) + if not np.isfinite(p).all(): + raise SystemExit("FAIL non-finite probability") + if ((p < 0) | (p > 1)).any(): + raise SystemExit("FAIL probability outside [0,1]") + result = { + "rows": int(len(p)), + "min_probability": float(p.min()), + "max_probability": float(p.max()), + "lt_0p01": int((p < .01).sum()), + "gt_0p99": int((p > .99).sum()), + "quantiles": {str(q): float(np.quantile(p, q)) for q in [0,.001,.01,.05,.5,.95,.99,.999,1]}, + } + print(json.dumps(result, indent=2)) + return result + + +def compare_predictions(a_path: Path, b_path: Path, tol: float) -> dict: + print("reference columns:", read_headers(a_path)) + print("runtime columns:", read_headers(b_path)) + a = pd.read_csv(a_path) + b = pd.read_csv(b_path) + if "response_id" not in a or "probability" not in a or "response_id" not in b or "probability" not in b: + raise SystemExit("FAIL comparison inputs need response_id,probability") + m = a[["response_id","probability"]].merge( + b[["response_id","probability"]], on="response_id", suffixes=("_a","_b"), validate="one_to_one" + ) + if len(m) != len(a) or len(m) != len(b): + raise SystemExit("FAIL comparison response ID sets differ") + d = np.abs(m.probability_a.to_numpy(float) - m.probability_b.to_numpy(float)) + result = {"rows": int(len(m)), "max_abs_difference": float(d.max(initial=0)), "tolerance": tol} + print(json.dumps(result, indent=2)) + if result["max_abs_difference"] > tol: + raise SystemExit("FAIL prediction parity") + return result + + +def independence(paths: list[Path], response_id: str, tol: float) -> dict: + vals = [] + for path in paths: + print(f"{path.name} columns:", read_headers(path)) + df = pd.read_csv(path) + hit = df.loc[df.response_id.astype(str) == str(response_id), "probability"] + if len(hit) != 1: + raise SystemExit(f"FAIL {path}: expected one row for {response_id}, got {len(hit)}") + vals.append(float(hit.iloc[0])) + spread = max(vals) - min(vals) + result = {"response_id": str(response_id), "probabilities": vals, "spread": spread, "tolerance": tol} + print(json.dumps(result, indent=2)) + if spread > tol: + raise SystemExit("FAIL sample independence") + return result + + +def self_test() -> None: + import tempfile + with tempfile.TemporaryDirectory() as td: + root = Path(td) + fmt = pd.DataFrame({"response_id":["a","b"], "probability":[0.5,0.5]}) + p = pd.DataFrame({"response_id":["a","b"], "probability":[0.2,0.8]}) + fmt.to_csv(root/"fmt.csv", index=False); p.to_csv(root/"p.csv", index=False); p.to_csv(root/"q.csv", index=False) + validate_output(root/"fmt.csv", root/"p.csv") + compare_predictions(root/"p.csv", root/"q.csv", 1e-8) + independence([root/"p.csv", root/"q.csv"], "a", 1e-8) + print("SELF TEST PASS") + + +def main() -> None: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("self-test") + p = sub.add_parser("output"); p.add_argument("--format", required=True); p.add_argument("--predictions", required=True) + p = sub.add_parser("parity"); p.add_argument("--reference", required=True); p.add_argument("--runtime", required=True); p.add_argument("--tol", type=float, default=1e-8) + p = sub.add_parser("independence"); p.add_argument("--response-id", required=True); p.add_argument("--predictions", nargs="+", required=True); p.add_argument("--tol", type=float, default=1e-8) + args = ap.parse_args() + if args.cmd == "self-test": self_test() + elif args.cmd == "output": validate_output(Path(args.format), Path(args.predictions)) + elif args.cmd == "parity": compare_predictions(Path(args.reference), Path(args.runtime), args.tol) + else: independence([Path(x) for x in args.predictions], args.response_id, args.tol) + +if __name__ == "__main__": + main() diff --git a/competitions/trace_the_ace/train_v74_runtime_assets.py b/competitions/trace_the_ace/train_v74_runtime_assets.py new file mode 100644 index 00000000..0e451863 --- /dev/null +++ b/competitions/trace_the_ace/train_v74_runtime_assets.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse,json,hashlib,joblib +from pathlib import Path +import numpy as np +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.model_selection import GroupKFold +from v74_semantic_objective_prior import load_training,semantic_prior_predict +from runtime_v74.v74_runtime_core import predict + +def fit_model(df,k=16,smooth=2.0): + global_p=float(df.target.mean()) + stats=df.groupby('learning_objective').target.agg(['sum','count']) + stats['p']=(stats['sum']+smooth*global_p)/(stats['count']+smooth) + objs=stats.index.astype(str).tolist() + vec=TfidfVectorizer(analyzer='char_wb',ngram_range=(3,5),min_df=1,sublinear_tf=True,norm='l2') + A=vec.fit_transform(objs) + return {'k':k,'smooth':smooth,'global_p':global_p,'vectorizer':vec,'A':A,'posterior':stats['p'].to_numpy(float),'mapped':{str(k):float(v) for k,v in stats['p'].items()},'counts':{str(k):float(v) for k,v in stats['count'].items()}} + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def main(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + y=f.target.to_numpy(int); sess=f.session_id.astype(str).to_numpy() + tr,va=next(iter(GroupKFold(5).split(np.zeros(len(y)),y,sess))) + ref,_=semantic_prior_predict(f.iloc[tr],f.iloc[va],k=16,smooth=2.0) + m=fit_model(f.iloc[tr]); got=predict(m,f.iloc[va].learning_objective.astype(str).tolist()) + md=float(np.max(np.abs(ref-got))) + if md>=1e-8: raise RuntimeError(f'parity failed {md}') + a.assets.mkdir(parents=True,exist_ok=True) + model=fit_model(f); mp=a.assets/'v74_model.joblib'; joblib.dump(model,mp,compress=3) + manifest={'candidate':'V74_PURE_HIERARCHICAL_OBJECTIVE_PRIOR','k':16,'smooth':2.0,'trust_denominator':10.0,'v115b_session_cold':0.5511484894117864,'v115b_objective_cold_stress':0.6013242442331039,'v115b_exact_support_rate':0.9975193886861314,'parity_max_abs_diff':md,'model_sha256':sha(mp)} + (a.assets/'manifest.json').write_text(json.dumps(manifest,indent=2)) + print(json.dumps(manifest,indent=2)) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--assets',type=Path,required=True);main(p.parse_args()) diff --git a/competitions/trace_the_ace/train_v75_runtime_assets.py b/competitions/trace_the_ace/train_v75_runtime_assets.py new file mode 100644 index 00000000..fe238650 --- /dev/null +++ b/competitions/trace_the_ace/train_v75_runtime_assets.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Fit the promoted V75 all-views model on all training rows and export runtime assets.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, SEED + +N_HASH = 2**18 +VIEW_ORDER = ["objective", "raw", "student", "local", "canonical", "terminal"] + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def run(args) -> None: + frame = load_training(args.features, args.labels).reset_index(drop=True) + cache = {} + view_rows, nums = [], [] + for i, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + views, num, _ = trajectory_views(cache[sid], str(row.learning_objective)) + view_rows.append(views) + nums.append(num) + if (i + 1) % 2500 == 0: + print("fitted-feature rows", i + 1) + + numeric = np.vstack(nums).astype(np.float64) + num_mean = numeric.mean(axis=0) + num_std = numeric.std(axis=0) + 1e-6 + z = (numeric - num_mean) / num_std + + hv = HashingVectorizer( + n_features=N_HASH, + alternate_sign=False, + norm="l2", + ngram_range=(1, 2), + lowercase=True, + ) + objective = hv.transform(["[OBJECTIVE] " + str(x) for x in frame.learning_objective]) + raw = hv.transform(["[RAW] " + v["raw"] for v in view_rows]) + student = hv.transform(["[STUDENT] " + v["student"] for v in view_rows]) + local = hv.transform(["[LOCAL] " + v["local"] for v in view_rows]) + canonical = hv.transform(["[STATE] " + v["canonical"] for v in view_rows]) + terminal = hv.transform(["[TERMINAL] " + v["terminal"] for v in view_rows]) + X = hstack([objective, raw, student, local, canonical, terminal, csr_matrix(z)], format="csr") + y = frame.target.to_numpy(dtype=int) + + model = LogisticRegression(C=0.25, max_iter=300, solver="liblinear", random_state=SEED) + model.fit(X, y) + + args.out_dir.mkdir(parents=True, exist_ok=True) + assets = args.out_dir / "v75_runtime_assets.npz" + np.savez_compressed( + assets, + coef=model.coef_.ravel().astype(np.float64), + intercept=np.asarray(model.intercept_, dtype=np.float64), + num_mean=num_mean.astype(np.float64), + num_std=num_std.astype(np.float64), + ) + manifest = { + "candidate": "V75_ALL_VIEWS", + "seed": SEED, + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "hash_features_per_view": N_HASH, + "view_order": VIEW_ORDER, + "numeric_features": int(numeric.shape[1]), + "total_features": int(X.shape[1]), + "logistic_C": 0.25, + "solver": "liblinear", + "validation": { + "session_cold_logloss": 0.5395443498930955, + "session_cold_worst_fold": 0.545097717258529, + "objective_cold_logloss": 0.5923494729977774, + "objective_cold_worst_fold": 0.6412260329460859, + }, + "assets_sha256": sha256_file(assets), + } + (args.out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print(json.dumps(manifest, indent=2)) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path, required=True) + p.add_argument("--labels", type=Path, required=True) + p.add_argument("--transcripts", type=Path, required=True) + p.add_argument("--out-dir", type=Path, required=True) + return p.parse_args() + +if __name__ == "__main__": + run(parse_args()) diff --git a/competitions/trace_the_ace/train_v97_runtime_assets.py b/competitions/trace_the_ace/train_v97_runtime_assets.py new file mode 100644 index 00000000..0314af86 --- /dev/null +++ b/competitions/trace_the_ace/train_v97_runtime_assets.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Train V75 + V94 RELATED assets for the fixed V97 support gate.""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + + +def pack_model(m): return m.coef_.ravel().astype(np.float64), np.asarray(m.intercept_,dtype=np.float64) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[];rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t);rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X0,y) + mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr,y) + Z=np.vstack(rz).astype(float); rmean=Z.mean(0); rstd=Z.std(0)+1e-6 + from v75_canonical_trajectory import trajectory_views + nums=[] + for _,r in f.iterrows(): nums.append(trajectory_views(cache[str(r.session_id)],str(r.learning_objective))[1]) + N=np.vstack(nums).astype(float); vmean=N.mean(0); vstd=N.std(0)+1e-6 + c0,b0=pack_model(m0); cr,br=pack_model(mr) + a.out_dir.mkdir(parents=True,exist_ok=True) + np.savez_compressed(a.out_dir/'v97_assets.npz',v75_coef=c0,v75_intercept=b0,v75_num_mean=vmean,v75_num_std=vstd, + related_coef=cr,related_intercept=br,related_num_mean=rmean,related_num_std=rstd) + counts=f.groupby('learning_objective').size().astype(int).to_dict() + manifest={'candidate':'V97_FIXED_EXACT_SUPPORT_GATE','unseen_weight':0.35,'seen_weight':0.0,'objective_key':'learning_objective', + 'objective_counts':{str(k):int(v) for k,v in counts.items()},'rows':int(len(f))} + (a.out_dir/'manifest.json').write_text(json.dumps(manifest,indent=2)); print(json.dumps({k:v for k,v in manifest.items() if 'counts' not in k},indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out-dir',type=Path,required=True);run(p.parse_args()) diff --git a/competitions/trace_the_ace/v100_nested_stacker.py b/competitions/trace_the_ace/v100_nested_stacker.py new file mode 100644 index 00000000..d0825090 --- /dev/null +++ b/competitions/trace_the_ace/v100_nested_stacker.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""V100 Phase A: leakage-safe calibrated meta-stacker. + +V99 asks which expert wins. V100 preserves probability/loss magnitude by learning +P(correct) directly from INNER-OOF V75/RELATED predictions plus runtime-visible +support/disagreement context. Outer objective-cold folds remain untouched. +""" +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript,tokens +from v75_canonical_trajectory import load_training,SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups,obj_family +from v94_related_control import segmented_control,build_control + +EPS=1e-5 + +def expert(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + +def counts(tr,va,v): + u,n=np.unique(v[tr],return_counts=True); d=dict(zip(u,n)); return np.asarray([d.get(v[i],0) for i in va],float) + +def meta(p0,pr,ec,fc,sc,turns,olen): + p0=np.asarray(p0); pr=np.asarray(pr); d=pr-p0 + return np.column_stack([p0,pr,d,np.abs(d),p0*pr,p0*p0,pr*pr, + np.abs(p0-.5),np.abs(pr-.5),np.log1p(ec),np.log1p(fc),np.log1p(sc),np.log1p(turns),np.log1p(olen)]) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in f.session_id.astype(str).unique()} + rt=[];rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t);rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + key=f.learning_objective.astype(str).to_numpy(); fam=f.learning_objective.astype(str).map(obj_family).astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + turns=np.asarray([len(cache[str(s)]) for s in sess],float); olen=np.asarray([len(tokens(x)) for x in key],float) + outer=folds_from_groups(obj); P0=np.zeros(len(y)); PR=np.zeros(len(y)); PS=np.zeros(len(y)); P97=np.zeros(len(y)); folds=[] + for k,(tr,va) in enumerate(outer,1): + print('OUTER',k,len(tr),len(va)); p0=expert(X0,y,tr,va); pr=expert(Xr,y,tr,va); P0[va]=p0;PR[va]=pr + inner=list(GroupKFold(3).split(np.zeros(len(tr)),y[tr],obj[tr])); M=np.zeros((len(tr),14)) + for j,(itl,ivl) in enumerate(inner,1): + it=tr[itl]; iv=tr[ivl]; q0=expert(X0,y,it,iv); qr=expert(Xr,y,it,iv) + M[ivl]=meta(q0,qr,counts(it,iv,key),counts(it,iv,fam),counts(it,iv,sess),turns[iv],olen[iv]); print(' inner',j,len(iv)) + stack=HistGradientBoostingClassifier(loss='log_loss',max_depth=2,max_iter=100,learning_rate=.04,min_samples_leaf=120,l2_regularization=2.0,random_state=SEED) + stack.fit(M,y[tr]) + MV=meta(p0,pr,counts(tr,va,key),counts(tr,va,fam),counts(tr,va,sess),turns[va],olen[va]); ps=np.clip(stack.predict_proba(MV)[:,1],EPS,1-EPS); PS[va]=ps + ec=counts(tr,va,key); w=np.where(ec==0,.35,0.0); p97=np.clip((1-w)*p0+w*pr,EPS,1-EPS);P97[va]=p97 + r={'fold':k,'v75':float(log_loss(y[va],p0)),'related':float(log_loss(y[va],pr)),'v97':float(log_loss(y[va],p97)),'v100':float(log_loss(y[va],ps))};folds.append(r);print(r) + ll0=float(log_loss(y,P0)); llr=float(log_loss(y,PR)); ll97=float(log_loss(y,P97)); lls=float(log_loss(y,PS)); gain=ll97-lls + verdict='PROMOTE_TO_FOUR_WORLD_V100' if gain>=.002 else ('REFINE_STACKER' if gain>=.0005 else 'SUPPRESS_THIS_STACKER') + out={'primary':'objective-cold nested calibrated stacker','v75':ll0,'related':llr,'v97':ll97,'v100_stacker':lls,'gain_vs_v97':gain,'folds':folds,'decision':{'verdict':verdict,'precommit':'promote only if nested objective-cold gain vs V97 >= 0.002'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',default='v100_nested_stacker.json');run(p.parse_args()) diff --git a/competitions/trace_the_ace/v101_objective_level_router.py b/competitions/trace_the_ace/v101_objective_level_router.py new file mode 100644 index 00000000..36956894 --- /dev/null +++ b/competitions/trace_the_ace/v101_objective_level_router.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""V101: nested objective-level applicability router. + +Residual from V99/V100: row-level gates failed while the endpoint oracle remains huge. +Hypothesis: applicability lives at the objective level, not the individual row. + +Outer objective-cold folds are untouched evaluation. Inner OOF expert predictions on +outer-train objectives are aggregated by objective. A small objective-level regressor +predicts a single RELATED blend weight for each unseen outer objective using only +runtime-visible objective text and unlabeled aggregate expert behavior. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack, csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression, Ridge +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript, tokens +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +GRID=np.array([0.0,0.15,0.25,0.35,0.45,0.60]) + + +def fit_expert(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + + +def obj_numeric(p0,pr,idx,turns,obj_len): + a=p0[idx]; b=pr[idx]; d=b-a + return np.array([ + len(idx), a.mean(), b.mean(), np.mean(np.abs(d)), np.std(d), + np.mean(np.abs(a-.5)), np.mean(np.abs(b-.5)), + np.mean(d>0), np.mean(np.abs(d)>.05), np.mean(np.abs(d)>.10), + np.mean(turns[idx]), np.mean(obj_len[idx]) + ],float) + + +def best_weight(y,p0,pr,idx): + best=(1e9,0.35) + for w in GRID: + q=np.clip((1-w)*p0[idx]+w*pr[idx],EPS,1-EPS) + ll=float(log_loss(y[idx],q,labels=[0,1])) + if ll=.002 else ('REFINE_OBJECTIVE_ROUTER' if gain>=.0005 else 'SUPPRESS_OBJECTIVE_ROUTER') + out={'primary':'nested objective-level applicability router','v75':ll0,'v97':ll97,'v101':ll101, + 'gain_vs_v97':gain,'diagnostic_endpoint_oracle':oracle,'folds':rows, + 'decision':{'verdict':verdict,'precommit':'promote only if objective-cold gain vs frozen V97 >= 0.002'}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v101_objective_level_router.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v102_applicability_resolution.py b/competitions/trace_the_ace/v102_applicability_resolution.py new file mode 100644 index 00000000..b117cdbb --- /dev/null +++ b/competitions/trace_the_ace/v102_applicability_resolution.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""V102: locate the resolution at which RELATED applicability actually lives. + +Diagnostic only. Uses untouched outer objective-cold folds to generate V75/RELATED +predictions once, then measures label-informed oracle blend ceilings at progressively +finer groupings. This does NOT define a deployable router; it decides what unit V103 +should model. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from v71_mastery_events import load_transcript, tokens +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +GRID=np.array([0.,.15,.25,.35,.45,.60,1.0]) + +def fit(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) + +def oracle_group(y,p0,pr,groups): + q=np.empty(len(y)); ws=[] + for g in np.unique(groups): + ix=np.where(groups==g)[0]; best=(1e99,0.,None) + for w in GRID: + z=(1-w)*p0[ix]+w*pr[ix]; v=ll(y[ix],z) + if v.03), np.mean(np.abs(d)>.06), np.mean(np.abs(d)>.10), + np.mean(np.abs(a-.5)), np.mean(np.abs(b-.5)), + np.mean(turns[idx]), np.mean(obj_len[idx]), float(n_objectives) + ],float) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1,flush=True) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy(); text=f.learning_objective.astype(str).to_numpy() + turns=np.asarray([len(cache[str(s)]) for s in sess],float) + obj_len=np.asarray([len(tokens(x)) for x in text],float) + + p0_all=np.zeros(len(y)); pr_all=np.zeros(len(y)); p97_all=np.zeros(len(y)); p103_all=np.zeros(len(y)) + folds=[] + for k,(tr,va) in enumerate(folds_from_groups(obj),1): + p0=fit_expert(X0,y,tr,va); pr=fit_expert(Xr,y,tr,va) + p0_all[va]=p0; pr_all[va]=pr + + # Inner objective-grouped OOF predictions on outer-train only. + ip0=np.zeros(len(tr)); ipr=np.zeros(len(tr)) + inner=GroupKFold(min(3,len(np.unique(obj[tr])))).split(np.zeros(len(tr)),y[tr],obj[tr]) + for itr_l,iva_l in inner: + itr=tr[itr_l]; iva=tr[iva_l] + ip0[iva_l]=fit_expert(X0,y,itr,iva); ipr[iva_l]=fit_expert(Xr,y,itr,iva) + + train_sessions=np.unique(sess[tr]); Z=[]; target=[] + for s in train_sessions: + loc=np.where(sess[tr]==s)[0] + nobj=len(np.unique(obj[tr][loc])) + Z.append(session_features(ip0,ipr,loc,turns[tr],obj_len[tr],nobj)) + target.append(best_weight(y[tr],ip0,ipr,loc)) + Z=np.vstack(Z); target=np.asarray(target) + mu=Z.mean(0); sd=Z.std(0)+1e-6 + reg=HistGradientBoostingRegressor(max_depth=2,max_iter=120,learning_rate=.04,min_samples_leaf=40, + l2_regularization=2.0,random_state=SEED).fit((Z-mu)/sd,target) + + w=np.zeros(len(va)) + for s in np.unique(sess[va]): + loc=np.where(sess[va]==s)[0] + nobj=len(np.unique(obj[va][loc])) + z=session_features(p0,pr,loc,turns[va],obj_len[va],nobj).reshape(1,-1) + ws=float(np.clip(reg.predict((z-mu)/sd)[0],0,.60)) + w[loc]=ws + q103=np.clip((1-w)*p0+w*pr,EPS,1-EPS); p103_all[va]=q103 + q97=np.clip(.65*p0+.35*pr,EPS,1-EPS); p97_all[va]=q97 + folds.append({'fold':k,'v97':float(log_loss(y[va],q97)),'v103':float(log_loss(y[va],q103)), + 'gain':float(log_loss(y[va],q97)-log_loss(y[va],q103)),'mean_weight':float(w.mean())}) + print(folds[-1],flush=True) + + ll97=float(log_loss(y,p97_all)); ll103=float(log_loss(y,p103_all)); gain=ll97-ll103 + l0=-(y*np.log(p0_all)+(1-y)*np.log(1-p0_all)); lr=-(y*np.log(pr_all)+(1-y)*np.log(1-pr_all)) + oracle=float(np.mean(np.minimum(l0,lr))) + verdict='PROMOTE_TO_FOUR_WORLD_V103' if gain>=.002 else ('REFINE_SESSION_ROUTER' if gain>=.0005 else 'SUPPRESS_SESSION_ROUTER') + out={'primary':'nested session-level applicability router','v97':ll97,'v103':ll103,'gain_vs_v97':gain, + 'row_endpoint_oracle':oracle,'folds':folds, + 'decision':{'verdict':verdict,'precommit':'promote only if objective-cold gain vs frozen V97 >= 0.002'}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v103_session_level_router.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v104_single_session_separator.py b/competitions/trace_the_ace/v104_single_session_separator.py new file mode 100644 index 00000000..ffaa339d --- /dev/null +++ b/competitions/trace_the_ace/v104_single_session_separator.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""V104: single-separator test for session-level applicability. + +V102 localized most oracle headroom to session resolution, while V103's multivariate +session regressor overfit/regressed. This test asks the cheapest next question: does one +runtime-visible session statistic lawfully separate sessions that want MORE vs LESS +RELATED than frozen V97's 0.35 blend? + +Selection is nested. Outer objective-cold folds are untouched. Inside each outer-train, +objective-grouped OOF expert predictions are used to choose exactly one feature, +threshold, direction, and two blend weights. The chosen rule is then applied unchanged +to the outer validation fold. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript, tokens +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +WEIGHTS=np.array([0.0,0.15,0.25,0.35,0.45,0.60]) +FEATURE_NAMES=['n_rows','p0_mean','pr_mean','abs_disagree_mean','disagree_std','signed_disagree_mean', + 'frac_abs_gt_03','frac_abs_gt_06','frac_abs_gt_10','p0_conf','pr_conf','turns','obj_len','n_objectives'] + + +def fit_expert(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) + +def sf(p0,pr,idx,turns,obj_len,nobj): + a=p0[idx]; b=pr[idx]; d=b-a + return np.array([len(idx),a.mean(),b.mean(),np.mean(np.abs(d)),np.std(d),np.mean(d), + np.mean(np.abs(d)>.03),np.mean(np.abs(d)>.06),np.mean(np.abs(d)>.10), + np.mean(np.abs(a-.5)),np.mean(np.abs(b-.5)),np.mean(turns[idx]),np.mean(obj_len[idx]),float(nobj)],float) + +def session_table(y,p0,pr,sess,obj,turns,obj_len,indices): + rows=[] + for s in np.unique(sess[indices]): + loc_global=indices[np.where(sess[indices]==s)[0]] + z=sf(p0,pr,loc_global,turns,obj_len,len(np.unique(obj[loc_global]))) + rows.append((s,loc_global,z)) + return rows + +def choose_rule(y,p0,pr,rows): + # Predeclared quantile thresholds; choose one feature + threshold + direction + two weights. + Z=np.vstack([r[2] for r in rows]) + best=(1e99,None) + for j,name in enumerate(FEATURE_NAMES): + vals=Z[:,j] + for q in (0.2,0.35,0.5,0.65,0.8): + t=float(np.quantile(vals,q)) + for direction in ('low_more','high_more'): + for w_more in (0.45,0.60): + for w_less in (0.0,0.15,0.25,0.35): + pred=np.empty(len(y)); mask=np.zeros(len(y),bool) + for _,ix,z in rows: + more=(z[j] <= t) if direction=='low_more' else (z[j] > t) + w=w_more if more else w_less + pred[ix]=(1-w)*p0[ix]+w*pr[ix]; mask[ix]=True + v=ll(y[mask],pred[mask]) + if v t) + w=rule['w_more'] if more else rule['w_less']; q[ix]=(1-w)*p0[ix]+w*pr[ix]; weights.extend([w]*len(ix)) + return q,np.asarray(weights) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1,flush=True) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy(); text=f.learning_objective.astype(str).to_numpy() + turns=np.asarray([len(cache[str(s)]) for s in sess],float); obj_len=np.asarray([len(tokens(x)) for x in text],float) + p97_all=np.zeros(len(y)); p104_all=np.zeros(len(y)); folds=[] + for k,(tr,va) in enumerate(folds_from_groups(obj),1): + p0=np.zeros(len(y)); pr=np.zeros(len(y)) + p0[va]=fit_expert(X0,y,tr,va); pr[va]=fit_expert(Xr,y,tr,va) + ip0=np.zeros(len(y)); ipr=np.zeros(len(y)) + inner=GroupKFold(min(3,len(np.unique(obj[tr])))).split(np.zeros(len(tr)),y[tr],obj[tr]) + for itr_l,iva_l in inner: + itr=tr[itr_l]; iva=tr[iva_l] + ip0[iva]=fit_expert(X0,y,itr,iva); ipr[iva]=fit_expert(Xr,y,itr,iva) + train_rows=session_table(y,ip0,ipr,sess,obj,turns,obj_len,tr) + _,rule=choose_rule(y,ip0,ipr,train_rows) + va_rows=session_table(y,p0,pr,sess,obj,turns,obj_len,va) + q=np.empty(len(y)); qva,w=apply_rule(p0,pr,va_rows,rule,len(y)); q[va]=qva[va] + q97=np.clip(.65*p0[va]+.35*pr[va],EPS,1-EPS); q104=np.clip(q[va],EPS,1-EPS) + p97_all[va]=q97; p104_all[va]=q104 + folds.append({'fold':k,'v97':ll(y[va],q97),'v104':ll(y[va],q104),'gain':ll(y[va],q97)-ll(y[va],q104), + 'rule':rule,'mean_weight':float(np.mean(w))}) + print(folds[-1],flush=True) + l97=ll(y,p97_all); l104=ll(y,p104_all); gain=l97-l104 + verdict='PROMOTE_TO_FOUR_WORLD_V104' if gain>=.002 else ('REFINE_SINGLE_SEPARATOR' if gain>=.0005 else 'SUPPRESS_SINGLE_SEPARATOR') + out={'primary':'nested single session separator','v97':l97,'v104':l104,'gain_vs_v97':gain,'folds':folds, + 'decision':{'verdict':verdict,'precommit':'promote only if objective-cold gain vs frozen V97 >= 0.002'}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v104_single_session_separator.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v105_prior_state_composition.py b/competitions/trace_the_ace/v105_prior_state_composition.py new file mode 100644 index 00000000..a7700adb --- /dev/null +++ b/competitions/trace_the_ace/v105_prior_state_composition.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""V105: restore the missing V74 objective-prior + V75 student-state composition. + +Repo reset hypothesis: V74 was promoted as a mandatory independent objective-difficulty +prior, but the V75->V104 lineage largely rebuilt V75 without it. Test the smallest +lawful composition before further routing work. + +For each outer validation world: + * V74 is fit only on outer-train and predicts outer-valid. + * V75 and RELATED are fit only on outer-train and predict outer-valid. + * Inner grouped OOF predictions on outer-train select a tiny convex grid. + * RELATED weight is permitted only where the exact objective has zero support in the + corresponding training fold. No test-batch aggregation or cross-response features. + +A deterministic mixed-support world combines unseen-objective rows with held-out-session +rows on otherwise seen objectives, so the gate is not judged only in pure objective-cold. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.cluster import KMeans +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v74_semantic_objective_prior import semantic_prior_predict +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups, obj_family, style_matrix +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +W74=np.array([0.,.10,.20,.30,.40,.50]) +WR=np.array([0.,.10,.20,.30,.40]) + + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) + +def fit_lr(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + +def unseen_mask(keys,tr,va): + seen=set(keys[tr].tolist()) + return np.asarray([keys[i] not in seen for i in va],bool) + +def compose(p75,p74,pr,unseen,w74,wr): + # RELATED is unavailable on supported objectives. Preserve convexity per row. + rw=np.where(unseen,wr,0.0) + base=np.maximum(0.0,1.0-w74-rw) + return np.clip(base*p75+w74*p74+rw*pr,EPS,1-EPS) + +def mixed_support_folds(obj,sess,n=5): + """Each fold has cold objectives plus session-held-out rows from remaining objectives.""" + uo=np.unique(obj); us=np.unique(sess) + of={x:i % n for i,x in enumerate(sorted(uo))} + sf={x:i % n for i,x in enumerate(sorted(us))} + out=[] + idx=np.arange(len(obj)) + for k in range(n): + cold=np.asarray([of[x]==k for x in obj]) + seen_session=np.asarray([(of[o]!=k and sf[s]==k) for o,s in zip(obj,sess)]) + va=idx[cold|seen_session]; tr=idx[~(cold|seen_session)] + out.append((tr,va)) + return out + +def inner_oof(f,X75,Xr,y,outer_tr,groups,support_key): + n=len(outer_tr); p75=np.zeros(n); p74=np.zeros(n); pr=np.zeros(n); uns=np.zeros(n,bool) + g=groups[outer_tr] + ns=min(3,len(np.unique(g))) + for itr_l,iva_l in GroupKFold(ns).split(np.zeros(n),y[outer_tr],g): + itr=outer_tr[itr_l]; iva=outer_tr[iva_l] + p75[iva_l]=fit_lr(X75,y,itr,iva) + pr[iva_l]=fit_lr(Xr,y,itr,iva) + p74[iva_l],_=semantic_prior_predict(f.iloc[itr],f.iloc[iva]) + uns[iva_l]=unseen_mask(support_key,itr,iva) + return p75,p74,pr,uns + +def select_weights(y,p75,p74,pr,uns): + best_base=(1e99,None); best_gate=(1e99,None) + for a in W74: + q=compose(p75,p74,pr,uns,a,0.0); v=ll(y,q) + if v.80: continue + q=compose(p75,p74,pr,uns,a,r); v=ll(y,q) + if v=.0015 and mixed>=.0010 and session>=-.0005 and g.min()>=-.0010) + out['decision']={'mean_gate_gain':float(g.mean()),'mean_prior_state_gain':float(gp.mean()),'mixed_support_gain':float(mixed), + 'worst_gate_gain':float(g.min()),'verdict':'PROMOTE_V105_COMPOSITION' if promote else 'DO_NOT_PROMOTE_V105', + 'precommit':'promote iff mean five-world gate gain >= .0015, mixed-support >= .0010, session-cold >= -.0005, no world worse than -.0010'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v105_prior_state_composition.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v106_fixed_prior_state_law.py b/competitions/trace_the_ace/v106_fixed_prior_state_law.py new file mode 100644 index 00000000..9663347a --- /dev/null +++ b/competitions/trace_the_ace/v106_fixed_prior_state_law.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""V106: independent confirmation of a fixed V105-derived runtime law. + +Precommitted after V105, before inspecting these fresh hash-fold results: + seen objective: 0.80 V75 + 0.20 V74 + unseen objective: 0.50 V75 + 0.20 V74 + 0.30 RELATED + +Comparator is frozen V97: + seen objective: 1.00 V75 + unseen objective: 0.65 V75 + 0.35 RELATED + +V106 deliberately changes validation geometry from V105's GroupKFold ordering to +stable SHA256 hash partitions. No weights are selected from V106 labels. +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from sklearn.cluster import KMeans +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v74_semantic_objective_prior import semantic_prior_predict +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import obj_family, style_matrix +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +W74=0.20 +WR=0.30 +V97_WR=0.35 +N=5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) +def hfold(x,salt='v106'): + b=(salt+'|'+str(x)).encode('utf-8') + return int(hashlib.sha256(b).hexdigest()[:12],16)%N + +def folds_hash(groups,salt): + g=np.asarray(groups).astype(str); idx=np.arange(len(g)); a=np.asarray([hfold(x,salt) for x in g]) + return [(idx[a!=k],idx[a==k]) for k in range(N)] + +def mixed_hash(obj,sess): + obj=np.asarray(obj).astype(str); sess=np.asarray(sess).astype(str); idx=np.arange(len(obj)) + of=np.asarray([hfold(x,'v106-mixed-obj') for x in obj]); sf=np.asarray([hfold(x,'v106-mixed-sess') for x in sess]) + out=[] + for k in range(N): + cold=of==k + session_seen=(of!=k)&(sf==k) + va=idx[cold|session_seen]; tr=idx[~(cold|session_seen)] + out.append((tr,va)) + return out + +def fit_lr(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def unseen_mask(keys,tr,va): + seen=set(keys[tr].tolist()); return np.asarray([keys[i] not in seen for i in va],bool) +def v97(p75,pr,u): + w=np.where(u,V97_WR,0.0); return np.clip((1-w)*p75+w*pr,EPS,1-EPS) +def v106(p75,p74,pr,u): + rw=np.where(u,WR,0.0); return np.clip((1-W74-rw)*p75+W74*p74+rw*pr,EPS,1-EPS) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1,flush=True) + X75=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + fam=f.learning_objective.astype(str).map(obj_family).astype(str).to_numpy() + style=KMeans(n_clusters=5,random_state=137,n_init=10).fit(style_matrix(f,cache)).labels_.astype(str) + worlds={ + 'objective_cold':folds_hash(obj,'v106-obj'), + 'session_cold':folds_hash(sess,'v106-sess'), + 'objective_family_cold':folds_hash(fam,'v106-fam'), + 'style_cold':folds_hash(style,'v106-style'), + 'mixed_support':mixed_hash(obj,sess), + } + out={'primary':'fixed V105-derived law on fresh hash folds','law':{'w74_all':W74,'wr_unseen':WR},'comparator':{'v97_wr_unseen':V97_WR},'worlds':{}} + gains=[] + for name,sp in worlds.items(): + P75=np.zeros(len(y)); P97=np.zeros(len(y)); P106=np.zeros(len(y)); U=np.zeros(len(y),bool); folds=[] + for k,(tr,va) in enumerate(sp,1): + if not len(va): continue + p75=fit_lr(X75,y,tr,va); pr=fit_lr(Xr,y,tr,va); p74,_=semantic_prior_predict(f.iloc[tr],f.iloc[va]) + u=unseen_mask(support,tr,va); q97=v97(p75,pr,u); q106=v106(p75,p74,pr,u) + P75[va]=p75; P97[va]=q97; P106[va]=q106; U[va]=u + row={'fold':k,'rows':int(len(va)),'unseen_fraction':float(u.mean()),'v75':ll(y[va],p75),'v97':ll(y[va],q97),'v106':ll(y[va],q106),'gain_vs_v97':ll(y[va],q97)-ll(y[va],q106)} + folds.append(row); print(name,row,flush=True) + used=P106>0 + rec={'rows':int(used.sum()),'unseen_fraction':float(U[used].mean()),'v75':ll(y[used],P75[used]),'v97':ll(y[used],P97[used]),'v106':ll(y[used],P106[used]),'gain_vs_v97':ll(y[used],P97[used])-ll(y[used],P106[used]),'folds':folds} + out['worlds'][name]=rec; gains.append(rec['gain_vs_v97']); print(name,'SUMMARY',rec,flush=True) + g=np.asarray(gains); mixed=out['worlds']['mixed_support']['gain_vs_v97']; session=out['worlds']['session_cold']['gain_vs_v97']; objg=out['worlds']['objective_cold']['gain_vs_v97'] + promote=(g.mean()>=.0005 and mixed>=.0005 and session>=0 and g.min()>=-.0005) + out['decision']={'mean_gain_vs_v97':float(g.mean()),'mixed_gain_vs_v97':float(mixed),'session_gain_vs_v97':float(session),'objective_gain_vs_v97':float(objg),'worst_gain_vs_v97':float(g.min()),'verdict':'BUILD_V106_RUNTIME' if promote else 'KEEP_V97','precommit':'build iff mean gain vs V97 >= .0005, mixed >= .0005, session >= 0, worst world >= -.0005'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v106_fixed_prior_state_law.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v107_support_conditioned_prior.py b/competitions/trace_the_ace/v107_support_conditioned_prior.py new file mode 100644 index 00000000..8c50e41f --- /dev/null +++ b/competitions/trace_the_ace/v107_support_conditioned_prior.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""V107: support-conditioned V74 prior on top of frozen V97. + +Hypothesis from V106: V74 helps supported/seen-objective regimes but hurts exact-unseen +objectives. Preserve V97 exactly on unseen objectives and add a small fixed V74 prior +only when the exact objective has fold-local training support. + +This is a fixed-law confirmation: no labels select weights. +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from sklearn.cluster import KMeans +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v74_semantic_objective_prior import semantic_prior_predict +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import obj_family, style_matrix +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +W_V97_UNSEEN=0.35 +W74_SEEN=0.20 + + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) +def fit_lr(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def bucket(x,n=5): + h=int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) + return h % n +def hash_folds(groups,n=5): + g=np.asarray(groups).astype(str); idx=np.arange(len(g)); out=[] + gb=np.asarray([bucket(x,n) for x in g]) + for k in range(n): + va=idx[gb==k]; tr=idx[gb!=k] + if len(va) and len(tr): out.append((tr,va)) + return out +def mixed_support_folds(obj,sess,n=5): + idx=np.arange(len(obj)); ob=np.asarray([bucket(x,n) for x in obj]); sb=np.asarray([bucket(x,n) for x in sess]); out=[] + for k in range(n): + cold=ob==k + seen_session=(ob!=k)&(sb==k) + va=idx[cold|seen_session]; tr=idx[~(cold|seen_session)] + if len(va) and len(tr): out.append((tr,va)) + return out +def unseen_mask(keys,tr,va): + seen=set(keys[tr].tolist()); return np.asarray([keys[i] not in seen for i in va],bool) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1,flush=True) + X75=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + fam=f.learning_objective.astype(str).map(obj_family).astype(str).to_numpy() + style=KMeans(n_clusters=5,random_state=137,n_init=10).fit(style_matrix(f,cache)).labels_.astype(str) + worlds={ + 'objective_cold':hash_folds(obj), + 'session_cold':hash_folds(sess), + 'objective_family_cold':hash_folds(fam), + 'style_cold':hash_folds(style), + 'mixed_support':mixed_support_folds(obj,sess), + } + out={'law':{'unseen':'V97 = .65 V75 + .35 RELATED','seen':'0.80 V75 + 0.20 V74'},'worlds':{}} + gains=[] + for name,sp in worlds.items(): + p97=np.zeros(len(y)); p107=np.zeros(len(y)); U=np.zeros(len(y),bool); covered=np.zeros(len(y),bool); folds=[] + for k,(tr,va) in enumerate(sp,1): + p75=fit_lr(X75,y,tr,va); pr=fit_lr(Xr,y,tr,va); p74,_=semantic_prior_predict(f.iloc[tr],f.iloc[va]) + uns=unseen_mask(support,tr,va); U[va]=uns; covered[va]=True + q97=np.where(uns,.65*p75+.35*pr,p75) + q107=np.where(uns,q97,.80*p75+.20*p74) + q97=np.clip(q97,EPS,1-EPS); q107=np.clip(q107,EPS,1-EPS) + p97[va]=q97; p107[va]=q107 + folds.append({'fold':k,'rows':int(len(va)),'unseen_fraction':float(uns.mean()),'v97':ll(y[va],q97),'v107':ll(y[va],q107),'gain':ll(y[va],q97)-ll(y[va],q107)}) + print(name,folds[-1],flush=True) + rec={'v97':ll(y[covered],p97[covered]),'v107':ll(y[covered],p107[covered]),'gain_vs_v97':ll(y[covered],p97[covered])-ll(y[covered],p107[covered]),'unseen_fraction':float(U[covered].mean()),'coverage':float(covered.mean()),'folds':folds} + out['worlds'][name]=rec; gains.append(rec['gain_vs_v97']); print(name,'SUMMARY',rec,flush=True) + g=np.asarray(gains,float); mixed=out['worlds']['mixed_support']['gain_vs_v97']; session=out['worlds']['session_cold']['gain_vs_v97']; objg=out['worlds']['objective_cold']['gain_vs_v97'] + promote=(g.mean()>=.0005 and mixed>=.0003 and session>=.0003 and objg>=-.0002 and g.min()>=-.0003) + out['decision']={'mean_gain_vs_v97':float(g.mean()),'mixed_support_gain':float(mixed),'session_cold_gain':float(session),'objective_cold_gain':float(objg),'worst_gain':float(g.min()),'verdict':'PROMOTE_V107_RUNTIME' if promote else 'KEEP_V97','precommit':'promote iff mean >= .0005, mixed >= .0003, session >= .0003, objective >= -.0002, worst >= -.0003'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v107_support_conditioned_prior.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v108_objective_transcript_interactions.py b/competitions/trace_the_ace/v108_objective_transcript_interactions.py new file mode 100644 index 00000000..9ef75b82 --- /dev/null +++ b/competitions/trace_the_ace/v108_objective_transcript_interactions.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""V108: explicit objective x transcript interactions dropped into frozen V97.""" +from __future__ import annotations +import argparse, hashlib, json +from collections import Counter +from pathlib import Path +import numpy as np +from scipy.sparse import hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from v71_mastery_events import load_transcript, tokens +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +EPS=1e-5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) +def bucket(x,n=5): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)%n +def folds(g,n=5): + g=np.asarray(g).astype(str); i=np.arange(len(g)); b=np.array([bucket(x,n) for x in g]); return [(i[b!=k],i[b==k]) for k in range(n) if np.any(b==k) and np.any(b!=k)] +def mixed(obj,sess,n=5): + i=np.arange(len(obj)); ob=np.array([bucket(x,n) for x in obj]); sb=np.array([bucket(x,n) for x in sess]); out=[] + for k in range(n): + m=(ob==k)|((ob!=k)&(sb==k)); + if np.any(m) and np.any(~m): out.append((i[~m],i[m])) + return out +def unseen(keys,tr,va): + s=set(keys[tr]); return np.array([keys[i] not in s for i in va],bool) +def fitp(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]); return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def role_terms(df,role,cap): + c=Counter(); [c.update(tokens(x)) for x in df.loc[df.role.astype(str).str.lower()==role,'content'].fillna('').astype(str)]; return [w for w,_ in c.most_common(cap)] +def cross_doc(df,obj): + ot=sorted(tokens(obj))[:16] or ['_objective_']; st=role_terms(df,'student',72); tt=role_terms(df,'tutor',48); p=[] + for o in ot: p += [f'O={o}|S={x}' for x in st]+[f'O={o}|T={x}' for x in tt] + return ' '.join(p) +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in f.session_id.astype(str).unique()}; rt=[]; rz=[]; docs=[] + for j,r in f.iterrows(): + s=str(r.session_id); o=str(r.learning_objective); t,z=segmented_control(cache[s],o,'related'); rt.append(t); rz.append(z); docs.append(cross_doc(cache[s],o)); + if (j+1)%2500==0: print('rows',j+1,flush=True) + X75=build_v75(f,cache); Xr=build_control(rt,rz); Xi=HashingVectorizer(n_features=2**20,alternate_sign=False,norm='l2',analyzer=str.split,lowercase=False).transform(docs); X=hstack([X75,Xi],format='csr') + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); sup=f.learning_objective.astype(str).to_numpy(); ses=f.session_id.astype(str).to_numpy(); worlds={'objective_cold':folds(obj),'session_cold':folds(ses),'mixed_support':mixed(obj,ses)}; out={'worlds':{}}; gains=[] + for name,sp in worlds.items(): + a0=np.zeros(len(y)); a1=np.zeros(len(y)); cov=np.zeros(len(y),bool); fs=[] + for k,(tr,va) in enumerate(sp,1): + p75=fitp(X75,y,tr,va); p108=fitp(X,y,tr,va); pr=fitp(Xr,y,tr,va); u=unseen(sup,tr,va); q0=np.where(u,.65*p75+.35*pr,p75); q1=np.where(u,.65*p108+.35*pr,p108); a0[va]=q0; a1[va]=q1; cov[va]=1; rec={'fold':k,'v97':ll(y[va],q0),'v108':ll(y[va],q1),'gain':ll(y[va],q0)-ll(y[va],q1)}; fs.append(rec); print(name,rec,flush=True) + rec={'v97':ll(y[cov],a0[cov]),'v108':ll(y[cov],a1[cov]),'gain_vs_v97':ll(y[cov],a0[cov])-ll(y[cov],a1[cov]),'folds':fs}; out['worlds'][name]=rec; gains.append(rec['gain_vs_v97']) + g=np.array(gains); out['decision']={'mean_gain':float(g.mean()),'verdict':'STRUCTURAL_PROMOTE_V108' if g.mean()>=.003 and g.min()>=0 and out['worlds']['mixed_support']['gain_vs_v97']>=.002 else 'SUPPRESS_V108','precommit':'mean>=.003, mixed>=.002, no negative world'}; Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v108.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v109_supervised_objective_semantics.py b/competitions/trace_the_ace/v109_supervised_objective_semantics.py new file mode 100644 index 00000000..a0c295ab --- /dev/null +++ b/competitions/trace_the_ace/v109_supervised_objective_semantics.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""V109: supervised semantic objective-difficulty model composed with frozen V97.""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +EPS=1e-5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) +def bucket(x,n=5): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)%n +def folds(g,n=5): + g=np.asarray(g).astype(str); i=np.arange(len(g)); b=np.array([bucket(x,n) for x in g]); return [(i[b!=k],i[b==k]) for k in range(n) if np.any(b==k) and np.any(b!=k)] +def mixed(obj,sess,n=5): + i=np.arange(len(obj)); ob=np.array([bucket(x,n) for x in obj]); sb=np.array([bucket(x,n) for x in sess]); out=[] + for k in range(n): + m=(ob==k)|((ob!=k)&(sb==k)); + if np.any(m) and np.any(~m): out.append((i[~m],i[m])) + return out +def unseen(keys,tr,va): + s=set(keys[tr]); return np.array([keys[i] not in s for i in va],bool) +def fitp(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]); return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def semantic_predict(text,y,tr,va): + trtxt=[text[i] for i in tr]; vatxt=[text[i] for i in va]; cw=TfidfVectorizer(analyzer='char_wb',ngram_range=(3,5),min_df=2,max_features=120000,sublinear_tf=True); ww=TfidfVectorizer(ngram_range=(1,2),min_df=2,max_features=50000,sublinear_tf=True); A=hstack([cw.fit_transform(trtxt),ww.fit_transform(trtxt)],format='csr'); B=hstack([cw.transform(vatxt),ww.transform(vatxt)],format='csr'); m=LogisticRegression(C=.8,max_iter=300,solver='liblinear',random_state=SEED).fit(A,y[tr]); return np.clip(m.predict_proba(B)[:,1],EPS,1-EPS) +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in f.session_id.astype(str).unique()}; rt=[]; rz=[] + for j,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (j+1)%2500==0: print('rows',j+1,flush=True) + X75=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int); text=f.learning_objective.fillna('').astype(str).tolist(); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); sup=f.learning_objective.astype(str).to_numpy(); ses=f.session_id.astype(str).to_numpy(); worlds={'objective_cold':folds(obj),'session_cold':folds(ses),'mixed_support':mixed(obj,ses)}; out={'law':'seen=.75 V75+.25 SEM; unseen=.85 V97+.15 SEM','worlds':{}}; gains=[] + for name,sp in worlds.items(): + a0=np.zeros(len(y)); a1=np.zeros(len(y)); cov=np.zeros(len(y),bool); fs=[] + for k,(tr,va) in enumerate(sp,1): + p75=fitp(X75,y,tr,va); pr=fitp(Xr,y,tr,va); ps=semantic_predict(text,y,tr,va); u=unseen(sup,tr,va); q0=np.where(u,.65*p75+.35*pr,p75); q1=np.where(u,.85*q0+.15*ps,.75*p75+.25*ps); a0[va]=q0; a1[va]=q1; cov[va]=1; rec={'fold':k,'v97':ll(y[va],q0),'v109':ll(y[va],q1),'semantic':ll(y[va],ps),'gain':ll(y[va],q0)-ll(y[va],q1)}; fs.append(rec); print(name,rec,flush=True) + rec={'v97':ll(y[cov],a0[cov]),'v109':ll(y[cov],a1[cov]),'gain_vs_v97':ll(y[cov],a0[cov])-ll(y[cov],a1[cov]),'folds':fs}; out['worlds'][name]=rec; gains.append(rec['gain_vs_v97']) + g=np.array(gains); out['decision']={'mean_gain':float(g.mean()),'verdict':'STRUCTURAL_PROMOTE_V109' if g.mean()>=.003 and g.min()>=0 and out['worlds']['mixed_support']['gain_vs_v97']>=.002 else 'SUPPRESS_V109','precommit':'mean>=.003, mixed>=.002, no negative world'}; Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v109.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v110_residual_collider_state_discovery.py b/competitions/trace_the_ace/v110_residual_collider_state_discovery.py new file mode 100644 index 00000000..e45a201f --- /dev/null +++ b/competitions/trace_the_ace/v110_residual_collider_state_discovery.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""V110: residual-collider mastery-state discovery. + +Freeze V97. Split objectives by SHA before grammar search. On discovery objectives, +produce objective-cold OOF V97 predictions, then search a small causal grammar of +ordered mastery-state machines. Select by grouped OOF log loss with a collider +constraint. Freeze the winning grammar. Verify on untouched objectives in two worlds: +(1) objective-cold/unseen support, (2) session-cold/supported support. + +Phase-change claim requires >= .010 log-loss gain in either untouched world, +non-inferiority in the other, and ordered-state advantage over an order ablation. +""" +from __future__ import annotations +import argparse, hashlib, itertools, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75, evidence_events +from v94_related_control import segmented_control, build_control + +EPS=1e-5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS),labels=[0,1])) +def logit(p): + p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p)) +def hb(x,n=5): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16)%n + +def fit_base(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def p97_predict(X75,Xr,y,tr,va,support): + p75=fit_base(X75,y,tr,va); pr=fit_base(Xr,y,tr,va) + seen=set(support[tr].tolist()); uns=np.asarray([support[i] not in seen for i in va],bool) + q=np.where(uns,.65*p75+.35*pr,p75) + return np.clip(q,EPS,1-EPS),uns + + +def sig(e): + s=(str(e['state'])+'|'+str(round(float(e['rel']),2))+'|'+str(e['assistance'])+'|'+str(e['q'])).encode('utf8','ignore') + return hashlib.sha1(s).hexdigest() +def state_vec(events,cfg,destroy_order=False): + decay,rel_power,assist_pen,error_pen=cfg + E=list(events) + if destroy_order: E=sorted(E,key=sig) + if not E: return np.zeros(14,float) + vals=[]; K=[]; k=0.; streak=0; best_streak=0; recoveries=0; regressions=0 + last_neg=-1; last_ind=-1; ind_after_neg=0; contradictions_after_ind=0 + for t,e in enumerate(E): + rel=max(.01,float(e['rel']))**rel_power + ass=float(e['assistance']); ind=1. if e['independent'] else 0. + if e['neg']: v=-error_pen*rel*(1.-.5*ass) + elif e['pos']: v=rel*(assist_pen+(1-assist_pen)*(1-ass))*(1.+.35*ind) + else: v=.05*rel*(1-ass) + k=decay*k+v; vals.append(v); K.append(k) + good=bool(e['pos'] and e['independent']) + if good: + streak+=1; best_streak=max(best_streak,streak); last_ind=t + if last_neg>=0 and t>last_neg: ind_after_neg+=1 + elif e['neg']: streak=0; last_neg=t + if t and E[t-1]['neg'] and e['pos']: recoveries+=1 + if t and E[t-1]['pos'] and e['neg']: regressions+=1 + if e['neg'] and last_ind>=0 and t>last_ind: contradictions_after_ind+=1 + V=np.asarray(vals,float); A=np.asarray(K,float); n=len(E) + suffix=0 + for e in reversed(E): + if e['pos'] and not e['neg']: suffix+=1 + else: break + post_neg = float(sum(1 for j,e in enumerate(E) if j>last_neg and e['pos'] and e['independent'])) if last_neg>=0 else float(sum(e['pos'] and e['independent'] for e in E)) + return np.asarray([ + A[-1],A.max(),A.min(),A.mean(),A[-1]-A[0],V[-1],V.sum(), + best_streak/n,suffix/n,recoveries/n,regressions/n,post_neg/n, + contradictions_after_ind/n,ind_after_neg/n + ],float) + + +def collider_mask(obj,p,y): + b=np.floor(np.clip(p,0,.999999)*20).astype(int); keys=np.asarray([f'{o}|{z}' for o,z in zip(obj,b)]) + keep=np.zeros(len(y),bool) + for k in np.unique(keys): + ix=np.where(keys==k)[0] + if len(ix)>=4 and len(np.unique(y[ix]))==2: keep[ix]=True + return keep +def meta_cv(P,S,y,groups,splits): + q=np.zeros(len(y)); covered=np.zeros(len(y),bool) + for tr,va in splits: + sc=StandardScaler().fit(S[tr]); ztr=sc.transform(S[tr]); zva=sc.transform(S[va]) + Xtr=np.c_[logit(P[tr]),ztr,ztr[:,0]*logit(P[tr])] + Xva=np.c_[logit(P[va]),zva,zva[:,0]*logit(P[va])] + m=LogisticRegression(C=.15,max_iter=300,solver='liblinear',random_state=SEED).fit(Xtr,y[tr]) + q[va]=np.clip(m.predict_proba(Xva)[:,1],EPS,1-EPS); covered[va]=True + return q,covered +def fit_meta(P,S,y): + sc=StandardScaler().fit(S); Z=sc.transform(S); X=np.c_[logit(P),Z,Z[:,0]*logit(P)] + m=LogisticRegression(C=.15,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y) + return sc,m +def apply_meta(sc,m,P,S): + Z=sc.transform(S); X=np.c_[logit(P),Z,Z[:,0]*logit(P)] + return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + print('features columns',list(f.columns),flush=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in np.unique(sess)} + rt=[]; rz=[]; EV=[] + for i,r in f.iterrows(): + sid=str(r.session_id); text=str(r.learning_objective) + t,z=segmented_control(cache[sid],text,'related'); rt.append(t); rz.append(z) + seg,_=choose_target_segment(cache[sid],text); EV.append(evidence_events(seg,text)) + if (i+1)%2500==0: print('rows',i+1,flush=True) + X75=build_v75(f,cache); Xr=build_control(rt,rz) + + verify=np.asarray([hb(x,5)==0 for x in obj]); disc=~verify + D=np.where(disc)[0]; V=np.where(verify)[0] + print('split',{'discovery_rows':len(D),'verification_rows':len(V),'verification_objectives':int(len(np.unique(obj[V])))},flush=True) + + gd=obj[D]; nsp=min(4,len(np.unique(gd))); raw=list(GroupKFold(nsp).split(np.zeros(len(D)),y[D],gd)) + splits=[]; P=np.zeros(len(D)) + for ltr,lva in raw: + tr=D[ltr]; va=D[lva]; q,_=p97_predict(X75,Xr,y,tr,va,support); P[lva]=q; splits.append((ltr,lva)) + base_disc=ll(y[D],P); C=collider_mask(obj[D],P,y[D]); print('discovery_v97',base_disc,'collider_rows',int(C.sum()),flush=True) + + configs=list(itertools.product([.50,.70,.85,.95],[.5,1.0,1.5],[.20,.45,.70],[.75,1.0,1.30])) + leaderboard=[]; best=None; bestS=None + for j,cfg in enumerate(configs): + S=np.vstack([state_vec(EV[i],cfg,False) for i in D]) + q,cov=meta_cv(P,S,y[D],gd,splits); gain=base_disc-ll(y[D][cov],q[cov]); cg=(ll(y[D][C],P[C])-ll(y[D][C],q[C])) if C.sum() else 0. + rec={'cfg':list(cfg),'gain':gain,'collider_gain':cg} + leaderboard.append(rec) + if cg>=0 and (best is None or gain>best['gain']): best=rec; bestS=S + if (j+1)%24==0: print('grammar',j+1,'best',best,flush=True) + if best is None: best=max(leaderboard,key=lambda r:r['gain']); bestS=np.vstack([state_vec(EV[i],tuple(best['cfg']),False) for i in D]) + cfg=tuple(best['cfg']); qord,cov=meta_cv(P,bestS,y[D],gd,splits) + Sab=np.vstack([state_vec(EV[i],cfg,True) for i in D]); qab,_=meta_cv(P,Sab,y[D],gd,splits) + ordered_gain=base_disc-ll(y[D],qord); ablated_gain=base_disc-ll(y[D],qab); causal=ordered_gain-ablated_gain + print('SELECTED',best,'ordered_gain',ordered_gain,'ablation_gain',ablated_gain,'causal',causal,flush=True) + + sc_u,meta_u=fit_meta(P,bestS,y[D]) + pV,unsV=p97_predict(X75,Xr,y,D,V,support); SV=np.vstack([state_vec(EV[i],cfg,False) for i in V]); qV=apply_meta(sc_u,meta_u,pV,SV) + verify_obj={'rows':int(len(V)),'unseen_fraction':float(unsV.mean()),'v97':ll(y[V],pV),'v110':ll(y[V],qV),'gain':ll(y[V],pV)-ll(y[V],qV)} + print('VERIFY_OBJECTIVE',verify_obj,flush=True) + + ns=min(4,len(np.unique(sess[D]))); sraw=list(GroupKFold(ns).split(np.zeros(len(D)),y[D],sess[D])); Ps=np.zeros(len(D)) + for ltr,lva in sraw: + tr=D[ltr]; va=D[lva]; qq,_=p97_predict(X75,Xr,y,tr,va,support); Ps[lva]=qq + sc_s,meta_s=fit_meta(Ps,bestS,y[D]) + vv=V[np.asarray([hb(sess[i],5)==0 for i in V])] + tr=np.where(np.asarray([hb(s,5)!=0 for s in sess]))[0] + pS,unsS=p97_predict(X75,Xr,y,tr,vv,support); SS=np.vstack([state_vec(EV[i],cfg,False) for i in vv]); qS=apply_meta(sc_s,meta_s,pS,SS) + verify_sess={'rows':int(len(vv)),'unseen_fraction':float(unsS.mean()),'v97':ll(y[vv],pS),'v110':ll(y[vv],qS),'gain':ll(y[vv],pS)-ll(y[vv],qS)} + print('VERIFY_SESSION',verify_sess,flush=True) + + gains=[verify_obj['gain'],verify_sess['gain']] + if max(gains)>=.010 and min(gains)>=-.001 and causal>=.001: verdict='PHASE_CHANGE_STATE_LIFT' + elif max(gains)>=.003 and min(gains)>=-.001: verdict='PROMISING_STATE_LIFT' + else: verdict='SUPPRESS_V110_GRAMMAR' + out={'protocol':'objective-hash discovery/untouched verification','selected':best,'discovery':{'v97':base_disc,'ordered_gain':ordered_gain,'order_ablation_gain':ablated_gain,'causal_order_gain':causal,'collider_rows':int(C.sum()),'top10':sorted(leaderboard,key=lambda r:r['gain'],reverse=True)[:10]},'verification':{'objective_cold':verify_obj,'session_supported':verify_sess},'decision':{'verdict':verdict,'precommit':'PHASE_CHANGE iff max untouched gain >= .010, other >= -.001, causal order gain >= .001; PROMISING iff max >= .003 and other >= -.001'}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v110_residual_collider_state_discovery.json'); run(p.parse_args()) + +# Trigger-only low-level ref update; experiment logic unchanged. +# Low-level trigger sequence 2. diff --git a/competitions/trace_the_ace/v111_fast_residual_screen.py b/competitions/trace_the_ace/v111_fast_residual_screen.py new file mode 100644 index 00000000..50420a32 --- /dev/null +++ b/competitions/trace_the_ace/v111_fast_residual_screen.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +"""Temporary isolated-runner shim: execute frozen V112 fast raw-observable screen.""" +from v112_fast_raw_observable_screen import main +import argparse +from pathlib import Path +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v111_fast_residual_screen.json');main(p.parse_args()) diff --git a/competitions/trace_the_ace/v112_fast_raw_observable_screen.py b/competitions/trace_the_ace/v112_fast_raw_observable_screen.py new file mode 100644 index 00000000..e3494bb6 --- /dev/null +++ b/competitions/trace_the_ace/v112_fast_raw_observable_screen.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""V112 FAST RAW-OBSERVABLE SCREEN. +Frozen screen only: one shared transcript/V97 pass, deterministic 2500-row discovery sample, +objective-grouped OOF. Tests whether information discarded by V71 exists in raw transcripts. +Escalate only >= .003 V97 gain; >= .010 is phase-change candidate. +""" +from __future__ import annotations +import argparse, json, re, hashlib +from pathlib import Path +import numpy as np +from scipy.sparse import hstack, csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler +from v110_residual_collider_state_discovery import hb, ll, logit, p97_predict +from v71_mastery_events import load_transcript, normalize_roles +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +EPS=1e-5 +MATH=re.compile(r"\d|[+\-*/=×÷<>%]|\b(?:half|quarter|third|decimal|fraction|percent|times|divide|multiply)\b",re.I) + +def h(x): + return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) + +def texts(df,obj): + d=normalize_roles(df).reset_index(drop=True); role=d.role_repaired.astype(str).tolist(); c=d.content.fillna('').astype(str).tolist() + student=' '.join(x for r,x in zip(role,c) if r=='student'); tutor=' '.join(x for r,x in zip(role,c) if r=='tutor'); full=' '.join(f'[{r}] {x}' for r,x in zip(role,c)) + seg,_=choose_target_segment(df,obj); sr=normalize_roles(seg).reset_index(drop=True); local=' '.join(f'[{r}] {x}' for r,x in zip(sr.role_repaired.astype(str),sr.content.fillna('').astype(str))) + last=' '.join(f'[{r}] {x}' for r,x in list(zip(role,c))[-8:]) + return student,tutor,full,local,last + +def numvec(df): + d=normalize_roles(df).reset_index(drop=True); role=d.role_repaired.astype(str).to_numpy(); c=d.content.fillna('').astype(str).tolist(); n=max(1,len(c)); stu=[x for r,x in zip(role,c) if r=='student']; tut=[x for r,x in zip(role,c) if r=='tutor'] + lens=np.array([len(x) for x in stu],float) if stu else np.zeros(1); words=np.array([len(x.split()) for x in stu],float) if stu else np.zeros(1) + return np.array([len(c),len(stu),len(tut),lens.mean(),lens.max(),words.mean(),np.mean([bool(MATH.search(x)) for x in stu]) if stu else 0,np.mean(d.role_changed),len(stu)/n,len(tut)/n],float) + +def sparse_oof(P,X,y,g): + q=np.zeros(len(y)); folds=GroupKFold(min(4,len(np.unique(g)))) + for tr,va in folds.split(np.zeros(len(y)),y,g): + m=LogisticRegression(C=.08,max_iter=180,solver='liblinear',random_state=SEED).fit(hstack([csr_matrix(logit(P[tr])[:,None]),X[tr]],format='csr'),y[tr]); q[va]=m.predict_proba(hstack([csr_matrix(logit(P[va])[:,None]),X[va]],format='csr'))[:,1] + return np.clip(q,EPS,1-EPS) +def dense_oof(P,X,y,g): + q=np.zeros(len(y)); folds=GroupKFold(min(4,len(np.unique(g)))) + for tr,va in folds.split(X,y,g): + sc=StandardScaler().fit(X[tr]); m=LogisticRegression(C=.15,max_iter=180,solver='liblinear',random_state=SEED).fit(np.c_[logit(P[tr]),sc.transform(X[tr])],y[tr]); q[va]=m.predict_proba(np.c_[logit(P[va]),sc.transform(X[va])])[:,1] + return np.clip(q,EPS,1-EPS) +def main(a): + f=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f.columns),flush=True) + obj0=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); cand=np.where(np.array([hb(x,5)!=0 for x in obj0]))[0]; ix=np.array(sorted(cand,key=lambda i:h(f.response_id.iloc[i]))[:a.rows]); f=f.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy(); cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[];rz=[]; T={k:[] for k in ['STUDENT','TUTOR','FULL','LOCAL','LAST8']}; N=[] + for _,r in f.iterrows(): + d=cache[str(r.session_id)]; t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z); vals=texts(d,str(r.learning_objective)); + for k,v in zip(T,vals): T[k].append(v) + N.append(numvec(d)) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P=np.zeros(len(f));splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(f)),y,obj)) + for tr,va in splits:P[va],_=p97_predict(X75,Xr,y,tr,va,support) + base=ll(y,P);out={'rows':len(f),'objectives':len(np.unique(obj)),'v97':base,'tests':{}} + hv=HashingVectorizer(n_features=2**16,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + for k,txt in T.items(): + X=hv.transform(txt);q=sparse_oof(P,X,y,obj);out['tests'][k]={'ll':ll(y,q),'gain':base-ll(y,q)} + Xn=np.vstack(N);q=dense_oof(P,Xn,y,obj);out['tests']['STRUCTURE']={'ll':ll(y,q),'gain':base-ll(y,q)} + X=hstack([hv.transform(T['STUDENT']),hv.transform(T['TUTOR']),hv.transform(T['LOCAL']),csr_matrix(StandardScaler().fit_transform(Xn))],format='csr');q=sparse_oof(P,X,y,obj);out['tests']['COMBINED']={'ll':ll(y,q),'gain':base-ll(y,q)} + ds=[] + for o in np.unique(obj): + z=np.where(obj==o)[0];a0=z[y[z]==0];a1=z[y[z]==1] + if len(a0) and len(a1): + p1=np.sort(P[a1]);ds.extend(float(np.min(np.abs(p1-P[i]))) for i in a0) + out['tight_collisions']={'basis':len(ds),'median_dp':float(np.median(ds)) if ds else None,'p10_dp':float(np.quantile(ds,.1)) if ds else None} + gains={k:v['gain'] for k,v in out['tests'].items()};win=max(gains,key=gains.get);g=gains[win];out['decision']={'winner':win,'winner_gain':g,'verdict':'PHASE_CHANGE_CANDIDATE' if g>=.01 else 'ESCALATE_RAW_OBSERVABLE' if g>=.003 else 'RAW_TRANSCRIPT_NOT_SEPARATING','rule':'Escalate >=.003; phase-change candidate >=.010; otherwise audit non-text metadata/test-regime/applicability.'} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v112_fast_raw_observable_screen.json');main(p.parse_args()) diff --git a/competitions/trace_the_ace/v113_applicability_regime_fast.py b/competitions/trace_the_ace/v113_applicability_regime_fast.py new file mode 100644 index 00000000..e4fc1762 --- /dev/null +++ b/competitions/trace_the_ace/v113_applicability_regime_fast.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""V113 frozen fast pass: applicability + regime fingerprint. +Primary: can sample-local non-text/support topology explain when RELATED beats V75? +Frozen: deterministic 2500 rows, objective-grouped 4-fold OOF, fixed HistGB gate. +Thresholds: phase >=.010 (and all folds nonnegative) or >=.008 + >=15% oracle recovery; +escalate >=.003 with >=3/4 positive folds and placebo <25% real gain; structured .001-.003 only +if a family ablation removes >=.001; otherwise suppress metadata router. +No cross-test aggregates enter prediction. +""" +from __future__ import annotations +import argparse,json,hashlib +from pathlib import Path +import numpy as np +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.model_selection import GroupKFold +from sklearn.metrics import log_loss,roc_auc_score +from v71_mastery_events import load_transcript,normalize_roles +from v75_canonical_trajectory import load_training,SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import obj_family +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,p97_predict,ll +EPS=1e-5 + +def H(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def lossrow(y,p): return -(y*np.log(np.clip(p,EPS,1))+(1-y)*np.log(np.clip(1-p,EPS,1))) +def counts(train_vals, eval_vals): + u,n=np.unique(train_vals,return_counts=True);d=dict(zip(u,n));return np.array([d.get(x,0) for x in eval_vals],float) +def transcript_meta(d,obj): + z=normalize_roles(d).reset_index(drop=True); roles=z.role_repaired.astype(str).to_numpy(); n=max(1,len(z)); + seg,_=choose_target_segment(d,obj); ns=len(seg) + stu=float(np.sum(roles=='student')); tut=float(np.sum(roles=='tutor')) + # timestamp duration only when parseable; otherwise 0. + dur=0. + for c in ['timestamp','time','created_at']: + if c in z.columns: + try: + t=np.array(np.asarray(__import__('pandas').to_datetime(z[c],errors='coerce').astype('int64')),float); good=t>0 + if good.sum()>1: dur=float((t[good].max()-t[good].min())/1e9) + except Exception: pass + break + start=0. + if ns and len(seg): + try: start=float(seg.index.min())/n + except Exception: start=0. + return np.array([len(z),stu,tut,stu/n,tut/n,ns,ns/n,start,np.log1p(max(dur,0.))],float) +def geometry(p0,pr): + d=pr-p0 + return np.c_[p0,pr,d,np.abs(d),np.abs(p0-.5),np.abs(pr-.5),np.minimum(p0,pr),np.maximum(p0,pr)] +def fit_gate(X,ywin,sw,tr,va): + m=HistGradientBoostingClassifier(max_depth=2,max_iter=70,learning_rate=.05,min_samples_leaf=80,l2_regularization=2.,random_state=SEED) + m.fit(X[tr],ywin[tr],sample_weight=sw[tr]); return m.predict_proba(X[va])[:,1] +def route(p0,pr,g): + # fixed conservative interpolation; no sweep + w=np.clip(.65*g,0,.65); return np.clip((1-w)*p0+w*pr,EPS,1-EPS) +def run(a): + f0=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy(); cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0] + ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]); f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy(); fam=np.array([obj_family(x) for x in key]); sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[];rz=[];meta=[] + for _,r in f.iterrows(): + d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z);meta.append(transcript_meta(d,str(r.learning_objective))) + X75=build_v75(f,cache);Xr=build_control(rt,rz); P0=np.zeros(len(f));PR=np.zeros(len(f)); fold=np.full(len(f),-1,int) + splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + # experts once, true outer OOF + for k,(tr,va) in enumerate(splits): + P0[va],_=p97_predict(X75,Xr,y,tr,va,key); # returns V97, so fit experts explicitly below is unavailable + # recover endpoints with same base learner via helper's ingredients: fixed V97 endpoint reconstruction impossible from p97 alone. + # use imported logistic expert fitter locally. + from sklearn.linear_model import LogisticRegression + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]); mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + base=np.where(np.array([np.sum(key[np.setdiff1d(np.arange(len(y)),np.where(fold==fold[i])[0])]==key[i]) for i in range(len(y))])==0,.65*P0+.35*PR,P0) + base_ll=ll(y,base); oracle=np.where(lossrow(y,PR)0,fc>0,np.divide(ec,fc+1.)] + G=geometry(P0,PR); I=np.c_[np.log1p(np.array([H(x)%997 for x in f.response_id.astype(str)])),np.array([H(x)%31 for x in sess])] + D=np.abs(PR-P0)[:,None]; SX=np.c_[D*support[:,:3],(PR-P0)[:,None]*support[:,:3],D*(support[:,3:5])] + allx=np.c_[G,support,M,SX] + mats={'GEOMETRY':G,'SUPPORT':support,'SESSION':M,'SUPPORT_X_DISAGREEMENT':SX,'ALL_APPLICABILITY':allx,'ID_PLACEBO':I} + win=(lossrow(y,PR)0))} + real=[x for x in families if x!='ID_PLACEBO']; winner=max(real,key=lambda n:tests[n]['gain']); gain=tests[winner]['gain']; rec=(gain/gap if gap>0 else 0.); placebo=tests['ID_PLACEBO']['gain'] + # family ablation criterion: ALL minus best non-all as observable diagnostic + ablation=max([tests[x]['gain'] for x in ['GEOMETRY','SUPPORT','SESSION','SUPPORT_X_DISAGREEMENT']]) + all_gain=tests['ALL_APPLICABILITY']['gain']; removal=all_gain-ablation + phase=(gain>=.010 and min(tests[winner]['fold_gains'])>=0) or (gain>=.008 and rec>=.15) + escalate=(gain>=.003 and tests[winner]['positive_folds']>=3 and placebo < .25*gain) + structured=(.001<=gain<.003 and abs(removal)>=.001) + verdict='PHASE_CHANGE_APPLICABILITY' if phase else 'ESCALATE_APPLICABILITY' if escalate else 'STRUCTURED_HINT' if structured else 'SUPPRESS_METADATA_ROUTER' + out={'rows':len(y),'objectives':len(np.unique(obj)),'v97':base_ll,'row_endpoint_oracle':oracle_ll,'oracle_gap':gap,'tests':tests,'winner':winner,'winner_gain':gain,'oracle_gap_recovered_fraction':rec,'all_vs_best_family_delta':removal,'decision':verdict,'precommit':{'phase':'>=.010 and all folds nonnegative OR >=.008 and >=15% oracle recovery','escalate':'>=.003, >=3/4 folds positive, placebo <25% real gain','structured':'.001-.003 only if family ablation >=.001','otherwise':'suppress metadata router'}} + # Optional label-free real-test fingerprint when test_features exists. + out['regime_fingerprint']={'status':'UNAVAILABLE_NO_TEST_FEATURES'} + if a.test_features and a.test_features.exists(): + te=__import__('pandas').read_csv(a.test_features); trkey=f0.learning_objective.astype(str).to_numpy(); trfam=np.array([obj_family(x) for x in trkey]); tek=te.learning_objective.astype(str).to_numpy(); tef=np.array([obj_family(x) for x in tek]) + ec=counts(trkey,tek);fc=counts(trfam,tef);out['regime_fingerprint']={'status':'AVAILABLE','rows':len(te),'exact_seen_rate':float(np.mean(ec>0)),'family_seen_rate':float(np.mean(fc>0)),'median_exact_support':float(np.median(ec)),'median_family_support':float(np.median(fc))} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--test-features',type=Path,default=None);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v113_applicability_regime_fast.json');run(p.parse_args()) diff --git a/competitions/trace_the_ace/v114_representation_applicability.py b/competitions/trace_the_ace/v114_representation_applicability.py new file mode 100644 index 00000000..c57707a3 --- /dev/null +++ b/competitions/trace_the_ace/v114_representation_applicability.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""V114 REPRESENTATION -> APPLICABILITY intervention. + +Question left by V112/V113: + V112: raw transcript views did not improve direct label prediction. + V113: geometry/support/session metadata did not recover the endpoint-oracle gap. + +V114 asks the missing cross: can richer row-level representation predict WHICH already-capable +endpoint (V75 or RELATED) should apply? This is an applicability target, not another label model. + +Frozen protocol: +- deterministic 2500-row sample (same hash rule as V112/V113) +- objective-grouped 4-fold outer OOF +- endpoints trained only on outer-train rows +- oracle-choice target formed per row from endpoint losses, used only inside outer-train for gate fit +- fixed conservative routing weight 0.65; no hyperparameter sweep +- families: geometry, objective semantics, raw transcript, objective+raw, full representation +- controls: response/session ID placebo; shuffled applicability target; flipped-route ablation + +Decision thresholds (precommitted before result): +- PHASE_CHANGE_REPRESENTATION: gain >= .010 and all folds nonnegative, OR gain >= .008 and >=15% oracle-gap recovery +- REPRESENTATION_REPAIR_FOUND: gain >= .003, >=3/4 positive folds, controls <25% real gain, flipped route <=0 +- STRUCTURED_REPRESENTATION_HINT: .001 <= gain < .003 and best family beats geometry by >=.001 +- otherwise REPRESENTATION_NOT_OBSERVED +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack, csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.ensemble import HistGradientBoostingClassifier +from v71_mastery_events import load_transcript, normalize_roles +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +from v110_residual_collider_state_discovery import hb, ll + +EPS=1e-5 + +def H(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def lossrow(y,p): + p=np.clip(p,EPS,1-EPS) + return -(y*np.log(p)+(1-y)*np.log(1-p)) +def geometry(p0,pr): + d=pr-p0 + return np.c_[p0,pr,d,np.abs(d),np.abs(p0-.5),np.abs(pr-.5),np.minimum(p0,pr),np.maximum(p0,pr)] +def transcript_views(df,obj): + d=normalize_roles(df).reset_index(drop=True) + roles=d.role_repaired.astype(str).tolist(); c=d.content.fillna('').astype(str).tolist() + stu=' '.join(x for r,x in zip(roles,c) if r=='student') + tut=' '.join(x for r,x in zip(roles,c) if r=='tutor') + full=' '.join(f'[{r}] {x}' for r,x in zip(roles,c)) + seg,_=choose_target_segment(df,obj); s=normalize_roles(seg).reset_index(drop=True) + local=' '.join(f'[{r}] {x}' for r,x in zip(s.role_repaired.astype(str),s.content.fillna('').astype(str))) + last=' '.join(f'[{r}] {x}' for r,x in list(zip(roles,c))[-8:]) + return stu,tut,full,local,last + +def route(p0,pr,g,flip=False): + if flip: g=1-g + w=np.clip(.65*g,0,.65) + return np.clip((1-w)*p0+w*pr,EPS,1-EPS) +def fit_dense_gate(X,win,sw,tr,va): + m=HistGradientBoostingClassifier(max_depth=2,max_iter=70,learning_rate=.05,min_samples_leaf=80,l2_regularization=2.,random_state=SEED) + m.fit(X[tr],win[tr],sample_weight=sw[tr]) + return m.predict_proba(X[va])[:,1] +def fit_sparse_gate(X,win,sw,tr,va,shuffle=False): + yt=win[tr].copy() + if shuffle: + rng=np.random.default_rng(SEED+len(tr)+len(va)); yt=yt[rng.permutation(len(yt))] + # geometry is already concatenated into X; fixed regularization, no sweep + m=LogisticRegression(C=.08,max_iter=220,solver='liblinear',random_state=SEED) + m.fit(X[tr],yt,sample_weight=sw[tr]) + return m.predict_proba(X[va])[:,1] +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True) + print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy() + cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0] + ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]) + f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + key=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[]; rz=[]; T={k:[] for k in ['STUDENT','TUTOR','FULL','LOCAL','LAST8']} + for _,r in f.iterrows(): + d=cache[str(r.session_id)] + t,z=segmented_control(d,str(r.learning_objective),'related'); rt.append(t); rz.append(z) + vals=transcript_views(d,str(r.learning_objective)) + for k,v in zip(T,vals): T[k].append(v) + X75=build_v75(f,cache); Xr=build_control(rt,rz) + P0=np.zeros(len(f)); PR=np.zeros(len(f)); fold=np.full(len(f),-1,int) + splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS) + PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS); fold[va]=k + # exact-support V97 reconstruction, same rule as V113 + base=np.zeros(len(y)) + allidx=np.arange(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]] + base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS) + base_ll=ll(y,base) + L0=lossrow(y,P0); LR=lossrow(y,PR); win=(LR0))} + shgain=float(base_ll-ll(y,shuffled)) + real=['OBJECTIVE_SEMANTICS','RAW_TRANSCRIPT','OBJECTIVE_X_RAW','FULL_REPRESENTATION'] + winner=max(real,key=lambda n:tests[n]['gain']); gain=tests[winner]['gain']; rec=gain/gap if gap>0 else 0. + flipped=route(P0,PR,gate_keep[winner],flip=True); flipped_gain=float(base_ll-ll(y,flipped)) + geometry_gain=tests['GEOMETRY']['gain']; idgain=tests['ID_PLACEBO']['gain']; control=max(idgain,shgain) + phase=(gain>=.010 and min(tests[winner]['fold_gains'])>=0) or (gain>=.008 and rec>=.15) + found=(gain>=.003 and tests[winner]['positive_folds']>=3 and control<.25*gain and flipped_gain<=0) + hint=(.001<=gain<.003 and gain-geometry_gain>=.001) + verdict='PHASE_CHANGE_REPRESENTATION' if phase else 'REPRESENTATION_REPAIR_FOUND' if found else 'STRUCTURED_REPRESENTATION_HINT' if hint else 'REPRESENTATION_NOT_OBSERVED' + out={ + 'rows':len(y),'objectives':len(np.unique(obj)),'v97':base_ll,'row_endpoint_oracle':oracle_ll,'oracle_gap':gap, + 'oracle_related_win_rate':float(np.mean(win)),'tests':tests,'winner':winner,'winner_gain':gain, + 'oracle_gap_recovered_fraction':rec,'controls':{'shuffled_applicability_gain':shgain,'id_placebo_gain':idgain,'flipped_winner_route_gain':flipped_gain}, + 'representation_increment_over_geometry':float(gain-geometry_gain),'decision':verdict, + 'precommit':{ + 'phase':'gain >=.010 and all folds nonnegative OR gain >=.008 and >=15% oracle recovery', + 'repair_found':'gain >=.003, >=3/4 positive folds, controls <25% real gain, flipped route <=0', + 'structured':'.001-.003 and representation beats geometry by >=.001', + 'otherwise':'representation not observed' + } + } + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--rows',type=int,default=2500); p.add_argument('--out',default='v114_representation_applicability.json'); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v115_collision_resolution_knn.py b/competitions/trace_the_ace/v115_collision_resolution_knn.py new file mode 100644 index 00000000..c9161b61 --- /dev/null +++ b/competitions/trace_the_ace/v115_collision_resolution_knn.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""V115 COLLISION-RESOLUTION AUDIT. +Orthogonal to V114's learned gate: use fixed kNN on held-out objectives to test whether widening +representation makes endpoint applicability locally identifiable. + +Frozen: same 2500 rows, same endpoint oracle, GroupKFold by objective, k=15, no sweep. +Families compare geometry-only against objective semantics, raw transcript, and their union. +A real representation repair should improve routing, increase weighted neighbor agreement, survive +shuffled-target control, and fail under flipped routing. +""" +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack,csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import NearestNeighbors +from sklearn.preprocessing import StandardScaler,normalize +from v75_canonical_trajectory import load_training,SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,ll +from v114_representation_applicability import H,lossrow,geometry,transcript_views,route,EPS + +def knn_gate_dense(X,win,sw,tr,va,k=15,shuffle=False): + sc=StandardScaler().fit(X[tr]); A=sc.transform(X[tr]); B=sc.transform(X[va]) + nn=NearestNeighbors(n_neighbors=min(k,len(tr)),metric='euclidean').fit(A); d,ix=nn.kneighbors(B) + yt=win[tr].copy() + if shuffle: + rng=np.random.default_rng(SEED+17+len(tr)); yt=yt[rng.permutation(len(yt))] + wt=sw[tr][ix]/(d+0.25); return np.sum(wt*yt[ix],axis=1)/np.sum(wt,axis=1) +def knn_gate_sparse(X,win,sw,tr,va,k=15,shuffle=False): + A=normalize(X[tr]); B=normalize(X[va]); nn=NearestNeighbors(n_neighbors=min(k,len(tr)),metric='cosine',algorithm='brute').fit(A); d,ix=nn.kneighbors(B) + yt=win[tr].copy() + if shuffle: + rng=np.random.default_rng(SEED+19+len(tr)); yt=yt[rng.permutation(len(yt))] + sim=np.maximum(1-d,0.01); wt=sw[tr][ix]*sim; return np.sum(wt*yt[ix],axis=1)/np.sum(wt,axis=1) +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy(); cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0] + ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]); f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}; rt=[];rz=[];T={k:[] for k in ['STUDENT','TUTOR','FULL','LOCAL','LAST8']} + for _,r in f.iterrows(): + d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z);vals=transcript_views(d,str(r.learning_objective)) + for k,v in zip(T,vals):T[k].append(v) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P0=np.zeros(len(f));PR=np.zeros(len(f));fold=np.full(len(f),-1,int);splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]);mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]);P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + allidx=np.arange(len(y));base=np.zeros(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]];base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS);base_ll=ll(y,base);L0=lossrow(y,P0);LR=lossrow(y,PR);win=(LR=.5)==win).astype(float) + tests[n]={'ll':float(ll(y,q)),'gain':float(base_ll-ll(y,q)),'fold_gains':fg,'positive_folds':int(np.sum(np.array(fg)>0)),'oracle_choice_accuracy':float(np.mean(correct)),'confidence_weighted_agreement':float(np.sum(conf*correct)/(np.sum(conf)+1e-12))} + real=['OBJECTIVE_SEMANTICS','RAW_TRANSCRIPT','OBJECTIVE_X_RAW'];winner=max(real,key=lambda n:tests[n]['gain']);gain=tests[winner]['gain'];rec=gain/gap if gap>0 else 0.;shgain=float(base_ll-ll(y,shuffled));flipped=route(P0,PR,gates[winner],flip=True);flipgain=float(base_ll-ll(y,flipped));geom=tests['GEOMETRY']['gain'] + phase=(gain>=.010 and min(tests[winner]['fold_gains'])>=0) or (gain>=.008 and rec>=.15);found=(gain>=.003 and tests[winner]['positive_folds']>=3 and shgain<.25*gain and flipgain<=0);hint=(.001<=gain<.003 and gain-geom>=.001) + out={'rows':len(y),'objectives':len(np.unique(obj)),'v97':base_ll,'row_endpoint_oracle':oracle_ll,'oracle_gap':gap,'tests':tests,'winner':winner,'winner_gain':gain,'oracle_gap_recovered_fraction':rec,'representation_increment_over_geometry':float(gain-geom),'controls':{'shuffled_applicability_gain':shgain,'flipped_winner_route_gain':flipgain},'decision':'PHASE_CHANGE_COLLISION_RESOLUTION' if phase else 'NONLINEAR_REPRESENTATION_REPAIR_FOUND' if found else 'STRUCTURED_COLLISION_HINT' if hint else 'COLLISIONS_NOT_RESOLVED','precommit':{'phase':'gain >=.010 all folds nonnegative OR >=.008 and >=15% oracle recovery','repair_found':'gain >=.003, >=3/4 folds positive, shuffled <25% gain, flipped <=0','structured':'.001-.003 and >=.001 over geometry','otherwise':'collisions not resolved'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v115_collision_resolution_knn.json');main(p.parse_args()) diff --git a/competitions/trace_the_ace/v115_reality_audit.py b/competitions/trace_the_ace/v115_reality_audit.py new file mode 100644 index 00000000..7aa4ef06 --- /dev/null +++ b/competitions/trace_the_ace/v115_reality_audit.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""V115 Reality Audit. + +Primary question: did we over-optimize objective-cold validation and underweight the +competition's natural new-session / same-objective regime? + +Frozen outputs: +- full 35,072-row 5-fold session-grouped OOF for pure V74, V75 and V97; +- pure V74 support/frequency stratification using fold-local objective support; +- per-fold gains and objective-frequency contribution to log loss; +- deterministic provider-proxy stratification from transcript structure only; +- objective-grouped stress results are reported as a secondary contrast, not a gate. + +No test labels, no cross-validation leakage, no cross-test aggregation. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript, normalize_roles +from v74_semantic_objective_prior import semantic_prior_predict +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS=1e-5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) + +def provider_proxy(df): + d=normalize_roles(df).reset_index(drop=True) + roles=d.role_repaired.astype(str).str.lower().to_numpy() + txt=d.content.fillna('').astype(str).tolist() + n=len(d); tut=int(np.sum(roles=='tutor')) + mean_words=float(np.mean([len(x.split()) for x in txt])) if txt else 0.0 + markers=sum(bool(re.search(r'\b(?:learning objective|learning goal|prior learning|i do|we do|you do|application|slide|lesson)\b',x,re.I)) for x in txt) + tsl_like=(n>=24) or (markers>=2) or (tut>=12 and mean_words>=8) + return 'TSL_LIKE' if tsl_like else 'EEDI_LIKE' + +def fit_logit(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + +def p97_endpoint(X75,Xr,y,key,tr,va): + p75=fit_logit(X75,y,tr,va); pr=fit_logit(Xr,y,tr,va) + vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)) + seen=np.array([d.get(x,0)>0 for x in key[va]]) + p=np.where(seen,p75,.65*p75+.35*pr) + return np.clip(p,EPS,1-EPS),p75,pr,np.array([d.get(x,0) for x in key[va]],float) + +def eval_session(frame,cache): + y=frame.target.to_numpy(int); key=frame.learning_objective.astype(str).to_numpy(); sess=frame.session_id.astype(str).to_numpy() + X75=build_v75(frame,cache) + rt=[];rz=[] + for _,r in frame.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t);rz.append(z) + Xr=build_control(rt,rz) + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,sess)) + p74=np.zeros(len(y));p75=np.zeros(len(y));p97=np.zeros(len(y));support=np.zeros(len(y));folds=[] + for k,(tr,va) in enumerate(splits): + ph,_=semantic_prior_predict(frame.iloc[tr],frame.iloc[va],k=16,smooth=2.0);p74[va]=ph + q,q75,qr,s=p97_endpoint(X75,Xr,y,key,tr,va);p97[va]=q;p75[va]=q75;support[va]=s + folds.append({'fold':k+1,'rows':len(va),'v74':ll(y[va],p74[va]),'v75':ll(y[va],p75[va]),'v97':ll(y[va],p97[va])}) + bins=[('ZERO',lambda s:s==0),('1_2',lambda s:(s>=1)&(s<=2)),('3_9',lambda s:(s>=3)&(s<=9)),('10_29',lambda s:(s>=10)&(s<=29)),('30_PLUS',lambda s:s>=30)] + strat={};rowloss=-(y*np.log(np.clip(p74,EPS,1))+(1-y)*np.log(np.clip(1-p74,EPS,1)));total=float(rowloss.sum()) + for name,fn in bins: + m=fn(support) + if m.any(): strat[name]={'rows':int(m.sum()),'share':float(m.mean()),'mean_support':float(support[m].mean()),'v74_ll':ll(y[m],p74[m]),'v75_ll':ll(y[m],p75[m]),'v97_ll':ll(y[m],p97[m]),'v74_loss_share':float(rowloss[m].sum()/total)} + return {'v74':ll(y,p74),'v75':ll(y,p75),'v97':ll(y,p97),'folds':folds,'support_strata':strat},(p74,p75,p97,support) + +def objective_stress(frame): + y=frame.target.to_numpy(int); grp=(frame.learning_objective_id if 'learning_objective_id' in frame else frame.learning_objective).astype(str).to_numpy();p=np.zeros(len(y)) + for tr,va in GroupKFold(5).split(np.zeros(len(y)),y,grp): p[va],_=semantic_prior_predict(frame.iloc[tr],frame.iloc[va],k=16,smooth=2.0) + return ll(y,p) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('features columns',list(f.columns),flush=True);print('rows',len(f),'sessions',f.session_id.nunique(),'objectives',f.learning_objective.nunique(),flush=True) + cache={};proxy={} + for i,sid in enumerate(f.session_id.astype(str).unique()): + d=load_transcript(a.transcripts/f'{sid}.csv');cache[sid]=d;proxy[sid]=provider_proxy(d) + if (i+1)%5000==0: print('loaded sessions',i+1,flush=True) + session,(p74,p75,p97,support)=eval_session(f,cache) + y=f.target.to_numpy(int);reg=np.array([proxy[str(s)] for s in f.session_id.astype(str)]);regimes={} + for r in ['EEDI_LIKE','TSL_LIKE']: + m=reg==r + if m.any(): regimes[r]={'rows':int(m.sum()),'sessions':int(f.loc[m,'session_id'].nunique()),'share':float(m.mean()),'v74':ll(y[m],p74[m]),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'v74_gain_vs_v75':ll(y[m],p75[m])-ll(y[m],p74[m])} + objstress=objective_stress(f);delta=session['v75']-session['v74'];supported=float(np.mean(support>0)) + verdict='PRIORITIZE_PURE_V74_RUNTIME' if delta>=.010 and supported>=.60 else ('V74_REAL_BUT_MIXED' if delta>=.003 else 'V74_NOT_PRIMARY') + out={'diagnostics':{'rows':len(f),'sessions':int(f.session_id.nunique()),'objectives':int(f.learning_objective.nunique()),'positive_rate':float(y.mean()),'session_exact_objective_support_rate':supported,'provider_proxy_is_heuristic':True},'session_cold':session,'objective_cold_v74_stress':objstress,'provider_proxy':regimes,'decision':{'verdict':verdict,'v74_gain_vs_v75_session':delta,'rule':'PRIORITIZE pure V74 if session-cold gain vs V75 >=.010 and >=60% validation rows have fold-local exact-objective support; objective-cold is secondary stress only.'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',default='v115_reality_audit.json');run(p.parse_args()) diff --git a/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py new file mode 100644 index 00000000..dba1706c --- /dev/null +++ b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v74_semantic_objective_prior import load_training, semantic_prior_predict +EPS=1e-5 +# trigger: pure V74 reality audit v2 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('columns',list(f.columns),flush=True) + y=f.target.to_numpy(int); sess=f.session_id.astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy() + grp=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + p=np.zeros(len(f)); support=np.zeros(len(f)); folds=[] + for k,(tr,va) in enumerate(GroupKFold(5).split(np.zeros(len(y)),y,sess),1): + ph,_=semantic_prior_predict(f.iloc[tr],f.iloc[va],k=16,smooth=2.0);p[va]=ph + vals,cts=np.unique(key[tr],return_counts=True);d=dict(zip(vals,cts));support[va]=np.array([d.get(x,0) for x in key[va]],float) + folds.append({'fold':k,'rows':len(va),'v74_ll':ll(y[va],ph),'support_rate':float(np.mean(support[va]>0))}) + session_ll=ll(y,p); support_rate=float(np.mean(support>0)) + bins=[('ZERO',support==0),('1_2',(support>=1)&(support<=2)),('3_9',(support>=3)&(support<=9)),('10_29',(support>=10)&(support<=29)),('30_PLUS',support>=30)] + strat={} + rowloss=-(y*np.log(np.clip(p,EPS,1))+(1-y)*np.log(np.clip(1-p,EPS,1)));tot=rowloss.sum() + for n,m in bins: + if m.any(): strat[n]={'rows':int(m.sum()),'share':float(m.mean()),'v74_ll':ll(y[m],p[m]),'loss_share':float(rowloss[m].sum()/tot),'mean_support':float(support[m].mean())} + po=np.zeros(len(f)) + for tr,va in GroupKFold(5).split(np.zeros(len(y)),y,grp): po[va],_=semantic_prior_predict(f.iloc[tr],f.iloc[va],k=16,smooth=2.0) + obj_ll=ll(y,po) + out={'rows':len(f),'sessions':int(f.session_id.nunique()),'objectives':int(f.learning_objective.nunique()),'positive_rate':float(y.mean()),'session_cold_v74':session_ll,'session_exact_objective_support_rate':support_rate,'session_folds':folds,'support_strata':strat,'objective_cold_v74_stress':obj_ll,'session_minus_objective_advantage':obj_ll-session_ll,'decision':{'verdict':'V74_SESSION_GEOMETRY_CONFIRMED' if session_ll<=.56 and support_rate>=.60 else 'V74_SESSION_GEOMETRY_NOT_CONFIRMED','rule':'Confirm if session-cold V74 <=.560 and fold-local exact objective support >=60%. Runtime promotion still requires comparison to verified incumbent/public test.'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--out',default='v115b_pure_v74_reality_audit.json');run(p.parse_args()) diff --git a/competitions/trace_the_ace/v116_row_alignment_alias_audit.py b/competitions/trace_the_ace/v116_row_alignment_alias_audit.py new file mode 100644 index 00000000..8a104326 --- /dev/null +++ b/competitions/trace_the_ace/v116_row_alignment_alias_audit.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""V116 ROW-ALIGNMENT / REPRESENTATION-ALIAS AUDIT. + +After V112/V113/V114/V115 fail to recover the large endpoint-oracle gap, test the upstream +hypothesis: distinct labelled responses may be mapped to the same session/objective state before +prediction. If identical representations contain different labels or different oracle endpoint +choices, no downstream router can resolve them without a new row-level alignment/state variable. + +This is an aggregate diagnostic only: no raw transcript content is emitted. +""" +from __future__ import annotations +import argparse,hashlib,json +from pathlib import Path +from collections import defaultdict +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from v75_canonical_trajectory import load_training,SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,ll +from v114_representation_applicability import H,lossrow,EPS + +def dense_row(X,i): + r=X[i] + if hasattr(r,'toarray'): return np.asarray(r.toarray()).ravel() + return np.asarray(r).ravel() +def rep_hash(x): + a=np.asarray(x,dtype=np.float64);a=np.nan_to_num(a,nan=0.,posinf=1e30,neginf=-1e30);a=np.round(a,10) + return hashlib.sha256(a.tobytes()).hexdigest() +def summarize_groups(groups,y,win,Lbase,Loracle): + collision=[]; mixed_y=[];mixed_w=[];gap=0.;rows=0 + for z in groups.values(): + if len(z)>1: + collision.extend(z);rows+=len(z) + yy=y[z];ww=win[z] + if len(np.unique(yy))>1:mixed_y.extend(z) + if len(np.unique(ww))>1: + mixed_w.extend(z);gap+=float(np.sum(Lbase[z]-Loracle[z])) + return {'groups_total':len(groups),'collision_groups':int(sum(len(z)>1 for z in groups.values())),'rows_in_collision_groups':len(set(collision)),'mixed_label_groups':int(sum(len(z)>1 and len(np.unique(y[z]))>1 for z in groups.values())),'rows_in_mixed_label_groups':len(set(mixed_y)),'mixed_oracle_choice_groups':int(sum(len(z)>1 and len(np.unique(win[z]))>1 for z in groups.values())),'rows_in_mixed_oracle_choice_groups':len(set(mixed_w)),'oracle_gap_sum_in_mixed_choice_groups':gap} +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True);print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy();cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0];ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]);f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int);obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy();key=f.learning_objective.astype(str).to_numpy();sess=f.session_id.astype(str).to_numpy();rid=f.response_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)};rt=[];rz=[] + for _,r in f.iterrows():d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P0=np.zeros(len(f));PR=np.zeros(len(f));fold=np.full(len(f),-1,int);splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]);mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]);P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + allidx=np.arange(len(y));base=np.zeros(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]];base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS);L0=lossrow(y,P0);LR=lossrow(y,PR);win=(LR1: + p=float(np.clip(np.mean(y[z]),EPS,1-EPS));const_nll+=float(np.sum(lossrow(y[z],np.full(len(z),p))));const_rows+=len(z) + out={'rows':len(y),'objectives':len(np.unique(obj)),'sessions':len(np.unique(sess)),'v97':float(ll(y,base)),'row_endpoint_oracle':float(ll(y,oracle)),'oracle_gap':float(ll(y,base)-ll(y,oracle)),'oracle_related_win_rate':float(np.mean(win)),'session_objective_aliases':summarize_groups(groups_so,y,win,Lbase,Loracle),'exact_feature_representation_aliases':summarize_groups(groups_rep,y,win,Lbase,Loracle),'endpoint_prediction_aliases':summarize_groups(groups_endpoint,y,win,Lbase,Loracle),'aggregate_alignment_evidence':{'transcript_columns':sorted(headers.keys()),'response_ids_found_exactly_anywhere':len(found),'response_id_anchor_rate':float(len(found)/len(rid)),'anchor_columns':{c:len(v) for c,v in found_cols.items()},'constant_within_rep_collision_empirical_nll':float(const_nll/const_rows) if const_rows else None,'constant_collision_rows':const_rows}} + so=out['session_objective_aliases'];rp=out['exact_feature_representation_aliases'];anchor=out['aggregate_alignment_evidence']['response_id_anchor_rate'] + if rp['mixed_oracle_choice_groups']>0 and rp['rows_in_mixed_oracle_choice_groups']>=.05*len(y):verdict='REPRESENTATION_ALIAS_CONFIRMED' + elif so['mixed_oracle_choice_groups']>0 and anchor<.5:verdict='ROW_ALIGNMENT_STATE_CANDIDATE' + else:verdict='NO_STRONG_ROW_ALIAS_EVIDENCE' + out['decision']=verdict;out['interpretation_rule']='Alias confirmed if exact X75+Xr collisions with mixed oracle choice cover >=5% rows; row-alignment candidate if session/objective aliases mix oracle choice and <50% response IDs anchor exactly in transcript.' + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v116_row_alignment_alias_audit.json');main(p.parse_args()) diff --git a/competitions/trace_the_ace/v117_oracle_information_audit.py b/competitions/trace_the_ace/v117_oracle_information_audit.py new file mode 100644 index 00000000..72a75476 --- /dev/null +++ b/competitions/trace_the_ace/v117_oracle_information_audit.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""V117 ORACLE INFORMATION AUDIT. + +Tests whether the large per-row V75-vs-RELATED endpoint oracle gap is evidence of a latent +applicability regime or simply the value of revealing the realized label. For binary log loss, +if two endpoint probabilities differ, the lower-loss endpoint is determined by the label and the +sign of (PR-P0). This audit quantifies that identity on the frozen V113 sample and compares it with +strictly label-free/cross-fitted endpoint selection. + +Frozen protocol: same deterministic 2500 rows, same objective-grouped 4-fold endpoint fits, +no result-dependent tuning. Meta selectors are trained only on outer-train rows. +""" +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.model_selection import GroupKFold +from v75_canonical_trajectory import load_training,SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,ll +from v114_representation_applicability import H,lossrow,geometry,route,EPS + +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True);print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy();cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0];ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]);f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int);obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy();key=f.learning_objective.astype(str).to_numpy();sess=f.session_id.astype(str).to_numpy();cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)};rt=[];rz=[] + for _,r in f.iterrows():d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P0=np.zeros(len(f));PR=np.zeros(len(f));fold=np.full(len(f),-1,int);splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]);mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]);P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + allidx=np.arange(len(y));base=np.zeros(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]];base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS);L0=lossrow(y,P0);LR=lossrow(y,PR);win=(LR0,d<0).astype(int);mask=~ties;identity=float(np.mean(win[mask]==clairvoyant[mask])) if np.any(mask) else 1. + # Counterfactual label flip: lower-loss endpoint must flip whenever endpoint probabilities differ. + yf=1-y;wf=(lossrow(yf,PR)0 else 0. + out={'rows':len(y),'objectives':len(np.unique(obj)),'endpoint_disagreement_rate':float(np.mean(mask)),'oracle_related_win_rate':float(np.mean(win)),'clairvoyant_choice_identity_rate':identity,'choice_flip_under_label_counterfactual_rate':flip_rate,'v97':float(base_ll),'row_label_oracle':float(oracle_ll),'oracle_gap':float(gap),'label_free_controls':endpoint_ll,'label_free_gains_vs_v97':gains,'best_label_free':best_label_free,'best_label_free_gain':best_gain,'best_label_free_oracle_gap_recovered_fraction':recovered} + out['decision']='ROW_ORACLE_IS_REALIZED_LABEL_INFORMATION' if identity>=.999 and flip_rate>=.999 else 'ORACLE_HAS_NONTRIVIAL_APPLICABILITY_STRUCTURE' + out['interpretation_rule']='If oracle choice matches label+endpoint-order identity and flips under counterfactual label >=99.9%, the row oracle is clairvoyant realized-outcome information; its raw gap must not be treated as recoverable capability without a separately validated label-free selector.' + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v117_oracle_information_audit.json');main(p.parse_args()) diff --git a/competitions/trace_the_ace/v118_hidden_test_geometry_probe.py b/competitions/trace_the_ace/v118_hidden_test_geometry_probe.py new file mode 100644 index 00000000..dbe2fcd1 --- /dev/null +++ b/competitions/trace_the_ace/v118_hidden_test_geometry_probe.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""V118 hidden test geometry probe. + +Frozen historical probe table supplied by actual DrivenData submissions on 2026-08-18. +Question: does smoke rank candidates like public, and what does V74 smoke imply? +No competition labels are used; only user-observed submission scores. +""" +import json +from pathlib import Path +import numpy as np +from scipy.stats import spearmanr, pearsonr + +PROBES = [ + ("V37", 0.4801, 0.7384), + ("V48", 0.5052, 0.6179), + ("V54", 0.5192, 0.6329), + ("V57", 0.5216, 0.6211), + ("V63", 0.4880, 0.6284), + ("V75", 0.4693, 0.6047), + ("V97", 0.4693, 0.6044), +] +V74_SMOKE = 0.5181 +V74_SESSION = 0.5511484894117864 +V74_OBJECTIVE = 0.6013242442331039 + +def metrics(rows): + s=np.array([r[1] for r in rows],float); p=np.array([r[2] for r in rows],float) + sp=spearmanr(s,p); pe=pearsonr(s,p) + b,a=np.polyfit(s,p,1) + pred=float(a+b*V74_SMOKE) + return {"n":len(rows),"spearman":float(sp.statistic),"spearman_p":float(sp.pvalue),"pearson":float(pe.statistic),"pearson_p":float(pe.pvalue),"ols_intercept":float(a),"ols_slope":float(b),"v74_public_projection":pred} + +def main(): + allm=metrics(PROBES) + loo=[] + for i,r in enumerate(PROBES): + m=metrics(PROBES[:i]+PROBES[i+1:]); loo.append({"excluded":r[0],**m}) + best=max(loo,key=lambda x:x["spearman"]) + nonout=[r for r in PROBES if r[0]!=best["excluded"]] + robust=metrics(nonout) + # Ranking only: lower score is better. + smoke_order=[r[0] for r in sorted(PROBES,key=lambda r:r[1])] + public_order=[r[0] for r in sorted(PROBES,key=lambda r:r[2])] + out={ + "probe_table":[{"model":m,"smoke":s,"public":p} for m,s,p in PROBES], + "v74":{"smoke":V74_SMOKE,"session_cold":V74_SESSION,"objective_cold":V74_OBJECTIVE}, + "all_probes":allm, + "leave_one_out":loo, + "most_influential_outlier":best["excluded"], + "robust_without_outlier":robust, + "smoke_rank":smoke_order, + "public_rank":public_order, + } + # Precommit: smoke is not an adequate optimization target if rank rho < .8 or if one probe changes rho by >= .2. + influence=best["spearman"]-allm["spearman"] + if allm["spearman"]<0.8 or influence>=0.2: + verdict="SMOKE_NOT_PUBLIC_PROXY" + else: + verdict="SMOKE_USABLE_PUBLIC_PROXY" + # A V74 full submission is suppressed if robust smoke->public projection is >= current V97 public 0.6044. + v74_decision="SUPPRESS_V74_FULL" if robust["v74_public_projection"]>=0.6044 else "V74_FULL_STILL_PLAUSIBLE" + out["decision"]={ + "verdict":verdict, + "outlier_influence_on_spearman":float(influence), + "v74_decision":v74_decision, + "rule":"Smoke proxy requires Spearman >=0.8 and no single-probe rho improvement >=0.2. Suppress V74 full if robust smoke->public projection is not better than V97 public 0.6044.", + "next":"Infer public-aligned validation geometry from historical candidate response vectors; do not optimize smoke directly." if verdict=="SMOKE_NOT_PUBLIC_PROXY" else "Use smoke as a secondary ranking probe." + } + Path("v118_hidden_test_geometry_probe.json").write_text(json.dumps(out,indent=2)) + print(json.dumps(out,indent=2)) +if __name__=="__main__": main() diff --git a/competitions/trace_the_ace/v119_public_anchor_geometry.py b/competitions/trace_the_ace/v119_public_anchor_geometry.py new file mode 100644 index 00000000..15d4ddcf --- /dev/null +++ b/competitions/trace_the_ace/v119_public_anchor_geometry.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# trigger +"""V119 public-anchor geometry search. + +Recoverable-probe experiment. Historical V37/V48/V54/V57/V63 packages are not +available, so this does NOT claim a seven-model public reconstruction. + +Question: which lawful validation strata reproduce the one directly executable +public anchor: V97 slightly beats V75 publicly (0.6044 vs 0.6047), while their +smoke scores tie (0.4693 vs 0.4693)? + +Freeze: +- deterministic 6000-row session sample; +- 5-fold session-grouped OOF; +- V75 and V97 endpoints exactly as current repo lineage; +- cells from support x provider-proxy x session-length x objective-frequency; +- promote a cell only if V97 beats V75 in >=4/5 folds and aggregate delta is + within 0.0005 of the public delta (-0.0003), with >=150 rows. + +No leaderboard labels are used for fitting predictions; public scores are used +only as an external geometry target after OOF predictions are frozen. +""" +from __future__ import annotations +import argparse,json,hashlib,re +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript,normalize_roles +from v75_canonical_trajectory import load_training,SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control + +EPS=1e-5; TARGET=-0.0003 + +def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) +def provider(df): + d=normalize_roles(df).reset_index(drop=True); roles=d.role_repaired.astype(str).str.lower().to_numpy(); txt=d.content.fillna('').astype(str).tolist(); n=len(d); tut=int(np.sum(roles=='tutor')); mw=float(np.mean([len(x.split()) for x in txt])) if txt else 0.; markers=sum(bool(re.search(r'\b(?:learning objective|learning goal|prior learning|i do|we do|you do|application|slide|lesson)\b',x,re.I)) for x in txt); return 'TSL' if (n>=24 or markers>=2 or (tut>=12 and mw>=8)) else 'EEDI' +def fit(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=250,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]); return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def bin_support(x): + return 'S0' if x==0 else 'S1_9' if x<10 else 'S10_29' if x<30 else 'S30P' +def qbin(x,cuts,prefix): return prefix+str(int(np.searchsorted(cuts,x,side='right'))) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f.columns),flush=True) + sessions=sorted(f.session_id.astype(str).unique(),key=hh); take=set(sessions[:min(a.sessions,len(sessions))]); f=f[f.session_id.astype(str).isin(take)].reset_index(drop=True) + y=f.target.to_numpy(int); sess=f.session_id.astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}; meta={s:(provider(cache[s]),len(cache[s])) for s in cache} + X75=build_v75(f,cache); rt=[];rz=[] + for _,r in f.iterrows(): t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z) + Xr=build_control(rt,rz); p75=np.zeros(len(f));p97=np.zeros(len(f));support=np.zeros(len(f));foldid=np.zeros(len(f),int) + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,sess)) + for k,(tr,va) in enumerate(splits): + q75=fit(X75,y,tr,va); qr=fit(Xr,y,tr,va); vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)); s=np.array([d.get(x,0) for x in key[va]],float); seen=s>0; q97=np.where(seen,q75,.65*q75+.35*qr); p75[va]=q75;p97[va]=q97;support[va]=s;foldid[va]=k + global_counts=dict(zip(*np.unique(key,return_counts=True))); olen=np.array([global_counts[x] for x in key],float); slen=np.array([meta[s][1] for s in sess],float); prov=np.array([meta[s][0] for s in sess]) + slcuts=np.quantile(slen,[.25,.5,.75]); ocuts=np.quantile(olen,[.25,.5,.75]); labels=[] + for i in range(len(f)): labels.append('|'.join([bin_support(support[i]),prov[i],qbin(slen[i],slcuts,'L'),qbin(olen[i],ocuts,'F')])) + labels=np.array(labels); rows=[] + for c in np.unique(labels): + m=labels==c + if m.sum()<150: continue + d=ll(y[m],p97[m])-ll(y[m],p75[m]); fg=[] + for k in range(5): + z=m&(foldid==k); fg.append(None if z.sum()<20 else ll(y[z],p97[z])-ll(y[z],p75[z])) + pos=sum(v is not None and v<0 for v in fg); err=abs(d-TARGET); rows.append({'cell':c,'rows':int(m.sum()),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'delta_v97_minus_v75':d,'target_error':err,'fold_deltas':fg,'v97_better_folds':pos,'qualified':bool(pos>=4 and err<=.0005)}) + rows.sort(key=lambda r:(not r['qualified'],r['target_error'],-r['rows'])) + overall={'v75':ll(y,p75),'v97':ll(y,p97),'delta':ll(y,p97)-ll(y,p75)}; q=[r for r in rows if r['qualified']]; verdict='PUBLIC_ANCHOR_CELL_FOUND' if q else 'CURRENT_SPLIT_GRAMMAR_NOT_PUBLIC_ALIGNED' + out={'rows':len(f),'sessions':len(np.unique(sess)),'public_anchor':{'v75':.6047,'v97':.6044,'target_delta':TARGET},'smoke_anchor':{'v75':.4693,'v97':.4693,'delta':0.0},'overall':overall,'top_cells':rows[:20],'decision':{'verdict':verdict,'qualified_cells':len(q),'rule':'Cell requires >=150 rows, V97 better in >=4/5 folds, and aggregate V97-V75 delta within 0.0005 of public -0.0003.','next':'Use qualified cell(s) as public-aligned validation basis only if stable; otherwise expand split grammar beyond support/provider/session-length/objective-frequency.'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--sessions',type=int,default=4000);p.add_argument('--out',default='v119_public_anchor_geometry.json');run(p.parse_args()) diff --git a/competitions/trace_the_ace/v120_objective_identity_audit.py b/competitions/trace_the_ace/v120_objective_identity_audit.py new file mode 100644 index 00000000..cfdacaee --- /dev/null +++ b/competitions/trace_the_ace/v120_objective_identity_audit.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""V120 objective identity audit. + +Tests whether learning_objective text and learning_objective_id define the same +identity relation. This is metadata-only and intentionally does not use labels +or transcripts. + +Primary questions: +1. Is id <-> text one-to-one on train? +2. Does exact support on the official test set differ when keyed by id vs text? +3. In session-held-out folds, how much does support geometry differ by key? +4. Are any discrepancies explainable by whitespace/case normalization only? +""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.model_selection import GroupKFold + + +def norm_text(x: object) -> str: + s = str(x) + s = s.strip().casefold() + s = re.sub(r"\s+", " ", s) + return s + + +def clean_id(x: object) -> str: + if pd.isna(x): + return "" + return str(x).strip() + + +def key_stats(df: pd.DataFrame) -> dict: + x = df[["learning_objective_id", "learning_objective"]].copy() + x["oid"] = x["learning_objective_id"].map(clean_id) + x["text"] = x["learning_objective"].astype(str) + x["norm"] = x["learning_objective"].map(norm_text) + + id_to_text = x.groupby("oid")["text"].nunique(dropna=False) + id_to_norm = x.groupby("oid")["norm"].nunique(dropna=False) + text_to_id = x.groupby("text")["oid"].nunique(dropna=False) + norm_to_id = x.groupby("norm")["oid"].nunique(dropna=False) + + pairs_exact = x[["oid", "text"]].drop_duplicates() + pairs_norm = x[["oid", "norm"]].drop_duplicates() + return { + "rows": int(len(x)), + "unique_ids": int(x.oid.nunique()), + "unique_texts": int(x.text.nunique()), + "unique_norm_texts": int(x.norm.nunique()), + "unique_id_text_pairs": int(len(pairs_exact)), + "unique_id_norm_pairs": int(len(pairs_norm)), + "ids_with_multiple_exact_texts": int((id_to_text > 1).sum()), + "ids_with_multiple_norm_texts": int((id_to_norm > 1).sum()), + "exact_texts_with_multiple_ids": int((text_to_id > 1).sum()), + "norm_texts_with_multiple_ids": int((norm_to_id > 1).sum()), + "max_exact_texts_per_id": int(id_to_text.max()) if len(id_to_text) else 0, + "max_ids_per_exact_text": int(text_to_id.max()) if len(text_to_id) else 0, + "max_norm_texts_per_id": int(id_to_norm.max()) if len(id_to_norm) else 0, + "max_ids_per_norm_text": int(norm_to_id.max()) if len(norm_to_id) else 0, + } + + +def support_against(train: pd.DataFrame, test: pd.DataFrame) -> dict: + tr_id = set(train.learning_objective_id.map(clean_id)) + tr_text = set(train.learning_objective.astype(str)) + tr_norm = set(train.learning_objective.map(norm_text)) + + te_id = test.learning_objective_id.map(clean_id) + te_text = test.learning_objective.astype(str) + te_norm = test.learning_objective.map(norm_text) + + seen_id = te_id.isin(tr_id).to_numpy(bool) + seen_text = te_text.isin(tr_text).to_numpy(bool) + seen_norm = te_norm.isin(tr_norm).to_numpy(bool) + + return { + "rows": int(len(test)), + "seen_by_id_rate": float(seen_id.mean()), + "seen_by_exact_text_rate": float(seen_text.mean()), + "seen_by_norm_text_rate": float(seen_norm.mean()), + "id_seen_text_unseen": int(np.sum(seen_id & ~seen_text)), + "text_seen_id_unseen": int(np.sum(seen_text & ~seen_id)), + "id_seen_norm_unseen": int(np.sum(seen_id & ~seen_norm)), + "norm_seen_id_unseen": int(np.sum(seen_norm & ~seen_id)), + "id_vs_exact_gate_disagreement_rate": float(np.mean(seen_id != seen_text)), + "id_vs_norm_gate_disagreement_rate": float(np.mean(seen_id != seen_norm)), + } + + +def session_cv_support(df: pd.DataFrame, folds: int = 4) -> dict: + groups = df.session_id.astype(str).to_numpy() + gkf = GroupKFold(n_splits=folds) + seen_id = np.zeros(len(df), dtype=bool) + seen_text = np.zeros(len(df), dtype=bool) + seen_norm = np.zeros(len(df), dtype=bool) + fold_rows = [] + + for fold, (tr, va) in enumerate(gkf.split(df, groups=groups), 1): + a = support_against(df.iloc[tr], df.iloc[va]) + trf = df.iloc[tr] + vaf = df.iloc[va] + tr_ids = set(trf.learning_objective_id.map(clean_id)) + tr_txt = set(trf.learning_objective.astype(str)) + tr_nrm = set(trf.learning_objective.map(norm_text)) + seen_id[va] = vaf.learning_objective_id.map(clean_id).isin(tr_ids) + seen_text[va] = vaf.learning_objective.astype(str).isin(tr_txt) + seen_norm[va] = vaf.learning_objective.map(norm_text).isin(tr_nrm) + fold_rows.append({"fold": fold, **a}) + + return { + "folds": folds, + "overall_seen_by_id_rate": float(seen_id.mean()), + "overall_seen_by_exact_text_rate": float(seen_text.mean()), + "overall_seen_by_norm_text_rate": float(seen_norm.mean()), + "id_vs_exact_gate_disagreement_rate": float(np.mean(seen_id != seen_text)), + "id_vs_norm_gate_disagreement_rate": float(np.mean(seen_id != seen_norm)), + "fold_details": fold_rows, + } + + +def examples(df: pd.DataFrame, limit: int = 12) -> dict: + x = df[["learning_objective_id", "learning_objective"]].copy() + x["oid"] = x.learning_objective_id.map(clean_id) + x["text"] = x.learning_objective.astype(str) + x["norm"] = x.learning_objective.map(norm_text) + + id_multi = ( + x.groupby("oid")["text"].agg(lambda s: sorted(set(s))) + .loc[lambda s: s.map(len) > 1] + .head(limit) + ) + text_multi = ( + x.groupby("text")["oid"].agg(lambda s: sorted(set(s))) + .loc[lambda s: s.map(len) > 1] + .head(limit) + ) + return { + "ids_with_multiple_texts": {str(k): v for k, v in id_multi.items()}, + "texts_with_multiple_ids": {str(k): v for k, v in text_multi.items()}, + } + + +def main(a: argparse.Namespace) -> None: + train = pd.read_csv(a.train_features) + required = {"response_id", "session_id", "learning_objective_id", "learning_objective"} + missing = required - set(train.columns) + if missing: + raise SystemExit(f"missing train columns: {sorted(missing)}") + + print("train columns", list(train.columns), flush=True) + out = { + "protocol": "V120_OBJECTIVE_IDENTITY_AUDIT", + "train": key_stats(train), + "session_cv_support": session_cv_support(train, folds=a.folds), + "examples": examples(train), + } + + if a.test_features is not None and a.test_features.exists(): + test = pd.read_csv(a.test_features) + missing = required - set(test.columns) + if missing: + raise SystemExit(f"missing test columns: {sorted(missing)}") + print("test columns", list(test.columns), flush=True) + out["test"] = key_stats(test) + out["official_test_support"] = support_against(train, test) + + tr = out["train"] + test_support = out.get("official_test_support", {}) + structurally_same = ( + tr["ids_with_multiple_norm_texts"] == 0 + and tr["norm_texts_with_multiple_ids"] == 0 + and test_support.get("id_vs_norm_gate_disagreement_rate", 0.0) == 0.0 + ) + if structurally_same: + verdict = "ID_TEXT_EQUIVALENT_FOR_SUPPORT" + else: + verdict = "ID_TEXT_NOT_EQUIVALENT" + + out["decision"] = { + "verdict": verdict, + "next": ( + "Kill identity hypothesis; move to semantic residual representation." + if structurally_same + else "Rerun support gate and validation keyed by canonical learning_objective_id before semantic escalation." + ), + } + + Path(a.out).write_text(json.dumps(out, indent=2, ensure_ascii=False)) + print(json.dumps(out, indent=2, ensure_ascii=False), flush=True) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--train-features", type=Path, required=True) + p.add_argument("--test-features", type=Path, default=None) + p.add_argument("--folds", type=int, default=4) + p.add_argument("--out", default="v120_objective_identity_audit.json") + main(p.parse_args()) diff --git a/competitions/trace_the_ace/v121_embed_batch1_transport.py b/competitions/trace_the_ace/v121_embed_batch1_transport.py new file mode 100644 index 00000000..08f96175 --- /dev/null +++ b/competitions/trace_the_ace/v121_embed_batch1_transport.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Infrastructure-only memory-safe embedding transport for frozen V121. + +No scientific representation, model, sample, ordering, folds, controls, or gates +are changed. Each text is embedded independently with the same frozen Jina model; +batch_size=1 only lowers peak ONNX attention memory. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from fastembed import TextEmbedding + +MODEL_NAME = "jinaai/jina-embeddings-v2-small-en" + +def embed1(model, seq): + arr = np.vstack(list(model.embed(seq, batch_size=1))).astype(np.float32) + if not np.isfinite(arr).all(): + raise RuntimeError("non-finite embedding") + return arr + +def main(a): + d=Path(a.dir) + texts=json.loads((d/'texts.json').read_text()) + n=len(texts['semantic']); shard=int(a.shard); shards=int(a.shards) + if shards < 1 or not (0 <= shard < shards): raise ValueError(f'invalid shard {shard}/{shards}') + start=(n*shard)//shards; end=(n*(shard+1))//shards + obj=texts['objective'][start:end]; sem=texts['semantic'][start:end] + print('frozen shard',shard,'of',shards,'rows',start,end,flush=True) + model=TextEmbedding(model_name=MODEL_NAME, threads=4) + E_obj=embed1(model,obj) + E_sem=embed1(model,sem) + if E_obj.shape[0] != end-start or E_sem.shape[0] != end-start: raise RuntimeError('row mismatch') + np.savez_compressed(Path(a.out),E_obj=E_obj,E_sem=E_sem,start=np.array(start),end=np.array(end),total=np.array(n),shard=np.array(shard),shards=np.array(shards)) + print('complete',shard,E_obj.shape,E_sem.shape,flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--dir',required=True); p.add_argument('--out',required=True); p.add_argument('--shard',type=int,required=True); p.add_argument('--shards',type=int,required=True); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v121_pretrained_semantic_residual.py b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py new file mode 100644 index 00000000..b537d627 --- /dev/null +++ b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""V121 PRETRAINED SEMANTIC RESIDUAL TEST. + +Qualitatively different from V112: fixed pretrained neural text embeddings rather +than hashed n-gram features. Tests whether semantic transcript/objective +representation adds row-local information beyond V97. + +Frozen intervention: +- deterministic 2500-row sample, same hashing convention as V112 +- pretrained jinaai/jina-embeddings-v2-small-en via FastEmbed 0.8.0 +- three representations: objective-only control, objective+local/recent/student + semantic intervention, and within-objective shuffled semantic ablation +- evaluate both objective-grouped and session-grouped 4-fold OOF +- evaluate a fixed hard-collision subset: an opposite-label same-objective row + exists within |p97_i-p97_j| <= 0.01 +- no hyperparameter sweep + +Precommit: +PHASE_CHANGE if semantic gain >= .003 on BOTH split geometries, semantic beats +within-objective shuffled by >= .002 on BOTH, and hard-collision gain is positive +on BOTH. Otherwise do not promote. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +from fastembed import TextEmbedding +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold + +from v110_residual_collider_state_discovery import hb, ll, logit, p97_predict +from v112_fast_raw_observable_screen import texts +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS = 1e-5 +MODEL_NAME = "jinaai/jina-embeddings-v2-small-en" + + +def stable_hash(x: object) -> int: + return int(hashlib.sha256(str(x).encode()).hexdigest()[:16], 16) + + +def build_semantic_text(objective: str, transcript_df) -> str: + student, _tutor, _full, local, last8 = texts(transcript_df, objective) + # Keep a bounded tail while preserving the target segment and most recent turns. + student_tail = student[-9000:] + local_tail = local[-12000:] + recent_tail = last8[-5000:] + return ( + f"learning objective: {objective}\n" + f"target tutoring context: {local_tail}\n" + f"recent turns: {recent_tail}\n" + f"student evidence: {student_tail}" + ) + + +def embed(model: TextEmbedding, seq: list[str]) -> np.ndarray: + # Infrastructure-only repair. batch_size=64 requested a ~66.5 GB attention + # buffer; batch_size=2 fit memory but twice hit the hosted-runner wall-clock + # shutdown during the same frozen embedding. batch_size=4 preserves model, + # texts, ordering and outputs while fitting the measured memory envelope and + # reducing runtime enough to finish on the hosted runner. + arr = np.vstack(list(model.embed(seq, batch_size=4))).astype(np.float32) + if not np.isfinite(arr).all(): + raise RuntimeError("non-finite embedding") + return arr + + +def p97_oof(X75, Xr, y, groups, support): + q = np.zeros(len(y), dtype=float) + splits = list(GroupKFold(min(4, len(np.unique(groups)))).split(np.zeros(len(y)), y, groups)) + for tr, va in splits: + q[va], _ = p97_predict(X75, Xr, y, tr, va, support) + return np.clip(q, EPS, 1 - EPS), splits + + +def residual_oof(P, E, y, splits): + q = np.zeros(len(y), dtype=float) + for tr, va in splits: + Xtr = np.c_[logit(P[tr]), E[tr]] + Xva = np.c_[logit(P[va]), E[va]] + m = LogisticRegression( + C=0.05, + max_iter=300, + solver="liblinear", + random_state=SEED, + ).fit(Xtr, y[tr]) + q[va] = m.predict_proba(Xva)[:, 1] + return np.clip(q, EPS, 1 - EPS) + + +def within_objective_shuffle(E: np.ndarray, objectives: np.ndarray) -> np.ndarray: + out = E.copy() + rng = np.random.default_rng(SEED + 121) + for o in np.unique(objectives): + z = np.where(objectives == o)[0] + if len(z) > 1: + out[z] = E[rng.permutation(z)] + return out + + +def collision_mask(P: np.ndarray, y: np.ndarray, objectives: np.ndarray, tol: float = 0.01) -> np.ndarray: + mask = np.zeros(len(y), dtype=bool) + for o in np.unique(objectives): + z = np.where(objectives == o)[0] + a0 = z[y[z] == 0] + a1 = z[y[z] == 1] + if len(a0) == 0 or len(a1) == 0: + continue + p0 = P[a0] + p1 = P[a1] + # sample is only 2500 rows, so explicit pairwise distances are small. + D = np.abs(p0[:, None] - p1[None, :]) + mask[a0[np.min(D, axis=1) <= tol]] = True + mask[a1[np.min(D, axis=0) <= tol]] = True + return mask + + +def eval_geometry(name, groups, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf): + P, splits = p97_oof(X75, Xr, y, groups, support) + Qobj = residual_oof(P, E_obj, y, splits) + Qsem = residual_oof(P, E_sem, y, splits) + Qsh = residual_oof(P, E_shuf, y, splits) + base = ll(y, P) + mask = collision_mask(P, y, objectives, tol=0.01) + out = { + "geometry": name, + "rows": int(len(y)), + "groups": int(len(np.unique(groups))), + "baseline_v97_ll": float(base), + "objective_only": {"ll": float(ll(y, Qobj)), "gain": float(base - ll(y, Qobj))}, + "semantic": {"ll": float(ll(y, Qsem)), "gain": float(base - ll(y, Qsem))}, + "semantic_shuffled_within_objective": {"ll": float(ll(y, Qsh)), "gain": float(base - ll(y, Qsh))}, + "semantic_minus_shuffle_gain": float(ll(y, Qsh) - ll(y, Qsem)), + "hard_collision": {"rows": int(mask.sum())}, + } + if mask.any(): + b = ll(y[mask], P[mask]) + s = ll(y[mask], Qsem[mask]) + sh = ll(y[mask], Qsh[mask]) + out["hard_collision"].update({ + "baseline_ll": float(b), + "semantic_ll": float(s), + "semantic_gain": float(b - s), + "shuffled_ll": float(sh), + "semantic_minus_shuffle_gain": float(sh - s), + }) + return out + + +def main(a): + f = load_training(a.features, a.labels).reset_index(drop=True) + print("features columns", list(f.columns), flush=True) + obj0 = (f.learning_objective_id if "learning_objective_id" in f else f.learning_objective).astype(str).to_numpy() + cand = np.where(np.array([hb(x, 5) != 0 for x in obj0]))[0] + ix = np.array(sorted(cand, key=lambda i: stable_hash(f.response_id.iloc[i]))[: a.rows]) + f = f.iloc[ix].reset_index(drop=True) + + y = f.target.to_numpy(int) + objectives = (f.learning_objective_id if "learning_objective_id" in f else f.learning_objective).astype(str).to_numpy() + support = f.learning_objective.astype(str).to_numpy() + sessions = f.session_id.astype(str).to_numpy() + + cache = {s: load_transcript(a.transcripts / f"{s}.csv") for s in np.unique(sessions)} + rt, rz = [], [] + sem_text, obj_text = [], [] + for i, r in f.iterrows(): + d = cache[str(r.session_id)] + t, z = segmented_control(d, str(r.learning_objective), "related") + rt.append(t) + rz.append(z) + obj_text.append(f"learning objective: {r.learning_objective}") + sem_text.append(build_semantic_text(str(r.learning_objective), d)) + if (i + 1) % 500 == 0: + print("prepared rows", i + 1, flush=True) + + X75 = build_v75(f, cache) + Xr = build_control(rt, rz) + + print("loading embedding model", MODEL_NAME, flush=True) + model = TextEmbedding(model_name=MODEL_NAME) + print("embedding objective control", flush=True) + E_obj = embed(model, obj_text) + print("embedding semantic intervention", flush=True) + E_sem = embed(model, sem_text) + E_shuf = within_objective_shuffle(E_sem, objectives) + print("embedding shapes", E_obj.shape, E_sem.shape, flush=True) + + results = { + "protocol": "V121_PRETRAINED_SEMANTIC_RESIDUAL", + "model": MODEL_NAME, + "rows": int(len(f)), + "objectives": int(len(np.unique(objectives))), + "sessions": int(len(np.unique(sessions))), + "precommit": { + "semantic_gain_each_geometry": 0.003, + "semantic_minus_shuffle_each_geometry": 0.002, + "hard_collision_gain_each_geometry": ">0", + "no_hyperparameter_sweep": True, + }, + } + results["objective_grouped"] = eval_geometry( + "objective_grouped", objectives, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf + ) + results["session_grouped"] = eval_geometry( + "session_grouped", sessions, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf + ) + + def passes(r): + return ( + r["semantic"]["gain"] >= 0.003 + and r["semantic_minus_shuffle_gain"] >= 0.002 + and r["hard_collision"].get("semantic_gain", -1.0) > 0.0 + ) + + ok_obj = passes(results["objective_grouped"]) + ok_sess = passes(results["session_grouped"]) + if ok_obj and ok_sess: + verdict = "PHASE_CHANGE_CANDIDATE" + nxt = "Promote pretrained semantic residual to larger frozen validation and public-probe packaging." + else: + verdict = "NO_ROBUST_SEMANTIC_PHASE_CHANGE" + nxt = "Treat remaining oracle gap as largely unidentifiable from supplied transcript/objective observables; pivot to validation geometry / assessment-process inference rather than more text feature search." + results["decision"] = { + "objective_grouped_pass": bool(ok_obj), + "session_grouped_pass": bool(ok_sess), + "verdict": verdict, + "next": nxt, + } + + Path(a.out).write_text(json.dumps(results, indent=2)) + print(json.dumps(results, indent=2), flush=True) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path, required=True) + p.add_argument("--labels", type=Path, required=True) + p.add_argument("--transcripts", type=Path, required=True) + p.add_argument("--rows", type=int, default=2500) + p.add_argument("--out", default="v121_pretrained_semantic_residual.json") + main(p.parse_args()) diff --git a/competitions/trace_the_ace/v121_staged_transport.py b/competitions/trace_the_ace/v121_staged_transport.py new file mode 100644 index 00000000..0e5f1ee5 --- /dev/null +++ b/competitions/trace_the_ace/v121_staged_transport.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Infrastructure-only staged transport for frozen V121. + +This module does not define new scientific features, models, folds, controls, or +gates. It serializes the exact V121 computation across short-lived hosted runner +jobs so preparation, embedding, and evaluation can complete independently. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +from fastembed import TextEmbedding +from scipy.sparse import load_npz, save_npz + +from v110_residual_collider_state_discovery import hb +from v121_pretrained_semantic_residual import ( + MODEL_NAME, + build_semantic_text, + embed, + eval_geometry, + stable_hash, + within_objective_shuffle, +) +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + + +def prepare(a): + out = Path(a.dir); out.mkdir(parents=True, exist_ok=True) + f = load_training(a.features, a.labels).reset_index(drop=True) + print('features columns', list(f.columns), flush=True) + obj0 = (f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + cand = np.where(np.array([hb(x, 5) != 0 for x in obj0]))[0] + ix = np.array(sorted(cand, key=lambda i: stable_hash(f.response_id.iloc[i]))[:a.rows]) + f = f.iloc[ix].reset_index(drop=True) + + y = f.target.to_numpy(int) + objectives = (f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support = f.learning_objective.astype(str).to_numpy() + sessions = f.session_id.astype(str).to_numpy() + + cache = {s: load_transcript(a.transcripts / f'{s}.csv') for s in np.unique(sessions)} + rt, rz, sem_text, obj_text = [], [], [], [] + for i, r in f.iterrows(): + d = cache[str(r.session_id)] + t, z = segmented_control(d, str(r.learning_objective), 'related') + rt.append(t); rz.append(z) + obj_text.append(f'learning objective: {r.learning_objective}') + sem_text.append(build_semantic_text(str(r.learning_objective), d)) + if (i + 1) % 500 == 0: + print('prepared rows', i + 1, flush=True) + + save_npz(out / 'X75.npz', build_v75(f, cache)) + save_npz(out / 'Xr.npz', build_control(rt, rz)) + np.savez_compressed(out / 'arrays.npz', y=y, objectives=objectives, support=support, sessions=sessions) + (out / 'texts.json').write_text(json.dumps({'objective': obj_text, 'semantic': sem_text})) + manifest = { + 'protocol': 'V121_PRETRAINED_SEMANTIC_RESIDUAL', 'rows': int(len(f)), + 'objectives': int(len(np.unique(objectives))), 'sessions': int(len(np.unique(sessions))), + 'response_ids_sha256': __import__('hashlib').sha256('\n'.join(f.response_id.astype(str)).encode()).hexdigest(), + } + (out / 'manifest.json').write_text(json.dumps(manifest, indent=2)) + print(json.dumps(manifest, indent=2), flush=True) + + +def do_embed(a): + d = Path(a.dir) + texts = json.loads((d / 'texts.json').read_text()) + n = len(texts['semantic']) + shards = int(a.shards) + shard = int(a.shard) + if shards < 1 or shard < 0 or shard >= shards: + raise ValueError(f'invalid shard {shard}/{shards}') + start = (n * shard) // shards + end = (n * (shard + 1)) // shards + obj_text = texts['objective'][start:end] + sem_text = texts['semantic'][start:end] + print('loading embedding model', MODEL_NAME, 'shard', shard, 'rows', start, end, flush=True) + model = TextEmbedding(model_name=MODEL_NAME) + print('embedding objective control shard', shard, flush=True) + E_obj = embed(model, obj_text) + print('embedding semantic intervention shard', shard, flush=True) + E_sem = embed(model, sem_text) + if E_obj.shape[0] != E_sem.shape[0] or E_obj.shape[0] != end - start: + raise RuntimeError('embedding row mismatch') + np.savez_compressed(Path(a.out), E_obj=E_obj, E_sem=E_sem, + start=np.array(start), end=np.array(end), total=np.array(n), + shard=np.array(shard), shards=np.array(shards)) + print('embedding shard complete', shard, start, end, E_obj.shape, E_sem.shape, flush=True) + + +def load_embeddings(path: Path): + if path.is_file(): + e = np.load(path, allow_pickle=False) + return e['E_obj'], e['E_sem'] + files = sorted(path.glob('**/v121_embeddings_shard_*.npz')) + if not files: + files = sorted(path.glob('**/*.npz')) + parts = [] + for f in files: + e = np.load(f, allow_pickle=False) + if 'start' not in e.files or 'end' not in e.files: + continue + parts.append((int(e['start']), int(e['end']), int(e['total']), e['E_obj'], e['E_sem'], f)) + if not parts: + raise RuntimeError(f'no embedding shards found under {path}') + parts.sort(key=lambda x: x[0]) + total = parts[0][2] + cursor = 0 + obj, sem = [], [] + for start, end, t, eo, es, f in parts: + if t != total or start != cursor or end - start != eo.shape[0] or eo.shape[0] != es.shape[0]: + raise RuntimeError(f'invalid embedding shard coverage at {f}: {start}:{end}, cursor={cursor}, total={t}') + obj.append(eo); sem.append(es); cursor = end + if cursor != total: + raise RuntimeError(f'incomplete embedding coverage: {cursor}/{total}') + E_obj = np.vstack(obj); E_sem = np.vstack(sem) + print('merged embedding shards', len(parts), E_obj.shape, E_sem.shape, flush=True) + return E_obj, E_sem + + +def evaluate(a): + d = Path(a.dir) + X75 = load_npz(d / 'X75.npz'); Xr = load_npz(d / 'Xr.npz') + z = np.load(d / 'arrays.npz', allow_pickle=False) + y=z['y']; objectives=z['objectives']; support=z['support']; sessions=z['sessions'] + E_obj, E_sem = load_embeddings(Path(a.embeddings)) + if len(y) != E_obj.shape[0] or len(y) != E_sem.shape[0]: + raise RuntimeError('evaluation embedding row mismatch') + E_shuf = within_objective_shuffle(E_sem, objectives) + manifest=json.loads((d/'manifest.json').read_text()) + results = { + 'protocol': 'V121_PRETRAINED_SEMANTIC_RESIDUAL', 'model': MODEL_NAME, + 'rows': int(len(y)), 'objectives': int(len(np.unique(objectives))), + 'sessions': int(len(np.unique(sessions))), + 'transport_manifest': manifest, + 'transport': {'embedding_shards_merged': True}, + 'precommit': {'semantic_gain_each_geometry': .003, + 'semantic_minus_shuffle_each_geometry': .002, + 'hard_collision_gain_each_geometry': '>0', + 'no_hyperparameter_sweep': True}, + } + results['objective_grouped'] = eval_geometry('objective_grouped', objectives, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf) + results['session_grouped'] = eval_geometry('session_grouped', sessions, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf) + def passes(r): + return r['semantic']['gain'] >= .003 and r['semantic_minus_shuffle_gain'] >= .002 and r['hard_collision'].get('semantic_gain', -1.) > 0 + ok_obj=passes(results['objective_grouped']); ok_sess=passes(results['session_grouped']) + if ok_obj and ok_sess: + verdict='PHASE_CHANGE_CANDIDATE'; nxt='Promote pretrained semantic residual to larger frozen validation and public-probe packaging.' + else: + verdict='NO_ROBUST_SEMANTIC_PHASE_CHANGE'; nxt='Treat remaining oracle gap as largely unidentifiable from supplied transcript/objective observables; pivot to validation geometry / assessment-process inference rather than more text feature search.' + results['decision']={'objective_grouped_pass': bool(ok_obj), 'session_grouped_pass': bool(ok_sess), 'verdict': verdict, 'next': nxt} + Path(a.out).write_text(json.dumps(results, indent=2)); print(json.dumps(results, indent=2), flush=True) + + +if __name__ == '__main__': + p=argparse.ArgumentParser(); sp=p.add_subparsers(dest='cmd', required=True) + q=sp.add_parser('prepare'); q.add_argument('--features',type=Path,required=True); q.add_argument('--labels',type=Path,required=True); q.add_argument('--transcripts',type=Path,required=True); q.add_argument('--rows',type=int,default=2500); q.add_argument('--dir',required=True) + q=sp.add_parser('embed'); q.add_argument('--dir',required=True); q.add_argument('--out',required=True); q.add_argument('--shard',type=int,default=0); q.add_argument('--shards',type=int,default=1) + q=sp.add_parser('evaluate'); q.add_argument('--dir',required=True); q.add_argument('--embeddings',required=True); q.add_argument('--out',required=True) + a=p.parse_args(); {'prepare':prepare,'embed':do_embed,'evaluate':evaluate}[a.cmd](a) diff --git a/competitions/trace_the_ace/v122_id_morphology_regime_audit.py b/competitions/trace_the_ace/v122_id_morphology_regime_audit.py new file mode 100644 index 00000000..9155a9f8 --- /dev/null +++ b/competitions/trace_the_ace/v122_id_morphology_regime_audit.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""V122 — metadata-only ID morphology regime audit. + +Question: do session/objective identifier string patterns encode a stable provider / assessment-process +regime that could explain leaderboard behavior? This is deliberately metadata-only and fast. + +Frozen protocol: +- inspect headers first +- join train features/labels by response_id +- evaluate char-ngram morphology of session_id, learning_objective_id, and both together +- two untouched geometries: GroupKFold by session_id and GroupKFold by learning_objective_id +- compare against intercept-only fold baseline +- shuffled-label control with same folds +- promotion only if a real ID family gains >= .003 log loss in BOTH geometries and exceeds shuffle by >= .002 +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.metrics import log_loss + +EPS=1e-5 +SEED=20260818 + +def fold_intercept(ytr,n): + p=float(np.clip(np.mean(ytr),EPS,1-EPS)); return np.full(n,p,float) + +def oof(texts,y,groups): + q=np.zeros(len(y),float); b=np.zeros(len(y),float) + hv=HashingVectorizer(analyzer='char',ngram_range=(2,5),n_features=2**15,alternate_sign=False,norm='l2',lowercase=False) + X=hv.transform(texts) + k=min(5,len(np.unique(groups))) + for tr,va in GroupKFold(k).split(X,y,groups): + b[va]=fold_intercept(y[tr],len(va)) + m=LogisticRegression(C=.15,max_iter=220,solver='liblinear',random_state=SEED) + m.fit(X[tr],y[tr]); q[va]=m.predict_proba(X[va])[:,1] + return float(log_loss(y,np.clip(q,EPS,1-EPS))), float(log_loss(y,np.clip(b,EPS,1-EPS))) + +def run(a): + print('train_features headers',list(pd.read_csv(a.features,nrows=0).columns),flush=True) + print('train_labels headers',list(pd.read_csv(a.labels,nrows=0).columns),flush=True) + f=pd.read_csv(a.features); l=pd.read_csv(a.labels) + f=f.merge(l,on='response_id',how='inner',validate='one_to_one') + y=f.is_correct.to_numpy(int) + sess=f.session_id.astype(str).to_numpy(); oid=f.learning_objective_id.astype(str).to_numpy() + families={ + 'SESSION_ID':np.array(['S:'+x for x in sess],object), + 'OBJECTIVE_ID':np.array(['O:'+x for x in oid],object), + 'SESSION_X_OBJECTIVE':np.array(['S:'+s+'|O:'+o for s,o in zip(sess,oid)],object), + } + geoms={'session_cold':sess,'objective_cold':oid} + rng=np.random.default_rng(SEED); ys=y.copy(); rng.shuffle(ys) + out={'rows':int(len(f)),'sessions':int(len(np.unique(sess))),'objectives':int(len(np.unique(oid))), 'families':{}, 'shuffle':{}} + gains=[] + for name,txt in families.items(): + out['families'][name]={} + out['shuffle'][name]={} + for gname,g in geoms.items(): + ll,base=oof(txt,y,g); sll,sbase=oof(txt,ys,g) + gain=base-ll; sgain=sbase-sll + out['families'][name][gname]={'ll':ll,'baseline_ll':base,'gain':gain} + out['shuffle'][name][gname]={'gain':sgain} + g1=out['families'][name]['session_cold']['gain']; g2=out['families'][name]['objective_cold']['gain'] + sh=max(out['shuffle'][name]['session_cold']['gain'],out['shuffle'][name]['objective_cold']['gain']) + gains.append((min(g1,g2)-sh,name,g1,g2,sh)) + gains.sort(reverse=True) + margin,name,g1,g2,sh=gains[0] + promote=(g1>=.003 and g2>=.003 and min(g1,g2)-sh>=.002) + out['decision']={'winner':name,'session_gain':g1,'objective_gain':g2,'max_shuffle_gain':sh,'margin':margin, + 'verdict':'ID_REGIME_SIGNAL' if promote else 'ID_MORPHOLOGY_NOT_DECISION_CHANGING', + 'rule':'Promote only if >=.003 gain in both geometries and >=.002 above shuffled control.'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--out',default='v122_id_morphology_regime_audit.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py new file mode 100644 index 00000000..b08d6008 --- /dev/null +++ b/competitions/trace_the_ace/v125_nested_calibration.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""V125: nested calibration residual over frozen V97. + +Question: is V97 leaving lawful log-loss improvement in probability calibration, +without adding new information or exploiting a particular validation geometry? + +Frozen protocol: +- deterministic 2500-row response-id sample; +- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported); +- 4-fold outer objective-grouped and session-grouped OOF; +- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold; +- intervention = one global Platt map sigmoid(a + b*logit(p97)); +- control = same map fit after deterministic shuffle of inner-OOF probabilities; +- no hyperparameter sweep. + +Promote only if calibration gains >= .001 log loss in BOTH geometries and beats +the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law. +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS=1e-5 + +def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) +def logit(p): + p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p)) + +def endpoint(X75,Xr,y,key,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + p75=m.predict_proba(X75[va])[:,1] + r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + pr=r.predict_proba(Xr[va])[:,1] + vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)) + seen=np.array([d.get(x,0)>0 for x in key[va]]) + return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS) + +def fit_cal(p,y): + m=LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y) + return m + +def geometry(name,groups,X75,Xr,y,key): + outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups)) + pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[] + for k,(tr,va) in enumerate(outer): + inner_groups=groups[tr] + inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups)) + pi=np.zeros(len(tr)) + for itr,iva in inn: + pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva]) + cal=fit_cal(pi,y[tr]) + rng=np.random.default_rng(SEED+125+k) + sh=fit_cal(pi[rng.permutation(len(pi))],y[tr]) + raw=endpoint(X75,Xr,y,key,tr,va) + q=cal.predict_proba(logit(raw)[:,None])[:,1] + qs=sh.predict_proba(logit(raw)[:,None])[:,1] + pb[va]=raw;pc[va]=q;ps[va]=qs + folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q), + 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]), + 'intercept':float(cal.intercept_[0])}) + base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps) + return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal, + 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds} + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('features columns',list(f.columns),flush=True) + ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows] + f=f.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy() + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[];rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z) + if (i+1)%500==0: print('prepared rows',i+1,flush=True) + X75=build_v75(f,cache);Xr=build_control(rt,rz) + ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key) + def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001 + verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION' + out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True}, + 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args()) diff --git a/competitions/trace_the_ace/v129_tutor_uptake.py b/competitions/trace_the_ace/v129_tutor_uptake.py new file mode 100644 index 00000000..7421b662 --- /dev/null +++ b/competitions/trace_the_ace/v129_tutor_uptake.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V129 TUTOR UPTAKE residual over frozen V97. + +Primary separator: whether the tutor's next utterance lexically/structurally takes +up the student's immediately preceding language. V75 raw hashing preserves words, +but not this cross-turn relation explicitly. + +Control: deterministically rotate tutor replies among student->tutor pairs inside +each session, preserving the exact student/tutor text multisets and pair count while +destroying local response alignment. No labels enter feature construction. No sweep. +""" +from __future__ import annotations +import argparse,csv,io,json,re,zipfile,hashlib +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from v110_residual_collider_state_discovery import ll,logit +from v121_pretrained_semantic_residual import p97_oof,collision_mask +from v75_canonical_trajectory import SEED + +TOK=re.compile(r"[a-z0-9]+(?:\.[0-9]+)?",re.I) +NEG=re.compile(r"\b(?:no|not quite|incorrect|wrong|careful|try again|almost|remember|instead|actually)\b",re.I) +POS=re.compile(r"\b(?:yes|yeah|correct|right|exactly|perfect|good|great|well done|that's it|thats it|you got it|spot on)\b",re.I) +QUESTION=re.compile(r"\?|\b(?:what|which|how|why|can you|could you|tell me|work out|calculate|solve|find)\b",re.I) +STOP={'the','a','an','and','or','to','of','in','on','for','with','is','are','be','as','by','from','this','that','these','those','you','your','we','it','its'} +EPS=1e-5 + +def toks(s): return {x for x in TOK.findall(str(s).lower()) if len(x)>1 and x not in STOP} +def jac(a,b): + A,B=toks(a),toks(b) + return len(A&B)/len(A|B) if A and B else 0.0 + +def pairs(rows): + out=[] + for i,r in enumerate(rows[:-1]): + if str(r.get('role','')).lower()!='student': continue + s=str(r.get('content','')) + j=None + for k in range(i+1,min(len(rows),i+4)): + if str(rows[k].get('role','')).lower()=='tutor': j=k; break + if str(rows[k].get('role','')).lower()=='student': break + if j is not None: out.append((s,str(rows[j].get('content','')))) + return out + +def feats(P,shift=0): + if not P: return np.zeros(10,float) + S=[p[0] for p in P]; T=[p[1] for p in P]; n=len(P) + if shift and n>1: T=T[shift%n:]+T[:shift%n] + o=np.array([jac(s,t) for s,t in zip(S,T)],float) + neg=np.array([bool(NEG.search(t)) for t in T]); pos=np.array([bool(POS.search(t)) for t in T]); q=np.array([bool(QUESTION.search(t)) for t in T]) + substantive=np.array([len(toks(s))>=2 or bool(re.search(r'\d|[=+\-/*×÷%]',s)) for s in S]) + def mean(mask): return float(o[mask].mean()) if mask.any() else 0.0 + return np.array([o.mean(),np.quantile(o,.75),np.quantile(o,.9),o.max(),mean(neg),mean(pos),mean(q),mean(substantive),float((o>0).mean()),np.log1p(n)],float) + +def residual_oof(P,X,y,splits): + q=np.zeros(len(y),float) + for tr,va in splits: + mu=X[tr].mean(0); sd=X[tr].std(0)+1e-6 + A=np.c_[logit(P[tr]),(X[tr]-mu)/sd]; B=np.c_[logit(P[va]),(X[va]-mu)/sd] + m=LogisticRegression(C=.05,max_iter=300,solver='liblinear',random_state=SEED).fit(A,y[tr]) + q[va]=m.predict_proba(B)[:,1] + return np.clip(q,EPS,1-EPS) + +def evalg(name,groups,X75,Xr,y,support,obj,Xreal,Xctrl): + P,splits=p97_oof(X75,Xr,y,groups,support); Q=residual_oof(P,Xreal,y,splits); C=residual_oof(P,Xctrl,y,splits) + b=ll(y,P); r=ll(y,Q); c=ll(y,C); mask=collision_mask(P,y,obj,.01) + out={'geometry':name,'baseline_v97_ll':b,'uptake':{'ll':r,'gain':b-r},'rotated_reply_control':{'ll':c,'gain':b-c},'uptake_minus_control_gain':c-r,'hard_collision':{'rows':int(mask.sum())}} + if mask.any(): + bb=ll(y[mask],P[mask]); rr=ll(y[mask],Q[mask]); cc=ll(y[mask],C[mask]) + out['hard_collision'].update({'baseline_ll':bb,'uptake_ll':rr,'uptake_gain':bb-rr,'control_ll':cc,'uptake_minus_control_gain':cc-rr}) + return out + +def main(a): + d=Path(a.dir); z=np.load(d/'arrays.npz',allow_pickle=True); y=z['y']; obj=z['objectives']; support=z['support']; sessions=z['sessions'] + X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz'); cache={} + with zipfile.ZipFile(a.archive) as za: + names=set(za.namelist()) + for sid in np.unique(sessions): + name=f'{sid}.csv' + if name not in names: raise RuntimeError('missing '+name) + with za.open(name) as f: rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline=''))) + P=pairs(rows); shift=1+(int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(P)-1)) if len(P)>1 else 0 + cache[str(sid)]=(feats(P,0),feats(P,shift)) + R=np.vstack([cache[str(s)][0] for s in sessions]); C=np.vstack([cache[str(s)][1] for s in sessions]) + res={'protocol':'V129_TUTOR_UPTAKE','rows':int(len(y)),'primary':'student -> next tutor lexical uptake','control':'deterministic within-session rotation of tutor replies','precommit':{'promote_gain_each_geometry':.0015,'phase_change_gain_each_geometry':.003,'real_minus_control_each_geometry':.001,'hard_collision_gain_each_geometry':'>0','no_parameter_sweep':True}} + res['objective_grouped']=evalg('objective_grouped',obj,X75,Xr,y,support,obj,R,C); res['session_grouped']=evalg('session_grouped',sessions,X75,Xr,y,support,obj,R,C) + def ok(x,t): return x['uptake']['gain']>=t and x['uptake_minus_control_gain']>=.001 and x['hard_collision'].get('uptake_gain',-1)>0 + po=ok(res['objective_grouped'],.0015); ps=ok(res['session_grouped'],.0015); ph=ok(res['objective_grouped'],.003) and ok(res['session_grouped'],.003) + verdict='PHASE_CHANGE_CANDIDATE' if ph else 'PROMOTE_TUTOR_UPTAKE_LAW' if po and ps else 'SUPPRESS_TUTOR_UPTAKE' + res['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':verdict} + Path(a.out).write_text(json.dumps(res,indent=2)); print(json.dumps(res,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--archive',required=True); p.add_argument('--dir',required=True); p.add_argument('--out',required=True); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v132_nested_applicability_gate.py b/competitions/trace_the_ace/v132_nested_applicability_gate.py new file mode 100644 index 00000000..16eb83bd --- /dev/null +++ b/competitions/trace_the_ace/v132_nested_applicability_gate.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""V132: leakage-safe learned applicability gate for V129 tutor uptake. + +This is derived from the Residual Constraint Graph, not a feature sweep. +V129 shows control-separated uptake signal only in a label-defined ambiguity regime; +V131 shows prediction density alone is too broad. V132 asks whether the missing +activation predicate is itself learnable from runtime-visible state. + +For each OUTER fold: +1. Produce INNER-OOF V97 predictions on the outer-training rows. +2. Produce INNER-OOF uptake and rotated-control corrections on those rows. +3. Define correction-benefit targets only on outer-training rows from per-row logloss. +4. Fit one frozen logistic applicability gate from runtime-visible features. +5. Fit the correction model on all outer-training INNER-OOF base predictions. +6. Apply both correction and gate to untouched outer validation rows. + +No validation labels enter feature construction, correction fitting, or gating. +No threshold/C/feature sweep. The identical procedure is run for the rotated-reply +control so an apparent benefit from generic second-stage selection is not enough. +""" +from __future__ import annotations +import argparse,csv,hashlib,io,json,zipfile +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold + +from v110_residual_collider_state_discovery import ll,logit,p97_predict +from v129_tutor_uptake import feats,pairs +from v75_canonical_trajectory import SEED + +EPS=1e-5 +GATE_C=.10 +CORR_C=.05 + + +def row_loss(y,p): + p=np.clip(np.asarray(p,float),EPS,1-EPS); y=np.asarray(y,int) + return -(y*np.log(p)+(1-y)*np.log(1-p)) + + +def crowd_features(P,obj): + """Runtime-visible geometry, no labels.""" + P=np.asarray(P,float); obj=np.asarray(obj,str) + nearest=np.ones(len(P),float); count=np.ones(len(P),float) + for o in np.unique(obj): + z=np.where(obj==o)[0]; count[z]=len(z) + if len(z)>1: + d=np.abs(P[z,None]-P[None,z]); np.fill_diagonal(d,np.inf) + nearest[z]=np.min(d,axis=1) + return np.c_[np.log1p(count),nearest,(nearest<=.01).astype(float)] + + +def gate_features(P,Q,R,obj): + P=np.asarray(P,float); Q=np.asarray(Q,float); R=np.asarray(R,float) + d=Q-P + # Every field is available at inference time once V97 and the frozen uptake + # correction have produced their probabilities. + return np.c_[P,Q,d,np.abs(d),np.abs(P-.5),logit(P),crowd_features(P,obj),R] + + +def residual_fit(P,R,y): + mu=R.mean(0); sd=R.std(0)+1e-6 + X=np.c_[logit(P),(R-mu)/sd] + m=LogisticRegression(C=CORR_C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y) + return mu,sd,m + + +def residual_apply(model,P,R): + mu,sd,m=model + X=np.c_[logit(P),(R-mu)/sd] + return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS) + + +def inner_oof(X75,Xr,y,groups,support,R): + nsp=min(3,len(np.unique(groups))) + splits=list(GroupKFold(nsp).split(np.zeros(len(y)),y,groups)) + P=np.zeros(len(y)); Q=np.zeros(len(y)) + # First get base OOF for all rows. + for tr,va in splits: + P[va],_=p97_predict(X75,Xr,y,tr,va,support) + # Then fit each residual correction on OOF base probabilities of its training + # rows and validate on the same untouched inner validation block. + for tr,va in splits: + mod=residual_fit(P[tr],R[tr],y[tr]) + Q[va]=residual_apply(mod,P[va],R[va]) + return np.clip(P,EPS,1-EPS),np.clip(Q,EPS,1-EPS) + + +def fit_gate(P,Q,R,obj,y): + X=gate_features(P,Q,R,obj) + target=(row_loss(y,Q)=.5 + out=np.asarray(P,float).copy(); out[take]=Q[take] + return np.clip(out,EPS,1-EPS),take + + +def eval_geometry(name,groups,X75,Xr,y,support,obj,R,C): + groups=np.asarray(groups); nsp=min(4,len(np.unique(groups))) + outer=list(GroupKFold(nsp).split(np.zeros(len(y)),y,groups)) + PB=np.zeros(len(y)); GR=np.zeros(len(y)); GC=np.zeros(len(y)); + takeR=np.zeros(len(y),bool); takeC=np.zeros(len(y),bool); foldrows=[] + for k,(tr,va) in enumerate(outer,1): + # Clean outer validation base prediction. + pva,_=p97_predict(X75,Xr,y,tr,va,support) + # Training state for gate/correction is itself OOF. + pin,qin=inner_oof(X75[tr],Xr[tr],y[tr],groups[tr],support[tr],R[tr]) + _pin_c,qin_c=inner_oof(X75[tr],Xr[tr],y[tr],groups[tr],support[tr],C[tr]) + # pin and _pin_c are deterministically identical; do not use labels from va. + gateR=fit_gate(pin,qin,R[tr],obj[tr],y[tr]); gateC=fit_gate(pin,qin_c,C[tr],obj[tr],y[tr]) + corrR=residual_fit(pin,R[tr],y[tr]); corrC=residual_fit(pin,C[tr],y[tr]) + qva=residual_apply(corrR,pva,R[va]); qva_c=residual_apply(corrC,pva,C[va]) + gr,tR=apply_gate(gateR,pva,qva,R[va],obj[va]); gc,tC=apply_gate(gateC,pva,qva_c,C[va],obj[va]) + PB[va]=pva; GR[va]=gr; GC[va]=gc; takeR[va]=tR; takeC[va]=tC + foldrows.append({'fold':k,'rows':int(len(va)),'base_ll':ll(y[va],pva),'gate_ll':ll(y[va],gr),'control_gate_ll':ll(y[va],gc),'train_benefit_rate':gateR['rate'],'take_rate':float(tR.mean())}) + b=ll(y,PB); r=ll(y,GR); c=ll(y,GC) + return { + 'geometry':name,'baseline_v97_ll':b, + 'nested_gate':{'ll':r,'gain':b-r,'take_rate':float(takeR.mean())}, + 'nested_rotated_control_gate':{'ll':c,'gain':b-c,'take_rate':float(takeC.mean())}, + 'real_minus_control_gain':c-r, + 'folds':foldrows, + } + + +def main(a): + d=Path(a.dir); z=np.load(d/'arrays.npz',allow_pickle=True) + y=z['y']; obj=z['objectives']; support=z['support']; sessions=z['sessions'] + X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz'); cache={} + with zipfile.ZipFile(a.archive) as za: + names=set(za.namelist()) + for sid in np.unique(sessions): + name=f'{sid}.csv' + if name not in names: raise RuntimeError('missing '+name) + with za.open(name) as f: rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline=''))) + P=pairs(rows); shift=1+(int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(P)-1)) if len(P)>1 else 0 + cache[str(sid)]=(feats(P,0),feats(P,shift)) + R=np.vstack([cache[str(s)][0] for s in sessions]); C=np.vstack([cache[str(s)][1] for s in sessions]) + out={'protocol':'V132_NESTED_APPLICABILITY_GATE','rows':int(len(y)), + 'hypothesis':'V129 contains local relational information but requires a learned runtime-visible ambiguity/applicability predicate', + 'precommit':{'outer_folds':4,'inner_folds':3,'gate_C':GATE_C,'correction_C':CORR_C,'gate_threshold':.5,'no_sweep':True, + 'promote_gain_each_geometry':.0005,'real_minus_control_each_geometry':.0005}} + out['objective_grouped']=eval_geometry('objective_grouped',obj,X75,Xr,y,support,obj,R,C) + out['session_grouped']=eval_geometry('session_grouped',sessions,X75,Xr,y,support,obj,R,C) + def ok(r): return r['nested_gate']['gain']>=.0005 and r['real_minus_control_gain']>=.0005 + po,ps=ok(out['objective_grouped']),ok(out['session_grouped']) + out['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':'PROMOTE_LEARNED_APPLICABILITY_GATE' if po and ps else 'SUPPRESS_LEARNED_UPTAKE_APPLICABILITY_FAMILY'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--archive',required=True); p.add_argument('--dir',required=True); p.add_argument('--out',required=True); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v133_verified_math_evidence.py b/competitions/trace_the_ace/v133_verified_math_evidence.py new file mode 100644 index 00000000..dde210dc --- /dev/null +++ b/competitions/trace_the_ace/v133_verified_math_evidence.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""V133: verifier-derived mathematical evidence over frozen V97. + +Constraint-derived hypothesis: current residual requires a new sample-local observable, +not another semantic/routing feature. V75 records tutor feedback but never independently +checks whether an explicit arithmetic student answer is mathematically correct. + +Primary separator: exact arithmetic question->student-answer verification. +Control: deterministically rotate student numeric answers among the same session's +verifiable questions, preserving question/answer marginals but destroying the relation. +No threshold/feature sweep. Each outer fold fits the residual only from inner-OOF V97 +predictions on outer-training rows, then evaluates untouched outer validation rows. +""" +from __future__ import annotations +import argparse,csv,hashlib,io,json,re,zipfile +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from v110_residual_collider_state_discovery import ll,logit,p97_predict +from v75_canonical_trajectory import SEED + +EPS=1e-5 +C=.05 +NUM=r"(-?\d+(?:\.\d+)?(?:\s*/\s*-?\d+(?:\.\d+)?)?)" +PATTERNS=[ + (re.compile(rf"{NUM}\s*(?:\+|plus)\s*{NUM}",re.I),lambda a,b:a+b), + (re.compile(rf"{NUM}\s*(?:-|minus)\s*{NUM}",re.I),lambda a,b:a-b), + (re.compile(rf"{NUM}\s*(?:x|×|\*|times|multiplied\s+by)\s*{NUM}",re.I),lambda a,b:a*b), + (re.compile(rf"{NUM}\s*(?:/|÷|divided\s+by)\s*{NUM}",re.I),lambda a,b:a/b if abs(b)>1e-12 else np.nan), +] +ANS_RE=re.compile(NUM) +QUESTION_RE=re.compile(r"\?|\b(?:what|calculate|work out|solve|how much|how many)\b",re.I) + +def number(s): + s=str(s).replace(' ','') + try: + if '/' in s: + a,b=s.split('/',1); b=float(b); return float(a)/b if abs(b)>1e-12 else np.nan + return float(s) + except Exception:return np.nan + +def expected(q): + q=str(q).replace(',','') + for p,op in PATTERNS: + m=p.search(q) + if m: + a,b=number(m.group(1)),number(m.group(2)) + if np.isfinite(a) and np.isfinite(b): + try:return float(op(a,b)) + except Exception:return np.nan + return np.nan + +def answer_value(a): + m=ANS_RE.search(str(a).replace(',','')) + return number(m.group(1)) if m else np.nan + +def events(rows): + out=[] + for i,r in enumerate(rows): + if str(r.get('role','')).lower()!='tutor':continue + q=str(r.get('content','')) + if not QUESTION_RE.search(q):continue + e=expected(q) + if not np.isfinite(e):continue + ai=None + for j in range(i+1,min(len(rows),i+6)): + role=str(rows[j].get('role','')).lower(); txt=str(rows[j].get('content','')) + if role=='student' and txt.strip(): ai=j; break + if role=='tutor' and QUESTION_RE.search(txt) and j>i+1: break + if ai is None:continue + av=answer_value(rows[ai].get('content','')) + if np.isfinite(av):out.append((e,av,ai/max(1,len(rows)-1))) + return out + +def vec(E,shift=0): + n=len(E) + if not n:return np.zeros(8,float) + ans=np.asarray([x[1] for x in E],float) + if shift and n>1: ans=np.roll(ans,shift) + exp=np.asarray([x[0] for x in E],float); rec=np.asarray([x[2] for x in E],float) + ok=np.isclose(ans,exp,rtol=1e-6,atol=1e-6).astype(float); bad=1-ok + w=np.exp(2*(rec-1)); w/=w.sum()+1e-12 + return np.asarray([np.log1p(n),ok.mean(),bad.mean(),ok[-1]-bad[-1],float((w*ok).sum()),float((w*bad).sum()),float(rec[ok>0].max()) if np.any(ok>0) else 0.,float(rec[bad>0].max()) if np.any(bad>0) else 0.],float) + +def inner_base(X75,Xr,y,groups,support): + nsp=min(3,len(np.unique(groups))); P=np.zeros(len(y)) + for tr,va in GroupKFold(nsp).split(np.zeros(len(y)),y,groups): P[va],_=p97_predict(X75,Xr,y,tr,va,support) + return np.clip(P,EPS,1-EPS) +def fit_res(P,R,y): + mu=R.mean(0); sd=R.std(0)+1e-6 + X=np.c_[logit(P),(R-mu)/sd] + m=LogisticRegression(C=C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y) + return mu,sd,m +def apply_res(mod,P,R): + mu,sd,m=mod; X=np.c_[logit(P),(R-mu)/sd] + return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS) +def eval_geom(name,groups,X75,Xr,y,support,R,A,covered): + groups=np.asarray(groups); PB=np.zeros(len(y)); QR=np.zeros(len(y)); QA=np.zeros(len(y)); rows=[] + for k,(tr,va) in enumerate(GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(len(y)),y,groups),1): + pva,_=p97_predict(X75,Xr,y,tr,va,support); pin=inner_base(X75[tr],Xr[tr],y[tr],groups[tr],support[tr]) + qr=apply_res(fit_res(pin,R[tr],y[tr]),pva,R[va]); qa=apply_res(fit_res(pin,A[tr],y[tr]),pva,A[va]) + PB[va]=pva;QR[va]=qr;QA[va]=qa + rows.append({'fold':k,'rows':int(len(va)),'base_ll':ll(y[va],pva),'verified_ll':ll(y[va],qr),'control_ll':ll(y[va],qa),'covered':int(covered[va].sum())}) + b,r,a=ll(y,PB),ll(y,QR),ll(y,QA); cov=np.asarray(covered,bool) + return {'geometry':name,'baseline_v97_ll':b,'verified':{'ll':r,'gain':b-r},'rotated_control':{'ll':a,'gain':b-a},'real_minus_control_gain':a-r,'coverage_fraction':float(cov.mean()),'covered_rows':int(cov.sum()),'covered_only':({'baseline_ll':ll(y[cov],PB[cov]),'verified_ll':ll(y[cov],QR[cov]),'gain':ll(y[cov],PB[cov])-ll(y[cov],QR[cov])} if cov.any() else None),'folds':rows} +def main(a): + d=Path(a.dir); z=np.load(d/'arrays.npz',allow_pickle=True); y=z['y']; obj=z['objectives']; support=z['support']; sessions=z['sessions'] + X75=load_npz(d/'X75.npz'); Xr=load_npz(d/'Xr.npz'); cache={} + with zipfile.ZipFile(a.archive) as za: + names=set(za.namelist()) + for sid in np.unique(sessions): + name=f'{sid}.csv' + if name not in names:raise RuntimeError('missing '+name) + with za.open(name) as f: rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline=''))) + E=events(rows); shift=(1+int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(E)-1)) if len(E)>1 else 0 + cache[str(sid)]=(vec(E,0),vec(E,shift),len(E)) + R=np.vstack([cache[str(s)][0] for s in sessions]); A=np.vstack([cache[str(s)][1] for s in sessions]); covered=np.asarray([cache[str(s)][2]>0 for s in sessions]) + out={'protocol':'V133_VERIFIED_MATH_EVIDENCE','rows':int(len(y)),'hypothesis':'independent arithmetic verification is a missing sample-local observable over V97','precommit':{'residual_C':C,'outer_folds':4,'inner_folds':3,'no_sweep':True,'promote_gain_each_geometry':.001,'real_minus_control_each_geometry':.0005,'phase_change_gain_each_geometry':.003}} + out['objective_grouped']=eval_geom('objective_grouped',obj,X75,Xr,y,support,R,A,covered) + out['session_grouped']=eval_geom('session_grouped',sessions,X75,Xr,y,support,R,A,covered) + def ok(x,t=.001):return x['verified']['gain']>=t and x['real_minus_control_gain']>=.0005 + po,ps=ok(out['objective_grouped']),ok(out['session_grouped']); phase=ok(out['objective_grouped'],.003) and ok(out['session_grouped'],.003) + out['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':'PHASE_CHANGE_VERIFIED_MATH_EVIDENCE' if phase else 'PROMOTE_VERIFIED_MATH_EVIDENCE' if po and ps else 'SUPPRESS_VERIFIED_MATH_EVIDENCE'} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--archive',required=True);p.add_argument('--dir',required=True);p.add_argument('--out',required=True);main(p.parse_args()) diff --git a/competitions/trace_the_ace/v134_verifier_feedback_contradiction.py b/competitions/trace_the_ace/v134_verifier_feedback_contradiction.py new file mode 100644 index 00000000..0477440b --- /dev/null +++ b/competitions/trace_the_ace/v134_verifier_feedback_contradiction.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V134: verifier x tutor-feedback contradiction over frozen V97. + +V133 closes arithmetic truth in isolation. V75, however, derives mastery states from +tutor feedback. V134 tests the missing relation: whether independent arithmetic truth +agrees with the tutor's subsequent positive/negative judgment. + +Control: deterministically rotate tutor judgments across verifiable events within each +session, preserving truth and feedback marginals while destroying their pairing. +No sweep. Outer validation untouched; residual fit uses inner-OOF V97 only. +""" +from __future__ import annotations +import argparse,csv,hashlib,io,json,re,zipfile +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from v110_residual_collider_state_discovery import ll,logit,p97_predict +from v75_canonical_trajectory import SEED,POS_RE,NEG_RE +from v133_verified_math_evidence import expected,answer_value,QUESTION_RE +EPS=1e-5; C=.05 + +def events(rows): + out=[] + for i,r in enumerate(rows): + if str(r.get('role','')).lower()!='tutor': continue + q=str(r.get('content','')) + if not QUESTION_RE.search(q): continue + ex=expected(q) + if not np.isfinite(ex): continue + ai=None + for j in range(i+1,min(len(rows),i+6)): + role=str(rows[j].get('role','')).lower(); txt=str(rows[j].get('content','')) + if role=='student' and txt.strip(): ai=j; break + if role=='tutor' and QUESTION_RE.search(txt) and j>i+1: break + if ai is None: continue + av=answer_value(rows[ai].get('content','')) + if not np.isfinite(av): continue + fb='' + for j in range(ai+1,min(len(rows),ai+6)): + if str(rows[j].get('role','')).lower()=='tutor': fb=str(rows[j].get('content','')); break + judge=1 if POS_RE.search(fb) and not NEG_RE.search(fb) else -1 if NEG_RE.search(fb) and not POS_RE.search(fb) else 0 + truth=1 if np.isclose(av,ex,rtol=1e-6,atol=1e-6) else -1 + out.append((truth,judge,ai/max(1,len(rows)-1))) + return out + +def vec(E,shift=0): + n=len(E) + if not n:return np.zeros(10,float) + truth=np.asarray([e[0] for e in E],float); judge=np.asarray([e[1] for e in E],float); rec=np.asarray([e[2] for e in E],float) + if shift and n>1: judge=np.roll(judge,shift) + judged=judge!=0; agree=judged&(truth==judge); contra=judged&(truth!=judge) + false_pos=(judge==1)&(truth==-1); false_neg=(judge==-1)&(truth==1) + w=np.exp(2*(rec-1));w/=w.sum()+1e-12 + def mean(x):return float(np.mean(x)) + return np.asarray([np.log1p(n),mean(judged),mean(agree),mean(contra),mean(false_pos),mean(false_neg),float((w*contra).sum()),float((w*agree).sum()),float(rec[contra].max()) if np.any(contra) else 0.,float(contra[-1])],float) + +def inner_base(X75,Xr,y,groups,support): + P=np.zeros(len(y)); nsp=min(3,len(np.unique(groups))) + for tr,va in GroupKFold(nsp).split(np.zeros(len(y)),y,groups):P[va],_=p97_predict(X75,Xr,y,tr,va,support) + return np.clip(P,EPS,1-EPS) +def fit_res(P,R,y): + mu=R.mean(0);sd=R.std(0)+1e-6;X=np.c_[logit(P),(R-mu)/sd] + m=LogisticRegression(C=C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y);return mu,sd,m +def apply_res(mod,P,R): + mu,sd,m=mod;X=np.c_[logit(P),(R-mu)/sd];return np.clip(m.predict_proba(X)[:,1],EPS,1-EPS) +def eval_geom(name,groups,X75,Xr,y,support,R,A,covered): + groups=np.asarray(groups);PB=np.zeros(len(y));QR=np.zeros(len(y));QA=np.zeros(len(y));folds=[] + for k,(tr,va) in enumerate(GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(len(y)),y,groups),1): + pva,_=p97_predict(X75,Xr,y,tr,va,support);pin=inner_base(X75[tr],Xr[tr],y[tr],groups[tr],support[tr]) + qr=apply_res(fit_res(pin,R[tr],y[tr]),pva,R[va]);qa=apply_res(fit_res(pin,A[tr],y[tr]),pva,A[va]);PB[va]=pva;QR[va]=qr;QA[va]=qa + folds.append({'fold':k,'base_ll':ll(y[va],pva),'contradiction_ll':ll(y[va],qr),'control_ll':ll(y[va],qa),'covered':int(covered[va].sum())}) + b,r,a=ll(y,PB),ll(y,QR),ll(y,QA);cov=np.asarray(covered,bool) + return {'geometry':name,'baseline_v97_ll':b,'contradiction':{'ll':r,'gain':b-r},'rotated_judgment_control':{'ll':a,'gain':b-a},'real_minus_control_gain':a-r,'coverage_fraction':float(cov.mean()),'covered_rows':int(cov.sum()),'covered_only':({'baseline_ll':ll(y[cov],PB[cov]),'contradiction_ll':ll(y[cov],QR[cov]),'gain':ll(y[cov],PB[cov])-ll(y[cov],QR[cov])} if cov.any() else None),'folds':folds} +def main(a): + d=Path(a.dir);z=np.load(d/'arrays.npz',allow_pickle=True);y=z['y'];obj=z['objectives'];support=z['support'];sessions=z['sessions'];X75=load_npz(d/'X75.npz');Xr=load_npz(d/'Xr.npz');cache={} + with zipfile.ZipFile(a.archive) as za: + names=set(za.namelist()) + for sid in np.unique(sessions): + name=f'{sid}.csv' + if name not in names:raise RuntimeError('missing '+name) + with za.open(name) as f:rows=list(csv.DictReader(io.TextIOWrapper(f,encoding='utf-8-sig',newline=''))) + E=events(rows);shift=(1+int(hashlib.sha256(str(sid).encode()).hexdigest()[:8],16)%max(1,len(E)-1)) if len(E)>1 else 0 + cache[str(sid)]=(vec(E,0),vec(E,shift),len(E)) + R=np.vstack([cache[str(s)][0] for s in sessions]);A=np.vstack([cache[str(s)][1] for s in sessions]);covered=np.asarray([cache[str(s)][2]>0 for s in sessions]) + out={'protocol':'V134_VERIFIER_FEEDBACK_CONTRADICTION','rows':int(len(y)),'hypothesis':'independent truth x tutor judgment contradiction is a missing sample-local relation over V97','precommit':{'residual_C':C,'outer_folds':4,'inner_folds':3,'no_sweep':True,'promote_gain_each_geometry':.001,'real_minus_control_each_geometry':.0005,'phase_change_gain_each_geometry':.003}} + out['objective_grouped']=eval_geom('objective_grouped',obj,X75,Xr,y,support,R,A,covered);out['session_grouped']=eval_geom('session_grouped',sessions,X75,Xr,y,support,R,A,covered) + def ok(x,t=.001):return x['contradiction']['gain']>=t and x['real_minus_control_gain']>=.0005 + po,ps=ok(out['objective_grouped']),ok(out['session_grouped']);phase=ok(out['objective_grouped'],.003) and ok(out['session_grouped'],.003) + out['decision']={'objective_pass':bool(po),'session_pass':bool(ps),'verdict':'PHASE_CHANGE_VERIFIER_FEEDBACK_CONTRADICTION' if phase else 'PROMOTE_VERIFIER_FEEDBACK_CONTRADICTION' if po and ps else 'SUPPRESS_VERIFIER_FEEDBACK_CONTRADICTION'} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--archive',required=True);p.add_argument('--dir',required=True);p.add_argument('--out',required=True);main(p.parse_args()) diff --git a/competitions/trace_the_ace/v135_nested_supported_stack.py b/competitions/trace_the_ace/v135_nested_supported_stack.py new file mode 100644 index 00000000..c09bcb94 --- /dev/null +++ b/competitions/trace_the_ace/v135_nested_supported_stack.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""V135: deployable nested stack for supported objectives. + +Constraint-derived from V105/V106/V97 lineage: +- V97 changes only unsupported objectives; public gain over V75 is tiny. +- V105 showed V74+V75+RELATED complementarity but selected weights on held-out folds. +- V125 closed pure calibration. + +V135 learns composition only from inner-OOF predictions on outer-training rows. +For any outer row whose objective has no support in outer training, prediction is EXACTLY +V97. Thus objective-cold cannot regress by construction. For supported rows, compare: +A0 V97 +A1 nested stack from V75+RELATED only +A2 nested stack from V75+RELATED+smoothed objective-difficulty prior. +No sweep. Objective prior smoothing alpha fixed at 10. +""" +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from scipy.sparse import load_npz +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from v110_residual_collider_state_discovery import EPS,ll,logit,fit_base,p97_predict +from v75_canonical_trajectory import SEED +ALPHA=10.0; C=.10 + +def prior_apply(y,tr,va,support): + g=float(np.mean(y[tr])); sums={}; counts={} + for i in tr: + k=str(support[i]); sums[k]=sums.get(k,0.)+float(y[i]); counts[k]=counts.get(k,0)+1 + p=np.empty(len(va)); c=np.empty(len(va)); seen=np.empty(len(va),bool) + for j,i in enumerate(va): + k=str(support[i]); n=counts.get(k,0); s=sums.get(k,0.); seen[j]=n>0; c[j]=n + p[j]=(s+ALPHA*g)/(n+ALPHA) + return np.clip(p,EPS,1-EPS),c,seen + +def components(X75,Xr,y,tr,va,support): + p75=fit_base(X75,y,tr,va); pr=fit_base(Xr,y,tr,va); pp,c,seen=prior_apply(y,tr,va,support) + p97=np.where(seen,p75,.65*p75+.35*pr) + return np.clip(p97,EPS,1-EPS),p75,pr,pp,c,seen + +def feats(p75,pr,pp,c,full=True): + xs=[logit(p75),logit(pr),logit(p75)-logit(pr)] + if full: xs += [logit(pp),np.log1p(c)] + return np.column_stack(xs) + +def fit_stack(X,y): + return LogisticRegression(C=C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y) + +def nested_geom(name,groups,X75,Xr,y,support): + groups=np.asarray(groups); n=len(y); p0=np.zeros(n); p1=np.zeros(n); p2=np.zeros(n); folds=[] + outer=list(GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(n),y,groups)) + for k,(tr,va) in enumerate(outer,1): + # Outer predictions/components are untouched. + q0,o75,orr,opp,oc,oseen=components(X75,Xr,y,tr,va,support) + # Inner-OOF components for stack training only. + ig=groups[tr]; inner=list(GroupKFold(min(3,len(np.unique(ig)))).split(np.zeros(len(tr)),y[tr],ig)) + ip75=np.zeros(len(tr)); ipr=np.zeros(len(tr)); ipp=np.zeros(len(tr)); ic=np.zeros(len(tr)); iseen=np.zeros(len(tr),bool) + for ltr,lva in inner: + atr=tr[ltr]; ava=tr[lva] + _,a75,ar,ap,ac,aseen=components(X75,Xr,y,atr,ava,support) + ip75[lva]=a75;ipr[lva]=ar;ipp[lva]=ap;ic[lva]=ac;iseen[lva]=aseen + # Learn only from examples where the objective was actually supported. + fitmask=iseen + if fitmask.sum()<50 or len(np.unique(y[tr][fitmask]))<2: + q1=q0.copy();q2=q0.copy() + else: + m1=fit_stack(feats(ip75[fitmask],ipr[fitmask],ipp[fitmask],ic[fitmask],False),y[tr][fitmask]) + m2=fit_stack(feats(ip75[fitmask],ipr[fitmask],ipp[fitmask],ic[fitmask],True),y[tr][fitmask]) + q1=q0.copy();q2=q0.copy() + if oseen.any(): + q1[oseen]=np.clip(m1.predict_proba(feats(o75[oseen],orr[oseen],opp[oseen],oc[oseen],False))[:,1],EPS,1-EPS) + q2[oseen]=np.clip(m2.predict_proba(feats(o75[oseen],orr[oseen],opp[oseen],oc[oseen],True))[:,1],EPS,1-EPS) + p0[va]=q0;p1[va]=q1;p2[va]=q2 + folds.append({'fold':k,'rows':int(len(va)),'supported_fraction':float(oseen.mean()),'v97_ll':ll(y[va],q0),'composition_ll':ll(y[va],q1),'full_stack_ll':ll(y[va],q2)}) + b,a1,a2=ll(y,p0),ll(y,p1),ll(y,p2) + return {'geometry':name,'v97_ll':b,'composition_only':{'ll':a1,'gain':b-a1},'full_stack':{'ll':a2,'gain':b-a2},'prior_incremental_gain':a1-a2,'folds':folds} + +def main(a): + d=Path(a.dir);z=np.load(d/'arrays.npz',allow_pickle=True);y=z['y'];obj=z['objectives'];support=z['support'];sessions=z['sessions'];X75=load_npz(d/'X75.npz');Xr=load_npz(d/'Xr.npz') + out={'protocol':'V135_NESTED_SUPPORTED_STACK','rows':int(len(y)),'hypothesis':'deployable inner-OOF composition of V75, RELATED, and objective difficulty improves supported-objective regime while exact-fallback preserves unsupported V97','precommit':{'outer_folds':4,'inner_folds':3,'stack_C':C,'prior_alpha':ALPHA,'no_sweep':True,'supported_session_gain':.001,'objective_noninferiority':.0001,'prior_incremental_gain':.0003}} + out['objective_grouped']=nested_geom('objective_grouped',obj,X75,Xr,y,support) + out['session_grouped']=nested_geom('session_grouped',sessions,X75,Xr,y,support) + O=out['objective_grouped'];S=out['session_grouped'] + comp=(S['composition_only']['gain']>=.001 and O['composition_only']['gain']>=-.0001) + full=(S['full_stack']['gain']>=.001 and O['full_stack']['gain']>=-.0001) + prior=full and S['prior_incremental_gain']>=.0003 + best='FULL_STACK' if full and S['full_stack']['gain']>=S['composition_only']['gain'] else 'COMPOSITION_ONLY' if comp else 'NONE' + out['decision']={'composition_pass':bool(comp),'full_stack_pass':bool(full),'objective_prior_causal':bool(prior),'preferred':best,'verdict':'PROMOTE_NESTED_SUPPORTED_'+best if best!='NONE' else 'SUPPRESS_NESTED_SUPPORTED_STACK'} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--dir',required=True);p.add_argument('--out',required=True);main(p.parse_args()) diff --git a/competitions/trace_the_ace/v136_full_v135_verification.py b/competitions/trace_the_ace/v136_full_v135_verification.py new file mode 100644 index 00000000..01f6abde --- /dev/null +++ b/competitions/trace_the_ace/v136_full_v135_verification.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""V136: full-data untouched verification of promoted V135. + +This is verification, not discovery. V135's operator and hyperparameters are frozen: +- V75 + RELATED + smoothed exact-objective difficulty prior (alpha=10) +- logistic stack C=.10 +- stack fit only from inner-OOF support-seen rows +- exact V97 fallback whenever target objective is unsupported by outer training +- 4 outer / 3 inner folds, no sweep + +V136 rebuilds V75 and RELATED from the full competition training corpus and evaluates +that exact operator on all training rows. Primary gate is session-grouped transfer; +objective-grouped is a structural non-regression audit and must remain exactly V97. +""" +from __future__ import annotations +import argparse,json,time +from pathlib import Path +import numpy as np +from v75_canonical_trajectory import load_training +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v135_nested_supported_stack import nested_geom,ALPHA,C + +def main(a): + t0=time.time() + f=load_training(a.features,a.labels).reset_index(drop=True) + print('FULL_ROWS',len(f),'COLUMNS',list(f.columns),flush=True) + y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support=f.learning_objective.astype(str).to_numpy() + sessions=f.session_id.astype(str).to_numpy() + cache={} + us=np.unique(sessions) + for j,sid in enumerate(us,1): + cache[str(sid)]=load_transcript(a.transcripts/f'{sid}.csv') + if j%2500==0: print('TRANSCRIPTS',j,'/',len(us),'elapsed',round(time.time()-t0,1),flush=True) + print('BUILD_V75',flush=True) + X75=build_v75(f,cache) + print('V75_SHAPE',X75.shape,'nnz',X75.nnz,'elapsed',round(time.time()-t0,1),flush=True) + rt=[];rz=[] + for i,r in f.iterrows(): + text,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related') + rt.append(text);rz.append(z) + if (i+1)%5000==0: print('RELATED_ROWS',i+1,'elapsed',round(time.time()-t0,1),flush=True) + Xr=build_control(rt,rz) + print('RELATED_SHAPE',Xr.shape,'nnz',Xr.nnz,'elapsed',round(time.time()-t0,1),flush=True) + + # Primary untouched verification world: unseen sessions, mostly support-seen objectives. + S=nested_geom('session_grouped_full',sessions,X75,Xr,y,support) + print('SESSION_RESULT',json.dumps(S,indent=2),flush=True) + # Structural safety audit: objective-held-out rows must use exact V97 fallback. + O=nested_geom('objective_grouped_full',obj,X75,Xr,y,support) + print('OBJECTIVE_RESULT',json.dumps(O,indent=2),flush=True) + + sg=S['full_stack']['gain']; og=O['full_stack']['gain']; inc=S['prior_incremental_gain'] + all_session_folds=all(x['full_stack_ll'] < x['v97_ll'] for x in S['folds']) + obj_exact=abs(og)<=1e-12 and all(abs(x['full_stack_ll']-x['v97_ll'])<=1e-12 for x in O['folds']) + verdict=('PROMOTE_V135_TO_RUNTIME' if sg>=.005 and inc>=.003 and all_session_folds and obj_exact + else 'RETAIN_V135_PARTIAL' if sg>=.002 and obj_exact + else 'SUPPRESS_V135_FULL_TRANSFER') + out={ + 'protocol':'V136_FULL_V135_VERIFICATION', + 'rows':int(len(f)),'sessions':int(len(np.unique(sessions))),'objectives':int(len(np.unique(obj))), + 'frozen_operator':{'stack_C':C,'prior_alpha':ALPHA,'outer_folds':4,'inner_folds':3,'unsupported_fallback':'exact_v97'}, + 'precommit':{'primary_session_gain':.005,'prior_incremental_gain':.003,'all_session_folds_improve':True,'objective_exact_nonregression':True}, + 'session_grouped':S,'objective_grouped':O, + 'decision':{'session_gain':float(sg),'prior_incremental_gain':float(inc),'all_session_folds_improve':bool(all_session_folds),'objective_exact_nonregression':bool(obj_exact),'verdict':verdict}, + 'elapsed_seconds':float(time.time()-t0) + } + Path(a.out).write_text(json.dumps(out,indent=2));print('FINAL',json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',default='v136_full_v135_verification.json');main(p.parse_args()) diff --git a/competitions/trace_the_ace/v145_v135_plus_evidence_family.py b/competitions/trace_the_ace/v145_v135_plus_evidence_family.py new file mode 100644 index 00000000..51373b74 --- /dev/null +++ b/competitions/trace_the_ace/v145_v135_plus_evidence_family.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""V145: closure-before-invention test over the strongest old unsummed capability family. + +Residual entering V145: +- V135 is lawful and externally smoke-positive, but its full-data gain is too small for top 3. +- simple calibration and 1D/2D/component applicability have been closed. +- V81/V84/V87 contain target-segment, instructional-phase, and student-evidence views that + are not present as separate experts in V135. + +Frozen separator: +A0 = exact nested V135 incumbent. +A1 = nested V135 + target-segment + phase + evidence experts. +C0 = matched-capacity alignment control: same extra experts, but their row alignment is + deterministically permuted within support-status strata before meta fitting/application. + +All base-expert predictions used by a meta learner are OOF inside the corresponding outer +training partition. No test aggregates, same-session labels, or hidden outcomes are used at +inference. The full-data gate requires >= .004 LL gain over V135 in BOTH session- and +objective-grouped worlds, plus A1 beating C0 in both worlds. +""" +from __future__ import annotations +import argparse, json, time +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.metrics import log_loss + +from v75_canonical_trajectory import load_training, trajectory_views, SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +from v81_target_segment_phase import choose_target_segment, phase_views, build_X +from v84_student_evidence import evidence_view, build_evidence +from v110_residual_collider_state_discovery import EPS, fit_base, logit +from v135_nested_supported_stack import prior_apply, fit_stack, feats as v135_feats + +STACK_C=.10 +GAIN_GATE=.004 + + +def ll(y,p): + return float(log_loss(y,np.clip(p,EPS,1-EPS))) + + +def fit_lr(X,y): + return LogisticRegression(C=STACK_C,max_iter=300,solver='liblinear',random_state=SEED).fit(X,y) + + +def extra_feats(p75,pr,pp,c,ps,pph,pe,seen): + """Two routing-compatible feature spaces: prior allowed only when support is seen.""" + common=np.column_stack([ + logit(p75),logit(pr),logit(p75)-logit(pr), + logit(ps),logit(pph),logit(pe), + logit(ps)-logit(p75),logit(pph)-logit(p75),logit(pe)-logit(p75), + ]) + full=np.column_stack([common,logit(pp),np.log1p(c)]) + return full,common + + +def deterministic_perm(n,seed): + rng=np.random.RandomState(seed) + return rng.permutation(n) + + +def permute_extras(ps,pph,pe,seen,seed): + a,b,c=np.asarray(ps).copy(),np.asarray(pph).copy(),np.asarray(pe).copy() + for j,maskval in enumerate([False,True]): + idx=np.flatnonzero(np.asarray(seen)==maskval) + if len(idx)>1: + p=deterministic_perm(len(idx),seed+97*j) + src=idx[p] + a[idx]=np.asarray(ps)[src]; b[idx]=np.asarray(pph)[src]; c[idx]=np.asarray(pe)[src] + return a,b,c + + +def base_components(X75,Xr,Xs,Xp,Xe,y,tr,va,support): + p75=fit_base(X75,y,tr,va) + pr=fit_base(Xr,y,tr,va) + ps=fit_base(Xs,y,tr,va) + pph=fit_base(Xp,y,tr,va) + pe=fit_base(Xe,y,tr,va) + pp,c,seen=prior_apply(y,tr,va,support) + p97=np.where(seen,p75,.65*p75+.35*pr) + return [np.clip(x,EPS,1-EPS) for x in [p97,p75,pr,pp,ps,pph,pe]],c,seen + + +def nested_world(name,groups,X75,Xr,Xs,Xp,Xe,y,support): + groups=np.asarray(groups); support=np.asarray(support); n=len(y) + p135=np.zeros(n); p145=np.zeros(n); pctl=np.zeros(n); folds=[] + outer=list(GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(n),y,groups)) + for k,(tr,va) in enumerate(outer,1): + q97,o75,orr,opp,os,oph,oe,oc,oseen = (*base_components(X75,Xr,Xs,Xp,Xe,y,tr,va,support)[0], *base_components(X75,Xr,Xs,Xp,Xe,y,tr,va,support)[1:]) + # NOTE: avoid recomputing the expensive fits; tuple assembly above is replaced below. + raise RuntimeError('UNREACHABLE_PLACEHOLDER') + return {} + + +def nested_world(name,groups,X75,Xr,Xs,Xp,Xe,y,support): + groups=np.asarray(groups); support=np.asarray(support); n=len(y) + p135=np.zeros(n); p145=np.zeros(n); pctl=np.zeros(n); folds=[] + outer=list(GroupKFold(min(4,len(np.unique(groups)))).split(np.zeros(n),y,groups)) + for k,(tr,va) in enumerate(outer,1): + comps,oc,oseen=base_components(X75,Xr,Xs,Xp,Xe,y,tr,va,support) + q97,o75,orr,opp,os,oph,oe=comps + + ig=groups[tr] + inner=list(GroupKFold(min(3,len(np.unique(ig)))).split(np.zeros(len(tr)),y[tr],ig)) + ip75=np.zeros(len(tr)); ipr=np.zeros(len(tr)); ipp=np.zeros(len(tr)); ic=np.zeros(len(tr)); + ips=np.zeros(len(tr)); ipph=np.zeros(len(tr)); ipe=np.zeros(len(tr)); iseen=np.zeros(len(tr),bool) + for ltr,lva in inner: + atr=tr[ltr]; ava=tr[lva] + cc,ac,aseen=base_components(X75,Xr,Xs,Xp,Xe,y,atr,ava,support) + _,a75,ar,ap,as_,aph,ae=cc + ip75[lva]=a75; ipr[lva]=ar; ipp[lva]=ap; ic[lva]=ac + ips[lva]=as_; ipph[lva]=aph; ipe[lva]=ae; iseen[lva]=aseen + + # Exact V135 baseline: supported stack trained only on inner-OOF support-seen rows; + # unsupported outer rows remain exact V97. + q135=q97.copy() + sm=iseen + if sm.sum()>=50 and len(np.unique(y[tr][sm]))>1: + m135=fit_stack(v135_feats(ip75[sm],ipr[sm],ipp[sm],ic[sm],True),y[tr][sm]) + if oseen.any(): + q135[oseen]=np.clip(m135.predict_proba(v135_feats(o75[oseen],orr[oseen],opp[oseen],oc[oseen],True))[:,1],EPS,1-EPS) + + # V145 intervention: support-specific meta learners. Seen support gets prior + extras; + # unseen support gets only sample-local experts (no objective prior). + fin_s,fin_u=extra_feats(ip75,ipr,ipp,ic,ips,ipph,ipe,iseen) + fout_s,fout_u=extra_feats(o75,orr,opp,oc,os,oph,oe,oseen) + q145=q135.copy() + if sm.sum()>=50 and len(np.unique(y[tr][sm]))>1: + ms=fit_lr(fin_s[sm],y[tr][sm]) + if oseen.any(): q145[oseen]=np.clip(ms.predict_proba(fout_s[oseen])[:,1],EPS,1-EPS) + um=~iseen + if um.sum()>=50 and len(np.unique(y[tr][um]))>1: + mu=fit_lr(fin_u[um],y[tr][um]) + if (~oseen).any(): q145[~oseen]=np.clip(mu.predict_proba(fout_u[~oseen])[:,1],EPS,1-EPS) + + # Alignment control with identical model capacity and marginals. + cis,ciph,cie=permute_extras(ips,ipph,ipe,iseen,SEED+1000*k) + cos,coph,coe=permute_extras(os,oph,oe,oseen,SEED+2000*k) + cin_s,cin_u=extra_feats(ip75,ipr,ipp,ic,cis,ciph,cie,iseen) + cout_s,cout_u=extra_feats(o75,orr,opp,oc,cos,coph,coe,oseen) + qctl=q135.copy() + if sm.sum()>=50 and len(np.unique(y[tr][sm]))>1: + cms=fit_lr(cin_s[sm],y[tr][sm]) + if oseen.any(): qctl[oseen]=np.clip(cms.predict_proba(cout_s[oseen])[:,1],EPS,1-EPS) + if um.sum()>=50 and len(np.unique(y[tr][um]))>1: + cmu=fit_lr(cin_u[um],y[tr][um]) + if (~oseen).any(): qctl[~oseen]=np.clip(cmu.predict_proba(cout_u[~oseen])[:,1],EPS,1-EPS) + + p135[va]=q135; p145[va]=q145; pctl[va]=qctl + folds.append({ + 'fold':k,'rows':int(len(va)),'supported_fraction':float(oseen.mean()), + 'v135_ll':ll(y[va],q135),'v145_ll':ll(y[va],q145),'control_ll':ll(y[va],qctl), + 'gain_v145_vs_v135':ll(y[va],q135)-ll(y[va],q145), + 'real_vs_control':ll(y[va],qctl)-ll(y[va],q145), + 'inner_supported':int(sm.sum()),'inner_unsupported':int(um.sum()) + }) + print(name,'FOLD',json.dumps(folds[-1]),flush=True) + b=ll(y,p135); a=ll(y,p145); c=ll(y,pctl) + return {'geometry':name,'v135_ll':b,'v145_ll':a,'control_ll':c, + 'gain_vs_v135':b-a,'real_vs_control':c-a,'folds':folds} + + +def main(a): + t0=time.time(); f=load_training(a.features,a.labels).reset_index(drop=True) + print('ROWS',len(f),'COLUMNS',list(f.columns),flush=True) + y=f.target.to_numpy(int); sessions=f.session_id.astype(str).to_numpy() + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support=f.learning_objective.astype(str).to_numpy() + + cache={}; us=np.unique(sessions) + for j,sid in enumerate(us,1): + cache[str(sid)]=load_transcript(a.transcripts/f'{sid}.csv') + if j%2500==0: print('TRANSCRIPTS',j,'/',len(us),'elapsed',round(time.time()-t0,1),flush=True) + + print('BUILD_V75',flush=True); X75=build_v75(f,cache) + rt=[]; rz=[]; seg_rows=[]; seg_nums=[]; phase_rows=[]; phase_nums=[]; ev_whole=[]; ev_seg=[]; ev_num=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; o=str(r.learning_objective) + tx,z=segmented_control(d,o,'related'); rt.append(tx); rz.append(z) + seg,_=choose_target_segment(d,o) + vs,ns,_=trajectory_views(seg,o); seg_rows.append(vs); seg_nums.append(ns) + pv,pn=phase_views(seg,o); phase_rows.append({**vs,**pv}); phase_nums.append(np.concatenate([ns,pn])) + ew,enw=evidence_view(d,o); es,ens=evidence_view(seg,o) + ev_whole.append(ew); ev_seg.append(es); ev_num.append(np.concatenate([enw,ens])) + if (i+1)%5000==0: print('VIEW_ROWS',i+1,'elapsed',round(time.time()-t0,1),flush=True) + Xr=build_control(rt,rz) + Xs=build_X(f,seg_rows,seg_nums,'SEG') + Xp=build_X(f,phase_rows,phase_nums,'PHASE') + Xe=build_evidence(f,ev_whole,ev_seg,ev_num) + for nm,X in [('V75',X75),('RELATED',Xr),('SEGMENT',Xs),('PHASE',Xp),('EVIDENCE',Xe)]: + print(nm,'SHAPE',X.shape,'NNZ',X.nnz,flush=True) + + S=nested_world('session_grouped',sessions,X75,Xr,Xs,Xp,Xe,y,support) + O=nested_world('objective_grouped',obj,X75,Xr,Xs,Xp,Xe,y,support) + pass_mag=(S['gain_vs_v135']>=GAIN_GATE and O['gain_vs_v135']>=GAIN_GATE) + pass_ctl=(S['real_vs_control']>0 and O['real_vs_control']>0) + all_pos=(all(x['gain_v145_vs_v135']>0 for x in S['folds']) and all(x['gain_v145_vs_v135']>0 for x in O['folds'])) + verdict='PROMOTE_V145_TO_SMOKE' if pass_mag and pass_ctl and all_pos else 'CLOSE_V81_V84_V87_FAMILY_FOR_TOP3' + out={'protocol':'V145_V135_PLUS_EVIDENCE_FAMILY','rows':int(len(f)), + 'residual':'V135 transfers externally but lacks top3 magnitude', + 'hypothesis':'target segment + instructional phase + evidence IR contain complementary sample-local information not already represented by V135', + 'precommit':{'gain_gate_each_geometry':GAIN_GATE,'real_beats_alignment_control_both':True,'all_outer_folds_positive':True,'no_sweep':True}, + 'session_grouped':S,'objective_grouped':O, + 'decision':{'magnitude_pass':bool(pass_mag),'control_pass':bool(pass_ctl),'all_folds_positive':bool(all_pos),'verdict':verdict}, + 'elapsed_seconds':float(time.time()-t0)} + Path(a.out).write_text(json.dumps(out,indent=2)); print('FINAL',json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v145_v135_plus_evidence_family.json'); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v152_transition_law_discovery.py b/competitions/trace_the_ace/v152_transition_law_discovery.py new file mode 100644 index 00000000..b09f0f90 --- /dev/null +++ b/competitions/trace_the_ace/v152_transition_law_discovery.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""V152: explicit transition-law discovery beyond V135. + +Verified-residual separator after V145 and V146-V151 close local composition, +routing, support, graph, and latent-session surrogates as top-3 magnitude routes. + +H1: the missing row-local distinction is the trajectory of canonical student-state +transitions, not merely transcript content/state counts. V75 retains ordered text but +its learner does not explicitly expose a transition grammar. + +Frozen discovery protocol: +- deterministic 2,500-row sample by SHA256(response_id); +- two protected geometries: session-grouped and objective-grouped, 4 outer folds; +- within each outer training partition, 3-fold grouped OOF creates all meta inputs; +- incumbent is exact V135 construction on the same rows; +- candidate adds one fixed transition expert from adjacent canonical states; +- controls: predictions-only meta, state-count-only expert, and deterministic + within-row shuffled-order transition expert; +- no sweep and no result-dependent feature selection. + +Discovery advances to full 35,072 only if transition gain >= .006 in BOTH geometries, +all outer folds are positive, and real transitions beat both count-only and shuffled-order +controls in both geometries. Discovery itself never authorizes a smoke submission. +""" +from __future__ import annotations +import argparse, hashlib, json, time +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold + +from v75_canonical_trajectory import load_training, extract_canonical_events, STATE_ORDER, SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +from v110_residual_collider_state_discovery import EPS, ll, logit, fit_base +from v135_nested_supported_stack import components, feats, fit_stack + +ROWS=2500 +STATES=list(STATE_ORDER.keys()) +S2I={s:i for i,s in enumerate(STATES)} +D=8 + 64 + 64 + 6 +C_META=.10 + + +def h64(x:str)->int: + return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) + + +def vec_from_states(states:list[str], shuffled:bool=False, seed_key:str='')->np.ndarray: + st=list(states) + if shuffled and len(st)>1: + rng=np.random.default_rng(h64(seed_key) & ((1<<63)-1)); rng.shuffle(st) + x=np.zeros(D,float) + # state counts + for s in st: x[S2I[s]] += 1.0 + # adjacent transition counts + recency weighted transition counts + off=8; woff=8+64 + den=max(1,len(st)-1) + reversals=0; same=0 + scores=[STATE_ORDER[s] for s in st] + for j,(a,b) in enumerate(zip(st[:-1],st[1:])): + k=S2I[a]*8+S2I[b]; x[off+k]+=1.0; x[woff+k]+=(j+1)/den + same += int(a==b) + if j and np.sign(scores[j]-scores[j-1]) != np.sign(scores[j+1]-scores[j]) and scores[j]!=scores[j-1] and scores[j+1]!=scores[j]: reversals += 1 + # compact sequence summaries + base=8+128 + x[base+0]=len(st) + x[base+1]=same + x[base+2]=reversals + x[base+3]=scores[-1]-scores[0] if len(scores)>=2 else 0.0 + x[base+4]=scores[-1] if scores else 0.0 + # longest monotone nondecreasing run + best=cur=1 if scores else 0 + for a,b in zip(scores[:-1],scores[1:]): + cur=cur+1 if b>=a else 1; best=max(best,cur) + x[base+5]=best + return x + + +def count_only(v:np.ndarray)->np.ndarray: + z=np.zeros_like(v); z[:8]=v[:8]; z[-6]=v[-6]; return z + + +def meta_base(q0,p75,pr,pp,c,seen): + return np.column_stack([logit(q0),logit(p75),logit(pr),logit(pp),np.log1p(c),seen.astype(float)]) + + +def meta_plus(base,p): + lp=logit(p); return np.column_stack([base,lp,lp-base[:,0]]) + + +def fit_meta(X,y): + return LogisticRegression(C=C_META,max_iter=400,solver='liblinear',random_state=SEED).fit(X,y) + + +def outer_components(X75,Xr,Xt,Xc,Xs,y,tr,va,support): + q0,p75,pr,pp,c,seen=components(X75,Xr,y,tr,va,support) + pt=fit_base(Xt,y,tr,va); pc=fit_base(Xc,y,tr,va); ps=fit_base(Xs,y,tr,va) + return q0,p75,pr,pp,c,seen,pt,pc,ps + + +def geometry(name,groups,X75,Xr,Xt,Xc,Xs,y,support): + n=len(y); groups=np.asarray(groups); outer=list(GroupKFold(4).split(np.zeros(n),y,groups)) + P135=np.zeros(n); PM=np.zeros(n); PT=np.zeros(n); PC=np.zeros(n); PS=np.zeros(n); folds=[] + for k,(tr,va) in enumerate(outer,1): + oq0,o75,orr,opp,oc,oseen,opt,opc,ops=outer_components(X75,Xr,Xt,Xc,Xs,y,tr,va,support) + ig=groups[tr]; inner=list(GroupKFold(min(3,len(np.unique(ig)))).split(np.zeros(len(tr)),y[tr],ig)) + iq0=np.zeros(len(tr)); i75=np.zeros(len(tr)); ir=np.zeros(len(tr)); ipp=np.zeros(len(tr)); ic=np.zeros(len(tr)); iseen=np.zeros(len(tr),bool) + it=np.zeros(len(tr)); ico=np.zeros(len(tr)); ish=np.zeros(len(tr)) + for ltr,lva in inner: + atr=tr[ltr]; ava=tr[lva] + z=outer_components(X75,Xr,Xt,Xc,Xs,y,atr,ava,support) + iq0[lva],i75[lva],ir[lva],ipp[lva],ic[lva],iseen[lva],it[lva],ico[lva],ish[lva]=z + # Exact V135 incumbent on supported rows. + q135=oq0.copy(); fitmask=iseen + if fitmask.sum()>=50 and len(np.unique(y[tr][fitmask]))==2: + m135=fit_stack(feats(i75[fitmask],ir[fitmask],ipp[fitmask],ic[fitmask],True),y[tr][fitmask]) + if oseen.any(): q135[oseen]=np.clip(m135.predict_proba(feats(o75[oseen],orr[oseen],opp[oseen],oc[oseen],True))[:,1],EPS,1-EPS) + ib=meta_base(iq0,i75,ir,ipp,ic,iseen); ob=meta_base(oq0,o75,orr,opp,oc,oseen) + # Same-capacity/nesting controls establish whether any gain is transition-specific. + mb=fit_meta(ib,y[tr]); mt=fit_meta(meta_plus(ib,it),y[tr]); mc=fit_meta(meta_plus(ib,ico),y[tr]); ms=fit_meta(meta_plus(ib,ish),y[tr]) + qm=np.clip(mb.predict_proba(ob)[:,1],EPS,1-EPS) + qt=np.clip(mt.predict_proba(meta_plus(ob,opt))[:,1],EPS,1-EPS) + qc=np.clip(mc.predict_proba(meta_plus(ob,opc))[:,1],EPS,1-EPS) + qs=np.clip(ms.predict_proba(meta_plus(ob,ops))[:,1],EPS,1-EPS) + P135[va]=q135; PM[va]=qm; PT[va]=qt; PC[va]=qc; PS[va]=qs + r={'fold':k,'rows':int(len(va)),'v135_ll':ll(y[va],q135),'predictions_only_ll':ll(y[va],qm),'transition_ll':ll(y[va],qt),'count_only_ll':ll(y[va],qc),'shuffled_order_ll':ll(y[va],qs)} + r['gain_transition_vs_v135']=r['v135_ll']-r['transition_ll']; r['gain_transition_vs_count']=r['count_only_ll']-r['transition_ll']; r['gain_transition_vs_shuffle']=r['shuffled_order_ll']-r['transition_ll'] + folds.append(r); print(name,'FOLD',json.dumps(r),flush=True) + out={'geometry':name,'v135_ll':ll(y,P135),'predictions_only_ll':ll(y,PM),'transition_ll':ll(y,PT),'count_only_ll':ll(y,PC),'shuffled_order_ll':ll(y,PS),'folds':folds} + out['gain_transition_vs_v135']=out['v135_ll']-out['transition_ll']; out['gain_transition_vs_predictions_only']=out['predictions_only_ll']-out['transition_ll']; out['gain_transition_vs_count']=out['count_only_ll']-out['transition_ll']; out['gain_transition_vs_shuffle']=out['shuffled_order_ll']-out['transition_ll']; out['all_folds_positive']=all(r['gain_transition_vs_v135']>0 for r in folds) + return out + + +def main(a): + t0=time.time(); f0=load_training(a.features,a.labels).reset_index(drop=True) + order=sorted(range(len(f0)),key=lambda i:h64(f0.response_id.iloc[i])); ix=np.array(order[:min(ROWS,len(order))]); f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); sessions=f.session_id.astype(str).to_numpy(); objectives=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy() + cache={}; us=np.unique(sessions) + for j,sid in enumerate(us,1): + cache[str(sid)]=load_transcript(a.transcripts/f'{sid}.csv') + if j%500==0: print('TRANSCRIPTS',j,'/',len(us),'elapsed',round(time.time()-t0,1),flush=True) + print('BUILD V75',flush=True); X75=build_v75(f,cache) + rt=[]; rz=[]; vr=[]; vc=[]; vs=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + txt,z=segmented_control(d,obj,'related'); rt.append(txt); rz.append(z) + st=[e.state for e in extract_canonical_events(d,obj)] + real=vec_from_states(st,False,str(r.response_id)); sham=vec_from_states(st,True,str(r.response_id)); vr.append(real); vc.append(count_only(real)); vs.append(sham) + Xr=build_control(rt,rz); Xt=csr_matrix(np.asarray(vr)); Xc=csr_matrix(np.asarray(vc)); Xs=csr_matrix(np.asarray(vs)) + print('SHAPES',X75.shape,Xr.shape,Xt.shape,'elapsed',round(time.time()-t0,1),flush=True) + out={'protocol':'V152_TRANSITION_LAW_DISCOVERY','rows':int(len(y)),'states':STATES,'feature_dim':D,'precommit':{'sample':'lowest SHA256(response_id), 2500 rows','outer_folds':4,'inner_folds':3,'no_sweep':True,'discovery_gain_both_geometries':.006,'all_outer_folds_positive':True,'must_beat_count_and_shuffle_both':True,'full_run_before_smoke':True}} + out['session_grouped']=geometry('session_grouped',sessions,X75,Xr,Xt,Xc,Xs,y,support) + out['objective_grouped']=geometry('objective_grouped',objectives,X75,Xr,Xt,Xc,Xs,y,support) + S=out['session_grouped']; O=out['objective_grouped'] + advance=(S['gain_transition_vs_v135']>=.006 and O['gain_transition_vs_v135']>=.006 and S['all_folds_positive'] and O['all_folds_positive'] and S['gain_transition_vs_count']>0 and O['gain_transition_vs_count']>0 and S['gain_transition_vs_shuffle']>0 and O['gain_transition_vs_shuffle']>0) + out['decision']={'verdict':'ADVANCE_V152_TO_FULL' if advance else 'CLOSE_TRANSITION_LAW_DISCOVERY','advance':bool(advance),'next':'Run frozen full-data V152 with same representation and gates.' if advance else 'Ratchet transition grammar negative; zoom out to a representation not expressible by existing canonical-event sequence.'} + out['elapsed_seconds']=time.time()-t0 + Path(a.out).write_text(json.dumps(out,indent=2)); print('FINAL',json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v152_transition_law_discovery.json'); main(p.parse_args()) diff --git a/competitions/trace_the_ace/v71_mastery_events.py b/competitions/trace_the_ace/v71_mastery_events.py new file mode 100644 index 00000000..f4395a1a --- /dev/null +++ b/competitions/trace_the_ace/v71_mastery_events.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Trace the Ace V71: objective-conditioned mastery events. + +This development experiment converts each tutoring transcript into a sequence of +question -> student response -> tutor feedback episodes, scores objective +relevance, estimates independence/hint/correction state, and evaluates whether +those mastery-state features add signal beyond a sparse lexical baseline. + +The script intentionally inspects CSV headers before making schema decisions. +It never uses information across test samples at inference time. +""" +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +SEED = 20260815 +TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[0-9]+)?") +MATH_RE = re.compile(r"(?:\d|[+\-*/=×÷<>]|\b(?:half|quarter|third|tenths?|hundredths?|thousandths?)\b)", re.I) +QUESTION_RE = re.compile(r"\?|\b(?:what|which|how|why|can you|could you|tell me|work out|calculate|solve|find)\b", re.I) +POS_RE = re.compile(r"\b(?:yes|yeah|correct|right|exactly|perfect|good|great|well done|that's it|thats it|you got it|spot on)\b", re.I) +NEG_RE = re.compile(r"\b(?:no|not quite|incorrect|wrong|careful|try again|almost|remember|instead|actually)\b", re.I) +HINT_RE = re.compile(r"\b(?:hint|remember|think about|what if|try|look at|start with|first step|help you)\b", re.I) +AGREE_RE = re.compile(r"^(?:yeah|yes|yep|okay|ok|mm+|mhm|uh huh|right|sure)[.! ]*$", re.I) + +STOP = { + "the","a","an","and","or","to","of","in","on","for","with","is","are","be","as","by","from", + "this","that","these","those","you","your","we","it","its","into","using","use","up","than","then" +} + + +def tokens(text: str) -> set[str]: + return {t for t in TOKEN_RE.findall(str(text).lower()) if len(t) > 1 and t not in STOP} + + +def is_substantive_answer(answer: str) -> bool: + """Keep concise mathematical answers while rejecting acknowledgement-only turns.""" + a = str(answer).strip() + if not a or AGREE_RE.match(a): + return False + if MATH_RE.search(a): + return True + return len(tokens(a)) >= 2 + + +def jaccard(a: set[str], b: set[str]) -> float: + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def char_ngram_overlap(a: str, b: str, n: int = 4) -> float: + def grams(s: str) -> set[str]: + s = re.sub(r"\s+", " ", s.lower()).strip() + return {s[i:i+n] for i in range(max(0, len(s)-n+1))} + ga, gb = grams(a), grams(b) + return jaccard(ga, gb) + + +def inspect_headers(path: Path) -> list[str]: + return list(pd.read_csv(path, nrows=0).columns) + + +@dataclass +class Episode: + q_idx: int + a_idx: int + f_idx: int | None + question: str + answer: str + feedback: str + relevance: float + feedback_pos: float + feedback_neg: float + hinted: float + answer_substantive: float + answer_agreement: float + recency: float + + +def normalize_roles(df: pd.DataFrame) -> pd.DataFrame: + out = df.copy() + roles = out["role"].astype(str).str.lower().tolist() + text = out["content"].fillna("").astype(str).tolist() + repaired = roles[:] + for i in range(len(out)-1): + a, b = text[i].strip(), text[i+1].strip() + if roles[i] == "student" and roles[i+1] == "tutor": + a_question = bool(QUESTION_RE.search(a)) and len(a.split()) >= 3 + b_short_answer = len(b.split()) <= 6 and not QUESTION_RE.search(b) + if a_question and b_short_answer: + repaired[i], repaired[i+1] = "tutor", "student" + out["role_repaired"] = repaired + out["role_changed"] = np.asarray(repaired) != np.asarray(roles) + return out + + +def extract_episodes(df: pd.DataFrame, objective: str) -> list[Episode]: + df = normalize_roles(df).reset_index(drop=True) + roles = df["role_repaired"].tolist() + content = df["content"].fillna("").astype(str).tolist() + objective_tokens = tokens(objective) + episodes: list[Episode] = [] + n = max(1, len(df)-1) + + for q_idx in range(len(df)-1): + if roles[q_idx] != "tutor" or not QUESTION_RE.search(content[q_idx]): + continue + a_idx = None + for j in range(q_idx+1, min(len(df), q_idx+5)): + if roles[j] == "student" and content[j].strip(): + a_idx = j + break + if roles[j] == "tutor" and QUESTION_RE.search(content[j]) and j > q_idx+1: + break + if a_idx is None: + continue + + f_idx = None + for j in range(a_idx+1, min(len(df), a_idx+5)): + if roles[j] == "tutor": + f_idx = j + break + q, a = content[q_idx], content[a_idx] + f = content[f_idx] if f_idx is not None else "" + local_text = q + " " + a + " " + f + rel = max(jaccard(tokens(local_text), objective_tokens), 0.5 * char_ngram_overlap(local_text, objective)) + pos = 1.0 if POS_RE.search(f) else 0.0 + neg = 1.0 if NEG_RE.search(f) else 0.0 + hint = 1.0 if HINT_RE.search(q) else 0.0 + agreement = 1.0 if AGREE_RE.match(a.strip()) else 0.0 + substantive = float(is_substantive_answer(a)) + recency = a_idx / n + episodes.append(Episode(q_idx,a_idx,f_idx,q,a,f,rel,pos,neg,hint,substantive,agreement,recency)) + return episodes + + +def mastery_features(df: pd.DataFrame, objective: str) -> tuple[np.ndarray, str, dict]: + eps = extract_episodes(df, objective) + changed = float(normalize_roles(df)["role_changed"].mean()) if len(df) else 0.0 + if not eps: + return np.zeros(24, dtype=np.float64), "", {"episodes":0,"role_repair_rate":changed} + rel = np.array([e.relevance for e in eps]) + weights = np.maximum(rel, 0.02) * np.exp(2.0 * (np.array([e.recency for e in eps]) - 1.0)) + pos = np.array([e.feedback_pos for e in eps]); neg = np.array([e.feedback_neg for e in eps]) + hint = np.array([e.hinted for e in eps]); sub = np.array([e.answer_substantive for e in eps]) + agr = np.array([e.answer_agreement for e in eps]); rec = np.array([e.recency for e in eps]) + independent_positive = pos * sub * (1.0-hint); corrected = neg * sub + k = max(1, min(8, len(eps))); top = np.argsort(rel)[-k:]; tail = np.argsort(rec)[-k:] + wsum = float(weights.sum()) + 1e-12 + feats = np.array([ + len(eps), rel.mean(), rel.max(), np.quantile(rel,0.75), pos.mean(), neg.mean(), hint.mean(), sub.mean(), agr.mean(), + independent_positive.mean(), corrected.mean(), float((weights*pos).sum()/wsum), float((weights*neg).sum()/wsum), + float((weights*independent_positive).sum()/wsum), float(pos[top].mean()), float(neg[top].mean()), + float(independent_positive[top].mean()), float(pos[tail].mean()), float(neg[tail].mean()), + float(independent_positive[tail].mean()), float(rec[pos>0].mean()) if np.any(pos>0) else 0.0, + float(rec[neg>0].mean()) if np.any(neg>0) else 0.0, changed, float(sum(e.feedback_pos-e.feedback_neg for e in eps[-5:])), + ], dtype=np.float64) + ranked = sorted(eps, key=lambda e: (e.relevance * (0.25 + 0.75*e.recency)), reverse=True)[:8] + text = " ".join(f"[Q]{e.question} [STUDENT]{e.answer} [FEEDBACK]{e.feedback}" for e in ranked) + meta = {"episodes":len(eps),"role_repair_rate":changed,"max_relevance":float(rel.max())} + return feats, text, meta + + +def load_transcript(path: Path) -> pd.DataFrame: + cols = inspect_headers(path) + required = {"session_id","utterance_id","role","content","timestamp"} + missing = required - set(cols) + if missing: raise ValueError(f"{path.name}: missing transcript columns {sorted(missing)}; got {cols}") + return pd.read_csv(path) + + +def build_frame(features_path: Path, labels_path: Path, transcript_dir: Path): + fcols = inspect_headers(features_path); lcols = inspect_headers(labels_path) + print("features columns", fcols); print("labels columns", lcols) + required_f = {"response_id","session_id","learning_objective"} + if not required_f.issubset(fcols): raise ValueError(f"features missing {sorted(required_f-set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: raise ValueError(f"labels need is_correct or correct; got {lcols}") + features = pd.read_csv(features_path); labels = pd.read_csv(labels_path) + return features.merge(labels[["response_id",target]], on="response_id", how="inner", validate="one_to_one").rename(columns={target:"target"}) + + +def fixed_group_folds(groups: Iterable[str], n_splits: int = 5): + groups = np.asarray(list(groups)); dummy = np.zeros(len(groups)) + return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)) + + +def fit_eval(X, y, folds, name: str): + oof = np.zeros(len(y), dtype=np.float64); rows = [] + for fold,(tr,va) in enumerate(folds,1): + m = LogisticRegression(C=0.35, max_iter=250, solver="liblinear", random_state=SEED) + m.fit(X[tr], y[tr]); p = np.clip(m.predict_proba(X[va])[:,1], 1e-5, 1-1e-5); oof[va] = p + rows.append({"fold":fold,"rows":len(va),"logloss":log_loss(y[va],p),"auc":roc_auc_score(y[va],p)}); print(name, rows[-1]) + return oof, rows + + +def run(args): + frame = build_frame(args.features, args.labels, args.transcripts) + cache: dict[str,pd.DataFrame] = {}; numeric, episode_text, meta = [], [], [] + for i,row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + f,t,m = mastery_features(cache[sid], str(row.learning_objective)); numeric.append(f); episode_text.append(t); meta.append(m) + if args.limit and i+1 >= args.limit: frame = frame.iloc[:i+1].copy(); break + numeric = np.vstack(numeric); episode_text = episode_text[:len(frame)]; y = frame.target.to_numpy(dtype=int) + hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1,2), lowercase=True) + objective_text = frame.learning_objective.fillna("").astype(str).tolist() + X_obj = hv.transform(["[OBJECTIVE] "+x for x in objective_text]); X_ep = hv.transform(["[EPISODES] "+x for x in episode_text]) + X_num = csr_matrix((numeric - numeric.mean(0)) / (numeric.std(0)+1e-6)); X_base = X_obj; X_full = hstack([X_obj,X_ep,X_num], format="csr") + session_folds = fixed_group_folds(frame.session_id, 5) + objective_folds = fixed_group_folds(frame.learning_objective_id if "learning_objective_id" in frame else frame.learning_objective, 5) + results = {} + for split,folds in [("session",session_folds),("objective",objective_folds)]: + p0,r0 = fit_eval(X_base,y,folds,f"baseline/{split}"); p1,r1 = fit_eval(X_full,y,folds,f"mastery/{split}") + results[split] = {"baseline_logloss":float(log_loss(y,p0)),"mastery_logloss":float(log_loss(y,p1)),"delta":float(log_loss(y,p1)-log_loss(y,p0)),"baseline_auc":float(roc_auc_score(y,p0)),"mastery_auc":float(roc_auc_score(y,p1)),"folds_baseline":r0,"folds_mastery":r1} + results["diagnostics"] = {"rows":len(frame),"sessions":int(frame.session_id.nunique()),"objectives":int(frame.learning_objective.nunique()),"mean_episode_count":float(np.mean([m["episodes"] for m in meta[:len(frame)]])),"mean_role_repair_rate":float(np.mean([m["role_repair_rate"] for m in meta[:len(frame)]]))} + print(json.dumps(results, indent=2)); args.out.parent.mkdir(parents=True, exist_ok=True); args.out.write_text(json.dumps(results, indent=2)) + + +def self_test(): + for x in ["42", "0.5", "3/4", "x = 6"]: assert is_substantive_answer(x), x + for x in ["yeah", "ok", "mhm"]: assert not is_substantive_answer(x), x + df = pd.DataFrame([["s","1","tutor","What is 6 times 7?","2026-01-01T00:00:00"],["s","2","student","42","2026-01-01T00:00:01"],["s","3","tutor","Exactly right, well done.","2026-01-01T00:00:02"],["s","4","tutor","Now what is 8 times 7?","2026-01-01T00:00:03"],["s","5","student","54","2026-01-01T00:00:04"],["s","6","tutor","Not quite, try again.","2026-01-01T00:00:05"]], columns=["session_id","utterance_id","role","content","timestamp"]) + f,t,m = mastery_features(df,"multiplying one-digit numbers"); assert m["episodes"] == 2; assert f[7] == 1.0; assert f[4] > 0 and f[5] > 0; assert "42" in t and "54" in t + print("SELF_TEST_PASS", json.dumps(m)) + + +def parse_args(): + p = argparse.ArgumentParser(); p.add_argument("--features", type=Path); p.add_argument("--labels", type=Path); p.add_argument("--transcripts", type=Path); p.add_argument("--out", type=Path, default=Path("v71_mastery_results.json")); p.add_argument("--limit", type=int, default=0); p.add_argument("--self-test", action="store_true"); return p.parse_args() + + +if __name__ == "__main__": + a = parse_args() + if a.self_test: self_test() + else: + if not (a.features and a.labels and a.transcripts): raise SystemExit("--features, --labels and --transcripts are required unless --self-test is used") + run(a) diff --git a/competitions/trace_the_ace/v72_supervision_audit.py b/competitions/trace_the_ace/v72_supervision_audit.py new file mode 100644 index 00000000..3526fe54 --- /dev/null +++ b/competitions/trace_the_ace/v72_supervision_audit.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Trace the Ace V72: audit hidden supervision in multi-objective tutoring sessions. + +This script measures two structural resources that a winning model can exploit: +(1) within-session label agreement / disagreement, which separates global session +state from objective-specific mastery; and (2) transcript micro-assessment density +from tutor-question -> student-answer -> tutor-feedback episodes. + +Only aggregate JSON is written. Raw competition text is never emitted. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd + +from v71_mastery_events import inspect_headers, load_transcript, extract_episodes + + +def load_training(features_path: Path, labels_path: Path) -> pd.DataFrame: + fcols = inspect_headers(features_path) + lcols = inspect_headers(labels_path) + print("features columns", fcols) + print("labels columns", lcols) + need = {"response_id", "session_id", "learning_objective"} + if not need.issubset(fcols): + raise ValueError(f"features missing {sorted(need-set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + f = pd.read_csv(features_path) + y = pd.read_csv(labels_path) + out = f.merge(y[["response_id", target]], on="response_id", validate="one_to_one") + return out.rename(columns={target: "target"}) + + +def pair_agreement(values: np.ndarray) -> tuple[int, int]: + n = len(values) + if n < 2: + return 0, 0 + total = n * (n - 1) // 2 + pos = int(values.sum()) + neg = n - pos + disagree = pos * neg + return total - disagree, total + + +def run(args) -> None: + df = load_training(args.features, args.labels) + if args.limit: + df = df.iloc[: args.limit].copy() + + sizes = df.groupby("session_id").size() + multi_ids = sizes[sizes > 1].index + multi = df[df.session_id.isin(multi_ids)] + + agree = total = homogeneous = mixed = 0 + session_means = [] + contrastive_pairs = 0 + for _, g in multi.groupby("session_id", sort=False): + vals = g.target.to_numpy(dtype=int) + a, t = pair_agreement(vals) + agree += a + total += t + homogeneous += int(vals.min() == vals.max()) + mixed += int(vals.min() != vals.max()) + session_means.append(float(vals.mean())) + pos, neg = int(vals.sum()), int(len(vals) - vals.sum()) + contrastive_pairs += pos * neg + + # Transcript micro-assessment density is sampled by unique session to keep this + # audit cheap enough for GitHub-hosted runners while remaining deterministic. + sample_ids = sorted(df.session_id.astype(str).unique())[: args.episode_sessions] + ep_counts = [] + feedback_pos = feedback_neg = substantive = 0 + for sid in sample_ids: + path = args.transcripts / f"{sid}.csv" + if not path.exists(): + continue + tdf = load_transcript(path) + # Use a neutral objective here: the purpose is density / weak-label audit, + # not objective relevance. + eps = extract_episodes(tdf, "") + ep_counts.append(len(eps)) + feedback_pos += sum(int(e.feedback_pos) for e in eps) + feedback_neg += sum(int(e.feedback_neg) for e in eps) + substantive += sum(int(e.answer_substantive) for e in eps) + + objective_counts = df.groupby("learning_objective").size().sort_values(ascending=False) + result = { + "rows": int(len(df)), + "sessions": int(df.session_id.nunique()), + "objectives": int(df.learning_objective.nunique()), + "positive_rate": float(df.target.mean()), + "multi_objective_sessions": int(len(multi_ids)), + "multi_objective_session_fraction": float(len(multi_ids) / max(1, df.session_id.nunique())), + "within_session_pair_agreement": float(agree / total) if total else None, + "homogeneous_multi_session_fraction": float(homogeneous / max(1, homogeneous + mixed)), + "mixed_multi_sessions": int(mixed), + "opposite_label_same_session_pairs": int(contrastive_pairs), + "median_session_label_mean": float(np.median(session_means)) if session_means else None, + "objectives_seen_once": int((objective_counts == 1).sum()), + "objectives_seen_at_most_5": int((objective_counts <= 5).sum()), + "top_10_objective_row_fraction": float(objective_counts.head(10).sum() / len(df)), + "episode_sessions_sampled": int(len(ep_counts)), + "mean_micro_assessments_per_sampled_session": float(np.mean(ep_counts)) if ep_counts else None, + "median_micro_assessments_per_sampled_session": float(np.median(ep_counts)) if ep_counts else None, + "positive_feedback_events": int(feedback_pos), + "negative_feedback_events": int(feedback_neg), + "substantive_student_response_events": int(substantive), + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test() -> None: + a, t = pair_agreement(np.array([1, 1, 0, 1])) + assert (a, t) == (3, 6) + a2, t2 = pair_agreement(np.array([1, 1, 1])) + assert (a2, t2) == (3, 3) + print("V72_SELF_TEST_PASS") + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v72_supervision_audit.json")) + p.add_argument("--episode-sessions", type=int, default=500) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels or not args.transcripts: + raise SystemExit("--features, --labels and --transcripts are required") + run(args) diff --git a/competitions/trace_the_ace/v73_contrastive_mastery.py b/competitions/trace_the_ace/v73_contrastive_mastery.py new file mode 100644 index 00000000..0a785a1e --- /dev/null +++ b/competitions/trace_the_ace/v73_contrastive_mastery.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Trace the Ace V73: same-session contrastive mastery model. + +V73 turns the structural observation behind V72 into a measurable model. +For each response it builds objective-conditioned mastery evidence from V71, +then trains two complementary learners inside each held-out split: + +1. row model: predicts correctness from objective text + mastery evidence; +2. contrastive model: on mixed-label training sessions, learns which of two + objectives in the SAME transcript is more likely to be correct. + +Because the contrastive examples cancel session-wide ability, their coefficient +vector is forced toward objective-specific mastery evidence. The final prediction +combines row and contrastive logits using an inner training split only; validation +labels are never used to choose the blend. + +The script inspects CSV headers before schema decisions and writes aggregate JSON +only. It does not use cross-test-sample information at inference time. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack, vstack +from scipy.special import expit, logit +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression, SGDClassifier +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold, GroupShuffleSplit + +from v71_mastery_events import ( + SEED, + build_frame, + fixed_group_folds, + load_transcript, + mastery_features, +) + + +def build_design(frame: pd.DataFrame, transcripts: Path): + cache: dict[str, pd.DataFrame] = {} + numeric, episode_text = [], [] + for _, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(transcripts / f"{sid}.csv") + f, t, _ = mastery_features(cache[sid], str(row.learning_objective)) + numeric.append(f) + episode_text.append(t) + + numeric = np.vstack(numeric) + mu = numeric.mean(axis=0) + sd = numeric.std(axis=0) + 1e-6 + X_num = csr_matrix((numeric - mu) / sd) + + hv = HashingVectorizer( + n_features=2**18, + alternate_sign=False, + norm="l2", + ngram_range=(1, 2), + lowercase=True, + ) + obj = frame.learning_objective.fillna("").astype(str).tolist() + X_obj = hv.transform(["[OBJECTIVE] " + x for x in obj]) + X_ep = hv.transform(["[EPISODES] " + x for x in episode_text]) + return hstack([X_obj, X_ep, X_num], format="csr") + + +def same_session_pairs(frame: pd.DataFrame, indices: np.ndarray, max_pairs: int = 50000): + """Return deterministic opposite-label pairs (positive_row, negative_row).""" + sub = frame.iloc[indices] + pairs: list[tuple[int, int]] = [] + # map original row index -> position in X subset later via explicit global indices + for _, g in sub.groupby("session_id", sort=True): + pos = g.index[g.target.to_numpy(dtype=int) == 1].tolist() + neg = g.index[g.target.to_numpy(dtype=int) == 0].tolist() + for p in pos: + for n in neg: + pairs.append((int(p), int(n))) + if len(pairs) >= max_pairs: + return pairs + return pairs + + +def pair_matrix(X, pairs: list[tuple[int, int]]): + """Balanced pairwise dataset: x_pos-x_neg => 1 and reverse => 0.""" + if not pairs: + return None, None + p = np.asarray([a for a, _ in pairs], dtype=int) + n = np.asarray([b for _, b in pairs], dtype=int) + d = X[p] - X[n] + Xp = vstack([d, -d], format="csr") + yp = np.r_[np.ones(len(pairs), dtype=int), np.zeros(len(pairs), dtype=int)] + return Xp, yp + + +def fit_pairwise(X, frame: pd.DataFrame, train_idx: np.ndarray, max_pairs: int): + pairs = same_session_pairs(frame, train_idx, max_pairs=max_pairs) + Xp, yp = pair_matrix(X, pairs) + if Xp is None or len(np.unique(yp)) < 2: + return None, len(pairs) + model = SGDClassifier( + loss="log_loss", + penalty="l2", + alpha=2e-5, + max_iter=60, + tol=1e-4, + random_state=SEED, + average=True, + ) + model.fit(Xp, yp) + return model, len(pairs) + + +def choose_blend(row_logit: np.ndarray, contrast: np.ndarray, y: np.ndarray) -> float: + """Choose contrast weight only on inner-training predictions.""" + best_a, best_loss = 0.0, float("inf") + for a in np.linspace(-0.30, 0.60, 19): + p = expit(row_logit + a * contrast) + loss = log_loss(y, np.clip(p, 1e-5, 1 - 1e-5)) + if loss < best_loss: + best_loss, best_a = float(loss), float(a) + return best_a + + +def fold_predict(X, frame: pd.DataFrame, tr: np.ndarray, va: np.ndarray, max_pairs: int): + y = frame.target.to_numpy(dtype=int) + + # Outer row model. + row = LogisticRegression(C=0.35, max_iter=300, solver="liblinear", random_state=SEED) + row.fit(X[tr], y[tr]) + row_va = np.clip(row.predict_proba(X[va])[:, 1], 1e-5, 1 - 1e-5) + + # Pairwise model uses only outer-training sessions. + pair, pair_count = fit_pairwise(X, frame, tr, max_pairs) + if pair is None: + return row_va, row_va, 0.0, pair_count + contrast_va = pair.decision_function(X[va]) + + # Learn blend weight on an inner session split, never outer validation. + gss = GroupShuffleSplit(n_splits=1, test_size=0.22, random_state=SEED) + inner_a_rel, inner_b_rel = next(gss.split(tr, y[tr], frame.session_id.iloc[tr])) + inner_a = tr[inner_a_rel] + inner_b = tr[inner_b_rel] + + inner_row = LogisticRegression(C=0.35, max_iter=300, solver="liblinear", random_state=SEED) + inner_row.fit(X[inner_a], y[inner_a]) + p_inner = np.clip(inner_row.predict_proba(X[inner_b])[:, 1], 1e-5, 1 - 1e-5) + inner_pair, _ = fit_pairwise(X, frame, inner_a, max(5000, max_pairs // 2)) + if inner_pair is None: + alpha = 0.0 + else: + c_inner = inner_pair.decision_function(X[inner_b]) + alpha = choose_blend(logit(p_inner), c_inner, y[inner_b]) + + p_blend = expit(logit(row_va) + alpha * contrast_va) + return row_va, np.clip(p_blend, 1e-5, 1 - 1e-5), alpha, pair_count + + +def evaluate_split(X, frame: pd.DataFrame, folds, name: str, max_pairs: int): + y = frame.target.to_numpy(dtype=int) + row_oof = np.zeros(len(frame), dtype=float) + blend_oof = np.zeros(len(frame), dtype=float) + details = [] + for k, (tr, va) in enumerate(folds, 1): + p0, p1, alpha, pair_count = fold_predict(X, frame, tr, va, max_pairs) + row_oof[va] = p0 + blend_oof[va] = p1 + rec = { + "fold": k, + "rows": int(len(va)), + "row_logloss": float(log_loss(y[va], p0)), + "contrastive_logloss": float(log_loss(y[va], p1)), + "delta": float(log_loss(y[va], p1) - log_loss(y[va], p0)), + "alpha": float(alpha), + "training_pairs": int(pair_count), + } + print(name, rec) + details.append(rec) + return { + "row_logloss": float(log_loss(y, row_oof)), + "contrastive_logloss": float(log_loss(y, blend_oof)), + "delta": float(log_loss(y, blend_oof) - log_loss(y, row_oof)), + "row_auc": float(roc_auc_score(y, row_oof)), + "contrastive_auc": float(roc_auc_score(y, blend_oof)), + "folds": details, + "alpha_mean": float(np.mean([d["alpha"] for d in details])), + "alpha_nonzero_folds": int(sum(abs(d["alpha"]) > 1e-12 for d in details)), + } + + +def run(args): + frame = build_frame(args.features, args.labels, args.transcripts) + if args.limit: + frame = frame.iloc[: args.limit].copy().reset_index(drop=True) + else: + frame = frame.reset_index(drop=True) + X = build_design(frame, args.transcripts) + + session_folds = fixed_group_folds(frame.session_id, 5) + objective_group = frame.learning_objective_id if "learning_objective_id" in frame else frame.learning_objective + objective_folds = fixed_group_folds(objective_group, 5) + + result = { + "session": evaluate_split(X, frame, session_folds, "session", args.max_pairs), + "objective": evaluate_split(X, frame, objective_folds, "objective", args.max_pairs), + "diagnostics": { + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "design_shape": [int(X.shape[0]), int(X.shape[1])], + }, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test(): + frame = pd.DataFrame({ + "session_id": ["a", "a", "b", "b", "c"], + "target": [1, 0, 1, 1, 0], + }) + pairs = same_session_pairs(frame, np.arange(len(frame)), max_pairs=20) + assert pairs == [(0, 1)], pairs + X = csr_matrix(np.array([[2.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 2.0], [0.0, 2.0]])) + Xp, yp = pair_matrix(X, pairs) + assert Xp.shape == (2, 2) + assert yp.tolist() == [1, 0] + assert np.allclose(Xp.toarray()[0], -Xp.toarray()[1]) + a = choose_blend(np.array([1.0, -1.0]), np.array([1.0, -1.0]), np.array([1, 0])) + assert a >= 0.0 + print("V73_SELF_TEST_PASS", {"pairs": len(pairs), "alpha": a}) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v73_contrastive_mastery.json")) + p.add_argument("--max-pairs", type=int, default=50000) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels or not args.transcripts: + raise SystemExit("--features, --labels and --transcripts are required") + run(args) diff --git a/competitions/trace_the_ace/v74_semantic_objective_prior.py b/competitions/trace_the_ace/v74_semantic_objective_prior.py new file mode 100644 index 00000000..c7b76697 --- /dev/null +++ b/competitions/trace_the_ace/v74_semantic_objective_prior.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Trace the Ace V74: leakage-safe hierarchical semantic objective prior. + +The objective distribution is extremely long-tailed. V74 estimates objective +difficulty with two levels of shrinkage inside each CV fold: + +1. exact objective posterior when the objective has training support; +2. semantic KNN posterior over objective descriptions for rare/unseen skills. + +This produces a calibrated difficulty prior that can later be combined with the +student-state/mastery branches. No validation labels are used in fitting. + +Defaults k=16, smooth=2.0 were promoted after a full 35,072-row stress grid in +which they improved both session-grouped and objective-cold log loss versus the +previous k=8, smooth=20.0 defaults. The trust denominator remains fixed at 10. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import inspect_headers + + +def load_training(features_path: Path, labels_path: Path) -> pd.DataFrame: + fcols = inspect_headers(features_path) + lcols = inspect_headers(labels_path) + print("features columns", fcols) + print("labels columns", lcols) + need = {"response_id", "session_id", "learning_objective"} + if not need.issubset(fcols): + raise ValueError(f"features missing {sorted(need-set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + f = pd.read_csv(features_path) + y = pd.read_csv(labels_path) + return f.merge(y[["response_id", target]], on="response_id", validate="one_to_one").rename(columns={target: "target"}) + + +def semantic_prior_predict(train: pd.DataFrame, valid: pd.DataFrame, k: int = 16, smooth: float = 2.0): + global_p = float(train.target.mean()) + stats = train.groupby("learning_objective").target.agg(["sum", "count"]) + stats["p"] = (stats["sum"] + smooth * global_p) / (stats["count"] + smooth) + + train_objs = stats.index.astype(str).tolist() + vec = TfidfVectorizer( + analyzer="char_wb", + ngram_range=(3, 5), + min_df=1, + sublinear_tf=True, + norm="l2", + ) + A = vec.fit_transform(train_objs) + B = vec.transform(valid.learning_objective.fillna("").astype(str).tolist()) + sims = cosine_similarity(B, A) + + kk = min(k, sims.shape[1]) + idx = np.argpartition(-sims, kth=kk - 1, axis=1)[:, :kk] + rows = np.arange(len(valid))[:, None] + w = sims[rows, idx] + neighbor_p = stats["p"].to_numpy()[idx] + sem = (w * neighbor_p).sum(axis=1) / (w.sum(axis=1) + 1e-9) + sem = np.where(w.sum(axis=1) > 1e-8, sem, global_p) + + # Pandas 3 can expose a read-only NumPy view from Series.to_numpy(). Copy + # explicitly because we fill unseen objectives below. + mapped = valid.learning_objective.map(stats["p"]).to_numpy(dtype=float).copy() + missing = np.isnan(mapped) + mapped[missing] = sem[missing] + counts = valid.learning_objective.map(stats["count"]).fillna(0).to_numpy(dtype=float) + + # Rare objectives borrow strength from semantically related skills; common + # objectives rely increasingly on their exact training posterior. + trust = counts / (counts + 10.0) + hierarchical = trust * mapped + (1.0 - trust) * sem + return np.clip(hierarchical, 1e-5, 1 - 1e-5), np.clip(sem, 1e-5, 1 - 1e-5) + + +def evaluate(df: pd.DataFrame, groups, k: int, smooth: float): + y = df.target.to_numpy(dtype=int) + p = np.zeros(len(df), dtype=float) + sem = np.zeros(len(df), dtype=float) + glob = np.zeros(len(df), dtype=float) + folds = [] + for fold, (tr, va) in enumerate(GroupKFold(5).split(df, y, groups), 1): + ph, ps = semantic_prior_predict(df.iloc[tr], df.iloc[va], k=k, smooth=smooth) + p[va], sem[va] = ph, ps + glob[va] = float(df.target.iloc[tr].mean()) + folds.append({ + "fold": fold, + "rows": int(len(va)), + "global_logloss": float(log_loss(y[va], glob[va])), + "hierarchical_logloss": float(log_loss(y[va], ph)), + "semantic_only_logloss": float(log_loss(y[va], ps)), + }) + return { + "global_logloss": float(log_loss(y, glob)), + "hierarchical_logloss": float(log_loss(y, p)), + "semantic_only_logloss": float(log_loss(y, sem)), + "hierarchical_auc": float(roc_auc_score(y, p)), + "delta_vs_global": float(log_loss(y, p) - log_loss(y, glob)), + "folds": folds, + } + + +def run(args): + df = load_training(args.features, args.labels) + if args.limit: + df = df.iloc[: args.limit].copy().reset_index(drop=True) + else: + df = df.reset_index(drop=True) + + objective_group = df.learning_objective_id if "learning_objective_id" in df else df.learning_objective + result = { + "session": evaluate(df, df.session_id, args.k, args.smooth), + "objective": evaluate(df, objective_group, args.k, args.smooth), + "diagnostics": { + "rows": int(len(df)), + "sessions": int(df.session_id.nunique()), + "objectives": int(df.learning_objective.nunique()), + "k": int(args.k), + "smooth": float(args.smooth), + }, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test(): + train = pd.DataFrame({ + "learning_objective": [ + "multiply decimals by ten", "multiply decimals by ten", + "write fractions as decimals", "write fractions as decimals", + "identify angles", "identify angles", + ], + "target": [1, 1, 0, 0, 1, 0], + }) + valid = pd.DataFrame({"learning_objective": ["multiplying a decimal by 10", "fractions written as decimals"]}) + p, s = semantic_prior_predict(train, valid, k=2, smooth=2) + assert len(p) == 2 and np.all(np.isfinite(p)) + assert p[0] > p[1], (p, s) + print("V74_SELF_TEST_PASS", p.tolist()) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--out", type=Path, default=Path("v74_semantic_objective_prior.json")) + p.add_argument("--k", type=int, default=16) + p.add_argument("--smooth", type=float, default=2.0) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels: + raise SystemExit("--features and --labels are required") + run(args) diff --git a/competitions/trace_the_ace/v75_canonical_trajectory.py b/competitions/trace_the_ace/v75_canonical_trajectory.py new file mode 100644 index 00000000..2477759d --- /dev/null +++ b/competitions/trace_the_ace/v75_canonical_trajectory.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Trace the Ace V75: canonical student-state trajectories. + +Goal: improve genuinely unseen log loss by removing nuisance variation while +preserving educational variation. This module converts raw tutoring dialogue into +multiple deterministic views plus a compact chronological event sequence. It +never deletes the raw view and never uses labels to construct features. + +The script inspects CSV headers before schema decisions and processes each +(session, objective) independently at inference time. +""" +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import ( + AGREE_RE, + HINT_RE, + NEG_RE, + POS_RE, + QUESTION_RE, + char_ngram_overlap, + inspect_headers, + jaccard, + load_transcript, + normalize_roles, + tokens, +) + +SEED = 20260815 +LOW_INFO_RE = re.compile( + r"^(?:hi|hello|hey|bye|goodbye|thanks|thank you|yeah|yes|yep|okay|ok|mm+|mhm|uh huh|right|sure|cool|great)[.! ]*$", + re.I, +) +SELF_CORRECT_RE = re.compile(r"\b(?:wait|sorry|actually|i mean|no,? it(?:'s| is)|let me change|correction)\b", re.I) +EXPLAIN_RE = re.compile(r"\b(?:because|so that|therefore|since|i know|the reason|which means)\b", re.I) +TRANSFER_RE = re.compile(r"\b(?:another|next one|different|now try|what about|similar|new example)\b", re.I) +ADMIN_RE = re.compile( + r"\b(?:can you hear me|internet|connection|camera|microphone|lesson today|how are you|good morning|good afternoon|see you|homework portal)\b", + re.I, +) +MATH_ANSWER_RE = re.compile( + r"(?:\d|[=+\-/*×÷%]|\b(?:half|quarter|third|tenths?|hundredths?|thousandths?)\b)", + re.I, +) + +MATH_REPLACEMENTS = ( + (re.compile(r"[−–—]"), "-"), + (re.compile(r"[×✕]"), " x "), + (re.compile(r"[÷]"), " / "), + (re.compile(r"\s+"), " "), +) + +STATE_ORDER = { + "UNRESOLVED_ERROR": -2.0, + "CORRECTED_BY_TUTOR": -1.0, + "AGREEMENT_ONLY": -0.25, + "NO_JUDGMENT": 0.0, + "CORRECT_AFTER_HINT": 0.75, + "SELF_CORRECT": 1.0, + "INDEPENDENT_CORRECT": 1.5, + "TRANSFER_SUCCESS": 2.0, +} + + +@dataclass +class CanonicalEvent: + state: str + relevance: float + recency: float + assistance: float + substantive: float + low_info: float + explanation: float + question: str + answer: str + feedback: str + + +def normalize_math(text: str) -> str: + s = str(text).strip() + for pattern, repl in MATH_REPLACEMENTS: + s = pattern.sub(repl, s) + # Standardize a few harmless surface variants while retaining original text in + # separate views. Avoid semantic rewriting of spoken numbers/fractions. + s = re.sub(r"(?<=\d)\s*%", "%", s) + s = re.sub(r"\s*([=+\-/*])\s*", r" \1 ", s) + return re.sub(r"\s+", " ", s).strip() + + +def low_information(text: str) -> bool: + s = str(text).strip() + return bool(LOW_INFO_RE.match(s) or ADMIN_RE.search(s)) + + +def substantive_answer(text: str) -> bool: + """Recognize real student work without penalizing short mathematical answers. + + A one-token response such as `42`, `0.5`, `3/4`, or `x=6` is high-value + evidence even though it has fewer lexical tokens than a verbal explanation. + Pure acknowledgements remain non-substantive. + """ + s = str(text).strip() + if not s or AGREE_RE.match(s): + return False + if MATH_ANSWER_RE.search(s): + return True + return len(tokens(s)) >= 2 + + +def role_repair_with_confidence(df: pd.DataFrame) -> pd.DataFrame: + """Retain original/repaired roles and attach conservative repair confidence.""" + repaired = normalize_roles(df) + conf = np.zeros(len(repaired), dtype=float) + changed = repaired["role_changed"].to_numpy(dtype=bool) + conf[changed] = 0.9 + # Short acknowledgements in suspiciously inverted local pairs are less certain. + for i in np.flatnonzero(changed): + txt = str(repaired.iloc[i]["content"]).strip() + if len(txt.split()) <= 2: + conf[i] = 0.75 + repaired["role_repair_confidence"] = conf + return repaired + + +def objective_relevance(question: str, answer: str, feedback: str, objective: str) -> float: + local = f"{question} {answer} {feedback}" + obj_tok = tokens(objective) + return float(max(jaccard(tokens(local), obj_tok), 0.5 * char_ngram_overlap(local, objective))) + + +def classify_state(question: str, answer: str, feedback: str) -> tuple[str, float]: + """Return canonical state and assistance level in [0, 1].""" + q, a, f = map(str, (question, answer, feedback)) + pos = bool(POS_RE.search(f)) + neg = bool(NEG_RE.search(f)) + hinted = bool(HINT_RE.search(q)) + agreement = bool(AGREE_RE.match(a.strip())) + substantive = substantive_answer(a) + self_correct = bool(SELF_CORRECT_RE.search(a)) + transfer = bool(TRANSFER_RE.search(q)) + + if neg and substantive: + return "UNRESOLVED_ERROR", 0.0 + if agreement and pos: + return "AGREEMENT_ONLY", 1.0 + if self_correct and pos and substantive: + return "SELF_CORRECT", 0.25 + if pos and substantive and hinted: + return "CORRECT_AFTER_HINT", 0.65 + if pos and substantive and transfer: + return "TRANSFER_SUCCESS", 0.0 + if pos and substantive: + return "INDEPENDENT_CORRECT", 0.0 + if neg or (hinted and agreement): + return "CORRECTED_BY_TUTOR", 1.0 + return "NO_JUDGMENT", float(hinted) + + +def extract_canonical_events(df: pd.DataFrame, objective: str) -> list[CanonicalEvent]: + d = role_repair_with_confidence(df).reset_index(drop=True) + roles = d["role_repaired"].astype(str).str.lower().tolist() + content = d["content"].fillna("").astype(str).tolist() + n = max(1, len(d) - 1) + out: list[CanonicalEvent] = [] + + for qi in range(len(d) - 1): + if roles[qi] != "tutor" or not QUESTION_RE.search(content[qi]): + continue + ai = None + for j in range(qi + 1, min(len(d), qi + 6)): + if roles[j] == "student" and content[j].strip(): + ai = j + break + if roles[j] == "tutor" and QUESTION_RE.search(content[j]) and j > qi + 1: + break + if ai is None: + continue + fi = None + for j in range(ai + 1, min(len(d), ai + 6)): + if roles[j] == "tutor": + fi = j + break + q = content[qi] + a = content[ai] + f = content[fi] if fi is not None else "" + state, assistance = classify_state(q, a, f) + rel = objective_relevance(q, a, f, objective) + substantive = float(substantive_answer(a)) + out.append( + CanonicalEvent( + state=state, + relevance=rel, + recency=ai / n, + assistance=assistance, + substantive=substantive, + low_info=float(low_information(a)), + explanation=float(bool(EXPLAIN_RE.search(a))), + question=normalize_math(q), + answer=normalize_math(a), + feedback=normalize_math(f), + ) + ) + return out + + +def trajectory_views(df: pd.DataFrame, objective: str) -> tuple[dict[str, str], np.ndarray, dict]: + d = role_repair_with_confidence(df).reset_index(drop=True) + events = extract_canonical_events(d, objective) + + raw = " ".join( + f"[{str(r.role).upper()}] {str(r.content)}" + for r in d[["role", "content"]].itertuples(index=False) + ) + student_only = " ".join( + normalize_math(str(r.content)) + for r in d[["role_repaired", "content"]].itertuples(index=False) + if str(r.role_repaired).lower() == "student" and not low_information(str(r.content)) + ) + + ranked = sorted(events, key=lambda e: e.relevance * (0.2 + 0.8 * e.recency), reverse=True) + local = " ".join( + f"[Q] {e.question} [S] {e.answer} [F] {e.feedback}" + for e in ranked[:12] + ) + canonical = " ".join( + f"[{e.state}] rel={e.relevance:.3f} rec={e.recency:.3f} assist={e.assistance:.2f}" + for e in events + ) + terminal_events = sorted(events, key=lambda e: (e.relevance * (0.25 + 0.75 * e.recency)), reverse=True)[:6] + terminal = " ".join( + f"[{e.state}] [S] {e.answer} [F] {e.feedback}" + for e in terminal_events + ) + + if events: + rel = np.array([e.relevance for e in events], dtype=float) + rec = np.array([e.recency for e in events], dtype=float) + assist = np.array([e.assistance for e in events], dtype=float) + subst = np.array([e.substantive for e in events], dtype=float) + low = np.array([e.low_info for e in events], dtype=float) + expl = np.array([e.explanation for e in events], dtype=float) + state_score = np.array([STATE_ORDER[e.state] for e in events], dtype=float) + w = np.maximum(rel, 0.02) * np.exp(2.5 * (rec - 1.0)) + w /= w.sum() + 1e-12 + top = np.argsort(rel * (0.25 + 0.75 * rec))[-min(6, len(events)):] + tail = np.argsort(rec)[-min(6, len(events)):] + positive = np.isin([e.state for e in events], ["INDEPENDENT_CORRECT", "SELF_CORRECT", "TRANSFER_SUCCESS", "CORRECT_AFTER_HINT"]).astype(float) + errors = np.isin([e.state for e in events], ["UNRESOLVED_ERROR", "CORRECTED_BY_TUTOR"]).astype(float) + independent = np.isin([e.state for e in events], ["INDEPENDENT_CORRECT", "SELF_CORRECT", "TRANSFER_SUCCESS"]).astype(float) + feats = np.array([ + len(events), rel.mean(), rel.max(), np.quantile(rel, 0.75), + rec.mean(), assist.mean(), subst.mean(), low.mean(), expl.mean(), + positive.mean(), errors.mean(), independent.mean(), + float((w * state_score).sum()), float((w * positive).sum()), float((w * errors).sum()), + float((w * independent).sum()), float(state_score[top].mean()), float(state_score[tail].mean()), + float(positive[top].mean()), float(errors[top].mean()), float(independent[top].mean()), + float(positive[tail].mean()), float(errors[tail].mean()), float(independent[tail].mean()), + float(np.max(rec[independent > 0])) if np.any(independent > 0) else 0.0, + float(np.max(rec[errors > 0])) if np.any(errors > 0) else 0.0, + float(np.sum(independent * (rel >= np.quantile(rel, 0.75)))), + float(np.sum(errors * (rel >= np.quantile(rel, 0.75)))), + ], dtype=float) + else: + feats = np.zeros(28, dtype=float) + + views = { + "raw": raw, + "student": student_only, + "local": local, + "canonical": canonical, + "terminal": terminal, + } + meta = { + "events": len(events), + "role_repair_rate": float(d["role_changed"].mean()) if len(d) else 0.0, + "student_chars": len(student_only), + "raw_chars": len(raw), + } + return views, feats, meta + + +def load_training(features: Path, labels: Path) -> pd.DataFrame: + fcols = inspect_headers(features) + lcols = inspect_headers(labels) + print("features columns", fcols) + print("labels columns", lcols) + need = {"response_id", "session_id", "learning_objective"} + if not need.issubset(fcols): + raise ValueError(f"features missing {sorted(need - set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + f = pd.read_csv(features) + y = pd.read_csv(labels) + return f.merge(y[["response_id", target]], on="response_id", validate="one_to_one").rename(columns={target: "target"}) + + +def folds_for(groups: pd.Series, n_splits: int = 5): + g = groups.astype(str).to_numpy() + dummy = np.zeros(len(g)) + return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, g)) + + +def oof_eval(X, y, folds, name: str): + pred = np.zeros(len(y), dtype=float) + per_fold = [] + for k, (tr, va) in enumerate(folds, 1): + model = LogisticRegression(C=0.25, max_iter=300, solver="liblinear", random_state=SEED) + model.fit(X[tr], y[tr]) + p = np.clip(model.predict_proba(X[va])[:, 1], 1e-5, 1 - 1e-5) + pred[va] = p + row = {"fold": k, "rows": len(va), "logloss": float(log_loss(y[va], p)), "auc": float(roc_auc_score(y[va], p))} + print(name, row) + per_fold.append(row) + return pred, per_fold + + +def run(args) -> None: + frame = load_training(args.features, args.labels) + if args.limit: + frame = frame.iloc[: args.limit].copy() + + cache: dict[str, pd.DataFrame] = {} + view_rows: list[dict[str, str]] = [] + nums, metas = [], [] + for i, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + v, n, m = trajectory_views(cache[sid], str(row.learning_objective)) + view_rows.append(v); nums.append(n); metas.append(m) + if (len(view_rows) % 2500) == 0: + print("canonicalized rows", len(view_rows)) + + numeric = np.vstack(nums) + y = frame.target.to_numpy(dtype=int) + hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1, 2), lowercase=True) + objective = hv.transform(["[OBJECTIVE] " + str(x) for x in frame.learning_objective]) + raw = hv.transform(["[RAW] " + v["raw"] for v in view_rows]) + student = hv.transform(["[STUDENT] " + v["student"] for v in view_rows]) + local = hv.transform(["[LOCAL] " + v["local"] for v in view_rows]) + canonical = hv.transform(["[STATE] " + v["canonical"] for v in view_rows]) + terminal = hv.transform(["[TERMINAL] " + v["terminal"] for v in view_rows]) + z = (numeric - numeric.mean(0)) / (numeric.std(0) + 1e-6) + num = csr_matrix(z) + + matrices = { + "objective_only": objective, + "raw": hstack([objective, raw], format="csr"), + "student": hstack([objective, student], format="csr"), + "local": hstack([objective, local, num], format="csr"), + "canonical": hstack([objective, canonical, terminal, num], format="csr"), + "all_views": hstack([objective, raw, student, local, canonical, terminal, num], format="csr"), + } + session_folds = folds_for(frame.session_id) + objective_groups = frame.learning_objective_id if "learning_objective_id" in frame.columns else frame.learning_objective + objective_folds = folds_for(objective_groups) + + results = {"diagnostics": { + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "mean_events": float(np.mean([m["events"] for m in metas])), + "mean_role_repair_rate": float(np.mean([m["role_repair_rate"] for m in metas])), + "mean_student_to_raw_char_ratio": float(np.mean([m["student_chars"] / max(1, m["raw_chars"]) for m in metas])), + }} + for split, folds in (("session", session_folds), ("objective", objective_folds)): + results[split] = {} + for name, X in matrices.items(): + p, pf = oof_eval(X, y, folds, f"{split}/{name}") + results[split][name] = { + "logloss": float(log_loss(y, p)), + "auc": float(roc_auc_score(y, p)), + "worst_fold_logloss": float(max(r["logloss"] for r in pf)), + "folds": pf, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(results, indent=2)) + print(json.dumps(results, indent=2)) + + +def self_test() -> None: + # Includes an obvious role inversion, a hinted success, an error, and a later + # independent transfer success. Raw view must remain available. + df = pd.DataFrame([ + ["s", "1", "student", "Hi, can you hear me?", "2026-01-01T00:00:00"], + ["s", "2", "tutor", "Yeah.", "2026-01-01T00:00:01"], + ["s", "3", "tutor", "Remember the 7 times table. What is 6 times 7?", "2026-01-01T00:00:02"], + ["s", "4", "student", "42", "2026-01-01T00:00:03"], + ["s", "5", "tutor", "Exactly right.", "2026-01-01T00:00:04"], + ["s", "6", "tutor", "What is 8 times 7?", "2026-01-01T00:00:05"], + ["s", "7", "student", "54", "2026-01-01T00:00:06"], + ["s", "8", "tutor", "Not quite, try again.", "2026-01-01T00:00:07"], + ["s", "9", "tutor", "Another one: what is 9 times 7?", "2026-01-01T00:00:08"], + ["s", "10", "student", "63 because nine sevens are sixty three", "2026-01-01T00:00:09"], + ["s", "11", "tutor", "Perfect, that's right.", "2026-01-01T00:00:10"], + ], columns=["session_id", "utterance_id", "role", "content", "timestamp"]) + views, feats, meta = trajectory_views(df, "multiplying one-digit numbers using the 7 times table") + ev = extract_canonical_events(df, "multiplying one-digit numbers using the 7 times table") + states = [e.state for e in ev] + assert substantive_answer("42") + assert substantive_answer("0.5") + assert substantive_answer("3/4") + assert not substantive_answer("yeah") + assert "CORRECT_AFTER_HINT" in states + assert "UNRESOLVED_ERROR" in states + assert "TRANSFER_SUCCESS" in states + assert "Hi, can you hear me?" in views["raw"] + assert "Hi, can you hear me?" not in views["student"] + assert feats.shape == (28,) + assert meta["role_repair_rate"] > 0 + print("V75_SELF_TEST_PASS", json.dumps({"states": states, **meta})) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v75_canonical_trajectory.json")) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels or not args.transcripts: + raise SystemExit("--features, --labels and --transcripts are required") + run(args) diff --git a/competitions/trace_the_ace/v76_unseen_validation.py b/competitions/trace_the_ace/v76_unseen_validation.py new file mode 100644 index 00000000..4b9d52f7 --- /dev/null +++ b/competitions/trace_the_ace/v76_unseen_validation.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Trace the Ace V76: frozen validation protocol for unseen log loss. + +This module builds deterministic, label-independent validation partitions that +stress the failure modes most likely to matter on a private leaderboard: +- session-cold transfer; +- exact-objective-cold transfer; +- semantic-family-cold transfer; +- rare-objective rows; +- long-tail objective rows. + +It intentionally does not choose splits using outcome labels. Predictions from +any candidate model can be scored against the same frozen partitions. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.cluster import KMeans +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import inspect_headers + +SEED = 20260815 + + +def stable_hash(text: str) -> int: + return int(hashlib.sha256(str(text).encode("utf-8")).hexdigest()[:16], 16) + + +def read_frame(features_path: Path, labels_path: Path | None = None) -> pd.DataFrame: + fcols = inspect_headers(features_path) + print("features columns", fcols) + required = {"response_id", "session_id", "learning_objective"} + if not required.issubset(fcols): + raise ValueError(f"features missing {sorted(required - set(fcols))}") + frame = pd.read_csv(features_path) + if labels_path is not None: + lcols = inspect_headers(labels_path) + print("labels columns", lcols) + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + labels = pd.read_csv(labels_path) + frame = frame.merge(labels[["response_id", target]], on="response_id", validate="one_to_one") + frame = frame.rename(columns={target: "target"}) + return frame + + +def assign_group_folds(groups: pd.Series, n_splits: int = 5) -> np.ndarray: + groups = groups.astype(str).to_numpy() + dummy = np.zeros(len(groups)) + fold_id = np.full(len(groups), -1, dtype=int) + for k, (_, va) in enumerate(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)): + fold_id[va] = k + assert np.all(fold_id >= 0) + return fold_id + + +def semantic_family_map(objectives: list[str], n_families: int = 32) -> dict[str, int]: + unique = sorted(set(map(str, objectives))) + if len(unique) <= 1: + return {x: 0 for x in unique} + k = max(2, min(n_families, len(unique))) + vec = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), min_df=1, sublinear_tf=True) + X = vec.fit_transform(unique) + model = KMeans(n_clusters=k, random_state=SEED, n_init=20) + labels = model.fit_predict(X) + # Canonicalize arbitrary KMeans cluster ids by the lexicographically first + # objective in each cluster so assignments remain auditable. + members: dict[int, list[str]] = {} + for obj, lab in zip(unique, labels): + members.setdefault(int(lab), []).append(obj) + ordered = sorted(members, key=lambda lab: min(members[lab])) + canon = {old: new for new, old in enumerate(ordered)} + return {obj: canon[int(lab)] for obj, lab in zip(unique, labels)} + + +def make_protocol(frame: pd.DataFrame, n_splits: int = 5, n_families: int = 32) -> tuple[pd.DataFrame, dict]: + out = frame[["response_id", "session_id", "learning_objective"]].copy() + objective_key = ( + frame["learning_objective_id"].astype(str) + if "learning_objective_id" in frame.columns + else frame["learning_objective"].astype(str) + ) + out["session_fold"] = assign_group_folds(frame.session_id, n_splits) + out["objective_fold"] = assign_group_folds(objective_key, n_splits) + + fam = semantic_family_map(frame.learning_objective.astype(str).tolist(), n_families) + out["semantic_family"] = frame.learning_objective.astype(str).map(fam).astype(int) + out["semantic_family_fold"] = assign_group_folds(out.semantic_family.astype(str), n_splits) + + counts = objective_key.value_counts() + out["objective_count"] = objective_key.map(counts).astype(int) + out["rare_le_5"] = out.objective_count <= 5 + out["rare_le_10"] = out.objective_count <= 10 + out["tail_le_20"] = out.objective_count <= 20 + out["singleton"] = out.objective_count == 1 + + # Stable row hash is useful for exact reproducibility/auditing but is not used + # as a feature or to choose outcomes. + out["row_hash"] = out.response_id.astype(str).map(lambda x: stable_hash(x) % (2**63 - 1)) + + protocol_bytes = out.sort_values("response_id").to_csv(index=False).encode("utf-8") + protocol_sha = hashlib.sha256(protocol_bytes).hexdigest() + summary = { + "rows": int(len(out)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "semantic_families": int(out.semantic_family.nunique()), + "rare_le_5_rows": int(out.rare_le_5.sum()), + "rare_le_10_rows": int(out.rare_le_10.sum()), + "tail_le_20_rows": int(out.tail_le_20.sum()), + "singleton_rows": int(out.singleton.sum()), + "session_fold_rows": out.session_fold.value_counts().sort_index().astype(int).to_dict(), + "objective_fold_rows": out.objective_fold.value_counts().sort_index().astype(int).to_dict(), + "semantic_family_fold_rows": out.semantic_family_fold.value_counts().sort_index().astype(int).to_dict(), + "protocol_sha256": protocol_sha, + } + return out, summary + + +def binary_logloss(y: np.ndarray, p: np.ndarray) -> float: + p = np.clip(np.asarray(p, dtype=float), 1e-6, 1 - 1e-6) + y = np.asarray(y, dtype=float) + return float(-np.mean(y * np.log(p) + (1 - y) * np.log(1 - p))) + + +def score_predictions(protocol: pd.DataFrame, labels: pd.DataFrame, predictions: pd.DataFrame) -> dict: + target_col = "target" if "target" in labels.columns else "is_correct" if "is_correct" in labels.columns else "correct" + prob_col = "probability" if "probability" in predictions.columns else "prediction" + m = protocol.merge(labels[["response_id", target_col]], on="response_id", validate="one_to_one") + m = m.merge(predictions[["response_id", prob_col]], on="response_id", validate="one_to_one") + y = m[target_col].to_numpy(dtype=float) + p = m[prob_col].to_numpy(dtype=float) + result = {"overall_logloss": binary_logloss(y, p)} + for col in ("rare_le_5", "rare_le_10", "tail_le_20", "singleton"): + mask = m[col].to_numpy(dtype=bool) + result[f"{col}_rows"] = int(mask.sum()) + result[f"{col}_logloss"] = binary_logloss(y[mask], p[mask]) if mask.any() else None + for fold_col in ("session_fold", "objective_fold", "semantic_family_fold"): + losses = [] + for k in sorted(m[fold_col].unique()): + mask = m[fold_col].to_numpy() == k + losses.append(binary_logloss(y[mask], p[mask])) + result[f"{fold_col}_losses"] = losses + result[f"{fold_col}_mean"] = float(np.mean(losses)) + result[f"{fold_col}_worst"] = float(np.max(losses)) + result[f"{fold_col}_std"] = float(np.std(losses)) + + confidence = np.maximum(p, 1 - p) + for q in (0.90, 0.95, 0.99): + threshold = float(np.quantile(confidence, q)) + mask = confidence >= threshold + result[f"confidence_top_{int((1-q)*100)}pct_rows"] = int(mask.sum()) + result[f"confidence_top_{int((1-q)*100)}pct_logloss"] = binary_logloss(y[mask], p[mask]) + return result + + +def self_test() -> None: + frame = pd.DataFrame({ + "response_id": [f"r{i}" for i in range(20)], + "session_id": [f"s{i//2}" for i in range(20)], + "learning_objective": [ + "multiply decimals", "multiply decimals", "divide decimals", "divide decimals", + "add fractions", "add fractions", "subtract fractions", "subtract fractions", + "place value tenths", "place value tenths", "place value hundredths", "place value hundredths", + "factor quadratics", "factor quadratics", "expand brackets", "expand brackets", + "compare money", "compare money", "order integers", "order integers", + ], + }) + p1, s1 = make_protocol(frame, n_splits=2, n_families=4) + p2, s2 = make_protocol(frame.sample(frac=1, random_state=3).reset_index(drop=True), n_splits=2, n_families=4) + # Cluster/fold assignments must be deterministic per response regardless of row order. + a = p1.set_index("response_id")[["session_fold", "objective_fold", "semantic_family"]].sort_index() + b = p2.set_index("response_id")[["session_fold", "objective_fold", "semantic_family"]].sort_index() + assert a.equals(b) + assert s1["rows"] == 20 and s1["semantic_families"] == 4 + print("V76_SELF_TEST_PASS", json.dumps({"protocol_sha256": s1["protocol_sha256"], "families": s1["semantic_families"]})) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--out-protocol", type=Path, default=Path("v76_validation_protocol.csv")) + p.add_argument("--out-summary", type=Path, default=Path("v76_validation_summary.json")) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features: + raise SystemExit("--features is required") + frame = read_frame(args.features, args.labels) + protocol, summary = make_protocol(frame) + args.out_protocol.parent.mkdir(parents=True, exist_ok=True) + protocol.to_csv(args.out_protocol, index=False) + args.out_summary.write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary, indent=2)) diff --git a/competitions/trace_the_ace/v77_incremental_mastery_stack.py b/competitions/trace_the_ace/v77_incremental_mastery_stack.py new file mode 100644 index 00000000..7466d5c7 --- /dev/null +++ b/competitions/trace_the_ace/v77_incremental_mastery_stack.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Trace the Ace V77: leakage-safe incremental mastery stack over V74. + +Primary question: do transcript-derived mastery features reduce unseen/session-cold +log loss *after* accounting for the strong V74 semantic objective prior? + +For each outer session fold: + 1. fit V74 only on outer-train and predict outer-valid; + 2. generate V74 predictions for outer-train via inner session-grouped OOF; + 3. fit residual correction models on outer-train using only inner-OOF V74 logits + plus mastery numeric and/or episode text features; + 4. apply the fitted correction to the outer-valid V74 logits. + +No outer-valid labels enter base or residual training. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import build_frame, load_transcript, mastery_features, inspect_headers +from v74_semantic_objective_prior import semantic_prior_predict + +SEED = 20260815 + + +def logit(p): + p = np.clip(np.asarray(p, dtype=float), 1e-6, 1 - 1e-6) + return np.log(p / (1 - p)) + + +def sigmoid(x): + return 1.0 / (1.0 + np.exp(-np.asarray(x, dtype=float))) + + +def fixed_group_folds(groups, n_splits=5): + groups = np.asarray(groups) + dummy = np.zeros(len(groups)) + return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)) + + +def inner_v74_oof(train_df: pd.DataFrame, n_splits: int = 4) -> np.ndarray: + groups = train_df.session_id.astype(str).to_numpy() + y = train_df.target.to_numpy(dtype=int) + p = np.zeros(len(train_df), dtype=float) + for tr, va in GroupKFold(n_splits=n_splits).split(train_df, y, groups): + ph, _ = semantic_prior_predict(train_df.iloc[tr], train_df.iloc[va]) + p[va] = ph + return np.clip(p, 1e-6, 1 - 1e-6) + + +def build_mastery(frame: pd.DataFrame, transcript_dir: Path): + cache = {} + numeric, episode_text, meta = [], [], [] + for i, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(transcript_dir / f"{sid}.csv") + f, t, m = mastery_features(cache[sid], str(row.learning_objective)) + numeric.append(f); episode_text.append(t); meta.append(m) + if (i + 1) % 5000 == 0: + print("mastery rows", i + 1) + return np.vstack(numeric), episode_text, meta + + +def standardize_train_valid(a_tr, a_va): + mu = a_tr.mean(axis=0) + sd = a_tr.std(axis=0) + 1e-6 + return (a_tr - mu) / sd, (a_va - mu) / sd + + +def fit_correction(y_tr, base_tr, base_va, X_tr_extra=None, X_va_extra=None, C=0.15): + base_logit_tr = csr_matrix(logit(base_tr).reshape(-1, 1)) + base_logit_va = csr_matrix(logit(base_va).reshape(-1, 1)) + Xtr = base_logit_tr if X_tr_extra is None else hstack([base_logit_tr, X_tr_extra], format="csr") + Xva = base_logit_va if X_va_extra is None else hstack([base_logit_va, X_va_extra], format="csr") + m = LogisticRegression(C=C, max_iter=400, solver="liblinear", random_state=SEED) + m.fit(Xtr, y_tr) + return np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6) + + +def run(args): + frame = build_frame(args.features, args.labels, args.transcripts).reset_index(drop=True) + if args.limit: + frame = frame.iloc[:args.limit].copy().reset_index(drop=True) + print("rows", len(frame), "sessions", frame.session_id.nunique(), "objectives", frame.learning_objective.nunique()) + + numeric, episode_text, meta = build_mastery(frame, args.transcripts) + y = frame.target.to_numpy(dtype=int) + hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1,2), lowercase=True) + X_ep_all = hv.transform(["[EPISODES] " + x for x in episode_text]) + + outer = fixed_group_folds(frame.session_id.astype(str).to_numpy(), 5) + preds = {k: np.zeros(len(frame), dtype=float) for k in ["v74", "v74_recal", "v74_num", "v74_ep", "v74_num_ep"]} + folds = [] + + for fold, (tr, va) in enumerate(outer, 1): + train_df, valid_df = frame.iloc[tr], frame.iloc[va] + base_tr = inner_v74_oof(train_df, n_splits=4) + base_va, _ = semantic_prior_predict(train_df, valid_df) + preds["v74"][va] = base_va + + num_tr, num_va = standardize_train_valid(numeric[tr], numeric[va]) + X_num_tr, X_num_va = csr_matrix(num_tr), csr_matrix(num_va) + X_ep_tr, X_ep_va = X_ep_all[tr], X_ep_all[va] + + preds["v74_recal"][va] = fit_correction(y[tr], base_tr, base_va) + preds["v74_num"][va] = fit_correction(y[tr], base_tr, base_va, X_num_tr, X_num_va) + preds["v74_ep"][va] = fit_correction(y[tr], base_tr, base_va, X_ep_tr, X_ep_va) + preds["v74_num_ep"][va] = fit_correction(y[tr], base_tr, base_va, hstack([X_num_tr, X_ep_tr], format="csr"), hstack([X_num_va, X_ep_va], format="csr")) + + row = {"fold": fold, "rows": int(len(va))} + for name, p in preds.items(): + row[name + "_logloss"] = float(log_loss(y[va], p[va])) + folds.append(row) + print(json.dumps(row)) + + summary = {} + base_ll = float(log_loss(y, preds["v74"])) + for name, p in preds.items(): + ll = float(log_loss(y, p)) + summary[name] = {"logloss": ll, "delta_vs_v74": ll - base_ll, "auc": float(roc_auc_score(y, p))} + + counts = frame.groupby("learning_objective_id" if "learning_objective_id" in frame else "learning_objective").response_id.transform("count").to_numpy() + slices = {} + for threshold in (5, 10, 20): + mask = counts <= threshold + slices[f"rare_le_{threshold}"] = {"rows": int(mask.sum())} + if mask.any(): + for name, p in preds.items(): + slices[f"rare_le_{threshold}"][name + "_logloss"] = float(log_loss(y[mask], p[mask])) + + result = { + "summary": summary, + "folds": folds, + "slices": slices, + "diagnostics": { + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "mean_episode_count": float(np.mean([m["episodes"] for m in meta])), + "mean_role_repair_rate": float(np.mean([m["role_repair_rate"] for m in meta])), + }, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test(): + p = np.array([0.2, 0.5, 0.8]) + assert np.allclose(sigmoid(logit(p)), p) + print("V77_SELF_TEST_PASS") + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v77_incremental_mastery_stack.json")) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + a = parse_args() + if a.self_test: + self_test() + else: + if not (a.features and a.labels and a.transcripts): + raise SystemExit("--features, --labels and --transcripts are required") + run(a) diff --git a/competitions/trace_the_ace/v78_seismic_semantic.py b/competitions/trace_the_ace/v78_seismic_semantic.py new file mode 100644 index 00000000..db5dddf6 --- /dev/null +++ b/competitions/trace_the_ace/v78_seismic_semantic.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""V78 seismic test: pretrained semantic interaction + learned episode mastery + V75 ensemble. + +Tests three model-class shifts under identical frozen group folds: +1) objective<->dialogue pretrained semantic interaction; +2) learned episode/trajectory mastery from semantic episode views; +3) nested-safe convex ensemble with the sparse V75 all-views model. + +Development-time model: sentence-transformers/all-MiniLM-L6-v2 (open, and preloaded in official runtime). +Only aggregate metrics are written. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler +from sentence_transformers import SentenceTransformer + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, SEED + + +def folds(groups, n=5): + g=groups.astype(str).to_numpy(); z=np.zeros(len(g)) + return list(GroupKFold(n_splits=n).split(z,z,g)) + +def sigmoid(x): + x=np.asarray(x,float); return 1/(1+np.exp(-np.clip(x,-40,40))) + +def encode(model, texts, batch=128): + return model.encode(texts,batch_size=batch,show_progress_bar=True,normalize_embeddings=True,convert_to_numpy=True).astype(np.float32) + +def dense_semantic_features(Eo, Es, El, Et, numeric): + cos_os=np.sum(Eo*Es,1,keepdims=True); cos_ol=np.sum(Eo*El,1,keepdims=True); cos_ot=np.sum(Eo*Et,1,keepdims=True) + cos_sl=np.sum(Es*El,1,keepdims=True); cos_lt=np.sum(El*Et,1,keepdims=True) + prod_ol=Eo*El; prod_ot=Eo*Et + return np.hstack([Eo,Es,El,Et,prod_ol,prod_ot,cos_os,cos_ol,cos_ot,cos_sl,cos_lt,numeric]).astype(np.float32) + +def build_v75_sparse(frame, view_rows, numeric): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + obj=hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective]) + raw=hv.transform(['[RAW] '+v['raw'] for v in view_rows]) + stu=hv.transform(['[STUDENT] '+v['student'] for v in view_rows]) + loc=hv.transform(['[LOCAL] '+v['local'] for v in view_rows]) + can=hv.transform(['[STATE] '+v['canonical'] for v in view_rows]) + ter=hv.transform(['[TERMINAL] '+v['terminal'] for v in view_rows]) + z=(numeric-numeric.mean(0))/(numeric.std(0)+1e-6) + return hstack([obj,raw,stu,loc,can,ter,csr_matrix(z)],format='csr') + +def eval_regime(Xv75, Xsem, y, split, name): + p75=np.zeros(len(y)); psem=np.zeros(len(y)); fold_rows=[] + for k,(tr,va) in enumerate(split,1): + m75=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xv75[tr],y[tr]) + a=np.clip(m75.predict_proba(Xv75[va])[:,1],1e-5,1-1e-5); p75[va]=a + sc=StandardScaler().fit(Xsem[tr]); A=sc.transform(Xsem[tr]); B=sc.transform(Xsem[va]) + ms=LogisticRegression(C=.05,max_iter=500,solver='liblinear',random_state=SEED).fit(A,y[tr]) + b=np.clip(ms.predict_proba(B)[:,1],1e-5,1-1e-5); psem[va]=b + fold_rows.append({'fold':k,'rows':len(va),'v75':float(log_loss(y[va],a)),'semantic':float(log_loss(y[va],b))}) + print(name,fold_rows[-1]) + grid=[]; best=None + for w in np.linspace(0,1,21): + p=np.clip((1-w)*p75+w*psem,1e-5,1-1e-5); ll=float(log_loss(y,p)) + row={'semantic_weight':float(w),'logloss':ll}; grid.append(row) + if best is None or lllate mastery +change, and tests an OOF blend with the sparse V75 trajectory champion. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler +from sentence_transformers import SentenceTransformer + +from v71_mastery_events import load_transcript, extract_episodes +from v75_canonical_trajectory import load_training, trajectory_views, SEED + + +def folds(groups, n=5): + g=np.asarray(groups.astype(str)); z=np.zeros(len(g)); return list(GroupKFold(n_splits=n).split(z,z,g)) + +def episode_text(e): + return f"Tutor question: {e.question} Student answer: {e.answer} Tutor feedback: {e.feedback}" + +def build_session_episodes(frame, transcript_dir): + sessions={} + for i,sid in enumerate(frame.session_id.astype(str).unique()): + df=load_transcript(transcript_dir/f'{sid}.csv') + eps=extract_episodes(df, '') + # cap at 32 episodes; keep both early and late coverage if exceptionally long + if len(eps)>32: + idx=np.unique(np.r_[np.arange(8),np.linspace(8,len(eps)-9,16,dtype=int),np.arange(len(eps)-8,len(eps))]) + eps=[eps[j] for j in idx] + sessions[sid]=eps + if (i+1)%2500==0: print('sessions',i+1) + return sessions + +def encode_episode_bank(model, sessions, batch): + flat=[]; spans={}; start=0 + for sid,eps in sessions.items(): + texts=[episode_text(e) for e in eps] + flat.extend(texts); spans[sid]=(start,start+len(texts)); start+=len(texts) + E=model.encode(flat or [' '],batch_size=batch,show_progress_bar=True,normalize_embeddings=True,convert_to_numpy=True).astype(np.float32) + return E,spans + +def retrieval_features(frame, sessions, Ebank, spans, Eobj, k=6): + rows=[]; agg=[] + for i,r in enumerate(frame.itertuples(index=False)): + sid=str(r.session_id); eps=sessions[sid]; a,b=spans[sid] + if not eps: + rows.append(np.zeros(24,np.float32)); agg.append(np.zeros(Eobj.shape[1],np.float32)); continue + E=Ebank[a:b]; sims=E@Eobj[i] + top=np.argsort(sims)[-min(k,len(eps)):][::-1] + st=sims[top]; w=np.exp(5*(st-st.max())); w=w/(w.sum()+1e-12) + agg.append((E[top]*w[:,None]).sum(0)) + pos=np.array([eps[j].feedback_pos for j in top],float); neg=np.array([eps[j].feedback_neg for j in top],float) + hint=np.array([eps[j].hinted for j in top],float); sub=np.array([eps[j].answer_substantive for j in top],float) + rec=np.array([eps[j].recency for j in top],float); independent=pos*sub*(1-hint) + # explicit early/final mastery among objective-relevant evidence + early=rec<=np.median(rec); late=~early + score=independent-neg-.35*hint + early_score=float(score[early].mean()) if early.any() else 0.; late_score=float(score[late].mean()) if late.any() else 0. + gain=late_score-early_score + feats=np.array([ + len(eps),len(top),st[0],st.mean(),st.min(),st.std() if len(st)>1 else 0., + pos.mean(),neg.mean(),hint.mean(),sub.mean(),independent.mean(), + float((w*pos).sum()),float((w*neg).sum()),float((w*independent).sum()), + early_score,late_score,gain,float(rec.mean()),float(rec[np.argmax(st)]), + float(score[-1]),float(score.max()),float(score.min()), + float(np.mean(st[rec>=.66])) if np.any(rec>=.66) else 0., + float(np.mean(st[rec<=.33])) if np.any(rec<=.33) else 0., + ],np.float32) + rows.append(feats) + return np.vstack(rows),np.vstack(agg) + +def build_v75(frame, transcript_dir): + cache={}; views=[]; nums=[] + for i,r in frame.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(transcript_dir/f'{sid}.csv') + v,n,_=trajectory_views(cache[sid],str(r.learning_objective)); views.append(v); nums.append(n) + if (i+1)%2500==0: print('v75 views',i+1) + nums=np.vstack(nums).astype(np.float64); z=(nums-nums.mean(0))/(nums.std(0)+1e-6) + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[ + hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective]), + hv.transform(['[RAW] '+v['raw'] for v in views]), + hv.transform(['[STUDENT] '+v['student'] for v in views]), + hv.transform(['[LOCAL] '+v['local'] for v in views]), + hv.transform(['[STATE] '+v['canonical'] for v in views]), + hv.transform(['[TERMINAL] '+v['terminal'] for v in views]), csr_matrix(z)] + return hstack(parts,format='csr') + +def eval_regime(X75,Xr,y,split,name): + p75=np.zeros(len(y)); pr=np.zeros(len(y)); fr=[] + for f,(tr,va) in enumerate(split,1): + m75=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + p75[va]=np.clip(m75.predict_proba(X75[va])[:,1],1e-5,1-1e-5) + sc=StandardScaler().fit(Xr[tr]); A=sc.transform(Xr[tr]); B=sc.transform(Xr[va]) + mr=LogisticRegression(C=.03,max_iter=500,solver='liblinear',random_state=SEED).fit(A,y[tr]) + pr[va]=np.clip(mr.predict_proba(B)[:,1],1e-5,1-1e-5) + fr.append({'fold':f,'v75':float(log_loss(y[va],p75[va])),'retrieval_gain':float(log_loss(y[va],pr[va]))}); print(name,fr[-1]) + grid=[]; best=None + for w in np.linspace(0,1,41): + p=(1-w)*p75+w*pr; ll=float(log_loss(y,p)); row={'retrieval_weight':float(w),'logloss':ll}; grid.append(row) + if best is None or ll float: + return float(max(jaccard(tokens(text), tokens(objective)), 0.5*char_ngram_overlap(text, objective))) + + +def split_segments(df: pd.DataFrame) -> list[tuple[int,int]]: + txt=df.content.fillna('').astype(str).tolist(); starts=[0] + for i,t in enumerate(txt): + if i>0 and BOUNDARY_RE.search(t): starts.append(i) + starts=sorted(set(starts)); segs=[] + for j,s in enumerate(starts): + e=starts[j+1] if j+1s: segs.append((s,e)) + return segs or [(0,len(df))] + + +def choose_target_segment(df: pd.DataFrame, objective: str) -> tuple[pd.DataFrame, dict]: + segs=split_segments(df); best=None + for s,e in segs: + part=df.iloc[s:e].copy(); text=' '.join(part.content.fillna('').astype(str)) + score=rel_text(text,objective) + # explicit goal language near the segment front is strong evidence + front=' '.join(part.content.fillna('').astype(str).head(20)) + goal_bonus=0.15 if GOAL_RE.search(front) else 0.0 + total=score+goal_bonus + row=(total,score,goal_bonus,s,e) + if best is None or row>best: best=row + _,score,bonus,s,e=best + return df.iloc[s:e].copy().reset_index(drop=True), {'segments':len(segs),'start':int(s),'end':int(e),'segment_fraction':float((e-s)/max(1,len(df))),'segment_relevance':float(score),'goal_bonus':float(bonus)} + + +def phase_for(text: str, current: str) -> str: + if GOAL_RE.search(text): return 'GOAL' + if PRIOR_RE.search(text): return 'PRIOR' + if APPLY_RE.search(text): return 'APPLICATION' + if INDEP_RE.search(text): return 'INDEPENDENT' + if GUIDED_RE.search(text): return 'GUIDED' + return current + + +def phase_views(seg: pd.DataFrame, objective: str) -> tuple[dict[str,str], np.ndarray]: + phase='OTHER'; buckets={k:[] for k in ['GOAL','PRIOR','GUIDED','INDEPENDENT','APPLICATION','OTHER']} + for r in seg[['role','content']].itertuples(index=False): + text=str(r.content); phase=phase_for(text,phase) + buckets[phase].append(f'[{str(r.role).upper()}] {text}') + texts={k:' '.join(v) for k,v in buckets.items()} + # Reuse V75 state extraction per phase so the abstraction stays comparable. + phase_num=[]; phase_state=[] + for k in ['PRIOR','GUIDED','INDEPENDENT','APPLICATION']: + sub=seg.iloc[0:0].copy() + if buckets[k]: + # recover rows by a simple phase replay + phase2='OTHER'; idx=[] + for i,r in enumerate(seg[['content']].itertuples(index=False)): + phase2=phase_for(str(r.content),phase2) + if phase2==k: idx.append(i) + if idx: sub=seg.iloc[idx].copy().reset_index(drop=True) + if len(sub): + v,n,_=trajectory_views(sub,objective); phase_num.append(n); phase_state.append(v['canonical']) + else: + phase_num.append(np.zeros(28,float)); phase_state.append('') + P=np.vstack(phase_num) + # Explicit learning-gain contrasts; first 28 are phase means collapsed pairwise. + pre=P[0]; guided=P[1]; indep=P[2]; app=P[3] + gain_ind=indep-pre; gain_app=app-pre + # compact summary scalars on key V75 dimensions: state score, positive/error/independent weighted-ish slots + key=[12,13,14,15,17,21,22,23,24,25] + scal=[] + for arr in [pre,guided,indep,app,gain_ind,gain_app]: scal.extend(arr[key].tolist()) + nums=np.asarray(scal,float) + views={ + 'phase_prior':texts['PRIOR'], 'phase_guided':texts['GUIDED'], + 'phase_independent':texts['INDEPENDENT'], 'phase_application':texts['APPLICATION'], + 'phase_states':' [PHASE] '.join(phase_state), + } + return views,nums + + +def folds(groups): + g=groups.astype(str).to_numpy(); z=np.zeros(len(g)); return list(GroupKFold(5).split(z,z,g)) + + +def build_X(frame, rows, nums, prefix): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform([f'[OBJECTIVE] {x}' for x in frame.learning_objective])] + keys=list(rows[0].keys()) + for k in keys: parts.append(hv.transform([f'[{prefix}_{k.upper()}] '+r[k] for r in rows])) + Z=np.vstack(nums).astype(float); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + parts.append(csr_matrix(Z)); return hstack(parts,format='csr') + + +def oof(X,y,split,name): + p=np.zeros(len(y)); fr=[] + for k,(tr,va) in enumerate(split,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + row={'fold':k,'rows':len(va),'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; fr.append(row); print(name,row) + return p,fr + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: f=f.iloc[:a.limit].copy().reset_index(drop=True) + whole_rows=[]; whole_nums=[]; seg_rows=[]; seg_nums=[]; phase_rows=[]; phase_nums=[]; meta=[]; cache={} + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + d=cache[sid]; vw,nw,_=trajectory_views(d,str(r.learning_objective)); whole_rows.append(vw); whole_nums.append(nw) + seg,m=choose_target_segment(d,str(r.learning_objective)); vs,ns,_=trajectory_views(seg,str(r.learning_objective)); seg_rows.append(vs); seg_nums.append(ns) + pv,pn=phase_views(seg,str(r.learning_objective)); phase_rows.append({**vs,**pv}); phase_nums.append(np.concatenate([ns,pn])); meta.append(m) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); grp=f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective; sp=folds(grp) + Xw=build_X(f,whole_rows,whole_nums,'WHOLE'); Xs=build_X(f,seg_rows,seg_nums,'SEG'); Xp=build_X(f,phase_rows,phase_nums,'PHASE') + pw,fw=oof(Xw,y,sp,'whole'); ps,fs=oof(Xs,y,sp,'segment'); pp,fp=oof(Xp,y,sp,'phase') + grid=[]; best=None + for ws in np.linspace(0,1,11): + for wp in np.linspace(0,1-ws,11): + ww=1-ws-wp; q=np.clip(ww*pw+ws*ps+wp*pp,1e-5,1-1e-5); ll=float(log_loss(y,q)); row={'whole_weight':float(ww),'segment_weight':float(ws),'phase_weight':float(wp),'logloss':ll} + grid.append(row); best=row if best is None or ll1 for m in meta]))}} + Path(a.out).write_text(json.dumps(result,indent=2)); print(json.dumps(result,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v81_target_segment_phase.json'); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v82_modernbert_supervised.py b/competitions/trace_the_ace/v82_modernbert_supervised.py new file mode 100644 index 00000000..3b8dcc78 --- /dev/null +++ b/competitions/trace_the_ace/v82_modernbert_supervised.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""V82: supervised ModernBERT on V81 multi-resolution mastery evidence. + +Primary question: does task-supervised language understanding materially improve +objective-cold log loss once the input representation exposes target segment, +instructional phase, assistance/state trajectory, and whole-session context? + +For CPU feasibility this experiment unfreezes the classifier/pooler plus the top +N transformer blocks. This is deliberately different from frozen embeddings: +task labels update the language model's upper representation layers. +""" +from __future__ import annotations +import argparse, json, random +from pathlib import Path +import numpy as np +import pandas as pd +import torch +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from torch.utils.data import Dataset, DataLoader +from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_linear_schedule_with_warmup + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, SEED +from v81_target_segment_phase import choose_target_segment, phase_views + +random.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED) + + +def clip_chars(s: str, n: int) -> str: + s=str(s) + if len(s)<=n: return s + # Preserve both start and terminal evidence. + a=n//2; return s[:a] + " [..CUT..] " + s[-(n-a):] + + +def make_text(df: pd.DataFrame, objective: str, max_chars: int=18000) -> tuple[str, dict]: + whole, whole_num, _ = trajectory_views(df, objective) + seg, meta = choose_target_segment(df, objective) + target, target_num, _ = trajectory_views(seg, objective) + phases, phase_num = phase_views(seg, objective) + + # Multi-resolution evidence. Whole context is compacted; target/independent/ + # application and canonical states get preferential budget. + parts = [ + f"[OBJECTIVE] {objective}", + f"[TARGET_SEGMENT] {clip_chars(target['raw'], 4200)}", + f"[TARGET_STUDENT] {clip_chars(target['student'], 2600)}", + f"[TARGET_CANONICAL] {clip_chars(target['canonical'], 2400)}", + f"[TARGET_TERMINAL] {clip_chars(target['terminal'], 2200)}", + f"[PRIOR] {clip_chars(phases.get('phase_prior',''), 1200)}", + f"[GUIDED] {clip_chars(phases.get('phase_guided',''), 1600)}", + f"[INDEPENDENT] {clip_chars(phases.get('phase_independent',''), 2200)}", + f"[APPLICATION] {clip_chars(phases.get('phase_application',''), 1800)}", + f"[PHASE_STATES] {clip_chars(phases.get('phase_states',''), 1800)}", + f"[WHOLE_CONTEXT] {clip_chars(whole['raw'], 3200)}", + f"[WHOLE_TERMINAL] {clip_chars(whole['terminal'], 1200)}", + ] + text="\n".join(parts) + if len(text)>max_chars: text=clip_chars(text,max_chars) + return text, meta + + +class TextDS(Dataset): + def __init__(self, texts, labels, tok, max_len): self.texts=texts; self.labels=labels; self.tok=tok; self.max_len=max_len + def __len__(self): return len(self.texts) + def __getitem__(self,i): + enc=self.tok(self.texts[i], truncation=True, max_length=self.max_len, padding=False) + enc={k:torch.tensor(v,dtype=torch.long) for k,v in enc.items()} + enc['labels']=torch.tensor(float(self.labels[i]),dtype=torch.float32) + return enc + + +def collate(tok): + def fn(rows): + labels=torch.stack([r.pop('labels') for r in rows]) + b=tok.pad(rows,padding=True,return_tensors='pt'); b['labels']=labels + return b + return fn + + +def unfreeze_top(model, top_blocks: int): + for p in model.parameters(): p.requires_grad=False + # Always train classification head / pooler-like named modules. + for n,p in model.named_parameters(): + ln=n.lower() + if any(x in ln for x in ['classifier','score','pooler']): p.requires_grad=True + # ModernBERT/HF encoder layer naming varies; discover indexed layer names. + layer_names=[] + for n,_ in model.named_parameters(): + bits=n.split('.') + for j,b in enumerate(bits[:-1]): + if b.isdigit() and any(x in '.'.join(bits[:j]).lower() for x in ['layer','layers','encoder']): + try: layer_names.append(int(b)) + except: pass + if layer_names: + mx=max(layer_names); cutoff=max(0,mx-top_blocks+1) + for n,p in model.named_parameters(): + bits=n.split('.') + idx=None + for j,b in enumerate(bits): + if b.isdigit() and any(x in '.'.join(bits[:j]).lower() for x in ['layer','layers','encoder']): idx=int(b); break + if idx is not None and idx>=cutoff: p.requires_grad=True + trainable=sum(p.numel() for p in model.parameters() if p.requires_grad) + total=sum(p.numel() for p in model.parameters()) + print('trainable_parameters',trainable,'total_parameters',total,'fraction',trainable/total) + return trainable,total + + +def predict(model, loader, device): + model.eval(); out=[]; ys=[] + with torch.no_grad(): + for b in loader: + y=b.pop('labels').numpy(); ys.extend(y.tolist()) + b={k:v.to(device) for k,v in b.items()} + z=model(**b).logits.squeeze(-1); out.extend(torch.sigmoid(z).cpu().numpy().tolist()) + return np.asarray(out),np.asarray(ys) + + +def run(a): + frame=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: frame=frame.iloc[:a.limit].copy().reset_index(drop=True) + cache={}; texts=[]; metas=[] + for i,r in frame.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + t,m=make_text(cache[sid],str(r.learning_objective),a.max_chars); texts.append(t); metas.append(m) + if (i+1)%1000==0: print('built_texts',i+1) + y=frame.target.to_numpy(np.float32) + grp=(frame.learning_objective_id if 'learning_objective_id' in frame else frame.learning_objective).astype(str).to_numpy() + folds=list(GroupKFold(5).split(np.zeros(len(y)),y,grp)) + selected=list(range(min(a.folds,len(folds)))) + tok=AutoTokenizer.from_pretrained(a.model, trust_remote_code=True) + device=torch.device('cuda' if torch.cuda.is_available() else 'cpu'); print('device',device) + results=[]; all_p=[]; all_y=[] + for fi in selected: + tr,va=folds[fi] + model=AutoModelForSequenceClassification.from_pretrained(a.model,num_labels=1,problem_type='regression',trust_remote_code=True) + unfreeze_top(model,a.top_blocks); model.to(device) + train_ds=TextDS([texts[i] for i in tr],y[tr],tok,a.max_len); val_ds=TextDS([texts[i] for i in va],y[va],tok,a.max_len) + train_dl=DataLoader(train_ds,batch_size=a.batch,shuffle=True,collate_fn=collate(tok),num_workers=0) + val_dl=DataLoader(val_ds,batch_size=a.eval_batch,shuffle=False,collate_fn=collate(tok),num_workers=0) + params=[p for p in model.parameters() if p.requires_grad] + opt=torch.optim.AdamW(params,lr=a.lr,weight_decay=.01) + steps=max(1,len(train_dl)*a.epochs); sched=get_linear_schedule_with_warmup(opt,max(1,int(.06*steps)),steps) + model.train(); seen=0 + for ep in range(a.epochs): + for b in train_dl: + labels=b.pop('labels').to(device) + b={k:v.to(device) for k,v in b.items()} + logits=model(**b).logits.squeeze(-1) + loss=torch.nn.functional.binary_cross_entropy_with_logits(logits,labels) + loss.backward(); torch.nn.utils.clip_grad_norm_(params,1.0); opt.step(); sched.step(); opt.zero_grad(set_to_none=True) + seen+=1 + if seen%50==0: print('fold',fi+1,'step',seen,'loss',float(loss.detach().cpu())) + p,yy=predict(model,val_dl,device); p=np.clip(p,1e-5,1-1e-5) + row={'fold':fi+1,'rows':len(va),'logloss':float(log_loss(yy,p)),'auc':float(roc_auc_score(yy,p))}; print('RESULT',row); results.append(row); all_p.extend(p.tolist()); all_y.extend(yy.tolist()) + del model; + if torch.cuda.is_available(): torch.cuda.empty_cache() + result={'model':a.model,'mode':'supervised_top_blocks','top_blocks':a.top_blocks,'epochs':a.epochs,'lr':a.lr,'max_len':a.max_len,'folds_run':len(selected),'fold_results':results,'pooled_logloss':float(log_loss(all_y,all_p)) if all_p else None,'pooled_auc':float(roc_auc_score(all_y,all_p)) if all_p else None,'segmentation':{'mean_fraction':float(np.mean([m['segment_fraction'] for m in metas])),'mean_segments':float(np.mean([m['segments'] for m in metas]))}} + Path(a.out).write_text(json.dumps(result,indent=2)); print(json.dumps(result,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v82_modernbert_supervised.json'); p.add_argument('--model',default='answerdotai/ModernBERT-large'); p.add_argument('--folds',type=int,default=1); p.add_argument('--epochs',type=int,default=1); p.add_argument('--top-blocks',type=int,default=2); p.add_argument('--lr',type=float,default=2e-5); p.add_argument('--max-len',type=int,default=768); p.add_argument('--max-chars',type=int,default=18000); p.add_argument('--batch',type=int,default=2); p.add_argument('--eval-batch',type=int,default=4); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v83_talkmove_supervised.py b/competitions/trace_the_ace/v83_talkmove_supervised.py new file mode 100644 index 00000000..ec6d7cb6 --- /dev/null +++ b/competitions/trace_the_ace/v83_talkmove_supervised.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""V83: task-supervised TalkMove-BERT on V81 multi-resolution evidence. + +R10 repair only: TalkMove-BERT ships with a 5-class head, while this experiment +uses a 1-logit binary head. Inject ignore_mismatched_sizes=True so the pretrained +encoder is retained and the task head is lawfully reinitialized. Hypothesis, +folds, representation and metrics remain unchanged. +""" +import argparse +from pathlib import Path +import v82_modernbert_supervised as base + +_orig = base.AutoModelForSequenceClassification.from_pretrained + +def _from_pretrained(*args, **kwargs): + kwargs['ignore_mismatched_sizes'] = True + return _orig(*args, **kwargs) + +base.AutoModelForSequenceClassification.from_pretrained = _from_pretrained + +if __name__=='__main__': + p=argparse.ArgumentParser() + p.add_argument('--features',type=Path,required=True) + p.add_argument('--labels',type=Path,required=True) + p.add_argument('--transcripts',type=Path,required=True) + p.add_argument('--out',default='v83_talkmove_supervised.json') + p.add_argument('--model',default='saroyehun/Talkmove-bert') + p.add_argument('--folds',type=int,default=1) + p.add_argument('--epochs',type=int,default=2) + p.add_argument('--top-blocks',type=int,default=4) + p.add_argument('--lr',type=float,default=2e-5) + p.add_argument('--max-len',type=int,default=512) + p.add_argument('--max-chars',type=int,default=14000) + p.add_argument('--batch',type=int,default=8) + p.add_argument('--eval-batch',type=int,default=16) + p.add_argument('--limit',type=int,default=0) + base.run(p.parse_args()) diff --git a/competitions/trace_the_ace/v84_student_evidence.py b/competitions/trace_the_ace/v84_student_evidence.py new file mode 100644 index 00000000..8c3606f2 --- /dev/null +++ b/competitions/trace_the_ace/v84_student_evidence.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""V84: student-evidence representation with tutor-language confound removed. + +Builds objective-cold OOF predictions from question + student answer + canonical +state/assistance tokens, but excludes raw tutor praise/feedback wording from the +student-evidence view. Compares against whole-session V75 and reports blend gain. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, extract_canonical_events, SEED +from v81_target_segment_phase import choose_target_segment + + +def folds(g): + z=np.zeros(len(g)); return list(GroupKFold(5).split(z,z,g.astype(str).to_numpy())) + +def evidence_view(df,obj): + ev=extract_canonical_events(df,obj) + chunks=[] + nums=[] + for e in ev: + rb='HIGH' if e.relevance>=.15 else 'MID' if e.relevance>=.05 else 'LOW' + ab='NONE' if e.assistance<=.1 else 'LOW' if e.assistance<=.5 else 'HIGH' + chunks.append(f'[STATE={e.state}] [REL={rb}] [ASSIST={ab}] [Q] {e.question} [STUDENT] {e.answer}') + nums.append([e.relevance,e.recency,e.assistance,e.substantive,e.explanation]) + if nums: + A=np.asarray(nums,float) + feat=np.concatenate([A.mean(0),A.max(0),A[-1],np.array([len(ev)],float)]) + else: feat=np.zeros(16,float) + return ' '.join(chunks),feat + +def build_v75(frame,rows,nums): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective])] + for k in ['raw','student','local','canonical','terminal']: + parts.append(hv.transform([f'[{k.upper()}] '+r[k] for r in rows])) + Z=np.vstack(nums); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6); parts.append(csr_matrix(Z)) + return hstack(parts,format='csr') +def build_evidence(frame,whole_text,seg_text,nums): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective]),hv.transform(['[WHOLE_EVIDENCE] '+x for x in whole_text]),hv.transform(['[TARGET_EVIDENCE] '+x for x in seg_text])] + Z=np.vstack(nums); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6); parts.append(csr_matrix(Z)) + return hstack(parts,format='csr') +def oof(X,y,sp,name): + p=np.zeros(len(y)); fr=[] + for k,(tr,va) in enumerate(sp,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + row={'fold':k,'rows':len(va),'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; print(name,row); fr.append(row) + return p,fr + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={}; vr=[]; vn=[]; wt=[]; st=[]; en=[] + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + d=cache[sid]; v,n,_=trajectory_views(d,str(r.learning_objective)); vr.append(v); vn.append(n) + seg,_=choose_target_segment(d,str(r.learning_objective)); w,wn=evidence_view(d,str(r.learning_objective)); s,sn=evidence_view(seg,str(r.learning_objective)); wt.append(w); st.append(s); en.append(np.concatenate([wn,sn])) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); grp=f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective; sp=folds(grp) + Xv=build_v75(f,vr,vn); Xe=build_evidence(f,wt,st,en); pv,fv=oof(Xv,y,sp,'v75'); pe,fe=oof(Xe,y,sp,'evidence') + grid=[]; best=None + for w in np.linspace(0,1,41): + q=np.clip((1-w)*pv+w*pe,1e-5,1-1e-5); ll=float(log_loss(y,q)); row={'evidence_weight':float(w),'logloss':ll}; grid.append(row); best=row if best is None or ll knowledge-state separator. + +Compares three objective-cold arms: +A0: V75 whole-session representation baseline. +A1: explicit objective-conditioned EvidenceEvent IR with assistance/independence tags. +A2: same EvidenceEvent IR with assistance/independence tags ablated. + +Primary decision: A1 must beat A0 by >=0.003 log loss and materially beat A2 to count +as a representation-level breakthrough. Otherwise retain as negative/conditional law. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript, tokens, jaccard, char_ngram_overlap +from v75_canonical_trajectory import load_training, trajectory_views, SEED +from v81_target_segment_phase import choose_target_segment, phase_for + +QUESTION_RE=re.compile(r"\?|\b(?:what|how|why|which|calculate|solve|find|show|explain|tell me|your turn|try|have a go)\b",re.I) +POS_RE=re.compile(r"\b(?:correct|right|yes|exactly|great|brilliant|well done|good job|nice|perfect)\b",re.I) +NEG_RE=re.compile(r"\b(?:not quite|incorrect|wrong|try again|check|remember|almost|no[, .])\b",re.I) +HINT_RE=re.compile(r"\b(?:remember|think about|hint|try|look at|first|start by|let's|lets|together|we can|I'll|i will|let me)\b",re.I) +SUPPLY_RE=re.compile(r"\b(?:the answer is|it is|equals|so .* is|that gives|we get|you should)\b",re.I) +ACK_RE=re.compile(r"^(?:ok(?:ay)?|yes|yeah|yep|no|nope|thanks?|thank you|got it|sure|right|mhm|uh huh|great|cool)[.! ]*$",re.I) + +PHASE_WEIGHT={'OTHER':.45,'GOAL':.25,'PRIOR':.35,'GUIDED':.55,'INDEPENDENT':1.0,'APPLICATION':1.1} + + +def rel(text,obj): + return float(max(jaccard(tokens(str(text)),tokens(str(obj))),.5*char_ngram_overlap(str(text),str(obj)))) + + +def substantive(s): + s=str(s).strip() + if not s or ACK_RE.match(s): return False + # preserve short numeric/math answers + if re.search(r"\d|[=+\-*/%]",s): return True + return len(tokens(s))>=2 + + +def phase_replay(df): + ph='OTHER'; out=[] + for t in df.content.fillna('').astype(str): + ph=phase_for(t,ph); out.append(ph) + return out + + +def evidence_events(df,obj): + d=df.reset_index(drop=True).copy(); phases=phase_replay(d) + events=[] + n=len(d) + for i,row in d.iterrows(): + if str(row.role).lower()!='student' or not substantive(row.content): continue + # nearest preceding tutor question/prompt within 4 turns + qidx=None + for j in range(i-1,max(-1,i-5),-1): + if str(d.iloc[j].role).lower()=='tutor' and QUESTION_RE.search(str(d.iloc[j].content)): + qidx=j; break + if qidx is None: continue + q=str(d.iloc[qidx].content); ans=str(row.content) + # immediate/near tutor feedback after response + feedback=''; fbidx=None + for j in range(i+1,min(n,i+4)): + if str(d.iloc[j].role).lower()=='tutor': feedback=str(d.iloc[j].content); fbidx=j; break + rq=rel(q,obj); ra=rel(ans,obj); relevance=max(rq,.4*ra) + # Assistance is derived only from tutor turns after previous student response and before this answer. + window=' '.join(str(d.iloc[j].content) for j in range(max(0,qidx-2),i) if str(d.iloc[j].role).lower()=='tutor') + supplied=bool(SUPPLY_RE.search(window)); hinted=bool(HINT_RE.search(window)) + assistance=1.0 if supplied else (.6 if hinted else 0.0) + independent=(phases[i] in ('INDEPENDENT','APPLICATION') and assistance<.3) + pos=bool(POS_RE.search(feedback)); neg=bool(NEG_RE.search(feedback)) + # canonical state; tutor feedback is auxiliary, not ground truth + if neg: state='UNRESOLVED_ERROR' + elif pos and assistance>=.6: state='CORRECT_AFTER_GUIDANCE' + elif pos and independent: state='INDEPENDENT_CORRECT' + elif pos: state='SUPPORTED_CORRECT' + else: state='UNJUDGED_RESPONSE' + events.append({'i':i,'phase':phases[i],'q':q,'a':ans,'feedback':feedback,'rel':relevance, + 'assistance':assistance,'independent':independent,'state':state, + 'position':i/max(1,n-1),'pos':pos,'neg':neg}) + return events + + +def render(events,obj,ablate=False): + keep=sorted(events,key=lambda e:(e['rel']*(.4+.6*e['position'])*PHASE_WEIGHT.get(e['phase'],.4)),reverse=True)[:16] + keep=sorted(keep,key=lambda e:e['i']) + rows=[] + for e in keep: + tags=[f"PHASE={e['phase']}",f"STATE={e['state']}",f"REL={e['rel']:.2f}",f"POS={int(e['pos'])}",f"NEG={int(e['neg'])}"] + if not ablate: tags += [f"ASSIST={e['assistance']:.1f}",f"INDEP={int(e['independent'])}"] + rows.append('['+' '.join(tags)+'] [Q] '+e['q']+' [STUDENT] '+e['a']) + return f"[OBJECTIVE] {obj}\n"+'\n'.join(rows) + + +def nums(events,ablate=False): + if not events: return np.zeros(22 if not ablate else 16,float) + E=events; rels=np.array([e['rel'] for e in E]); pos=np.array([e['pos'] for e in E],float); neg=np.array([e['neg'] for e in E],float) + ind=np.array([e['independent'] for e in E],float); ass=np.array([e['assistance'] for e in E],float); positions=np.array([e['position'] for e in E]) + app=np.array([e['phase']=='APPLICATION' for e in E],float); late=positions>=.6 + base=[len(E),rels.mean(),rels.max(),pos.mean(),neg.mean(),positions[pos>0].max() if pos.any() else 0, + positions[neg>0].max() if neg.any() else 0,pos[late].mean() if late.any() else 0,neg[late].mean() if late.any() else 0, + app.mean(),(pos*rels).sum()/(rels.sum()+1e-6),(neg*rels).sum()/(rels.sum()+1e-6), + float(any(e['state']=='UNRESOLVED_ERROR' for e in E[-3:])),float(any(e['state']=='INDEPENDENT_CORRECT' for e in E[-3:])), + sum(e['state']=='INDEPENDENT_CORRECT' for e in E),sum(e['state']=='CORRECT_AFTER_GUIDANCE' for e in E)] + if ablate: return np.asarray(base,float) + extra=[ass.mean(),ass[-3:].mean() if len(ass)>=3 else ass.mean(),ind.mean(),ind[late].mean() if late.any() else 0, + (pos*ind*rels).sum()/(rels.sum()+1e-6),(neg*(1-ass)*rels).sum()/(rels.sum()+1e-6)] + return np.asarray(base+extra,float) + + +def build_sparse(texts,Z,prefix): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform([f'[{prefix}] '+x for x in texts]); Z=np.vstack(Z); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def build_v75(frame,transcripts): + rows=[]; ns=[] + for _,r in frame.iterrows(): + v,n,_=trajectory_views(transcripts[str(r.session_id)],str(r.learning_objective)); rows.append(v); ns.append(n) + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform([f'[OBJECTIVE] {x}' for x in frame.learning_objective])] + for k in rows[0].keys(): parts.append(hv.transform([f'[{k.upper()}] '+r[k] for r in rows])) + Z=np.vstack(ns); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6); parts.append(csr_matrix(Z)) + return hstack(parts,format='csr') + + +def oof(X,y,splits,name): + p=np.zeros(len(y)); fs=[] + for k,(tr,va) in enumerate(splits,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + row={'fold':k,'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; print(name,row); fs.append(row) + return p,fs + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: f=f.iloc[:a.limit].copy().reset_index(drop=True) + cache={} + for sid in f.session_id.astype(str).unique(): cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + full_text=[]; abl_text=[]; full_num=[]; abl_num=[]; meta=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; seg,m=choose_target_segment(d,str(r.learning_objective)); ev=evidence_events(seg,str(r.learning_objective)) + full_text.append(render(ev,str(r.learning_objective),False)); abl_text.append(render(ev,str(r.learning_objective),True)) + full_num.append(nums(ev,False)); abl_num.append(nums(ev,True)); meta.append({'events':len(ev),**m}) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + X0=build_v75(f,cache); X1=build_sparse(full_text,full_num,'EVIDENCE'); X2=build_sparse(abl_text,abl_num,'EVIDENCE_ABL') + p0,f0=oof(X0,y,sp,'A0_v75'); p1,f1=oof(X1,y,sp,'A1_evidence'); p2,f2=oof(X2,y,sp,'A2_ablation') + ll0=float(log_loss(y,p0)); ll1=float(log_loss(y,p1)); ll2=float(log_loss(y,p2)) + # Also test whether evidence is orthogonal to V75; fixed transparent grid only. + blends=[]; best=None + for w in np.linspace(0,1,21): + q=np.clip((1-w)*p0+w*p1,1e-5,1-1e-5); ll=float(log_loss(y,q)); row={'evidence_weight':float(w),'logloss':ll}; blends.append(row) + if best is None or ll=.003 and causal>=.001: decision='REPRESENTATION_BREAKTHROUGH' + elif gain>=.001 and causal>0: decision='PROMISING_PARTIAL' + elif best['logloss']<=ll0-.001: decision='ORTHOGONAL_SIGNAL_ONLY' + else: decision='REJECT_OR_REFINE_R5' + out={'primary':'objective-cold','A0_v75':ll0,'A1_evidence':ll1,'A2_ablation':ll2,'gain_vs_A0':gain,'causal_assistance_gain':causal, + 'best_blend':best,'decision':decision,'folds':{'A0':f0,'A1':f1,'A2':f2}, + 'event_stats':{'mean_events':float(np.mean([m['events'] for m in meta])),'zero_event_fraction':float(np.mean([m['events']==0 for m in meta]))}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v85_evidence_state.json'); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v86_knowledge_state_dynamics.py b/competitions/trace_the_ace/v86_knowledge_state_dynamics.py new file mode 100644 index 00000000..134fa9b8 --- /dev/null +++ b/competitions/trace_the_ace/v86_knowledge_state_dynamics.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""V86: RGRS sequential knowledge-state dynamics separator. + +Hypothesis: the missing object is not merely an EvidenceEvent multiset but the +objective-conditioned *state trajectory* induced by ordered evidence. + +Arms (objective-cold, frozen folds/model family): +A0: EvidenceEvent multiset representation (same event extractor as V85). +A1: ordered EvidenceEvents + explicit cumulative/decayed knowledge-state dynamics. +A2: causal ablation: same events/state formulas with event order canonically sorted + by content signature, destroying observed temporal order while preserving the + event multiset and most marginal features. + +A1 must beat A0 materially and A2 must lose the gain to support an ordering/state +representation claim. Otherwise retain a negative/conditional law. +""" +from __future__ import annotations +import argparse, json, hashlib +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, PHASE_WEIGHT + + +def event_value(e): + """Signed mastery evidence, deliberately simple and inspectable.""" + rel=float(e['rel']); assist=float(e['assistance']); phase=PHASE_WEIGHT.get(e['phase'],.4) + independent=1.0 if e['independent'] else 0.0 + if e['neg']: + # An unassisted, relevant error is strong negative evidence. + return - rel * phase * (0.55 + 0.45*(1-assist)) + if e['pos']: + # Correctness after strong assistance is weaker than independent production. + return rel * phase * (0.25 + 0.75*(1-assist)) * (1.0 + 0.25*independent) + # Unjudged substantive production is weak evidence, not zero. + return 0.08 * rel * phase * (1-assist) + + +def signature(e): + s=(str(e['q'])+'\x1f'+str(e['a'])+'\x1f'+str(e['state'])).encode('utf-8','ignore') + return hashlib.sha1(s).hexdigest() + + +def state_features(events, destroy_order=False): + if not events: + return np.zeros(46,float), '[NO_EVENTS]' + E=list(events) + if destroy_order: + # Canonical deterministic order: same event multiset, observed chronology removed. + E=sorted(E,key=signature) + vals=[]; ks=[]; fast=[]; slow=[] + k=0.0; kf=0.0; kslo=0.0 + prev_state='START'; transitions=[] + for t,e in enumerate(E): + v=event_value(e); vals.append(v) + # bounded additive state + two recency scales + k=np.tanh(0.78*np.arctanh(np.clip(k,-.999,.999)) + v) + kf=.55*kf + v + kslo=.88*kslo + v + ks.append(k); fast.append(kf); slow.append(kslo) + transitions.append(prev_state+'>'+str(e['state'])); prev_state=str(e['state']) + V=np.asarray(vals); K=np.asarray(ks); F=np.asarray(fast); S=np.asarray(slow) + n=len(E); q=max(1,n//4) + pos=np.asarray([e['pos'] for e in E],float); neg=np.asarray([e['neg'] for e in E],float) + ind=np.asarray([e['independent'] for e in E],float); ass=np.asarray([e['assistance'] for e in E],float) + rel=np.asarray([e['rel'] for e in E],float) + app=np.asarray([e['phase']=='APPLICATION' for e in E],float) + # Transition features target the educational trajectory directly. + err_to_ind=0; guide_to_ind=0; recovery=0; regress=0 + for a,b in zip(E[:-1],E[1:]): + if a['neg'] and b['independent'] and b['pos']: err_to_ind+=1 + if a['assistance']>=.6 and b['independent'] and b['pos']: guide_to_ind+=1 + if a['neg'] and b['pos']: recovery+=1 + if a['pos'] and b['neg']: regress+=1 + last_pos=max([i for i,e in enumerate(E) if e['pos']],default=-1)/(max(1,n-1)) + last_neg=max([i for i,e in enumerate(E) if e['neg']],default=-1)/(max(1,n-1)) + last_ind=max([i for i,e in enumerate(E) if e['independent'] and e['pos']],default=-1)/(max(1,n-1)) + feats=[ + n, V.mean(), V.sum(), V[-1], V[:q].mean(), V[-q:].mean(), + K[-1], K.max(), K.min(), K.mean(), K[-q:].mean(), + F[-1], F.max(), F.min(), S[-1], S.max(), S.min(), + float(K[-1]-K[0]), float(F[-1]-F[0]), float(S[-1]-S[0]), + pos.mean(), neg.mean(), ind.mean(), ass.mean(), rel.mean(), app.mean(), + float((pos*ind*rel).sum()), float((neg*(1-ass)*rel).sum()), + err_to_ind, guide_to_ind, recovery, regress, + last_pos,last_neg,last_ind, + float(any(e['neg'] for e in E[-3:])), + float(any(e['independent'] and e['pos'] for e in E[-3:])), + float(sum(e['independent'] and e['pos'] for e in E[-5:])), + float(sum(e['neg'] for e in E[-5:])), + float(np.polyfit(np.arange(n),K,1)[0] if n>1 else 0), + float(np.polyfit(np.arange(n),V,1)[0] if n>1 else 0), + float(np.std(V)),float(np.std(K)), + float(np.mean(np.abs(np.diff(K))) if n>1 else 0), + float(np.max(np.abs(np.diff(K))) if n>1 else 0), + float(sum(t=='UNRESOLVED_ERROR>INDEPENDENT_CORRECT' for t in transitions)), + ] + # Render ordered state path so sparse model can exploit categorical transitions too. + rows=[] + for i,(e,v,kv) in enumerate(zip(E,V,K)): + rows.append(f"[T={i} PHASE={e['phase']} STATE={e['state']} ASSIST={e['assistance']:.1f} INDEP={int(e['independent'])} REL={e['rel']:.2f} DV={v:.2f} K={kv:.2f}]") + return np.asarray(feats,float), ' '.join(rows) + + +def build(texts,Z,prefix): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform([f'[{prefix}] '+x for x in texts]); Z=np.vstack(Z); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def oof(X,y,sp,name): + p=np.zeros(len(y)); folds=[] + for k,(tr,va) in enumerate(sp,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + r={'fold':k,'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; print(name,r); folds.append(r) + return p,folds + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: f=f.iloc[:a.limit].copy().reset_index(drop=True) + cache={}; base_t=[]; base_n=[]; seq_t=[]; seq_n=[]; abl_t=[]; abl_n=[]; counts=[] + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + seg,_=choose_target_segment(cache[sid],str(r.learning_objective)); ev=evidence_events(seg,str(r.learning_objective)); counts.append(len(ev)) + base_t.append(render(ev,str(r.learning_objective),False)); base_n.append(nums(ev,False)) + z,t=state_features(ev,False); za,ta=state_features(ev,True) + seq_t.append(f"[OBJECTIVE] {r.learning_objective} [STATE_PATH] {t} [EVENTS] {base_t[-1]}"); seq_n.append(np.r_[base_n[-1],z]) + abl_t.append(f"[OBJECTIVE] {r.learning_objective} [STATE_PATH_ORDER_ABLATED] {ta} [EVENTS] {base_t[-1]}"); abl_n.append(np.r_[base_n[-1],za]) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + X0=build(base_t,base_n,'EVENT_MULTISET'); X1=build(seq_t,seq_n,'STATE_ORDERED'); X2=build(abl_t,abl_n,'STATE_ORDER_ABLATED') + p0,f0=oof(X0,y,sp,'A0_multiset'); p1,f1=oof(X1,y,sp,'A1_ordered_state'); p2,f2=oof(X2,y,sp,'A2_order_ablation') + ll0=float(log_loss(y,p0)); ll1=float(log_loss(y,p1)); ll2=float(log_loss(y,p2)) + gain=ll0-ll1; causal=ll2-ll1 + # Check orthogonality if state helps only in blend. + best=None + for w in np.linspace(0,1,21): + q=np.clip((1-w)*p0+w*p1,1e-5,1-1e-5); ll=float(log_loss(y,q)) + if best is None or ll=.003 and causal>=.001: decision='SEQUENTIAL_STATE_BREAKTHROUGH' + elif gain>=.001 and causal>0: decision='PROMISING_SEQUENTIAL_STATE' + elif best['logloss']<=ll0-.001: decision='ORTHOGONAL_SEQUENCE_SIGNAL' + else: decision='ORDER_NOT_CAUSAL_OR_REFINE_R5' + out={'primary':'objective-cold','A0_event_multiset':ll0,'A1_ordered_state':ll1,'A2_order_ablation':ll2, + 'gain_vs_A0':gain,'causal_order_gain':causal,'best_blend':best,'decision':decision, + 'event_stats':{'mean_events':float(np.mean(counts)),'zero_event_fraction':float(np.mean(np.asarray(counts)==0))}, + 'folds':{'A0':f0,'A1':f1,'A2':f2}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v86_knowledge_state_dynamics.json'); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v87_rgrs_composition.py b/competitions/trace_the_ace/v87_rgrs_composition.py new file mode 100644 index 00000000..cd452cd9 --- /dev/null +++ b/competitions/trace_the_ace/v87_rgrs_composition.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""V87: RGRS composition-before-invention test. + +R7 hypothesis: V81 target/phase structure and V84 student-evidence IR are individually +incomplete but complementary to V75 whole-session context. Build all four OOF predictors +on identical objective-cold folds, then choose convex blend weights for each held-out fold +using ONLY the other four folds' OOF predictions. + +This removes the optimistic full-OOF blend-weight selection used in exploratory V81/V84. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views +from v81_target_segment_phase import choose_target_segment, phase_views, build_X, oof +from v84_student_evidence import evidence_view, build_evidence + + +def simplex_grid(step=.1): + vals=np.arange(0,1+1e-9,step) + for w0 in vals: + for w1 in vals: + for w2 in vals: + s=w0+w1+w2 + if s>1+1e-9: continue + w3=1-s + yield np.array([w0,w1,w2,w3],float) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={}; whole_rows=[]; whole_nums=[]; seg_rows=[]; seg_nums=[]; phase_rows=[]; phase_nums=[]; ev_whole=[]; ev_seg=[]; ev_num=[] + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + d=cache[sid]; obj=str(r.learning_objective) + vw,nw,_=trajectory_views(d,obj); whole_rows.append(vw); whole_nums.append(nw) + seg,_=choose_target_segment(d,obj); vs,ns,_=trajectory_views(seg,obj); seg_rows.append(vs); seg_nums.append(ns) + pv,pn=phase_views(seg,obj); phase_rows.append({**vs,**pv}); phase_nums.append(np.concatenate([ns,pn])) + ew,enw=evidence_view(d,obj); es,ens=evidence_view(seg,obj); ev_whole.append(ew); ev_seg.append(es); ev_num.append(np.concatenate([enw,ens])) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); grp=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,grp)) + Xw=build_X(f,whole_rows,whole_nums,'WHOLE'); Xs=build_X(f,seg_rows,seg_nums,'SEG'); Xp=build_X(f,phase_rows,phase_nums,'PHASE'); Xe=build_evidence(f,ev_whole,ev_seg,ev_num) + pw,_=oof(Xw,y,sp,'whole'); ps,_=oof(Xs,y,sp,'segment'); pp,_=oof(Xp,y,sp,'phase'); pe,_=oof(Xe,y,sp,'evidence') + P=np.column_stack([pw,ps,pp,pe]); names=['whole','segment','phase','evidence'] + # Cross-fit the blend: each outer fold's weights are selected on the other 4 folds only. + final=np.zeros(len(y)); fold_rows=[] + all_idx=np.arange(len(y)) + for k,(_,va) in enumerate(sp,1): + trmeta=np.setdiff1d(all_idx,va,assume_unique=False) + best=None + for w in simplex_grid(a.step): + q=np.clip(P[trmeta]@w,1e-5,1-1e-5); ll=float(log_loss(y[trmeta],q)) + if best is None or ll=.003 else ('R7_PROMISING' if gain>=.001 else 'R7_INSUFFICIENT')} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v87_rgrs_composition.json'); p.add_argument('--step',type=float,default=.1); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v88_crossfit_evidence_composition.py b/competitions/trace_the_ace/v88_crossfit_evidence_composition.py new file mode 100644 index 00000000..9181031e --- /dev/null +++ b/competitions/trace_the_ace/v88_crossfit_evidence_composition.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""V88: cross-fitted V75 + ablated EvidenceEvent composition. + +RGRS response to V85: +- assistance/independence tags were not causal (A1 ~= A2), so remove them; +- EvidenceEvents were weak standalone but strongly complementary to V75. + +This test asks whether that composition survives leakage-resistant weight selection. +For each held-out objective-cold fold, choose the blend weight using only OOF +predictions from the other four folds, then score the untouched fold. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={} + for sid in f.session_id.astype(str).unique(): + cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + + texts=[]; z=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)] + seg,_=choose_target_segment(d,str(r.learning_objective)) + ev=evidence_events(seg,str(r.learning_objective)) + texts.append(render(ev,str(r.learning_objective),ablate=True)) + z.append(nums(ev,ablate=True)) + if (i+1)%2500==0: print('rows',i+1) + + y=f.target.to_numpy(int) + groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + + X0=build_v75(f,cache) + X2=build_sparse(texts,z,'EVIDENCE_ABL') + p0,_=oof(X0,y,splits,'V75') + pe,_=oof(X2,y,splits,'EVIDENCE_ABL') + + fold_id=np.empty(len(y),int) + for k,(_,va) in enumerate(splits): fold_id[va]=k + + grid=np.linspace(0,0.8,33) + q=np.zeros(len(y)); selected=[] + for k,(_,va) in enumerate(splits): + tune=np.where(fold_id!=k)[0] + best=None + for w in grid: + pt=np.clip((1-w)*p0[tune]+w*pe[tune],1e-5,1-1e-5) + ll=float(log_loss(y[tune],pt)) + if best is None or ll=.003 else ('R7_PARTIAL' if gain>=.001 else 'REJECT_GLOBAL_BLEND_GAIN') + out={'primary':'objective-cold-crossfitted','v75_logloss':ll0,'evidence_ablation_logloss':lle, + 'crossfit_blend_logloss':llq,'crossfit_gain_vs_v75':gain,'decision':decision, + 'selected_by_fold':selected,'mean_evidence_weight':float(np.mean([x['evidence_weight'] for x in selected]))} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v88_crossfit_evidence_composition.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v89_relative_ability_composition.py b/competitions/trace_the_ace/v89_relative_ability_composition.py new file mode 100644 index 00000000..ddbf177f --- /dev/null +++ b/competitions/trace_the_ace/v89_relative_ability_composition.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V89: target mastery relative to within-session student ability. + +RGRS hypothesis: V88 captures target-local evidence but not the student's general +baseline competence expressed elsewhere in the same transcript. Build a sample- +local, non-target ability view and test whether it adds orthogonal signal to +V75 + EvidenceEvents under objective-cold cross-fitted composition. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment, split_segments +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof + +POS=re.compile(r"\b(?:correct|right|exactly|great|brilliant|well done|good job|perfect|yes)\b",re.I) +NEG=re.compile(r"\b(?:incorrect|wrong|not quite|try again|check|almost|no[, .])\b",re.I) + + +def non_target_ability(df,obj): + target,meta=choose_target_segment(df,obj) + ts,te=meta['start'],meta['end'] + # preserve all transcript rows outside selected target segment + rest=df.iloc[list(range(0,ts))+list(range(te,len(df)))].copy().reset_index(drop=True) + if not len(rest): rest=df.iloc[0:0].copy() + ev=evidence_events(rest,obj) + # objective relevance is intentionally not required here: this is general ability. + students=rest[rest.role.astype(str).str.lower().eq('student')].content.fillna('').astype(str).tolist() if len(rest) else [] + tutors=rest[rest.role.astype(str).str.lower().eq('tutor')].content.fillna('').astype(str).tolist() if len(rest) else [] + pos=sum(bool(POS.search(x)) for x in tutors); neg=sum(bool(NEG.search(x)) for x in tutors) + substantive=sum(bool(re.search(r"\d|[=+\-*/%]",x)) or len(x.split())>=2 for x in students) + explain=sum(any(k in x.lower() for k in ['because','so ','therefore','i think','first','then']) for x in students) + math=sum(bool(re.search(r"\d|[=+\-*/%]",x)) for x in students) + n=max(1,len(students)); fbn=max(1,pos+neg) + z=np.asarray([ + len(rest),len(students),len(tutors),substantive/n,explain/n,math/n, + pos/fbn,neg/fbn,(pos-neg)/fbn,len(ev),meta['segment_fraction'],meta['segments'] + ],float) + # compact text: student production + tutor verdict tokens, no target segment content + text=' [STUDENT] '.join(students[-80:]) + verdict=' '.join('[POS]' if POS.search(x) else '[NEG]' if NEG.search(x) else '' for x in tutors[-100:]) + return f'[GENERAL_STUDENT] {text} [GENERAL_VERDICTS] {verdict}',z + + +def build_ability(texts,z): + hv=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform(texts); Z=np.vstack(z); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={} + for sid in f.session_id.astype(str).unique(): cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + et=[]; ez=[]; at=[]; az=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + seg,_=choose_target_segment(d,obj); ev=evidence_events(seg,obj) + et.append(render(ev,obj,ablate=True)); ez.append(nums(ev,ablate=True)) + t,z=non_target_ability(d,obj); at.append(t); az.append(z) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)); fold=np.empty(len(y),int) + for k,(_,va) in enumerate(sp): fold[va]=k + X0=build_v75(f,cache); Xe=build_sparse(et,ez,'EVIDENCE_ABL'); Xa=build_ability(at,az) + p0,_=oof(X0,y,sp,'V75'); pe,_=oof(Xe,y,sp,'EVIDENCE'); pa,_=oof(Xa,y,sp,'ABILITY') + q=np.zeros(len(y)); sel=[] + grid=np.arange(0,0.61,0.1) + for k,(_,va) in enumerate(sp): + tune=np.where(fold!=k)[0]; best=None + for we in grid: + for wa in grid: + if we+wa>.8: continue + p=np.clip((1-we-wa)*p0[tune]+we*pe[tune]+wa*pa[tune],1e-5,1-1e-5) + ll=float(log_loss(y[tune],p)) + if best is None or ll=.0015 else ('R5_WEAK' if llg-llr>=.0005 else 'R5_REJECT_STYLE_REGIMES')} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v90_regime_gated.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v91_disagreement_frontier.py b/competitions/trace_the_ace/v91_disagreement_frontier.py new file mode 100644 index 00000000..e0658f81 --- /dev/null +++ b/competitions/trace_the_ace/v91_disagreement_frontier.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V91: disagreement-frontier applicability separator. + +RGRS EXPEDITION after V88 R7 composition win. +Question: when V75 and EvidenceEvents disagree, can sample-local observable information +predict which expert should receive more weight? + +All base predictions are objective-cold OOF. For each held-out fold, the router is fit only +on the other folds. Reports a fixed/global cross-fit blend, routed blend, and oracle headroom. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof + + +def row_features(df, seg, ev, p0, pe): + n=max(1,len(df)); ns=max(1,len(seg)) + roles=df.role.fillna('').astype(str).str.lower() + stu=(roles=='student').sum(); tut=(roles=='tutor').sum() + texts=df.content.fillna('').astype(str) + student_text=' '.join(df.loc[roles=='student','content'].fillna('').astype(str)) + math_chars=sum(c.isdigit() or c in '=+-*/%' for c in student_text) + chars=max(1,len(student_text)) + rels=np.array([e['rel'] for e in ev],float) if ev else np.zeros(0) + positions=np.array([e['position'] for e in ev],float) if ev else np.zeros(0) + states=[e['state'] for e in ev] + return np.array([ + p0, pe, pe-p0, abs(pe-p0), abs(p0-.5), abs(pe-.5), + len(df), len(seg), ns/n, stu/n, tut/n, len(ev), + float(rels.mean()) if len(rels) else 0., float(rels.max()) if len(rels) else 0., + float(positions.mean()) if len(positions) else 0., + float(sum(s=='INDEPENDENT_CORRECT' for s in states)), + float(sum(s=='UNRESOLVED_ERROR' for s in states)), + math_chars/chars, + float(np.mean([len(x) for x in texts])) if len(texts) else 0., + ],float) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + texts=[]; zz=[]; stored=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; seg,m=choose_target_segment(d,str(r.learning_objective)); ev=evidence_events(seg,str(r.learning_objective)) + texts.append(render(ev,str(r.learning_objective),ablate=True)); zz.append(nums(ev,ablate=True)); stored.append((d,seg,ev,m)) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + p0,_=oof(build_v75(f,cache),y,splits,'V75') + pe,_=oof(build_sparse(texts,zz,'EVIDENCE_ABL'),y,splits,'EVIDENCE') + X=np.vstack([row_features(*stored[i][:3],p0[i],pe[i]) for i in range(len(y))]) + # normalize within each training partition only below + fold_id=np.empty(len(y),int) + for k,(_,va) in enumerate(splits): fold_id[va]=k + q_fixed=np.zeros(len(y)); q_route=np.zeros(len(y)); selected=[] + grid=np.linspace(0,0.8,33) + for k,(_,va) in enumerate(splits): + tr=np.where(fold_id!=k)[0] + # cross-fit fixed blend reference + best=min(((float(log_loss(y[tr],np.clip((1-w)*p0[tr]+w*pe[tr],1e-5,1-1e-5))),float(w)) for w in grid), key=lambda z:z[0]) + wf=best[1]; q_fixed[va]=np.clip((1-wf)*p0[va]+wf*pe[va],1e-5,1-1e-5) + # expert-win label: lower per-row log loss, equivalent to closer probability to realized label in log space + l0=-(y[tr]*np.log(np.clip(p0[tr],1e-6,1))+(1-y[tr])*np.log(np.clip(1-p0[tr],1e-6,1))) + le=-(y[tr]*np.log(np.clip(pe[tr],1e-6,1))+(1-y[tr])*np.log(np.clip(1-pe[tr],1e-6,1))) + z=(le1 else float('nan') + selected.append({'fold':k+1,'fixed_weight':wf,'mean_routed_weight':float(wr.mean()),'router_win_auc':auc,'fixed_ll':float(log_loss(y[va],q_fixed[va])),'routed_ll':float(log_loss(y[va],q_route[va]))}) + # oracle only quantifies headroom; never admissible + choose_e=np.where(y==1,pe>p0,pe=.001 else ('R5_WEAK' if llf-llr>0 else 'REJECT_ROUTER_CURRENT_FEATURES')} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v91_disagreement_frontier.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v92_latent_state_decomposition.py b/competitions/trace_the_ace/v92_latent_state_decomposition.py new file mode 100644 index 00000000..a9bee12d --- /dev/null +++ b/competitions/trace_the_ace/v92_latent_state_decomposition.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""V92: latent student-state decomposition. + +Hypothesis: the transcript is a noisy measurement instrument. Predict post-test +correctness from (a) whole-session V75, (b) non-target local ability, (c) target +EvidenceEvents, and (d) objective difficulty, with explicit latent contrasts. +All base predictions are objective-cold OOF; the meta-combiner is cross-fitted +across the same held-out objective folds. Ablations test which latent components +actually pay rent. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof +from v89_relative_ability_composition import non_target_ability, build_ability + +EPS=1e-5 + +def logit(p): + p=np.clip(np.asarray(p,float),EPS,1-EPS) + return np.log(p/(1-p)) + +def objective_matrix(texts): + w=HashingVectorizer(n_features=2**16,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + c=HashingVectorizer(n_features=2**16,alternate_sign=False,norm='l2',analyzer='char_wb',ngram_range=(3,5),lowercase=True) + return hstack([w.transform(texts),c.transform(texts)],format='csr') + +def base_meta(p0,pa,pe,pd): + l0,la,le,ld=map(logit,[p0,pa,pe,pd]) + return np.c_[ + l0,la,le,ld, + la-ld, # ability relative to objective difficulty + le-la, # target-specific deviation from general ability + le-ld, # target evidence relative to difficulty + np.abs(le-la), + np.abs(la-ld), + l0-la, + l0-le, + la*ld, + le*la, + ] + +def crossfit_meta(y,splits,p0,pa,pe,pd,cols=None,C=0.1): + X=base_meta(p0,pa,pe,pd) + if cols is not None: X=X[:,cols] + q=np.zeros(len(y)); fold_rows=[] + for k,(tr,va) in enumerate(splits): + # The base inputs are themselves OOF predictions. Meta fit is restricted + # to other objective-cold folds and scored on untouched held-out objectives. + m=LogisticRegression(C=C,max_iter=1000,solver='lbfgs',random_state=SEED) + m.fit(X[tr],y[tr]); q[va]=m.predict_proba(X[va])[:,1] + fold_rows.append({'fold':k+1,'logloss':float(log_loss(y[va],np.clip(q[va],EPS,1-EPS)))}) + return np.clip(q,EPS,1-EPS),fold_rows + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + et=[]; ez=[]; at=[]; az=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + seg,_=choose_target_segment(d,obj); ev=evidence_events(seg,obj) + et.append(render(ev,obj,ablate=True)); ez.append(nums(ev,ablate=True)) + t,z=non_target_ability(d,obj); at.append(t); az.append(z) + if (i+1)%2500==0: print('rows',i+1) + + y=f.target.to_numpy(int) + groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + + X0=build_v75(f,cache) + Xa=build_ability(at,az) + Xe=build_sparse(et,ez,'EVIDENCE_ABL') + Xd=objective_matrix(f.learning_objective.fillna('').astype(str).tolist()) + p0,_=oof(X0,y,splits,'V75') + pa,_=oof(Xa,y,splits,'ABILITY') + pe,_=oof(Xe,y,splits,'TARGET') + pd,_=oof(Xd,y,splits,'DIFFICULTY') + + # Full latent comparison and causal component ablations. + full,folds=crossfit_meta(y,splits,p0,pa,pe,pd) + # Column definitions from base_meta: 0 V75,1 ability,2 target,3 difficulty, + # 4 ability-difficulty,5 target-ability,6 target-difficulty,... + no_ability,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,2,3,6,10]) + no_difficulty,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,1,2,5,7,9,10,12]) + no_target,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,1,3,4,8,9,11]) + linear_only,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,1,2,3]) + + # Reproduce V89-style cross-fitted convex composition as a strong control. + fold=np.empty(len(y),int) + for k,(_,va) in enumerate(splits): fold[va]=k + blend=np.zeros(len(y)); selected=[] + grid=np.arange(0,0.61,0.1) + for k,(_,va) in enumerate(splits): + tune=np.where(fold!=k)[0]; best=None + for we in grid: + for wa in grid: + if we+wa>.8: continue + p=np.clip((1-we-wa)*p0[tune]+we*pe[tune]+wa*pa[tune],EPS,1-EPS) + ll=float(log_loss(y[tune],p)) + if best is None or ll=.002 and sum(v>0 for v in causal.values())>=2 else ('PARTIAL' if scores['gain_vs_v89']>0 else 'REJECT') + out={'primary':'objective-cold-crossfitted','scores':scores,'causal_ablation_values':causal,'folds':folds,'v89_selected':selected,'decision':decision} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v92_latent_state_decomposition.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v93_shift_robust_validation.py b/competitions/trace_the_ace/v93_shift_robust_validation.py new file mode 100644 index 00000000..9ebc4442 --- /dev/null +++ b/competitions/trace_the_ace/v93_shift_robust_validation.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""V93: distribution-shift validation + robust blend selection. + +Goal: objective-cold CV materially over-predicts V75 public performance. Rather +than tune to leaderboard scores, construct several lawful train-only stress +worlds and choose V75/relative-ability blend weights that remain good across +all of them. + +Worlds: + * objective-cold -- unseen learning objectives + * session-cold -- unseen tutoring sessions + * objective-family-cold -- related objective wording held together + * style-cold -- transcript structural regimes held together + +The public leaderboard is NOT used to fit predictions or weights. +""" +from __future__ import annotations +import argparse, json, re, hashlib +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from sklearn.cluster import KMeans + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75, oof +from v89_relative_ability_composition import non_target_ability, build_ability + +STOP={"the","a","an","to","of","and","with","using","in","on","for","by","from","within","up","simple"} + +def obj_family(s:str)->str: + toks=[x for x in re.findall(r"[a-z]+",str(s).lower()) if x not in STOP] + # intentionally coarse: hold semantically similar surface families together + return "|".join(toks[:3]) if toks else "EMPTY" + +def style_matrix(f,cache): + rows=[] + for _,r in f.iterrows(): + d=cache[str(r.session_id)] + roles=d.role.astype(str).str.lower() if 'role' in d else np.array([]) + contents=d.content.fillna('').astype(str).tolist() if 'content' in d else [] + n=max(1,len(d)); st=sum(x=='student' for x in roles); tu=sum(x=='tutor' for x in roles) + words=sum(len(x.split()) for x in contents); chars=sum(len(x) for x in contents) + math=sum(bool(re.search(r"\d|[=+\-*/%]",x)) for x in contents) + questions=sum('?' in x for x in contents) + rows.append([len(d),st/n,tu/n,words/n,chars/n,math/n,questions/n]) + X=np.asarray(rows,float); return (X-X.mean(0))/(X.std(0)+1e-6) + +def folds_from_groups(groups,n=5): + g=np.asarray(groups).astype(str) + k=min(n,len(np.unique(g))) + if k<2: raise ValueError('need at least two groups') + return list(GroupKFold(k).split(np.zeros(len(g)),np.zeros(len(g)),g)) + +def eval_world(name,X0,Xa,y,splits,grid): + p0,_=oof(X0,y,splits,f'{name}:V75') + pa,_=oof(Xa,y,splits,f'{name}:ABILITY') + ll0=float(log_loss(y,p0)); lla=float(log_loss(y,pa)) + scores=[] + for w in grid: + p=np.clip((1-w)*p0+w*pa,1e-5,1-1e-5) + scores.append({'w':float(w),'ll':float(log_loss(y,p))}) + best=min(scores,key=lambda z:z['ll']) + return {'name':name,'v75':ll0,'ability':lla,'best':best,'curve':scores},p0,pa + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + at=[]; az=[] + for i,r in f.iterrows(): + t,z=non_target_ability(cache[str(r.session_id)],str(r.learning_objective)); at.append(t); az.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xa=build_ability(at,az); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy(); fam=f.learning_objective.astype(str).map(obj_family).to_numpy() + SX=style_matrix(f,cache); km=KMeans(n_clusters=5,random_state=137,n_init=10).fit(SX); style=km.labels_.astype(str) + worlds={ + 'objective_cold':folds_from_groups(obj), + 'session_cold':folds_from_groups(sess), + 'objective_family_cold':folds_from_groups(fam), + 'style_cold':folds_from_groups(style), + } + grid=np.linspace(0,0.7,29); results={}; curves={} + for name,sp in worlds.items(): + r,_,_=eval_world(name,X0,Xa,y,sp,grid); results[name]=r; curves[name]={x['w']:x['ll'] for x in r['curve']} + print(name,'V75',r['v75'],'ABILITY',r['ability'],'BEST',r['best']) + # Robust weight: minimize worst excess loss relative to each world's own best. + robust=[] + for w in grid: + exc=[]; raw=[] + for name,r in results.items(): + ll=curves[name][float(w)]; raw.append(ll); exc.append(ll-r['best']['ll']) + robust.append({'w':float(w),'worst_excess':float(max(exc)),'mean_excess':float(np.mean(exc)), + 'worst_logloss':float(max(raw)),'mean_logloss':float(np.mean(raw))}) + choice=min(robust,key=lambda z:(z['worst_excess'],z['mean_excess'],z['mean_logloss'])) + out={'primary':'shift-robust-validation','worlds':results,'robust_choice':choice, + 'objective_cold_best_weight':results['objective_cold']['best']['w'], + 'note':'No leaderboard score used in model fitting, split construction, or weight selection.'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v93_shift_robust.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v94_related_control.py b/competitions/trace_the_ace/v94_related_control.py new file mode 100644 index 00000000..629885f1 --- /dev/null +++ b/competitions/trace_the_ace/v94_related_control.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""V94: smallest separator for the V89/V93 applicability residual. + +Question: does non-target student evidence help only when it is a *comparable +control* for the target objective? + +A0 GLOBAL : all non-target transcript evidence (V89 representation). +A1 RELATED : only the most objective-related non-target lesson segments. +A2 DISTANT : deliberately least-related non-target segments (causal ablation). + +Opposing discriminators: + * objective-cold -- V93 says GLOBAL ability helps strongly. + * session-cold -- V93 says GLOBAL ability should be suppressed. + +Promotion requires RELATED to preserve objective-cold gain while reducing the +session-cold penalty, and DISTANT must not reproduce the same effect. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment, split_segments, rel_text +from v85_evidence_state import build_v75, oof, evidence_events +from v89_relative_ability_composition import non_target_ability, build_ability, POS, NEG +from v93_shift_robust_validation import folds_from_groups + + +def _ability_from_rows(rows: pd.DataFrame, obj: str, meta: dict, tag: str): + if rows is None or not len(rows): + rows = pd.DataFrame(columns=['role','content']) + roles = rows.role.astype(str).str.lower() if 'role' in rows else pd.Series([], dtype=str) + contents = rows.content.fillna('').astype(str) if 'content' in rows else pd.Series([], dtype=str) + students = contents[roles.eq('student')].tolist() + tutors = contents[roles.eq('tutor')].tolist() + pos = sum(bool(POS.search(x)) for x in tutors) + neg = sum(bool(NEG.search(x)) for x in tutors) + substantive = sum(bool(re.search(r"\d|[=+\-*/%]",x)) or len(x.split()) >= 2 for x in students) + explain = sum(any(k in x.lower() for k in ['because','so ','therefore','i think','first','then']) for x in students) + math = sum(bool(re.search(r"\d|[=+\-*/%]",x)) for x in students) + n=max(1,len(students)); fbn=max(1,pos+neg) + ev = evidence_events(rows.reset_index(drop=True), obj) if len(rows) else [] + z=np.asarray([ + len(rows),len(students),len(tutors),substantive/n,explain/n,math/n, + pos/fbn,neg/fbn,(pos-neg)/fbn,len(ev), + float(meta.get('chosen_mean_rel',0.0)),float(meta.get('chosen_max_rel',0.0)), + float(meta.get('available_segments',0)),float(meta.get('chosen_segments',0)), + ],float) + text=' [STUDENT] '.join(students[-80:]) + verdict=' '.join('[POS]' if POS.search(x) else '[NEG]' if NEG.search(x) else '' for x in tutors[-100:]) + return f'[{tag}] {text} [VERDICTS] {verdict}', z + + +def segmented_control(df: pd.DataFrame, obj: str, mode: str): + _, tm = choose_target_segment(df, obj) + ts,te=int(tm['start']),int(tm['end']) + cand=[] + for s,e in split_segments(df): + # exclude any segment overlapping the selected target segment + if not (e <= ts or s >= te): + continue + part=df.iloc[s:e].copy().reset_index(drop=True) + txt=' '.join(part.content.fillna('').astype(str)) + rel=float(rel_text(txt,obj)) + cand.append((rel,s,e,part)) + if not cand: + rest=df.iloc[list(range(0,ts))+list(range(te,len(df)))].copy().reset_index(drop=True) + return _ability_from_rows(rest,obj,{'available_segments':0,'chosen_segments':0},mode.upper()) + cand=sorted(cand,key=lambda x:(x[0],x[1])) + # Keep about half the non-target segments, capped to three, to make RELATED and + # DISTANT equal-budget representations rather than a length confound. + k=max(1,min(3,int(np.ceil(len(cand)/2)))) + chosen = cand[-k:] if mode=='related' else cand[:k] + rows=pd.concat([x[3] for x in sorted(chosen,key=lambda z:z[1])],ignore_index=True) + rels=[x[0] for x in chosen] + meta={'available_segments':len(cand),'chosen_segments':len(chosen), + 'chosen_mean_rel':float(np.mean(rels)),'chosen_max_rel':float(np.max(rels))} + return _ability_from_rows(rows,obj,meta,mode.upper()) + + +def build_control(texts, nums): + hv=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform(texts); Z=np.vstack(nums).astype(float); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def eval_arm(name, X0, Xa, y, sp): + p0,_=oof(X0,y,sp,name+':V75'); pa,_=oof(Xa,y,sp,name+':ABILITY') + grid=np.linspace(0,0.6,25); curve=[] + for w in grid: + q=np.clip((1-w)*p0+w*pa,1e-5,1-1e-5) + curve.append({'w':float(w),'ll':float(log_loss(y,q))}) + best=min(curve,key=lambda z:z['ll']) + return {'v75':float(log_loss(y,p0)),'ability':float(log_loss(y,pa)),'best':best, + 'gain':float(log_loss(y,p0)-best['ll'])} + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + gt=[];gz=[];rt=[];rz=[];dt=[];dz=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + t,z=non_target_ability(d,obj); gt.append(t); gz.append(z) + t,z=segmented_control(d,obj,'related'); rt.append(t); rz.append(z) + t,z=segmented_control(d,obj,'distant'); dt.append(t); dz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xg=build_ability(gt,gz); Xr=build_control(rt,rz); Xd=build_control(dt,dz) + y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy() + worlds={'objective_cold':folds_from_groups(obj),'session_cold':folds_from_groups(sess)} + out={'primary':'related-control-separator','worlds':{}} + for wn,sp in worlds.items(): + out['worlds'][wn]={ + 'global':eval_arm(wn+':GLOBAL',X0,Xg,y,sp), + 'related':eval_arm(wn+':RELATED',X0,Xr,y,sp), + 'distant':eval_arm(wn+':DISTANT',X0,Xd,y,sp), + } + o=out['worlds']['objective_cold']; s=out['worlds']['session_cold'] + obj_preserve=o['related']['gain'] >= 0.75*max(1e-9,o['global']['gain']) + session_repair=s['related']['best']['ll'] <= s['global']['best']['ll']-0.0005 + causal=o['related']['best']['ll'] <= o['distant']['best']['ll']-0.0005 + if obj_preserve and session_repair and causal: + verdict='PROMOTE_RELATED_CONTROL' + elif o['related']['best']['ll'] < o['global']['best']['ll'] and causal: + verdict='PARTIAL_R5_REFINE' + else: + verdict='REJECT_RELATEDNESS_OBSERVABLE' + out['decision']={'objective_gain_preserved':bool(obj_preserve),'session_penalty_repaired':bool(session_repair), + 'causal_vs_distant':bool(causal),'verdict':verdict} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v94_related_control.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v95_objective_support_activation.py b/competitions/trace_the_ace/v95_objective_support_activation.py new file mode 100644 index 00000000..aaf0242e --- /dev/null +++ b/competitions/trace_the_ace/v95_objective_support_activation.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""V95: objective-support activation law for the V94 RELATED ability specialist. + +Residual +-------- +V94 showed that RELATED relative-ability evidence helps objective-cold validation +but should receive zero weight in session-cold validation. The proposed missing +applicability variable is epistemic support for the target objective. + +Separator +--------- +Take one deterministic objective-cold outer fold. For every held-out objective, +reserve a deterministic support pool and a disjoint fixed evaluation set. Reveal +nested amounts of labelled target-objective support (0, 1, 2, 4, 8, 16, 32+ per +objective), refit V75 and the V94 RELATED expert, and score the *same* evaluation +rows at every support level. + +Prediction +---------- +Optimal RELATED blend weight should be high at zero support and decline toward +zero as target-objective support increases. + +Cheap causal ablation +--------------------- +At support 8 and 32+, shuffle only the newly revealed support labels before +fitting the RELATED expert. Extra rows without valid target information should +not reproduce a lawful support benefit. + +No leaderboard score or hidden-test outcome is used anywhere in construction, +fitting, weighting, or the promotion decision. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +from scipy.stats import spearmanr +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control + + +LEVELS = [0, 1, 2, 4, 8, 16, "32+"] +GRID = np.linspace(0.0, 0.6, 25) + + +def stable_key(obj: str, response_id: str) -> str: + return hashlib.sha256(f"V95|{SEED}|{obj}|{response_id}".encode()).hexdigest() + + +def make_fixed_support_design(frame, val_idx, objective): + """Return disjoint nested support pools and one fixed evaluation index. + + Up to half of each held-out objective (capped at 64 rows) is reserved as its + support pool. The complement is scored at every support level, eliminating + changing-evaluation-set confounding. Small objectives saturate naturally; + the result records realised support counts for every level. + """ + val_idx = np.asarray(val_idx, dtype=int) + response = ( + frame.response_id.astype(str).to_numpy() + if "response_id" in frame + else np.arange(len(frame)).astype(str) + ) + pools = {} + eval_parts = [] + for g in sorted(np.unique(objective[val_idx])): + idx = val_idx[objective[val_idx] == g] + idx = np.asarray(sorted(idx, key=lambda i: stable_key(str(g), response[i])), dtype=int) + pool_n = min(64, len(idx) // 2) + pools[str(g)] = idx[:pool_n] + eval_parts.append(idx[pool_n:]) + eval_idx = np.concatenate(eval_parts) if eval_parts else np.array([], dtype=int) + return pools, np.asarray(sorted(eval_idx), dtype=int) + + +def support_indices(pools, level): + out = [] + counts = [] + for g in sorted(pools): + pool = pools[g] + k = len(pool) if level == "32+" else min(int(level), len(pool)) + counts.append(k) + if k: + out.append(pool[:k]) + idx = np.concatenate(out) if out else np.array([], dtype=int) + return np.asarray(sorted(idx), dtype=int), np.asarray(counts, dtype=int) + + +def fit_predict(X, y, train_idx, eval_idx, y_train_override=None): + yy = y[train_idx] if y_train_override is None else np.asarray(y_train_override, dtype=int) + m = LogisticRegression( + C=0.25, + max_iter=300, + solver="liblinear", + random_state=SEED, + ).fit(X[train_idx], yy) + return np.clip(m.predict_proba(X[eval_idx])[:, 1], 1e-5, 1 - 1e-5) + + +def best_blend(y, p0, pa): + curve = [] + for w in GRID: + q = np.clip((1 - w) * p0 + w * pa, 1e-5, 1 - 1e-5) + curve.append({"w": float(w), "ll": float(log_loss(y, q))}) + return min(curve, key=lambda z: z["ll"]), curve + + +def realised_support_summary(counts): + if not len(counts): + return {"min": 0, "median": 0.0, "mean": 0.0, "max": 0, "objectives": 0} + return { + "min": int(np.min(counts)), + "median": float(np.median(counts)), + "mean": float(np.mean(counts)), + "max": int(np.max(counts)), + "objectives": int(len(counts)), + } + + +def run(a): + f = load_training(a.features, a.labels).reset_index(drop=True) + cache = { + sid: load_transcript(a.transcripts / f"{sid}.csv") + for sid in f.session_id.astype(str).unique() + } + + related_text, related_num = [], [] + for i, r in f.iterrows(): + d = cache[str(r.session_id)] + t, z = segmented_control(d, str(r.learning_objective), "related") + related_text.append(t) + related_num.append(z) + if (i + 1) % 2500 == 0: + print("rows", i + 1) + + X0 = build_v75(f, cache) + Xr = build_control(related_text, related_num) + y = f.target.to_numpy(int) + obj = ( + f.learning_objective_id + if "learning_objective_id" in f + else f.learning_objective + ).astype(str).to_numpy() + + # Cheapest sufficient causal world: the first deterministic GroupKFold + # objective-cold split, held fixed for every support dose. + base_train, heldout = folds_from_groups(obj)[0] + pools, eval_idx = make_fixed_support_design(f, heldout, obj) + if not len(eval_idx): + raise RuntimeError("V95 fixed evaluation set is empty") + + eval_y = y[eval_idx] + results = [] + predictions = {} + support_by_level = {} + + for level in LEVELS: + sup_idx, counts = support_indices(pools, level) + train_idx = np.concatenate([np.asarray(base_train, dtype=int), sup_idx]) + p0 = fit_predict(X0, y, train_idx, eval_idx) + pr = fit_predict(Xr, y, train_idx, eval_idx) + b, curve = best_blend(eval_y, p0, pr) + ll0 = float(log_loss(eval_y, p0)) + llr = float(log_loss(eval_y, pr)) + label = str(level) + results.append({ + "support": label, + "realised_support_per_objective": realised_support_summary(counts), + "support_rows_total": int(len(sup_idx)), + "eval_rows": int(len(eval_idx)), + "v75": ll0, + "related_ability": llr, + "best": b, + "gain_vs_v75": float(ll0 - b["ll"]), + "blend_curve": curve, + }) + predictions[label] = (p0, pr) + support_by_level[label] = (sup_idx, counts) + print("SUPPORT", label, "V75", ll0, "RELATED", llr, "BEST", b) + + # Information-destruction ablation: same revealed rows and class marginal, + # but support labels are deterministically shuffled. Only RELATED is refit; + # this asks whether valid labelled support, rather than row count alone, + # improves the specialist representation. + ablations = {} + rng = np.random.RandomState(SEED + 95) + for level in (8, "32+"): + label = str(level) + sup_idx, _ = support_by_level[label] + train_idx = np.concatenate([np.asarray(base_train, dtype=int), sup_idx]) + yy = y[train_idx].copy() + nbase = len(base_train) + if len(sup_idx) > 1: + yy[nbase:] = yy[nbase:][rng.permutation(len(sup_idx))] + pr_bad = fit_predict(Xr, y, train_idx, eval_idx, y_train_override=yy) + normal_pr = predictions[label][1] + ablations[label] = { + "normal_related_ll": float(log_loss(eval_y, normal_pr)), + "shuffled_support_related_ll": float(log_loss(eval_y, pr_bad)), + "valid_information_gain": float(log_loss(eval_y, pr_bad) - log_loss(eval_y, normal_pr)), + } + print("ABLATION", label, ablations[label]) + + weights = np.asarray([r["best"]["w"] for r in results], dtype=float) + # Use realised median support, with 32+ naturally reflecting the whole fixed pool. + dose = np.asarray([r["realised_support_per_objective"]["median"] for r in results], dtype=float) + rho = float(spearmanr(dose, weights).statistic) if len(np.unique(dose)) > 1 else 0.0 + near_monotone_steps = int(np.sum(np.diff(weights) <= 0.025 + 1e-12)) + possible_steps = len(weights) - 1 + delta = float(weights[0] - weights[-1]) + low_gain = float(results[0]["gain_vs_v75"]) + high_weight = float(weights[-1]) + + clean = ( + near_monotone_steps >= possible_steps - 1 + and rho <= -0.75 + and delta >= 0.15 + and low_gain >= 0.003 + and high_weight <= 0.15 + ) + partial = rho <= -0.50 and delta >= 0.10 and low_gain >= 0.002 + if clean: + verdict = "PROMOTE_OBJECTIVE_SUPPORT_ACTIVATION" + elif partial: + verdict = "R5_REFINE_EFFECTIVE_SUPPORT" + else: + verdict = "SUPPRESS_OBJECTIVE_SUPPORT" + + out = { + "primary": "objective-support-activation-law", + "design": { + "outer_world": "first deterministic objective-cold GroupKFold split", + "support_levels": [str(x) for x in LEVELS], + "fixed_eval_rows": int(len(eval_idx)), + "heldout_objectives": int(len(pools)), + "support_pool_rule": "stable hash order; reserve up to half/objective capped at 64; score fixed complement", + "note": "No leaderboard score or hidden-test outcome used in fitting, weighting, or decision.", + }, + "support_response": results, + "ablations": ablations, + "decision": { + "spearman_support_vs_weight": rho, + "near_monotone_steps": near_monotone_steps, + "possible_steps": possible_steps, + "weight_drop_zero_to_32plus": delta, + "zero_support_gain_vs_v75": low_gain, + "high_support_weight": high_weight, + "verdict": verdict, + "precommit": { + "promote": "near-monotone (<=1 tolerance step), rho<=-0.75, weight drop>=0.15, zero-support gain>=0.003, 32+ weight<=0.15", + "refine": "rho<=-0.50, weight drop>=0.10, zero-support gain>=0.002", + "otherwise": "suppress raw objective support and seek another observable", + }, + }, + } + Path(a.out).write_text(json.dumps(out, indent=2)) + print(json.dumps(out, indent=2)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path, required=True) + p.add_argument("--labels", type=Path, required=True) + p.add_argument("--transcripts", type=Path, required=True) + p.add_argument("--out", default="v95_objective_support_activation.json") + run(p.parse_args()) diff --git a/competitions/trace_the_ace/v96_effective_support_separator.py b/competitions/trace_the_ace/v96_effective_support_separator.py new file mode 100644 index 00000000..1e52570e --- /dev/null +++ b/competitions/trace_the_ace/v96_effective_support_separator.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""V96: refine V95 raw support into an effective-support observable. + +V95 established a causal regime transition: RELATED ability is valuable with +little target-objective support and should be suppressed once V75 has enough +support. Raw dose was too crude because many held-out objectives saturate early. + +This separator keeps V95's fixed evaluation rows and nested support intervention, +but asks which observable explains the transition better: + COUNT realised labelled rows for the objective; + COVERAGE realised rows / available support-pool rows (saturation); + BALANCE labelled support containing both classes; + CAPACITY available support-pool size (objective prevalence control). + +For every dose we fit V75 and RELATED exactly as in V95, then score per-objective +losses on the same evaluation rows. For each observable we fit a tiny deterministic +threshold router: below threshold use the globally selected V95 blend weight for +that dose, above threshold use V75. Thresholds are selected on a deterministic +half of held-out objectives and evaluated on the other half. This is a routing +separator, not a leaderboard fit. +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control +from v95_objective_support_activation import make_fixed_support_design, support_indices, fit_predict, best_blend, LEVELS + + +def split_objectives(objs): + def key(g): return hashlib.sha256(f'V96|{SEED}|{g}'.encode()).hexdigest() + s=sorted(objs,key=key); return set(s[::2]),set(s[1::2]) + +def ll(y,p): return float(log_loss(y,np.clip(p,1e-5,1-1e-5))) + +def route_eval(y,p0,pa,groups,metric,fit_objs,test_objs,w): + vals=np.array([metric[str(g)] for g in groups],float) + uniq=np.unique([metric[g] for g in fit_objs if g in metric]) + if len(uniq)>20: cuts=np.unique(np.quantile(uniq,np.linspace(0,1,21))) + else: cuts=uniq + candidates=[-1e-12]+[float(x) for x in cuts]+[float(np.max(uniq)+1e-9)] if len(uniq) else [0.0] + fit=np.array([str(g) in fit_objs for g in groups]); test=np.array([str(g) in test_objs for g in groups]) + rows=[] + for t in candidates: + # low effective support => ability blend; high => V75 + use=vals<=t; q=np.where(use,(1-w)*p0+w*pa,p0) + rows.append((t,ll(y[fit],q[fit]),ll(y[test],q[test]),float(np.mean(use[test])))) + best=min(rows,key=lambda z:z[1]) + return {'threshold':best[0],'fit_ll':best[1],'test_ll':best[2],'test_ability_fraction':best[3], + 'test_v75':ll(y[test],p0[test]),'test_global_blend':ll(y[test],((1-w)*p0+w*pa)[test]), + 'gain_vs_v75':ll(y[test],p0[test])-best[2]} + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + base,held=folds_from_groups(obj)[0]; pools,ev=make_fixed_support_design(f,held,obj); ey=y[ev]; eg=obj[ev] + fit_objs,test_objs=split_objectives(sorted(pools)) + out={'primary':'effective-support-separator','objective_router_split':{'fit':len(fit_objs),'test':len(test_objs)},'levels':[]} + for level in LEVELS: + si,counts=support_indices(pools,level); tr=np.concatenate([np.asarray(base,int),si]) + p0=fit_predict(X0,y,tr,ev); pa=fit_predict(Xr,y,tr,ev); b,_=best_blend(ey,p0,pa); w=float(b['w']) + count={}; coverage={}; balance={}; capacity={} + for g in pools: + pool=pools[g]; k=len(pool) if level=='32+' else min(int(level),len(pool)); chosen=pool[:k] + count[g]=float(k); capacity[g]=float(len(pool)); coverage[g]=float(k/max(1,len(pool))) + balance[g]=float(len(np.unique(y[chosen]))>=2) if k else 0.0 + arms={ + 'count':route_eval(ey,p0,pa,eg,count,fit_objs,test_objs,w), + 'coverage':route_eval(ey,p0,pa,eg,coverage,fit_objs,test_objs,w), + 'balance':route_eval(ey,p0,pa,eg,balance,fit_objs,test_objs,w), + 'capacity':route_eval(ey,p0,pa,eg,capacity,fit_objs,test_objs,w), + } + out['levels'].append({'support':str(level),'global_weight':w,'arms':arms}) + print('LEVEL',level,'W',w,{k:round(v['gain_vs_v75'],6) for k,v in arms.items()}) + # Decision uses only levels where V95 had a nonzero specialist weight. + useful=[x for x in out['levels'] if x['global_weight']>0] + means={k:float(np.mean([x['arms'][k]['gain_vs_v75'] for x in useful])) for k in ('count','coverage','balance','capacity')} + wins={k:int(sum(x['arms'][k]['test_ll'] <= min(v['test_ll'] for v in x['arms'].values())+1e-12 for x in useful)) for k in means} + best=max(means,key=means.get) + # Promotion needs positive held-out routing value and superiority to raw count. + if best!='count' and means[best]>=0.001 and means[best]>=means['count']+0.0005: + verdict='PROMOTE_'+best.upper()+'_AS_EFFECTIVE_SUPPORT' + elif max(means.values())>=0.0005: + verdict='R5_REFINE_COMPOSITE_SUPPORT' + else: + verdict='SUPPRESS_SIMPLE_SUPPORT_ROUTING' + out['decision']={'mean_test_gain_vs_v75':means,'wins':wins,'best_observable':best,'verdict':verdict, + 'precommit':'promote non-count observable iff held-out mean gain>=.001 and >= raw-count gain+.0005; else refine if any >=.0005'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v96_effective_support.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v97_support_gate.py b/competitions/trace_the_ace/v97_support_gate.py new file mode 100644 index 00000000..157510b7 --- /dev/null +++ b/competitions/trace_the_ace/v97_support_gate.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""V97: submission-sprint exact-support gate. + +Frozen law from V93-V96: RELATED ability is a specialist for objectives with no +training support; V75 dominates when exact objective support is present. +This test does not tune the law. It evaluates the precommitted runtime rule: + count_train(objective) == 0 -> 0.35 RELATED + 0.65 V75 + otherwise -> V75 +across four shift worlds using fold-local training counts only. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75, oof +from v93_shift_robust_validation import folds_from_groups, obj_family, style_matrix +from v94_related_control import segmented_control, build_control +from sklearn.cluster import KMeans + +W_UNSEEN = 0.35 +EPS = 1e-5 + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related') + rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support_key=f.learning_objective.astype(str).to_numpy() # always runtime-visible + sess=f.session_id.astype(str).to_numpy(); fam=f.learning_objective.astype(str).map(obj_family).to_numpy() + SX=style_matrix(f,cache); style=KMeans(n_clusters=5,random_state=137,n_init=10).fit(SX).labels_.astype(str) + worlds={'objective_cold':folds_from_groups(obj),'session_cold':folds_from_groups(sess), + 'objective_family_cold':folds_from_groups(fam),'style_cold':folds_from_groups(style)} + out={'primary':'fixed-exact-support-runtime-gate','law':{'unseen_weight':W_UNSEEN,'seen_weight':0.0,'predicate':'fold-local exact learning_objective text training count == 0'},'worlds':{}} + gains=[]; regress=[] + for name,sp in worlds.items(): + p0,_=oof(X0,y,sp,name+':V75'); pr,_=oof(Xr,y,sp,name+':RELATED') + q=np.zeros(len(y)); unseen=np.zeros(len(y),bool) + for tr,va in sp: + counts={g:int(n) for g,n in zip(*np.unique(support_key[tr],return_counts=True))} + m=np.array([counts.get(str(support_key[i]),0)==0 for i in va],bool); unseen[va]=m + w=np.where(m,W_UNSEEN,0.0) + q[va]=np.clip((1-w)*p0[va]+w*pr[va],EPS,1-EPS) + ll0=float(log_loss(y,p0)); llq=float(log_loss(y,q)); gain=ll0-llq + rec={'v75':ll0,'support_gate':llq,'gain_vs_v75':gain,'ability_fraction':float(unseen.mean()), + 'unseen_rows':int(unseen.sum()),'seen_rows':int((~unseen).sum())} + out['worlds'][name]=rec; gains.append(gain); regress.append(max(0.0,-gain)); print(name,rec) + mean_gain=float(np.mean(gains)); worst_reg=float(max(regress)); obj_gain=out['worlds']['objective_cold']['gain_vs_v75'] + promote=obj_gain>=0.0025 and mean_gain>=0.0007 and worst_reg<=0.0005 + out['decision']={'objective_gain':obj_gain,'mean_gain_four_worlds':mean_gain,'worst_world_regression':worst_reg, + 'verdict':'BUILD_SUBMISSION_NOW' if promote else 'STOP_DO_NOT_SUBMIT', + 'precommit':'build iff objective gain >=.0025, four-world mean gain >=.0007, worst regression <=.0005'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + if a.require_promote and not promote: raise SystemExit(42) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v97_support_gate.json'); p.add_argument('--require-promote',action='store_true'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v98_finishing_sweep.py b/competitions/trace_the_ace/v98_finishing_sweep.py new file mode 100644 index 00000000..a90d3e7d --- /dev/null +++ b/competitions/trace_the_ace/v98_finishing_sweep.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""V98: tiny support/ability finishing sweep against frozen V97. + +No new representation is fit. V75 and RELATED are computed once per frozen shift +world, then a small predeclared family of runtime-visible exact-support laws is +scored. V97 (count==0 -> 0.35 RELATED) is the immutable baseline. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.cluster import KMeans +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75, oof +from v93_shift_robust_validation import folds_from_groups, obj_family, style_matrix +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +V97=(0.35, 0, 0.0) # unseen weight, low-support threshold, low-support weight +# Deliberately tiny family. Each tuple = (w0, low_threshold, wlow). +CANDIDATES=[ + (0.35,0,0.0), + (0.45,0,0.0),(0.50,0,0.0),(0.60,0,0.0), + (0.45,1,0.15),(0.50,1,0.15),(0.60,1,0.15), + (0.45,2,0.15),(0.50,2,0.25),(0.60,2,0.25), + (0.50,4,0.15),(0.60,4,0.25),(0.60,4,0.35), +] + +def name(c): + w0,t,wl=c + return f'w0={w0:.2f}|n1to{t}={wl:.2f}' if t else f'w0={w0:.2f}|seen=0' + +def weights_for_counts(counts,c): + w0,t,wl=c + counts=np.asarray(counts) + w=np.zeros(len(counts),float) + w[counts==0]=w0 + if t: + w[(counts>=1)&(counts<=t)]=wl + return w + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related') + rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + key=f.learning_objective.astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy(); fam=f.learning_objective.astype(str).map(obj_family).to_numpy() + SX=style_matrix(f,cache); style=KMeans(n_clusters=5,random_state=137,n_init=10).fit(SX).labels_.astype(str) + worlds={'objective_cold':folds_from_groups(obj),'session_cold':folds_from_groups(sess), + 'objective_family_cold':folds_from_groups(fam),'style_cold':folds_from_groups(style)} + out={'primary':'V98 tiny finishing sweep','v97_baseline':name(V97),'candidates':[name(c) for c in CANDIDATES],'worlds':{}} + by_candidate={name(c):[] for c in CANDIDATES} + for wn,sp in worlds.items(): + p0,_=oof(X0,y,sp,wn+':V75'); pr,_=oof(Xr,y,sp,wn+':RELATED') + fold_counts=np.zeros(len(y),int) + for tr,va in sp: + vals,ns=np.unique(key[tr],return_counts=True); d=dict(zip(vals,ns)) + fold_counts[va]=[int(d.get(str(key[i]),0)) for i in va] + rec={'v75':float(log_loss(y,p0)),'laws':{}} + for c in CANDIDATES: + w=weights_for_counts(fold_counts,c) + q=np.clip((1-w)*p0+w*pr,EPS,1-EPS) + ll=float(log_loss(y,q)); gain=rec['v75']-ll + rec['laws'][name(c)]={'logloss':ll,'gain_vs_v75':gain,'mean_weight':float(w.mean())} + by_candidate[name(c)].append((wn,ll,gain)) + out['worlds'][wn]=rec + print(wn,'V75',rec['v75'],'V97',rec['laws'][name(V97)]) + v97_world={wn:out['worlds'][wn]['laws'][name(V97)]['logloss'] for wn in worlds} + ranking=[] + for c in CANDIDATES: + n=name(c); vals=by_candidate[n] + deltas=[v97_world[wn]-ll for wn,ll,_ in vals] + gains=[g for _,_,g in vals] + ranking.append({'law':n,'mean_gain_vs_v97':float(np.mean(deltas)), + 'objective_gain_vs_v97':float(deltas[0]), + 'worst_world_delta_vs_v97':float(min(deltas)), + 'mean_gain_vs_v75':float(np.mean(gains))}) + eligible=[r for r in ranking if r['objective_gain_vs_v97']>=0 and r['worst_world_delta_vs_v97']>=-0.0005] + best=max(eligible,key=lambda r:r['mean_gain_vs_v97']) if eligible else next(r for r in ranking if r['law']==name(V97)) + promote=(best['law']!=name(V97) and best['mean_gain_vs_v97']>=0.0005 and best['worst_world_delta_vs_v97']>=-0.0005) + out['ranking']=sorted(ranking,key=lambda r:r['mean_gain_vs_v97'],reverse=True) + out['decision']={'best':best,'verdict':'PROMOTE_V98_FINISHER' if promote else 'KEEP_V97', + 'precommit':'promote only if mean gain vs frozen V97 >=0.0005, objective nonnegative, worst-world delta >=-0.0005'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out['decision'],indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v98_finishing_sweep.json'); run(p.parse_args()) diff --git a/competitions/trace_the_ace/v99_oof_expert_gate.py b/competitions/trace_the_ace/v99_oof_expert_gate.py new file mode 100644 index 00000000..0aed347d --- /dev/null +++ b/competitions/trace_the_ace/v99_oof_expert_gate.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""V99 Phase A: leakage-safe learned expert applicability gate. + +Question: can runtime-visible evidence predict when RELATED beats V75 on objective-cold +rows, recovering material V91 oracle complementarity beyond frozen V97? + +Discipline: +- Outer objective-cold folds are untouched evaluation. +- Gate training features use INNER out-of-fold expert predictions only. +- Outer expert predictions are produced by models fit only on outer-train rows. +- No leaderboard/test labels or outer-validation labels enter fitting or tuning. +- One fixed conservative gate architecture and blend law; no sweep. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript, tokens +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups, obj_family +from v94_related_control import segmented_control, build_control + +EPS=1e-5 +GATE_SCALE=.65 + + +def fit_expert(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + + +def counts_for(train_idx, eval_idx, values): + u,n=np.unique(values[train_idx],return_counts=True); d=dict(zip(u,n)) + return np.asarray([d.get(values[i],0) for i in eval_idx],float) + + +def gate_features(p0,pr,exact_count,family_count,session_count,turns,obj_len): + p0=np.asarray(p0); pr=np.asarray(pr) + # Every feature is available at runtime before observing the target label. + return np.column_stack([ + p0,pr,np.abs(pr-p0),pr-p0, + np.abs(p0-.5),np.abs(pr-.5), + np.log1p(exact_count),np.log1p(family_count),np.log1p(session_count), + np.log1p(turns),np.log1p(obj_len), + ]) + + +def row_loss(y,p): + return -(y*np.log(p)+(1-y)*np.log(1-p)) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related') + rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + key=f.learning_objective.astype(str).to_numpy() + fam=f.learning_objective.astype(str).map(obj_family).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy() + turns=np.asarray([len(cache[str(s)]) for s in sess],float) + obj_len=np.asarray([len(tokens(x)) for x in key],float) + outer=folds_from_groups(obj) + + p0_all=np.zeros(len(y)); pr_all=np.zeros(len(y)); pg_all=np.zeros(len(y)); pv97_all=np.zeros(len(y)) + fold_rows=[] + for k,(tr,va) in enumerate(outer,1): + print('OUTER',k,'train',len(tr),'val',len(va)) + p0=fit_expert(X0,y,tr,va); pr=fit_expert(Xr,y,tr,va) + p0_all[va]=p0; pr_all[va]=pr + + # Inner OOF expert predictions on outer-train only. + inner_groups=obj[tr]; n_inner=min(3,len(np.unique(inner_groups))) + inner=GroupKFold(n_inner).split(np.zeros(len(tr)),y[tr],inner_groups) + ip0=np.zeros(len(tr)); ipr=np.zeros(len(tr)); G=np.zeros((len(tr),11)) + for j,(itr_local,iva_local) in enumerate(inner,1): + itr=tr[itr_local]; iva=tr[iva_local] + q0=fit_expert(X0,y,itr,iva); qr=fit_expert(Xr,y,itr,iva) + ip0[iva_local]=q0; ipr[iva_local]=qr + ec=counts_for(itr,iva,key); fc=counts_for(itr,iva,fam); sc=counts_for(itr,iva,sess) + G[iva_local]=gate_features(q0,qr,ec,fc,sc,turns[iva],obj_len[iva]) + print(' inner',j,'n',len(iva)) + + l0=row_loss(y[tr],ip0); lr=row_loss(y[tr],ipr) + gy=(lr0 else 0.0 + verdict='PROMOTE_TO_FOUR_WORLD_V99' if gain>=.002 else ('REFINE_GATE' if gain>=.0005 else 'SUPPRESS_THIS_GATE') + out={'primary':'objective-cold nested applicability gate','v75':ll0,'related':llr,'v97':ll97,'v99_gate':llg, + 'gain_vs_v97':gain,'diagnostic_endpoint_oracle':llor,'oracle_gap_from_v97':oracle_gap, + 'oracle_gap_recovered_fraction':recovery,'gate_scale':GATE_SCALE,'folds':fold_rows, + 'decision':{'verdict':verdict,'precommit':'promote to four-world only if nested objective-cold gain vs V97 >= 0.002'}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v99_oof_expert_gate.json'); run(p.parse_args())