diff --git a/baseline/README.md b/baseline/README.md index 0daa1a9..67e8353 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -30,6 +30,7 @@ The audit and qualification documents are: | **MNIST / MLP3** | 55k optimization / 5k validation; official 10k test monitoring-only | `784 → 512 → 512 → 10`, ReLU | SGD + Nesterov, AdamW, Muon + auxiliary AdamW | [`notebooks/MNIST_MLP3_Baseline_Comparison.ipynb`](notebooks/MNIST_MLP3_Baseline_Comparison.ipynb) | | **CIFAR-10 / small ViT** | 45k optimization / 5k validation; official 10k test monitoring-only | 4×4 patches, width 192, 6 blocks, 3 heads | SGD + Nesterov, AdamW, Muon + auxiliary AdamW | [`notebooks/CIFAR10_ViT_Optimizer_Baselines.ipynb`](notebooks/CIFAR10_ViT_Optimizer_Baselines.ipynb) | | **One-head nanoGPT / FineWeb-Edu** | Pinned `sample-10BT`; exact document-disjoint 80M / 1M / 1M GPT-2-BPE splits | 1 block, 1 head, width 128, context 256 | SGD + Nesterov, AdamW, Muon + auxiliary AdamW | [`nanogpt_one_head/README.md`](nanogpt_one_head/README.md) | +| **NGB v4 / FineWeb-Edu** | Same pinned document-disjoint corpus | Separate 1×1 control and 4-block/4-head width-128 model | Two-epoch tuned SGD, AdamW, and Muon protocols | [`ngb/README.md`](ngb/README.md) | | **nanochat d12** | Native pinned nanochat data/tokenizer pipeline | 12 layers, width 768, context 2048 | Native nanochat Muon + AdamW | [`notebooks/NanoChat_D12_Reference_Baseline.ipynb`](notebooks/NanoChat_D12_Reference_Baseline.ipynb) | | **nanochat mac_d4** | Separately cached reduced nanochat preparation | 4 layers, width 256, context 512 | Same pinned upstream optimizer mathematics | Same notebook; auto-selected on MPS/CPU | @@ -56,35 +57,28 @@ report a `mac_d4` result as d12. neighborhood for the exact architecture, data, initialization, budget, optimizer implementation, and runtime policy. -## Environment and persistent paths +## Environment and runtime paths -```bash -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install -e '.[experiment]' -``` - -When installing from the repository root, use: +Use the currently active conda environment; do not create a repository venv. +From the repository root: ```bash python -m pip install -e './baseline[experiment]' +python -m pip install -e './baseline/nanogpt_one_head[dev]' ``` -Set persistent locations before running long jobs: - -```bash -export RG_BASELINE_DATA_DIR="$HOME/rg-optimizer-data" -export RG_BASELINE_RUN_ROOT="$HOME/rg-optimizer-runs" -``` - -The isolated one-head nanoGPT suite uses: +All local data and long-running experiment artifacts use explicit `/tmp` roots: ```bash -export RG_NANOGPT_ONE_HEAD_ROOT="$HOME/rg-nanogpt-one-head" +export RG_BASELINE_DATA_DIR=/tmp/rg-optimizer-data +export RG_BASELINE_RUN_ROOT=/tmp/rg-optimizer-runs +export RG_NANOGPT_ONE_HEAD_ROOT=/tmp/rg-nanogpt-one-head +export RG_NGB_ROOT=/tmp/rg-ngb +export RG_NGB_DATA_ROOT=/tmp/rg-nanogpt-one-head/data ``` -Do not put long-running results in `/tmp`. +NGB v4 keeps its results under `/tmp/rg-ngb/results/` and reuses the +verified FineWeb-Edu token cache under `/tmp/rg-nanogpt-one-head/data`. ## 1. MNIST / MLP3 diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/__init__.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/__init__.py index aa8108b..22ae6b0 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/__init__.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/__init__.py @@ -1,25 +1,33 @@ -"""One-block, one-head nanoGPT optimizer baselines on pinned FineWeb-Edu.""" +"""Shared nanoGPT optimizer-baseline runtime for v3 and NGB v4.""" from .analysis import ( MATRIX_COLORS, OPTIMIZER_COLORS, OPTIMIZER_LABELS, + discover_complete_seeds, + discover_matched_complete_seeds, final_test_summary, load_epoch_metrics, load_layer_metrics, load_metrics, load_spectral_summary, load_test_results, + mean_ci95, + paired_test_differences, plot_epoch_metric, plot_layer_metric, plot_spectral_optimizer_summary, + run_diagnostics_table, run_status_table, + summarize_run_diagnostics, ) from .config import ( SUPPORTED_OPTIMIZERS, canonical_seeds, + expected_transformer_matrix_count, load_config, roots, + run_slug, ) from .data import prepare_fineweb_edu from .model import GPT, GPTConfig @@ -39,6 +47,9 @@ "SUPPORTED_OPTIMIZERS", "canonical_seeds", "choose_device", + "discover_complete_seeds", + "discover_matched_complete_seeds", + "expected_transformer_matrix_count", "final_test_summary", "load_config", "load_epoch_metrics", @@ -46,13 +57,18 @@ "load_metrics", "load_spectral_summary", "load_test_results", + "mean_ci95", + "paired_test_differences", "plot_epoch_metric", "plot_layer_metric", "plot_spectral_optimizer_summary", "prepare_fineweb_edu", "roots", "run_all_replicates", + "run_diagnostics_table", "run_one", "run_optimizer_replicates", + "run_slug", "run_status_table", + "summarize_run_diagnostics", ] diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/analysis.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/analysis.py index 621c4b4..927a195 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/analysis.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/analysis.py @@ -1,5 +1,6 @@ from __future__ import annotations +from itertools import combinations import json import math from pathlib import Path @@ -10,7 +11,7 @@ import pandas as pd from .config import SUPPORTED_OPTIMIZERS -from .training import run_directory, run_is_complete +from .run_utils import run_directory, run_is_complete OPTIMIZER_LABELS = { "sgd_momentum": "SGD + Nesterov", @@ -18,14 +19,12 @@ "muon": "Muon + auxiliary AdamW", } -# Okabe-Ito color-blind-safe optimizer palette. OPTIMIZER_COLORS = { "sgd_momentum": "#0072B2", "adamw": "#D55E00", "muon": "#009E73", } -# Matrix identity remains fixed across every optimizer notebook. MATRIX_COLORS = { "W_Q": "#0072B2", "W_K": "#E69F00", @@ -35,6 +34,13 @@ "W_MLP_OUT": "#56B4E9", } +_BLOCK_LINESTYLES = ( + "-", + "--", + "-.", + ":", +) + _T_975 = { 1: 12.7062047364, 2: 4.3026527297, @@ -46,6 +52,25 @@ 8: 2.3060041352, 9: 2.2621571629, 10: 2.2281388520, + 11: 2.2009851601, + 12: 2.1788128297, + 13: 2.1603686565, + 14: 2.1447866879, + 15: 2.1314495456, + 16: 2.1199052992, + 17: 2.1098155778, + 18: 2.1009220402, + 19: 2.0930240544, + 20: 2.0859634473, + 21: 2.0796138447, + 22: 2.0738730679, + 23: 2.0686576104, + 24: 2.0638985616, + 25: 2.0595385528, + 26: 2.0555294386, + 27: 2.0518305165, + 28: 2.0484071418, + 29: 2.0452296421, } @@ -76,7 +101,10 @@ def mean_ci95(values: Iterable[float]) -> dict[str, float]: } sd = float(array.std(ddof=1)) sem = sd / math.sqrt(n) - critical = _T_975.get(n - 1, 1.9599639845) + critical = _T_975.get( + n - 1, + 1.9599639845, + ) half = critical * sem return { "n": n, @@ -89,41 +117,159 @@ def mean_ci95(values: Iterable[float]) -> dict[str, float]: } +def _available_seed_dirs( + results_root: str | Path, + optimizer: str, +) -> tuple[int, ...]: + root = Path(results_root) / str(optimizer) + seeds: list[int] = [] + if not root.is_dir(): + return () + for path in root.glob("seed_*"): + if not path.is_dir(): + continue + try: + seeds.append( + int(path.name.removeprefix("seed_")) + ) + except ValueError: + continue + return tuple(sorted(set(seeds))) + + +def discover_complete_seeds( + results_root: str | Path, + optimizer: str, +) -> tuple[int, ...]: + return tuple( + seed + for seed in _available_seed_dirs( + results_root, + optimizer, + ) + if run_is_complete( + results_root, + optimizer, + seed, + ) + ) + + +def discover_matched_complete_seeds( + results_root: str | Path, + *, + optimizers: Sequence[str] = SUPPORTED_OPTIMIZERS, +) -> tuple[int, ...]: + sets = [ + set( + discover_complete_seeds( + results_root, + optimizer, + ) + ) + for optimizer in optimizers + ] + if not sets: + return () + return tuple(sorted(set.intersection(*sets))) + + def run_status_table( results_root: str | Path, *, optimizers: Sequence[str] = SUPPORTED_OPTIMIZERS, - seeds: Sequence[int] = (1337, 2027, 4099), + seeds: Sequence[int] | None = None, ) -> pd.DataFrame: + selected = ( + tuple(int(seed) for seed in seeds) + if seeds is not None + else tuple( + sorted( + { + seed + for optimizer in optimizers + for seed in _available_seed_dirs( + results_root, + optimizer, + ) + } + ) + ) + ) rows = [] for optimizer in optimizers: - for seed in seeds: - run_dir = run_directory(results_root, optimizer, seed) - completion_path = run_dir / "run_complete.json" + for seed in selected: + run_dir = run_directory( + results_root, + optimizer, + seed, + ) + completion_path = ( + run_dir / "run_complete.json" + ) payload = ( - json.loads(completion_path.read_text(encoding="utf-8")) + json.loads( + completion_path.read_text( + encoding="utf-8" + ) + ) if completion_path.is_file() else {} ) rows.append( { "optimizer": optimizer, - "optimizer_label": OPTIMIZER_LABELS.get(optimizer, optimizer), + "optimizer_label": ( + OPTIMIZER_LABELS.get( + optimizer, + optimizer, + ) + ), "seed": int(seed), - "complete": bool(payload.get("completed", False)), - "steps": payload.get("optimizer_steps", np.nan), - "final_test_loss": payload.get("final_test_loss", np.nan), - "final_test_bleu": payload.get("final_test_bleu", np.nan), + "complete": run_is_complete( + results_root, + optimizer, + seed, + ), + "steps": payload.get( + "optimizer_steps", + np.nan, + ), + "best_validation_step": payload.get( + "best_validation_step", + np.nan, + ), + "best_validation_loss": payload.get( + "best_validation_loss", + np.nan, + ), + "final_test_loss": payload.get( + "final_test_loss", + np.nan, + ), + "final_test_accuracy": payload.get( + "final_test_accuracy", + np.nan, + ), "run_dir": str(run_dir), } ) return pd.DataFrame(rows) -def _require_complete(results_root: str | Path, optimizer: str, seed: int) -> None: - if not run_is_complete(results_root, optimizer, seed): +def _require_complete( + results_root: str | Path, + optimizer: str, + seed: int, +) -> None: + if not run_is_complete( + results_root, + optimizer, + seed, + ): raise FileNotFoundError( - f"missing completed run for optimizer={optimizer} seed={seed}: " + "missing completed run for " + f"optimizer={optimizer} seed={seed}: " f"{run_directory(results_root, optimizer, seed)}" ) @@ -140,20 +286,40 @@ def _load_csvs( for optimizer in optimizers: for seed in seeds: if require_complete: - _require_complete(results_root, optimizer, seed) - path = run_directory(results_root, optimizer, seed) / relative_path + _require_complete( + results_root, + optimizer, + seed, + ) + path = ( + run_directory( + results_root, + optimizer, + seed, + ) + / relative_path + ) if not path.is_file(): if require_complete: raise FileNotFoundError(path) continue frame = pd.read_csv(path) frame["optimizer"] = optimizer - frame["optimizer_label"] = OPTIMIZER_LABELS.get(optimizer, optimizer) + frame["optimizer_label"] = ( + OPTIMIZER_LABELS.get( + optimizer, + optimizer, + ) + ) frame["seed"] = int(seed) frames.append(frame) if not frames: return pd.DataFrame() - return pd.concat(frames, ignore_index=True, sort=False) + return pd.concat( + frames, + ignore_index=True, + sort=False, + ) def load_metrics( @@ -172,8 +338,14 @@ def load_metrics( ) if frame.empty: return frame - return frame.sort_values(["optimizer", "seed", "step"]).drop_duplicates( - ["optimizer", "seed", "step"], keep="last" + return ( + frame.sort_values( + ["optimizer", "seed", "step"] + ) + .drop_duplicates( + ["optimizer", "seed", "step"], + keep="last", + ) ) @@ -193,8 +365,22 @@ def load_epoch_metrics( ) if frame.empty: return frame - return frame.sort_values(["optimizer", "seed", "nominal_epoch"]).drop_duplicates( - ["optimizer", "seed", "nominal_epoch"], keep="last" + return ( + frame.sort_values( + [ + "optimizer", + "seed", + "nominal_epoch", + ] + ) + .drop_duplicates( + [ + "optimizer", + "seed", + "nominal_epoch", + ], + keep="last", + ) ) @@ -214,8 +400,25 @@ def load_layer_metrics( ) if frame.empty: return frame - return frame.sort_values(["optimizer", "seed", "epoch", "matrix_type"]).drop_duplicates( - ["optimizer", "seed", "step", "matrix_name"], keep="last" + return ( + frame.sort_values( + [ + "optimizer", + "seed", + "epoch", + "block", + "matrix_type", + ] + ) + .drop_duplicates( + [ + "optimizer", + "seed", + "step", + "matrix_name", + ], + keep="last", + ) ) @@ -235,8 +438,14 @@ def load_spectral_summary( ) if frame.empty: return frame - return frame.sort_values(["optimizer", "seed", "epoch"]).drop_duplicates( - ["optimizer", "seed", "step"], keep="last" + return ( + frame.sort_values( + ["optimizer", "seed", "epoch"] + ) + .drop_duplicates( + ["optimizer", "seed", "step"], + keep="last", + ) ) @@ -249,22 +458,50 @@ def load_test_results( rows = [] for optimizer in optimizers: for seed in seeds: - _require_complete(results_root, optimizer, seed) - path = run_directory(results_root, optimizer, seed) / "test_results.json" - payload = json.loads(path.read_text(encoding="utf-8")) - for checkpoint in ("final", "validation_selected"): + _require_complete( + results_root, + optimizer, + seed, + ) + path = ( + run_directory( + results_root, + optimizer, + seed, + ) + / "test_results.json" + ) + payload = json.loads( + path.read_text(encoding="utf-8") + ) + for checkpoint in ( + "final", + "validation_selected", + ): values = payload[checkpoint] rows.append( { "optimizer": optimizer, - "optimizer_label": OPTIMIZER_LABELS[optimizer], + "optimizer_label": ( + OPTIMIZER_LABELS[ + optimizer + ] + ), "seed": int(seed), "checkpoint": checkpoint, "step": int(values["step"]), - "test_loss": float(values["loss"]), - "test_perplexity": float(values["perplexity"]), - "test_accuracy": float(values["accuracy"]), - "test_bleu": float(values["bleu"]), + "test_loss": float( + values["loss"] + ), + "test_perplexity": float( + values["perplexity"] + ), + "test_accuracy": float( + values["accuracy"] + ), + "test_bleu": float( + values["bleu"] + ), } ) return pd.DataFrame(rows) @@ -279,12 +516,32 @@ def summarize_by_epoch( ) -> pd.DataFrame: rows = [] keys = [*group, x] - subset = frame[[*keys, "seed", metric]].copy() - subset[metric] = pd.to_numeric(subset[metric], errors="coerce") - for values, group_frame in subset.groupby(keys, sort=True): - values_tuple = values if isinstance(values, tuple) else (values,) - row = dict(zip(keys, values_tuple, strict=True)) - row.update(mean_ci95(group_frame[metric])) + subset = frame[ + [*keys, "seed", metric] + ].copy() + subset[metric] = pd.to_numeric( + subset[metric], + errors="coerce", + ) + for values, group_frame in subset.groupby( + keys, + sort=True, + ): + values_tuple = ( + values + if isinstance(values, tuple) + else (values,) + ) + row = dict( + zip( + keys, + values_tuple, + strict=True, + ) + ) + row.update( + mean_ci95(group_frame[metric]) + ) rows.append(row) return pd.DataFrame(rows) @@ -300,42 +557,75 @@ def plot_epoch_metric( ): figure, axis = plt.subplots(figsize=(9, 5)) for optimizer in optimizers: - subset = frame[frame["optimizer"] == optimizer] + subset = frame[ + frame["optimizer"] == optimizer + ] if subset.empty: continue - for _, seed_frame in subset.groupby("seed"): + for _, seed_frame in subset.groupby( + "seed" + ): axis.plot( seed_frame[x], seed_frame[metric], - color=OPTIMIZER_COLORS[optimizer], + color=OPTIMIZER_COLORS[ + optimizer + ], alpha=0.20, linewidth=0.9, ) - summary = summarize_by_epoch(subset, metric, x=x) + summary = summarize_by_epoch( + subset, + metric, + x=x, + ) axis.plot( summary[x], summary["mean"], - color=OPTIMIZER_COLORS[optimizer], + color=OPTIMIZER_COLORS[ + optimizer + ], linewidth=2.0, - label=OPTIMIZER_LABELS[optimizer], + label=OPTIMIZER_LABELS[ + optimizer + ], ) axis.fill_between( summary[x], summary["ci95_lower"], summary["ci95_upper"], - color=OPTIMIZER_COLORS[optimizer], + color=OPTIMIZER_COLORS[ + optimizer + ], alpha=0.16, ) - axis.set_xlabel(x.replace("_", " ").title()) - axis.set_ylabel(metric.replace("_", " ").title()) - axis.set_title(title or f"{metric}: mean and 95% Student-t CI across seeds") + axis.set_xlabel( + x.replace("_", " ").title() + ) + axis.set_ylabel( + metric.replace("_", " ").title() + ) + axis.set_title( + title + or ( + f"{metric}: mean and 95% " + "Student-t CI across seeds" + ) + ) axis.grid(alpha=0.25) axis.legend(frameon=False) figure.tight_layout() if output is not None: output = Path(output) - output.parent.mkdir(parents=True, exist_ok=True) - figure.savefig(output, dpi=170, bbox_inches="tight") + output.parent.mkdir( + parents=True, + exist_ok=True, + ) + figure.savefig( + output, + dpi=170, + bbox_inches="tight", + ) return figure, axis @@ -347,51 +637,119 @@ def plot_layer_metric( title: str | None = None, output: str | Path | None = None, ): - subset = frame[frame["optimizer"] == optimizer].copy() + """Plot each block separately so blocks are never treated as replicates.""" + + subset = frame[ + frame["optimizer"] == optimizer + ].copy() if subset.empty: - raise ValueError(f"no layer data for optimizer={optimizer}") - figure, axis = plt.subplots(figsize=(10, 5.5)) - for matrix_type in MATRIX_COLORS: - matrix = subset[subset["matrix_type"] == matrix_type] - if matrix.empty: - continue - summary = summarize_by_epoch( - matrix, - metric, - x="epoch", - group=("matrix_type",), + raise ValueError( + f"no layer data for optimizer={optimizer}" ) - axis.plot( - summary["epoch"], - summary["mean"], - color=MATRIX_COLORS[matrix_type], - linewidth=2.0, - marker="o", - markersize=3, - label=matrix_type, + blocks = tuple( + sorted( + int(value) + for value in subset["block"].unique() ) - axis.fill_between( - summary["epoch"], - summary["ci95_lower"], - summary["ci95_upper"], - color=MATRIX_COLORS[matrix_type], - alpha=0.13, - ) - if metric == "alpha": - axis.axhline(2.0, color="black", linestyle="--", linewidth=1.0, label="alpha = 2") - if metric == "ERG_gap": - axis.axhline(0.0, color="black", linestyle="--", linewidth=1.0) - axis.set_xlabel("Epoch") - axis.set_ylabel(metric) - axis.set_title(title or f"{OPTIMIZER_LABELS[optimizer]} layer {metric}") - axis.grid(alpha=0.25) - axis.legend(frameon=False, ncol=2) + ) + figure, axes = plt.subplots( + 1, + len(blocks), + figsize=(5.2 * len(blocks), 5.2), + sharex=True, + sharey=True, + squeeze=False, + ) + for axis, block in zip( + axes[0], + blocks, + strict=True, + ): + block_frame = subset[ + subset["block"].astype(int) + == int(block) + ] + for matrix_type in MATRIX_COLORS: + matrix = block_frame[ + block_frame["matrix_type"] + == matrix_type + ] + if matrix.empty: + continue + summary = summarize_by_epoch( + matrix, + metric, + x="epoch", + group=( + "block", + "matrix_type", + "matrix_name", + ), + ) + axis.plot( + summary["epoch"], + summary["mean"], + color=MATRIX_COLORS[ + matrix_type + ], + linewidth=2.0, + marker="o", + markersize=3, + label=matrix_type, + ) + axis.fill_between( + summary["epoch"], + summary["ci95_lower"], + summary["ci95_upper"], + color=MATRIX_COLORS[ + matrix_type + ], + alpha=0.13, + ) + if metric == "alpha": + axis.axhline( + 2.0, + color="black", + linestyle="--", + linewidth=1.0, + label="alpha = 2", + ) + if metric == "ERG_gap": + axis.axhline( + 0.0, + color="black", + linestyle="--", + linewidth=1.0, + ) + axis.set_xlabel("Epoch") + axis.set_title(f"Block {block}") + axis.grid(alpha=0.25) + axes[0][0].set_ylabel(metric) + axes[0][-1].legend( + frameon=False, + ncol=2, + fontsize=8, + ) + figure.suptitle( + title + or ( + f"{OPTIMIZER_LABELS[optimizer]} " + f"layer {metric}" + ) + ) figure.tight_layout() if output is not None: output = Path(output) - output.parent.mkdir(parents=True, exist_ok=True) - figure.savefig(output, dpi=170, bbox_inches="tight") - return figure, axis + output.parent.mkdir( + parents=True, + exist_ok=True, + ) + figure.savefig( + output, + dpi=170, + bbox_inches="tight", + ) + return figure, axes def plot_spectral_optimizer_summary( @@ -406,20 +764,359 @@ def plot_spectral_optimizer_summary( metric=metric, x="epoch", optimizers=optimizers, - title=f"WeightWatcher {metric}: mean and 95% Student-t CI", + title=( + f"WeightWatcher {metric}: mean and " + "95% Student-t CI" + ), output=output, ) -def final_test_summary(test_results: pd.DataFrame) -> pd.DataFrame: +def _summary_row( + *, + optimizer: str, + checkpoint: str, + metric: str, + stats: dict[str, float], + interval_method: str, +) -> dict[str, object]: + return { + "optimizer": optimizer, + "optimizer_label": ( + OPTIMIZER_LABELS[optimizer] + ), + "checkpoint": checkpoint, + "metric": metric, + "interval_method": interval_method, + **stats, + } + + +def final_test_summary( + test_results: pd.DataFrame, +) -> pd.DataFrame: + """Summarize test metrics with a positive perplexity interval. + + Perplexity is reported as ``exp(mean loss)`` with the loss-space + Student-t interval exponentiated. This avoids impossible negative + perplexity bounds. + """ + + rows: list[dict[str, object]] = [] + for ( + optimizer, + checkpoint, + ), group in test_results.groupby( + ["optimizer", "checkpoint"] + ): + loss_stats = mean_ci95( + group["test_loss"] + ) + rows.append( + _summary_row( + optimizer=optimizer, + checkpoint=checkpoint, + metric="test_loss", + stats=loss_stats, + interval_method=( + "arithmetic_student_t" + ), + ) + ) + ppl_stats = { + "n": loss_stats["n"], + "mean": math.exp( + loss_stats["mean"] + ), + "sd": np.nan, + "sem": np.nan, + "ci95_half_width": np.nan, + "ci95_lower": math.exp( + loss_stats["ci95_lower"] + ), + "ci95_upper": math.exp( + loss_stats["ci95_upper"] + ), + } + rows.append( + _summary_row( + optimizer=optimizer, + checkpoint=checkpoint, + metric="test_perplexity", + stats=ppl_stats, + interval_method=( + "exp_test_loss_student_t" + ), + ) + ) + for metric in ( + "test_accuracy", + "test_bleu", + ): + rows.append( + _summary_row( + optimizer=optimizer, + checkpoint=checkpoint, + metric=metric, + stats=mean_ci95( + group[metric] + ), + interval_method=( + "arithmetic_student_t" + ), + ) + ) + return pd.DataFrame(rows) + + +def paired_test_differences( + test_results: pd.DataFrame, + *, + optimizers: Sequence[str] = SUPPORTED_OPTIMIZERS, +) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + for checkpoint in sorted( + test_results["checkpoint"].unique() + ): + selected = test_results[ + test_results["checkpoint"] + == checkpoint + ] + for left, right in combinations( + optimizers, + 2, + ): + for metric in ( + "test_loss", + "test_accuracy", + "test_bleu", + ): + left_frame = selected[ + selected["optimizer"] == left + ][["seed", metric]].rename( + columns={metric: "left"} + ) + right_frame = selected[ + selected["optimizer"] == right + ][["seed", metric]].rename( + columns={metric: "right"} + ) + paired = left_frame.merge( + right_frame, + on="seed", + validate="one_to_one", + ) + stats = mean_ci95( + paired["left"] + - paired["right"] + ) + rows.append( + { + "checkpoint": checkpoint, + "left_optimizer": left, + "right_optimizer": right, + "contrast": ( + f"{OPTIMIZER_LABELS[left]} " + f"- {OPTIMIZER_LABELS[right]}" + ), + "metric": metric, + "interval_method": ( + "paired_student_t" + ), + **stats, + } + ) + loss_left = selected[ + selected["optimizer"] == left + ][["seed", "test_loss"]].rename( + columns={"test_loss": "left"} + ) + loss_right = selected[ + selected["optimizer"] == right + ][["seed", "test_loss"]].rename( + columns={"test_loss": "right"} + ) + paired_loss = loss_left.merge( + loss_right, + on="seed", + validate="one_to_one", + ) + log_ratio = mean_ci95( + paired_loss["left"] + - paired_loss["right"] + ) + rows.append( + { + "checkpoint": checkpoint, + "left_optimizer": left, + "right_optimizer": right, + "contrast": ( + f"{OPTIMIZER_LABELS[left]} " + f"/ {OPTIMIZER_LABELS[right]}" + ), + "metric": ( + "test_perplexity_ratio" + ), + "interval_method": ( + "exp_paired_loss_student_t" + ), + "n": log_ratio["n"], + "mean": math.exp( + log_ratio["mean"] + ), + "sd": np.nan, + "sem": np.nan, + "ci95_half_width": np.nan, + "ci95_lower": math.exp( + log_ratio["ci95_lower"] + ), + "ci95_upper": math.exp( + log_ratio["ci95_upper"] + ), + } + ) + return pd.DataFrame(rows) + + +def run_diagnostics_table( + metrics: pd.DataFrame, + test_results: pd.DataFrame, +) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + selected_lookup = ( + test_results[ + test_results["checkpoint"] + == "validation_selected" + ] + .set_index(["optimizer", "seed"]) + ) + final_lookup = ( + test_results[ + test_results["checkpoint"] + == "final" + ] + .set_index(["optimizer", "seed"]) + ) + for ( + optimizer, + seed, + ), run in metrics.groupby( + ["optimizer", "seed"], + sort=True, + ): + ordered = run.sort_values("step") + finite = ordered[ + np.isfinite( + pd.to_numeric( + ordered["val_loss"], + errors="coerce", + ) + ) + ] + if finite.empty: + continue + best = finite.loc[ + finite["val_loss"].idxmin() + ] + final = finite.iloc[-1] + key = (optimizer, int(seed)) + selected_test = selected_lookup.loc[key] + final_test = final_lookup.loc[key] + clip_values = pd.to_numeric( + ordered.loc[ + ordered["step"].astype(int) > 0, + "gradient_clipped", + ], + errors="coerce", + ) + update_ratio = pd.to_numeric( + ordered["update_to_weight_ratio"], + errors="coerce", + ) + rows.append( + { + "optimizer": optimizer, + "optimizer_label": ( + OPTIMIZER_LABELS[optimizer] + ), + "seed": int(seed), + "best_validation_step": int( + selected_test["step"] + ), + "observed_best_validation_step": int( + best["step"] + ), + "best_validation_loss": float( + best["val_loss"] + ), + "final_validation_loss": float( + final["val_loss"] + ), + "final_minus_best_validation_loss": ( + float(final["val_loss"]) + - float(best["val_loss"]) + ), + "best_validation_accuracy": float( + best["val_accuracy"] + ), + "final_validation_accuracy": float( + final["val_accuracy"] + ), + "selected_test_loss": float( + selected_test["test_loss"] + ), + "selected_test_accuracy": float( + selected_test[ + "test_accuracy" + ] + ), + "final_test_loss": float( + final_test["test_loss"] + ), + "final_test_accuracy": float( + final_test["test_accuracy"] + ), + "max_update_to_weight_ratio": float( + update_ratio.max() + ), + "evaluation_snapshot_clip_fraction": ( + float(clip_values.mean()) + if clip_values.notna().any() + else np.nan + ), + } + ) + return pd.DataFrame(rows) + + +def summarize_run_diagnostics( + diagnostics: pd.DataFrame, +) -> pd.DataFrame: + metrics = ( + "best_validation_loss", + "final_validation_loss", + "final_minus_best_validation_loss", + "selected_test_loss", + "selected_test_accuracy", + "final_test_loss", + "final_test_accuracy", + "max_update_to_weight_ratio", + "evaluation_snapshot_clip_fraction", + ) rows = [] - for (optimizer, checkpoint), group in test_results.groupby(["optimizer", "checkpoint"]): - for metric in ("test_loss", "test_perplexity", "test_accuracy", "test_bleu"): + for optimizer, group in diagnostics.groupby( + "optimizer", + sort=True, + ): + for metric in metrics: rows.append( { "optimizer": optimizer, - "optimizer_label": OPTIMIZER_LABELS[optimizer], - "checkpoint": checkpoint, + "optimizer_label": ( + OPTIMIZER_LABELS[ + optimizer + ] + ), "metric": metric, **mean_ci95(group[metric]), } diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/completion.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/completion.py index fa62bc0..ba8d313 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/completion.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/completion.py @@ -1,4 +1,4 @@ -"""Validation for completed one-head nanoGPT experiment directories.""" +"""Validation for completed nanoGPT baseline experiment directories.""" from __future__ import annotations @@ -31,7 +31,7 @@ class CompletedRunValidationError(RuntimeError): def _fail(message: str) -> NoReturn: raise CompletedRunValidationError( - "completed one-head nanoGPT run is stale or inconsistent: " + "completed nanoGPT run is stale or inconsistent: " + message + ". Use a new results directory or rerun with explicit overwrite." ) @@ -53,7 +53,7 @@ def _read_csv(path: Path) -> pd.DataFrame: except Exception as exc: _fail(f"could not read {path}: {exc}") if frame.empty: - _fail(f" {path} is empty") + _fail(f"{path} is empty") return frame @@ -71,10 +71,16 @@ def _expect(observed: Any, expected: Any, label: str) -> None: ) -def _step_tuple(frame: pd.DataFrame, label: str) -> tuple[int, ...]: +def _step_tuple( + frame: pd.DataFrame, + label: str, +) -> tuple[int, ...]: if "step" not in frame.columns: _fail(f"{label} has no step column") - values = pd.to_numeric(frame["step"], errors="coerce").to_numpy(dtype=float) + values = pd.to_numeric( + frame["step"], + errors="coerce", + ).to_numpy(dtype=float) if not np.isfinite(values).all(): _fail(f"{label} contains non-finite step values") rounded = np.rint(values) @@ -88,7 +94,11 @@ def _step_tuple(frame: pd.DataFrame, label: str) -> tuple[int, ...]: def _load_checkpoint(path: Path) -> dict[str, Any]: try: - payload = torch.load(path, map_location="cpu", weights_only=False) + payload = torch.load( + path, + map_location="cpu", + weights_only=False, + ) except Exception as exc: _fail(f"could not load checkpoint {path}: {exc}") if not isinstance(payload, dict): @@ -96,6 +106,21 @@ def _load_checkpoint(path: Path) -> dict[str, Any]: return payload +def _manifest_matrix_count( + manifest: dict[str, Any], +) -> int: + model = manifest.get("model") + if not isinstance(model, dict): + _fail("manifest has no model configuration") + n_layer = _as_int( + model.get("n_layer"), + "manifest model.n_layer", + ) + if n_layer < 1: + _fail("manifest model.n_layer must be positive") + return 6 * n_layer + + def validate_completed_run( run_dir: str | Path, *, @@ -103,13 +128,10 @@ def validate_completed_run( expected_optimizer: str | None = None, expected_seed: int | None = None, expected_total_steps: int | None = None, + expected_matrix_count: int | None = None, verify_checkpoints: bool = True, ) -> dict[str, Any]: - """Validate a completed run before it is skipped or analyzed. - - Expected values supplied by the current configuration make this a stale-run - guard. Without them, the function still verifies internal consistency. - """ + """Validate a completed run before it is skipped or analyzed.""" root = Path(run_dir) missing = [ @@ -125,18 +147,26 @@ def validate_completed_run( manifest = _read_json(root / "manifest.json") test_results = _read_json(root / "test_results.json") if completion.get("completed") is not True: - _fail("run_complete.json does not declare completed=true") + _fail( + "run_complete.json does not declare completed=true" + ) - recorded_fingerprint = str(completion.get("fingerprint", "")) + recorded_fingerprint = str( + completion.get("fingerprint", "") + ) fingerprint = ( str(expected_fingerprint) if expected_fingerprint is not None else recorded_fingerprint ) if not fingerprint or not recorded_fingerprint: - _fail("the completion record has no protocol fingerprint") + _fail( + "the completion record has no protocol fingerprint" + ) - recorded_optimizer = str(completion.get("optimizer", "")) + recorded_optimizer = str( + completion.get("optimizer", "") + ) optimizer = ( str(expected_optimizer) if expected_optimizer is not None @@ -145,10 +175,18 @@ def validate_completed_run( if not optimizer or not recorded_optimizer: _fail("the completion record has no optimizer") - recorded_seed = _as_int(completion.get("seed"), "completion seed") - seed = int(expected_seed) if expected_seed is not None else recorded_seed + recorded_seed = _as_int( + completion.get("seed"), + "completion seed", + ) + seed = ( + int(expected_seed) + if expected_seed is not None + else recorded_seed + ) recorded_steps = _as_int( - completion.get("optimizer_steps"), "completion optimizer_steps" + completion.get("optimizer_steps"), + "completion optimizer_steps", ) total_steps = ( int(expected_total_steps) @@ -160,58 +198,145 @@ def validate_completed_run( "completion best_validation_step", ) - _expect(recorded_fingerprint, fingerprint, "completion fingerprint") - _expect(recorded_optimizer, optimizer, "completion optimizer") + _expect( + recorded_fingerprint, + fingerprint, + "completion fingerprint", + ) + _expect( + recorded_optimizer, + optimizer, + "completion optimizer", + ) _expect(recorded_seed, seed, "completion seed") - _expect(recorded_steps, total_steps, "completion optimizer_steps") + _expect( + recorded_steps, + total_steps, + "completion optimizer_steps", + ) _expect( str(manifest.get("protocol_fingerprint", "")), fingerprint, "manifest fingerprint", ) - _expect(str(manifest.get("optimizer", "")), optimizer, "manifest optimizer") - _expect(_as_int(manifest.get("seed"), "manifest seed"), seed, "manifest seed") _expect( - _as_int(manifest.get("max_steps"), "manifest max_steps"), + str(manifest.get("optimizer", "")), + optimizer, + "manifest optimizer", + ) + _expect( + _as_int(manifest.get("seed"), "manifest seed"), + seed, + "manifest seed", + ) + _expect( + _as_int( + manifest.get("max_steps"), + "manifest max_steps", + ), total_steps, "manifest max_steps", ) + manifest_matrix_count = _manifest_matrix_count( + manifest + ) + matrix_count = ( + int(expected_matrix_count) + if expected_matrix_count is not None + else manifest_matrix_count + ) + if matrix_count < 1: + _fail("expected matrix count must be positive") + _expect( + manifest_matrix_count, + matrix_count, + "manifest transformer matrix count", + ) + final_test = test_results.get("final") - selected_test = test_results.get("validation_selected") - if not isinstance(final_test, dict) or not isinstance(selected_test, dict): - _fail("test_results.json lacks final or validation_selected results") + selected_test = test_results.get( + "validation_selected" + ) + if not isinstance(final_test, dict) or not isinstance( + selected_test, + dict, + ): + _fail( + "test_results.json lacks final or " + "validation_selected results" + ) _expect( - _as_int(final_test.get("step"), "final test step"), + _as_int( + final_test.get("step"), + "final test step", + ), total_steps, "final test step", ) _expect( - _as_int(selected_test.get("step"), "selected test step"), + _as_int( + selected_test.get("step"), + "selected test step", + ), best_step, "selected test step", ) metrics = _read_csv(root / "metrics.csv") - epoch_metrics = _read_csv(root / "epoch_metrics.csv") - layers = _read_csv(root / "spectral" / "layers.csv") - summary = _read_csv(root / "spectral" / "summary.csv") - metric_steps = _step_tuple(metrics, "metrics.csv") - epoch_steps = _step_tuple(epoch_metrics, "epoch_metrics.csv") - summary_steps = _step_tuple(summary, "spectral/summary.csv") + epoch_metrics = _read_csv( + root / "epoch_metrics.csv" + ) + layers = _read_csv( + root / "spectral" / "layers.csv" + ) + summary = _read_csv( + root / "spectral" / "summary.csv" + ) + metric_steps = _step_tuple( + metrics, + "metrics.csv", + ) + epoch_steps = _step_tuple( + epoch_metrics, + "epoch_metrics.csv", + ) + summary_steps = _step_tuple( + summary, + "spectral/summary.csv", + ) for label, steps in ( ("metrics.csv", metric_steps), ("epoch_metrics.csv", epoch_steps), ): - if 0 not in steps or total_steps not in steps or max(steps) != total_steps: - _fail(f"{label} does not span step zero through {total_steps}") + if ( + 0 not in steps + or total_steps not in steps + or max(steps) != total_steps + ): + _fail( + f"{label} does not span step zero " + f"through {total_steps}" + ) if "test_monitoring_only" not in epoch_metrics.columns: - _fail("epoch_metrics.csv has no test_monitoring_only column") - policy = pd.to_numeric(epoch_metrics["test_monitoring_only"], errors="coerce") - if policy.isna().any() or not policy.astype(int).eq(1).all(): - _fail("epoch_metrics.csv violates the monitoring-only test policy") + _fail( + "epoch_metrics.csv has no " + "test_monitoring_only column" + ) + policy = pd.to_numeric( + epoch_metrics["test_monitoring_only"], + errors="coerce", + ) + if ( + policy.isna().any() + or not policy.astype(int).eq(1).all() + ): + _fail( + "epoch_metrics.csv violates the " + "monitoring-only test policy" + ) required_layer_columns = { "step", @@ -220,35 +345,74 @@ def validate_completed_run( "ERG_gap", "num_traps", } - missing_columns = required_layer_columns.difference(layers.columns) + missing_columns = required_layer_columns.difference( + layers.columns + ) if missing_columns: _fail( "spectral/layers.csv is missing columns " + ", ".join(sorted(missing_columns)) ) - if layers.duplicated(["step", "matrix_name"]).any(): - _fail("spectral/layers.csv has duplicate step/matrix rows") + if layers.duplicated( + ["step", "matrix_name"] + ).any(): + _fail( + "spectral/layers.csv has duplicate " + "step/matrix rows" + ) layer_steps = _step_tuple( - layers[["step"]].drop_duplicates().sort_values("step"), + layers[["step"]] + .drop_duplicates() + .sort_values("step"), "spectral/layers.csv", ) - if set(summary_steps) != set(epoch_steps) or set(layer_steps) != set(epoch_steps): - _fail("spectral steps do not match epoch_metrics.csv") - if not layers.groupby("step")["matrix_name"].nunique().eq(6).all(): - _fail("spectral/layers.csv does not contain six matrices per epoch") + if ( + set(summary_steps) != set(epoch_steps) + or set(layer_steps) != set(epoch_steps) + ): + _fail( + "spectral steps do not match " + "epoch_metrics.csv" + ) + layer_counts = ( + layers.groupby("step")["matrix_name"] + .nunique() + ) + if not layer_counts.eq(matrix_count).all(): + _fail( + "spectral/layers.csv does not contain " + f"{matrix_count} matrices per epoch" + ) if "n_matrices" not in summary.columns: - _fail("spectral/summary.csv has no n_matrices column") - matrix_counts = pd.to_numeric(summary["n_matrices"], errors="coerce") - if matrix_counts.isna().any() or not matrix_counts.astype(int).eq(6).all(): - _fail("spectral/summary.csv does not report six matrices per epoch") + _fail( + "spectral/summary.csv has no n_matrices column" + ) + matrix_counts = pd.to_numeric( + summary["n_matrices"], + errors="coerce", + ) + if ( + matrix_counts.isna().any() + or not matrix_counts.astype(int) + .eq(matrix_count) + .all() + ): + _fail( + "spectral/summary.csv does not report " + f"{matrix_count} matrices per epoch" + ) if "checkpoint_path" not in epoch_metrics.columns: - _fail("epoch_metrics.csv has no checkpoint_path column") + _fail( + "epoch_metrics.csv has no checkpoint_path column" + ) recorded_checkpoint_paths = [ Path(str(value)) for value in epoch_metrics["checkpoint_path"] ] - if len(recorded_checkpoint_paths) != len(set(epoch_steps)): + if len(recorded_checkpoint_paths) != len( + set(epoch_steps) + ): _fail( "epoch checkpoint inventory does not match " "epoch_metrics.csv" @@ -257,67 +421,131 @@ def validate_completed_run( for recorded in recorded_checkpoint_paths: candidate = recorded if not candidate.is_file(): - candidate = root / "epoch_checkpoints" / recorded.name - if not candidate.is_file() or candidate.stat().st_size == 0: + candidate = ( + root + / "epoch_checkpoints" + / recorded.name + ) + if ( + not candidate.is_file() + or candidate.stat().st_size == 0 + ): _fail( - f"missing epoch checkpoint recorded by " + "missing epoch checkpoint recorded by " f"epoch_metrics.csv: {recorded}" ) - resolved_checkpoint_paths.append(candidate.resolve()) + resolved_checkpoint_paths.append( + candidate.resolve() + ) if len(resolved_checkpoint_paths) != len( set(resolved_checkpoint_paths) ): _fail( - "epoch_metrics.csv references duplicate epoch " - "checkpoints" + "epoch_metrics.csv references duplicate " + "epoch checkpoints" ) if verify_checkpoints: try: - best_loss = float(completion.get("best_validation_loss")) + best_loss = float( + completion.get( + "best_validation_loss" + ) + ) except (TypeError, ValueError): - _fail("run_complete.json has invalid best_validation_loss") + _fail( + "run_complete.json has invalid " + "best_validation_loss" + ) for filename, expected_step in ( - ("checkpoint_latest.pt", total_steps), - ("checkpoint_final.pt", total_steps), - ("checkpoint_best.pt", best_step), + ( + "checkpoint_latest.pt", + total_steps, + ), + ( + "checkpoint_final.pt", + total_steps, + ), + ( + "checkpoint_best.pt", + best_step, + ), ): - payload = _load_checkpoint(root / filename) + payload = _load_checkpoint( + root / filename + ) _expect( - str(payload.get("fingerprint", "")), + str( + payload.get( + "fingerprint", + "", + ) + ), fingerprint, f"{filename} fingerprint", ) _expect( - str(payload.get("optimizer_name", "")), + str( + payload.get( + "optimizer_name", + "", + ) + ), optimizer, f"{filename} optimizer", ) _expect( - _as_int(payload.get("seed"), f"{filename} seed"), + _as_int( + payload.get("seed"), + f"{filename} seed", + ), seed, f"{filename} seed", ) _expect( - _as_int(payload.get("step"), f"{filename} step"), + _as_int( + payload.get("step"), + f"{filename} step", + ), expected_step, f"{filename} step", ) _expect( _as_int( - payload.get("best_validation_step"), - f"{filename} best_validation_step", + payload.get( + "best_validation_step" + ), + ( + f"{filename} " + "best_validation_step" + ), ), best_step, - f"{filename} best_validation_step", + ( + f"{filename} " + "best_validation_step" + ), ) try: - stored_loss = float(payload.get("best_validation_loss")) + stored_loss = float( + payload.get( + "best_validation_loss" + ) + ) except (TypeError, ValueError): - _fail(f"{filename} has invalid best_validation_loss") + _fail( + f"{filename} has invalid " + "best_validation_loss" + ) if not math.isclose( - stored_loss, best_loss, rel_tol=1e-12, abs_tol=1e-12 + stored_loss, + best_loss, + rel_tol=1e-12, + abs_tol=1e-12, ): - _fail(f"{filename} best_validation_loss does not match completion") + _fail( + f"{filename} best_validation_loss " + "does not match completion" + ) return completion diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/config.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/config.py index 8e26450..fafb58d 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/config.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/config.py @@ -5,15 +5,70 @@ import json import os from pathlib import Path +import re from typing import Any import yaml SUPPORTED_OPTIMIZERS = ("sgd_momentum", "adamw", "muon") DEFAULT_ROOT = Path("/tmp/rg-nanogpt-one-head") +_RUN_SLUG_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") -def roots() -> dict[str, Path]: +def run_slug(cfg: dict[str, Any]) -> str: + protocol = cfg.get("protocol", {}) + value = str(protocol.get("run_slug", "")).strip() + if not value: + value = str(protocol.get("name", "reference")).strip() + if not _RUN_SLUG_PATTERN.fullmatch(value): + raise ValueError( + "protocol.run_slug must contain only letters, numbers, dot, " + "underscore, and hyphen" + ) + return value + + +def roots(cfg: dict[str, Any] | None = None) -> dict[str, Path]: + """Resolve isolated v3 or NGB-v4 runtime roots under ``/tmp``. + + Version-3 callers retain the historical ``RG_NANOGPT_ONE_HEAD_*`` + environment contract. Version-4 configurations use the separate + ``RG_NGB_*`` contract and a required run slug, preventing artifact reuse + across protocols or architectures. + """ + + version = int(cfg.get("protocol", {}).get("version", 0)) if cfg else 0 + if cfg is not None and version >= 4: + protocol = cfg["protocol"] + root = Path( + os.environ.get( + "RG_NGB_ROOT", + protocol.get("storage_root", "/tmp/rg-ngb"), + ) + ) + data = Path( + os.environ.get( + "RG_NGB_DATA_ROOT", + protocol.get( + "data_root", + "/tmp/rg-nanogpt-one-head/data", + ), + ) + ) + results_base = Path( + os.environ.get("RG_NGB_RESULTS_ROOT", root / "results") + ) + plots_base = Path( + os.environ.get("RG_NGB_PLOTS_ROOT", root / "plots") + ) + slug = run_slug(cfg) + return { + "root": root, + "data": data, + "results": results_base / slug, + "plots": plots_base / slug, + } + root = Path(os.environ.get("RG_NANOGPT_ONE_HEAD_ROOT", DEFAULT_ROOT)) return { "root": root, @@ -55,21 +110,25 @@ def validate_config(cfg: dict[str, Any]) -> None: raise ValueError(f"missing configuration section: {section}") protocol = cfg["protocol"] - if int(protocol.get("version", 0)) < 1: + version = int(protocol.get("version", 0)) + if version < 1: raise ValueError("protocol.version must be positive") + if version >= 4: + run_slug(cfg) + for key in ("storage_root", "data_root"): + value = protocol.get(key) + if value is None: + continue + path = Path(str(value)) + if not path.is_absolute() or path.parts[:2] != ("/", "tmp"): + raise ValueError( + f"protocol.{key} must be an absolute /tmp path" + ) model = cfg["model"] for key in ("vocab_size", "block_size", "n_layer", "n_head", "n_embd"): if int(model[key]) < 1: raise ValueError(f"model.{key} must be positive") - if int(model["n_head"]) != 1: - raise ValueError( - "this experiment is intentionally fixed to exactly one attention head" - ) - if int(model["n_layer"]) != 1: - raise ValueError( - "this experiment is intentionally fixed to one transformer block" - ) if int(model["n_embd"]) % int(model["n_head"]) != 0: raise ValueError("model.n_embd must be divisible by model.n_head") if not 0.0 <= float(model.get("dropout", 0.0)) < 1.0: @@ -94,9 +153,15 @@ def validate_config(cfg: dict[str, Any]) -> None: ): if float(training[key]) <= 0: raise ValueError(f"training.{key} must be positive") + if float(training["epoch_interval"]) > float(training["target_epochs"]): + raise ValueError( + "training.epoch_interval cannot exceed training.target_epochs" + ) seeds = [int(seed) for seed in training["seeds"]] if not seeds or len(set(seeds)) != len(seeds): raise ValueError("training.seeds must contain unique values") + if any(seed < 0 for seed in seeds): + raise ValueError("training.seeds must be nonnegative") if float(training["grad_clip"]) < 0: raise ValueError("training.grad_clip must be nonnegative") @@ -231,7 +296,7 @@ def warmup_steps(profile: dict[str, Any], total_steps: int) -> int: def epoch_step_map( cfg: dict[str, Any], train_tokens: int | None = None ) -> dict[int, float]: - """Map preregistered nominal epochs, including epoch zero, to optimizer steps.""" + """Map preregistered nominal epochs, including epoch zero, to steps.""" train_tokens = int(train_tokens or cfg["dataset"]["train_tokens"]) total_steps = max_steps(cfg, train_tokens) @@ -259,6 +324,10 @@ def epoch_step_map( return dict(sorted(result.items())) +def expected_transformer_matrix_count(cfg: dict[str, Any]) -> int: + return 6 * int(cfg["model"]["n_layer"]) + + def protocol_fingerprint( cfg: dict[str, Any], *, diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/data.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/data.py index 63015c4..62e04c7 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/data.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/data.py @@ -320,7 +320,7 @@ def main() -> None: parser.add_argument("--force", action="store_true") args = parser.parse_args() cfg = load_config(args.config) - output = Path(args.output_dir) if args.output_dir else roots()["data"] + output = Path(args.output_dir) if args.output_dir else roots(cfg)["data"] prepare_fineweb_edu(cfg, output, force=args.force) diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/model.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/model.py index eb696bd..2fd7509 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/model.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/model.py @@ -20,8 +20,8 @@ class GPTConfig: tie_weights: bool = True def __post_init__(self) -> None: - if self.n_layer != 1 or self.n_head != 1: - raise ValueError("the reference architecture is fixed to one block and one attention head") + if self.n_layer < 1 or self.n_head < 1: + raise ValueError("n_layer and n_head must be positive") if self.n_embd % self.n_head != 0: raise ValueError("n_embd must be divisible by n_head") if self.block_size < 2 or self.vocab_size < 2 or self.n_embd < 1: @@ -37,7 +37,13 @@ def __init__(self, width: int, bias: bool) -> None: self.bias = nn.Parameter(torch.zeros(width)) if bias else None def forward(self, x: torch.Tensor) -> torch.Tensor: - return F.layer_norm(x, self.weight.shape, self.weight, self.bias, 1e-5) + return F.layer_norm( + x, + self.weight.shape, + self.weight, + self.bias, + 1e-5, + ) class CausalSelfAttention(nn.Module): @@ -46,18 +52,38 @@ def __init__(self, cfg: GPTConfig) -> None: self.n_head = cfg.n_head self.n_embd = cfg.n_embd self.dropout = float(cfg.dropout) - self.q_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) - self.k_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) - self.v_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) - self.out_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) + self.q_proj = nn.Linear( + cfg.n_embd, cfg.n_embd, bias=cfg.bias + ) + self.k_proj = nn.Linear( + cfg.n_embd, cfg.n_embd, bias=cfg.bias + ) + self.v_proj = nn.Linear( + cfg.n_embd, cfg.n_embd, bias=cfg.bias + ) + self.out_proj = nn.Linear( + cfg.n_embd, cfg.n_embd, bias=cfg.bias + ) self.resid_dropout = nn.Dropout(cfg.dropout) def forward(self, x: torch.Tensor) -> torch.Tensor: batch, sequence, channels = x.shape head_width = channels // self.n_head - q = self.q_proj(x).view(batch, sequence, self.n_head, head_width).transpose(1, 2) - k = self.k_proj(x).view(batch, sequence, self.n_head, head_width).transpose(1, 2) - v = self.v_proj(x).view(batch, sequence, self.n_head, head_width).transpose(1, 2) + q = ( + self.q_proj(x) + .view(batch, sequence, self.n_head, head_width) + .transpose(1, 2) + ) + k = ( + self.k_proj(x) + .view(batch, sequence, self.n_head, head_width) + .transpose(1, 2) + ) + v = ( + self.v_proj(x) + .view(batch, sequence, self.n_head, head_width) + .transpose(1, 2) + ) y = F.scaled_dot_product_attention( q, k, @@ -66,19 +92,31 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: dropout_p=self.dropout if self.training else 0.0, is_causal=True, ) - y = y.transpose(1, 2).contiguous().view(batch, sequence, channels) + y = ( + y.transpose(1, 2) + .contiguous() + .view(batch, sequence, channels) + ) return self.resid_dropout(self.out_proj(y)) class MLP(nn.Module): def __init__(self, cfg: GPTConfig) -> None: super().__init__() - self.fc = nn.Linear(cfg.n_embd, 4 * cfg.n_embd, bias=cfg.bias) - self.proj = nn.Linear(4 * cfg.n_embd, cfg.n_embd, bias=cfg.bias) + self.fc = nn.Linear( + cfg.n_embd, 4 * cfg.n_embd, bias=cfg.bias + ) + self.proj = nn.Linear( + 4 * cfg.n_embd, cfg.n_embd, bias=cfg.bias + ) self.dropout = nn.Dropout(cfg.dropout) def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.dropout(self.proj(F.gelu(self.fc(x), approximate="tanh"))) + return self.dropout( + self.proj( + F.gelu(self.fc(x), approximate="tanh") + ) + ) class Block(nn.Module): @@ -98,34 +136,59 @@ class GPT(nn.Module): def __init__(self, cfg: GPTConfig) -> None: super().__init__() self.cfg = cfg - self.token_embedding = nn.Embedding(cfg.vocab_size, cfg.n_embd) - self.position_embedding = nn.Embedding(cfg.block_size, cfg.n_embd) + self.token_embedding = nn.Embedding( + cfg.vocab_size, cfg.n_embd + ) + self.position_embedding = nn.Embedding( + cfg.block_size, cfg.n_embd + ) self.drop = nn.Dropout(cfg.dropout) - self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layer)]) + self.blocks = nn.ModuleList( + [Block(cfg) for _ in range(cfg.n_layer)] + ) self.ln_f = LayerNorm(cfg.n_embd, cfg.bias) - self.lm_head = nn.Linear(cfg.n_embd, cfg.vocab_size, bias=False) + self.lm_head = nn.Linear( + cfg.n_embd, cfg.vocab_size, bias=False + ) if cfg.tie_weights: self.lm_head.weight = self.token_embedding.weight self.apply(self._init_module) residual_std = 0.02 / math.sqrt(2 * cfg.n_layer) for block in self.blocks: - nn.init.normal_(block.attn.out_proj.weight, mean=0.0, std=residual_std) - nn.init.normal_(block.mlp.proj.weight, mean=0.0, std=residual_std) + nn.init.normal_( + block.attn.out_proj.weight, + mean=0.0, + std=residual_std, + ) + nn.init.normal_( + block.mlp.proj.weight, + mean=0.0, + std=residual_std, + ) @staticmethod def _init_module(module: nn.Module) -> None: if isinstance(module, (nn.Linear, nn.Embedding)): - nn.init.normal_(module.weight, mean=0.0, std=0.02) + nn.init.normal_( + module.weight, + mean=0.0, + std=0.02, + ) if isinstance(module, nn.Linear) and module.bias is not None: nn.init.zeros_(module.bias) def hidden_states(self, idx: torch.Tensor) -> torch.Tensor: _, sequence = idx.shape if sequence > self.cfg.block_size: - raise ValueError("input sequence exceeds model.block_size") + raise ValueError( + "input sequence exceeds model.block_size" + ) positions = torch.arange(sequence, device=idx.device) - x = self.drop(self.token_embedding(idx) + self.position_embedding(positions)) + x = self.drop( + self.token_embedding(idx) + + self.position_embedding(positions) + ) for block in self.blocks: x = block(x) return self.ln_f(x) @@ -138,36 +201,53 @@ def forward( logits = self.lm_head(self.hidden_states(idx)) loss = None if targets is not None: - loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1)) + loss = F.cross_entropy( + logits.reshape(-1, logits.size(-1)), + targets.reshape(-1), + ) return logits, loss def next_token_logits(self, idx: torch.Tensor) -> torch.Tensor: - # Apply the expensive vocabulary projection only to the final position. hidden = self.hidden_states(idx)[:, -1:, :] return self.lm_head(hidden) @torch.inference_mode() - def generate_greedy(self, prompts: torch.Tensor, max_new_tokens: int) -> torch.Tensor: + def generate_greedy( + self, + prompts: torch.Tensor, + max_new_tokens: int, + ) -> torch.Tensor: if prompts.ndim != 2: - raise ValueError("prompts must be [batch, sequence]") + raise ValueError( + "prompts must be [batch, sequence]" + ) if max_new_tokens < 0: - raise ValueError("max_new_tokens must be nonnegative") + raise ValueError( + "max_new_tokens must be nonnegative" + ) idx = prompts for _ in range(int(max_new_tokens)): idx_cond = idx[:, -self.cfg.block_size :] logits = self.next_token_logits(idx_cond) - next_token = logits[:, -1, :].argmax(dim=-1, keepdim=True) + next_token = logits[:, -1, :].argmax( + dim=-1, + keepdim=True, + ) idx = torch.cat((idx, next_token), dim=1) return idx def parameter_count(self) -> int: - return sum(parameter.numel() for parameter in self.parameters()) + return sum( + parameter.numel() + for parameter in self.parameters() + ) def transformer_matrix_items( model: GPT, ) -> list[tuple[str, str, int, torch.Tensor]]: - """Return the six transformer matrices used by WeightWatcher and Muon.""" + """Return six WeightWatcher/Muon matrices per transformer block.""" + items: list[tuple[str, str, int, torch.Tensor]] = [] for block_index, block in enumerate(model.blocks): matrices = ( @@ -179,5 +259,12 @@ def transformer_matrix_items( ("W_MLP_OUT", block.mlp.proj.weight), ) for matrix_type, weight in matrices: - items.append((f"L{block_index:02d}_{matrix_type}", matrix_type, block_index, weight)) + items.append( + ( + f"L{block_index:02d}_{matrix_type}", + matrix_type, + block_index, + weight, + ) + ) return items diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/spectral.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/spectral.py index 7417468..bcd425c 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/spectral.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/spectral.py @@ -32,7 +32,7 @@ class WeightMatrixHolder(nn.Module): - """CPU-only Linear view of the six one-block transformer matrices.""" + """CPU-only Linear view of all transformer matrices.""" def __init__(self, model: GPT) -> None: super().__init__() @@ -198,7 +198,8 @@ def run_weightwatcher( import weightwatcher as ww except ImportError as exc: raise RuntimeError( - "WeightWatcher is required; run scripts/setup_mac.sh" + "WeightWatcher is required; install baseline/nanogpt_one_head " + "in the active conda environment" ) from exc run_dir = Path(run_dir) diff --git a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/training.py b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/training.py index fd3cf8c..ea66254 100644 --- a/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/training.py +++ b/baseline/nanogpt_one_head/src/rg_nanogpt_one_head/training.py @@ -25,7 +25,7 @@ def run_optimizer_replicates( prepare_data: bool = True, progress: bool = True, ) -> list[Path]: - resolved = roots() + resolved = roots(cfg) data_root = Path(data_root or resolved["data"]) results_root = Path(results_root or resolved["results"]) selected_seeds = tuple(int(seed) for seed in (seeds or canonical_seeds(cfg))) @@ -61,7 +61,7 @@ def run_all_replicates( overwrite: bool = False, progress: bool = True, ) -> list[Path]: - resolved = roots() + resolved = roots(cfg) data_root = Path(data_root or resolved["data"]) results_root = Path(results_root or resolved["results"]) prepare_fineweb_edu(cfg, data_root) diff --git a/baseline/nanogpt_one_head/tests/test_completion.py b/baseline/nanogpt_one_head/tests/test_completion.py index d61f1cd..b4725b9 100644 --- a/baseline/nanogpt_one_head/tests/test_completion.py +++ b/baseline/nanogpt_one_head/tests/test_completion.py @@ -139,6 +139,7 @@ def _write_completed_run(results_root: Path, cfg: dict) -> tuple[Path, str, int] "optimizer": OPTIMIZER, "seed": SEED, "max_steps": total_steps, + "model": cfg["model"], "warmup_steps": warmup_steps(profile, total_steps), "protocol_fingerprint": fingerprint, } diff --git a/baseline/nanogpt_one_head/tests/test_ngb_v4.py b/baseline/nanogpt_one_head/tests/test_ngb_v4.py new file mode 100644 index 0000000..44e049f --- /dev/null +++ b/baseline/nanogpt_one_head/tests/test_ngb_v4.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pathlib import Path +import runpy + +_SOURCE = ( + Path(__file__).resolve().parents[2] + / "ngb" + / "tests" + / "test_ngb.py" +) +_NAMESPACE = runpy.run_path(str(_SOURCE)) +for _name, _value in _NAMESPACE.items(): + if _name.startswith("test_"): + globals()[_name] = _value diff --git a/baseline/nanogpt_one_head/tests/test_one_head.py b/baseline/nanogpt_one_head/tests/test_one_head.py index 80374de..d28a5e3 100644 --- a/baseline/nanogpt_one_head/tests/test_one_head.py +++ b/baseline/nanogpt_one_head/tests/test_one_head.py @@ -415,7 +415,11 @@ def test_tiny_cpu_training_writes_restart_and_epoch_artifacts( def test_notebooks_are_valid_and_expose_requested_metrics(): - notebook_paths = sorted((EXPERIMENT_ROOT / "notebooks").glob("*.ipynb")) + notebook_paths = sorted( + path + for path in (EXPERIMENT_ROOT / "notebooks").glob("*.ipynb") + if not path.name.endswith(".out.ipynb") + ) assert [path.name for path in notebook_paths] == [ "01_sgd_momentum_baseline.ipynb", "02_adamw_baseline.ipynb", diff --git a/baseline/ngb/.gitignore b/baseline/ngb/.gitignore new file mode 100644 index 0000000..a1875f8 --- /dev/null +++ b/baseline/ngb/.gitignore @@ -0,0 +1,3 @@ +.ipynb_checkpoints/ +__pycache__/ +*.pyc diff --git a/baseline/ngb/QUALIFICATION.md b/baseline/ngb/QUALIFICATION.md new file mode 100644 index 0000000..37d5fdf --- /dev/null +++ b/baseline/ngb/QUALIFICATION.md @@ -0,0 +1,48 @@ +# NGB v4 qualification plan + +The committed v4 optimizer profiles are conservative center candidates derived +from the v3 three-seed trajectories. They are not declared globally optimal. +Protected test metrics and WeightWatcher diagnostics must not select a profile. + +## Stage 1 — instability screen + +Use seeds `1337,2027`, which exposed the largest v3 adaptive-optimizer +bifurcation. Run temporary 0.5-epoch configurations in separate result roots. + +Candidate neighborhoods: + +| Optimizer | Screen | +|---|---| +| SGD + Nesterov | peak LR `{0.03, 0.05}`; floor `1%` of peak; decay `0.01` | +| AdamW | peak LR `{2e-4, 3e-4}`; decay `{0.05, 0.10}`; floor `1e-5` | +| Muon matrices | peak LR `{0.005, 0.01}`; decay `{0.01, 0.02}`; floor `{1e-4, 2e-4}` | +| Muon auxiliary AdamW | peak LR `{2e-4, 3e-4}`; decay `{0.01, 0.10}`; floor `1e-5` | + +Reject a candidate if a run becomes nonfinite, if final validation loss exceeds +its observed minimum by more than `0.25`, or if update-to-weight trajectories +show discontinuous excursions. + +## Stage 2 — two-epoch qualification + +Promote surviving candidates to seeds `1337,2027,4099` and two +corpus-equivalent epochs. Rank strictly by: + +1. mean best validation cross-entropy; +2. worst-seed best validation cross-entropy; +3. seed standard deviation; +4. final-minus-best validation-loss drift. + +Do not use alpha, ERG gap, trap counts, BLEU, test loss, or test accuracy to +select the optimizer profile. + +## Stage 3 — frozen matched replication + +After locking one profile per optimizer, run the same eight seeds for every arm: + +```text +1337, 2027, 4099, 5003, 6007, 7013, 8017, 9011 +``` + +Report final and validation-selected task metrics with run-level 95% Student-t +intervals and matched-seed paired contrasts. Perplexity intervals must be +obtained by exponentiating the corresponding loss-space interval. diff --git a/baseline/ngb/README.md b/baseline/ngb/README.md new file mode 100644 index 0000000..fd5fa72 --- /dev/null +++ b/baseline/ngb/README.md @@ -0,0 +1,210 @@ +# NGB v4 — nanoGPT baselines + +`baseline/ngb` is the version-4 nanoGPT experiment family. It is separate from +`baseline/nanogpt_one_head`, so the existing v3 one-epoch runs and checked-in +comparison notebook remain intact. + +NGB contains two preregistered architectures: + +1. **v4 one-head control** — one block, one attention head, width 128. +2. **v4 small 4×4 model** — four blocks, four attention heads, width 128. + +Both use the same pinned, document-disjoint FineWeb-Edu corpus and the same +fixed train/validation/test/BLEU probes. Test metrics remain monitoring-only. +Validation cross-entropy selects `checkpoint_best.pt`. + +## Why v4 exists + +The v3 experiment used one corpus-equivalent epoch and optimizer profiles +transferred almost directly from much larger nanoGPT/Muon training regimes. +The checked-in comparison showed: + +- SGD was reproducible but still improving near the one-epoch boundary. +- AdamW and Muon reached better validation-selected checkpoints but exhibited + large seed-dependent final-checkpoint drift. +- The adaptive profiles used a high peak learning rate relative to the + 8,192-token effective batch. +- The one-head model was 96.55% tied token embedding / output head by parameter + count, so Muon's auxiliary AdamW controlled most of the model. + +NGB v4 therefore changes both the horizon and the optimizer centers rather than +merely doubling the old schedule. + +## Protocols + +| Field | v4 one-head | v4 small 4×4 | +|---|---:|---:| +| Blocks | 1 | 4 | +| Attention heads | 1 | 4 | +| Embedding width | 128 | 128 | +| Context length | 256 | 256 | +| Parameters | 6,662,656 | 7,253,248 | +| Muon matrix parameters | 196,608 | 786,432 | +| Training tokens | about 160M | about 160M | +| Corpus-equivalent epochs | 2.0 | 2.0 | +| Reporting interval | 0.25 epoch | 0.25 epoch | +| Default seeds | 1337, 2027, 4099 | 1337, 2027, 4099 | + +The 4×4 architecture has only about 9% more total parameters than the one-head +control because both models are dominated by the tied GPT-2 vocabulary +embedding. It nevertheless has four times as many hidden transformer matrices +and materially greater contextual capacity. + +## Tuned optimizer centers + +### One-head v4 + +| Optimizer | Peak LR | Floor | Warm-up | Weight decay | +|---|---:|---:|---:|---:| +| SGD + Nesterov | 0.05 | 5e-4 | 5% | 0.01 | +| AdamW | 3e-4 | 1e-5 | 2.5% | 0.10 | +| Muon matrices | 0.01 | 1e-4 | 2.5% | 0.02 | +| Muon auxiliary AdamW | 2e-4 | 1e-5 | 2.5% | 0.10 | + +### Small 4×4 v4 + +The adaptive profiles are the same. The SGD peak is reduced to `0.03`, with a +`3e-4` floor, for the deeper residual stack. + +These are conservative v4 center profiles, not a claim that a validation-only +grid search has already proved global optimality. The protocol is designed to +train stably enough that the full two-epoch trajectory is scientifically +interpretable. + +## Storage + +No NGB command defaults to a home directory. + +```text +/tmp/rg-ngb/ + results/ + v4_one_head/ + v4_small_4x4/ + plots/ + v4_one_head/ + v4_small_4x4/ +``` + +The prepared v3 corpus can be reused directly: + +```text +/tmp/rg-nanogpt-one-head/data +``` + +Set a different explicit `/tmp` path through `RG_NGB_DATA_ROOT` when needed. + +## Install in the active conda environment + +From the repository root: + +```bash +cd /tmp/rg_optimizers/baseline/ngb +python -m pip install -e ../nanogpt_one_head +export PYTORCH_ENABLE_MPS_FALLBACK=1 +export RG_NGB_ROOT=/tmp/rg-ngb +export RG_NGB_DATA_ROOT=/tmp/rg-nanogpt-one-head/data +``` + +Prepare or verify the shared corpus: + +```bash +python -m rg_nanogpt_one_head.data \ + --config configs/v4_one_head.yaml \ + --output-dir /tmp/rg-nanogpt-one-head/data +``` + +## Run v4 one-head + +```bash +python -m rg_nanogpt_one_head.training \ + --config configs/v4_one_head.yaml \ + --optimizer all \ + --data-root /tmp/rg-nanogpt-one-head/data \ + --results-root /tmp/rg-ngb/results/v4_one_head \ + --device auto +``` + +## Run the small 4×4 model + +```bash +python -m rg_nanogpt_one_head.training \ + --config configs/v4_small_4x4.yaml \ + --optimizer all \ + --data-root /tmp/rg-nanogpt-one-head/data \ + --results-root /tmp/rg-ngb/results/v4_small_4x4 \ + --device auto +``` + +## Eight matched seeds + +After the canonical three, add five more matched seeds with: + +```bash +python -m rg_nanogpt_one_head.training \ + --config configs/v4_one_head.yaml \ + --optimizer all \ + --seeds 5003,6007,7013,8017,9011 \ + --data-root /tmp/rg-nanogpt-one-head/data \ + --results-root /tmp/rg-ngb/results/v4_one_head \ + --device auto +``` + +Use the same seed list and corresponding result root for `v4_small_4x4.yaml`. + +## Notebooks + +```text +notebooks/01_run_v4_one_head.ipynb +notebooks/02_compare_v4_one_head.ipynb +notebooks/03_run_v4_small_4x4.ipynb +notebooks/04_compare_v4_small_4x4.ipynb +notebooks/05_compare_v4_architectures.ipynb +notebooks/06_compare_v3_v4_one_head.ipynb +``` + +The comparison notebooks discover the intersection of completed seeds across +all three optimizers. They therefore use three, eight, or any later matched +seed count without editing the notebook. + +They report: + +- final and validation-selected metrics with run-level 95% Student-t intervals; +- perplexity intervals obtained by exponentiating the loss-space interval; +- matched-seed paired optimizer contrasts; +- best validation step and final-minus-best validation-loss drift; +- maximum update-to-weight ratio and evaluation-snapshot clipping rate; +- optimizer configuration tables; +- optimizer-level and block-resolved WeightWatcher trajectories. + +Block-resolved plots compute uncertainty across seeds for each matrix. Blocks +are never treated as additional statistical replicates. + +Run all NGB notebooks with the active conda kernel: + +```bash +cd /tmp/rg_optimizers/baseline/ngb + +for nb in \ + 01_run_v4_one_head \ + 02_compare_v4_one_head \ + 03_run_v4_small_4x4 \ + 04_compare_v4_small_4x4 \ + 05_compare_v4_architectures \ + 06_compare_v3_v4_one_head +do + papermill \ + "notebooks/${nb}.ipynb" \ + "notebooks/${nb}.out.ipynb" \ + -k python3 +done +``` + +The architecture-comparison notebook must run after both three-optimizer suites +are complete. + +## Validation + +```bash +PYTHONPATH=../nanogpt_one_head/src \ + python -m pytest -q tests +``` diff --git a/baseline/ngb/configs/v4_one_head.yaml b/baseline/ngb/configs/v4_one_head.yaml new file mode 100644 index 0000000..3ac9d27 --- /dev/null +++ b/baseline/ngb/configs/v4_one_head.yaml @@ -0,0 +1,105 @@ +protocol: + name: ngb_v4_one_head_fineweb + version: 4 + storage_root: /tmp/rg-ngb + data_root: /tmp/rg-nanogpt-one-head/data + run_slug: v4_one_head + description: Tuned two-epoch one-block, one-head nanoGPT baselines on pinned FineWeb-Edu. + +dataset: + name: HuggingFaceFW/fineweb-edu + config: sample-10BT + split: train + revision: 593b3a867298afb8ce42625a270ef20ddcad28f9 + tokenizer: gpt2 + train_tokens: 80000000 + val_tokens: 1000000 + test_tokens: 1000000 + +model: + vocab_size: 50257 + block_size: 256 + n_layer: 1 + n_head: 1 + n_embd: 128 + dropout: 0.0 + bias: false + tie_weights: true + +training: + seeds: [1337, 2027, 4099] + batch_size: 4 + grad_accum_steps: 8 + target_epochs: 2.0 + epoch_interval: 0.25 + eval_interval_steps: 500 + eval_batches: 64 + checkpoint_interval_steps: 500 + grad_clip: 1.0 + +optimizer_profiles: + sgd_momentum: + display_name: SGD + Nesterov momentum, v4 tuned + family: sgd + learning_rate: 0.05 + min_learning_rate: 0.0005 + warmup_fraction: 0.05 + schedule: warmup_cosine + momentum: 0.90 + dampening: 0.0 + nesterov: true + weight_decay: 0.01 + + adamw: + display_name: AdamW, v4 tuned + family: adamw + learning_rate: 0.0003 + min_learning_rate: 0.00001 + warmup_fraction: 0.025 + schedule: warmup_cosine + beta1: 0.90 + beta2: 0.95 + epsilon: 1.0e-8 + weight_decay: 0.10 + + muon: + display_name: Muon + auxiliary AdamW, v4 tuned + family: muon + matrix_learning_rate: 0.01 + matrix_min_learning_rate: 0.0001 + aux_learning_rate: 0.0002 + aux_min_learning_rate: 0.00001 + warmup_fraction: 0.025 + schedule: warmup_cosine + momentum: 0.95 + nesterov: true + newton_schulz_steps: 5 + muon_epsilon: 1.0e-7 + matrix_weight_decay: 0.02 + beta1: 0.90 + beta2: 0.95 + epsilon: 1.0e-8 + aux_weight_decay: 0.10 + +evaluation: + train_probe_seed: 21001 + validation_probe_seed: 22001 + test_probe_seed: 23001 + bleu_probe_seed: 24001 + bleu_examples: 64 + bleu_prompt_tokens: 64 + bleu_continuation_tokens: 32 + bleu_batch_size: 4 + +weightwatcher: + enabled: true + ERG: true + randomize: true + strict: true + min_evals: 20 + +runtime: + matmul_precision: high + mps_fallback: true + deterministic_algorithms: false + empty_mps_cache_after_weightwatcher: true diff --git a/baseline/ngb/configs/v4_small_4x4.yaml b/baseline/ngb/configs/v4_small_4x4.yaml new file mode 100644 index 0000000..b192149 --- /dev/null +++ b/baseline/ngb/configs/v4_small_4x4.yaml @@ -0,0 +1,105 @@ +protocol: + name: ngb_v4_small_4x4_fineweb + version: 4 + storage_root: /tmp/rg-ngb + data_root: /tmp/rg-nanogpt-one-head/data + run_slug: v4_small_4x4 + description: Tuned two-epoch four-block, four-head small nanoGPT baselines on pinned FineWeb-Edu. + +dataset: + name: HuggingFaceFW/fineweb-edu + config: sample-10BT + split: train + revision: 593b3a867298afb8ce42625a270ef20ddcad28f9 + tokenizer: gpt2 + train_tokens: 80000000 + val_tokens: 1000000 + test_tokens: 1000000 + +model: + vocab_size: 50257 + block_size: 256 + n_layer: 4 + n_head: 4 + n_embd: 128 + dropout: 0.0 + bias: false + tie_weights: true + +training: + seeds: [1337, 2027, 4099] + batch_size: 4 + grad_accum_steps: 8 + target_epochs: 2.0 + epoch_interval: 0.25 + eval_interval_steps: 500 + eval_batches: 64 + checkpoint_interval_steps: 500 + grad_clip: 1.0 + +optimizer_profiles: + sgd_momentum: + display_name: SGD + Nesterov momentum, v4 tuned + family: sgd + learning_rate: 0.03 + min_learning_rate: 0.0003 + warmup_fraction: 0.05 + schedule: warmup_cosine + momentum: 0.90 + dampening: 0.0 + nesterov: true + weight_decay: 0.01 + + adamw: + display_name: AdamW, v4 tuned + family: adamw + learning_rate: 0.0003 + min_learning_rate: 0.00001 + warmup_fraction: 0.025 + schedule: warmup_cosine + beta1: 0.90 + beta2: 0.95 + epsilon: 1.0e-8 + weight_decay: 0.10 + + muon: + display_name: Muon + auxiliary AdamW, v4 tuned + family: muon + matrix_learning_rate: 0.01 + matrix_min_learning_rate: 0.0001 + aux_learning_rate: 0.0002 + aux_min_learning_rate: 0.00001 + warmup_fraction: 0.025 + schedule: warmup_cosine + momentum: 0.95 + nesterov: true + newton_schulz_steps: 5 + muon_epsilon: 1.0e-7 + matrix_weight_decay: 0.02 + beta1: 0.90 + beta2: 0.95 + epsilon: 1.0e-8 + aux_weight_decay: 0.10 + +evaluation: + train_probe_seed: 21001 + validation_probe_seed: 22001 + test_probe_seed: 23001 + bleu_probe_seed: 24001 + bleu_examples: 64 + bleu_prompt_tokens: 64 + bleu_continuation_tokens: 32 + bleu_batch_size: 4 + +weightwatcher: + enabled: true + ERG: true + randomize: true + strict: true + min_evals: 20 + +runtime: + matmul_precision: high + mps_fallback: true + deterministic_algorithms: false + empty_mps_cache_after_weightwatcher: true diff --git a/baseline/ngb/notebooks/01_run_v4_one_head.ipynb b/baseline/ngb/notebooks/01_run_v4_one_head.ipynb new file mode 100644 index 0000000..70b4930 --- /dev/null +++ b/baseline/ngb/notebooks/01_run_v4_one_head.ipynb @@ -0,0 +1,144 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bcb1ae85", + "metadata": {}, + "source": [ + "# NGB v4 one-head tuned two-epoch baselines\n", + "\n", + "Runs the separate v4 one-block/one-head SGD, AdamW, and Muon protocols.\n", + "\n", + "The test split is monitoring-only. Validation cross-entropy selects the best\n", + "checkpoint. Runtime data and results remain under explicit `/tmp` paths.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70af919e", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "CONFIG_PATH = \"configs/v4_one_head.yaml\"\n", + "OPTIMIZER = \"all\"\n", + "SEEDS = \"\"\n", + "DEVICE = \"auto\"\n", + "DATA_ROOT = \"/tmp/rg-nanogpt-one-head/data\"\n", + "NGB_STORAGE_ROOT = \"/tmp/rg-ngb\"\n", + "FORCE_DATA = False\n", + "OVERWRITE = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4dfe4a1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os\n", + "import sys\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "candidates = [cwd, cwd.parent, cwd / \"baseline\" / \"ngb\"]\n", + "NGB_ROOT_DIR = next(\n", + " (path for path in candidates if (path / \"configs\" / \"v4_one_head.yaml\").is_file()),\n", + " None,\n", + ")\n", + "if NGB_ROOT_DIR is None:\n", + " raise FileNotFoundError(\"Run from baseline/ngb or the repository root\")\n", + "RUNTIME_SRC = NGB_ROOT_DIR.parent / \"nanogpt_one_head\" / \"src\"\n", + "if str(RUNTIME_SRC) not in sys.path:\n", + " sys.path.insert(0, str(RUNTIME_SRC))\n", + "\n", + "from rg_nanogpt_one_head import (\n", + " SUPPORTED_OPTIMIZERS,\n", + " canonical_seeds,\n", + " load_config,\n", + " prepare_fineweb_edu,\n", + " run_all_replicates,\n", + " run_optimizer_replicates,\n", + " run_slug,\n", + " run_status_table,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f324a98", + "metadata": {}, + "outputs": [], + "source": [ + "config_path = (NGB_ROOT_DIR / CONFIG_PATH).resolve()\n", + "cfg = load_config(config_path)\n", + "selected_seeds = (\n", + " tuple(int(value.strip()) for value in SEEDS.split(\",\") if value.strip())\n", + " if SEEDS.strip()\n", + " else canonical_seeds(cfg)\n", + ")\n", + "if not selected_seeds or len(set(selected_seeds)) != len(selected_seeds):\n", + " raise ValueError(\"SEEDS must contain unique integers\")\n", + "if OPTIMIZER != \"all\" and OPTIMIZER not in SUPPORTED_OPTIMIZERS:\n", + " raise ValueError(f\"OPTIMIZER must be all or one of {SUPPORTED_OPTIMIZERS}\")\n", + "\n", + "data_root = Path(DATA_ROOT)\n", + "results_root = Path(NGB_STORAGE_ROOT) / \"results\" / run_slug(cfg)\n", + "plots_root = Path(NGB_STORAGE_ROOT) / \"plots\" / run_slug(cfg)\n", + "for directory in (data_root, results_root, plots_root):\n", + " directory.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(\"config:\", config_path)\n", + "print(\"data:\", data_root)\n", + "print(\"results:\", results_root)\n", + "print(\"seeds:\", selected_seeds)\n", + "display(pd.DataFrame([cfg[\"model\"]]))\n", + "display(pd.DataFrame.from_dict(cfg[\"optimizer_profiles\"], orient=\"index\"))\n", + "\n", + "prepare_fineweb_edu(cfg, data_root, force=FORCE_DATA)\n", + "common = dict(\n", + " cfg=cfg,\n", + " config_path=config_path,\n", + " seeds=selected_seeds,\n", + " data_root=data_root,\n", + " results_root=results_root,\n", + " device=DEVICE,\n", + " resume=not OVERWRITE,\n", + " overwrite=OVERWRITE,\n", + " progress=True,\n", + ")\n", + "if OPTIMIZER == \"all\":\n", + " run_dirs = run_all_replicates(**common)\n", + "else:\n", + " run_dirs = run_optimizer_replicates(\n", + " optimizer_name=OPTIMIZER,\n", + " prepare_data=False,\n", + " **common,\n", + " )\n", + "print(\"run directories:\", len(run_dirs))\n", + "display(run_status_table(results_root, optimizers=SUPPORTED_OPTIMIZERS, seeds=selected_seeds))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (active conda environment)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/ngb/notebooks/02_compare_v4_one_head.ipynb b/baseline/ngb/notebooks/02_compare_v4_one_head.ipynb new file mode 100644 index 0000000..4df0e18 --- /dev/null +++ b/baseline/ngb/notebooks/02_compare_v4_one_head.ipynb @@ -0,0 +1,244 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c74edd06", + "metadata": {}, + "source": [ + "# NGB v4 one-head comparison\n", + "\n", + "This notebook discovers the intersection of **completed matched seeds** across\n", + "SGD, AdamW, and Muon. It reports final and validation-selected metrics, paired\n", + "optimizer contrasts, late-horizon drift, corrected perplexity intervals, and\n", + "block-resolved WeightWatcher trajectories. Blocks are never statistical\n", + "replicates.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5f10d3d", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "CONFIG_PATH = \"configs/v4_one_head.yaml\"\n", + "SEEDS = \"\"\n", + "NGB_STORAGE_ROOT = \"/tmp/rg-ngb\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ebbf2b6", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os\n", + "import sys\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "candidates = [cwd, cwd.parent, cwd / \"baseline\" / \"ngb\"]\n", + "NGB_ROOT_DIR = next(\n", + " (path for path in candidates if (path / \"configs\" / \"v4_one_head.yaml\").is_file()),\n", + " None,\n", + ")\n", + "if NGB_ROOT_DIR is None:\n", + " raise FileNotFoundError(\"Run from baseline/ngb or the repository root\")\n", + "RUNTIME_SRC = NGB_ROOT_DIR.parent / \"nanogpt_one_head\" / \"src\"\n", + "if str(RUNTIME_SRC) not in sys.path:\n", + " sys.path.insert(0, str(RUNTIME_SRC))\n", + "\n", + "import json\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "from rg_nanogpt_one_head import (\n", + " OPTIMIZER_COLORS,\n", + " OPTIMIZER_LABELS,\n", + " SUPPORTED_OPTIMIZERS,\n", + " discover_matched_complete_seeds,\n", + " final_test_summary,\n", + " load_config,\n", + " load_epoch_metrics,\n", + " load_layer_metrics,\n", + " load_metrics,\n", + " load_spectral_summary,\n", + " load_test_results,\n", + " paired_test_differences,\n", + " plot_epoch_metric,\n", + " plot_layer_metric,\n", + " plot_spectral_optimizer_summary,\n", + " run_diagnostics_table,\n", + " run_slug,\n", + " run_status_table,\n", + " summarize_run_diagnostics,\n", + ")\n", + "\n", + "pd.set_option(\"display.max_rows\", None)\n", + "pd.set_option(\"display.max_columns\", None)\n", + "pd.set_option(\"display.width\", None)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fff10377", + "metadata": {}, + "outputs": [], + "source": [ + "config_path = (NGB_ROOT_DIR / CONFIG_PATH).resolve()\n", + "cfg = load_config(config_path)\n", + "results_root = Path(NGB_STORAGE_ROOT) / \"results\" / run_slug(cfg)\n", + "plot_root = Path(NGB_STORAGE_ROOT) / \"plots\" / run_slug(cfg) / \"comparison\"\n", + "plot_root.mkdir(parents=True, exist_ok=True)\n", + "optimizers = tuple(SUPPORTED_OPTIMIZERS)\n", + "seeds = (\n", + " tuple(int(value.strip()) for value in SEEDS.split(\",\") if value.strip())\n", + " if SEEDS.strip()\n", + " else discover_matched_complete_seeds(results_root, optimizers=optimizers)\n", + ")\n", + "if not seeds:\n", + " raise RuntimeError(f\"No complete matched optimizer seeds under {results_root}\")\n", + "print(\"config:\", config_path)\n", + "print(\"results:\", results_root)\n", + "print(\"matched complete seeds:\", seeds)\n", + "display(run_status_table(results_root, optimizers=optimizers, seeds=seeds))\n", + "\n", + "config_rows = []\n", + "for optimizer in optimizers:\n", + " config_rows.append({\n", + " \"optimizer\": optimizer,\n", + " \"optimizer_label\": OPTIMIZER_LABELS[optimizer],\n", + " **cfg[\"optimizer_profiles\"][optimizer],\n", + " })\n", + "config_table = pd.DataFrame(config_rows)\n", + "display(pd.DataFrame([{\"run_slug\": run_slug(cfg), **cfg[\"model\"], **cfg[\"training\"]}]))\n", + "display(config_table)\n", + "config_table.to_csv(plot_root / \"optimizer_configurations.csv\", index=False)\n", + "\n", + "metrics = load_metrics(results_root, optimizers=optimizers, seeds=seeds)\n", + "epoch_metrics = load_epoch_metrics(results_root, optimizers=optimizers, seeds=seeds)\n", + "layer_metrics = load_layer_metrics(results_root, optimizers=optimizers, seeds=seeds)\n", + "spectral_summary = load_spectral_summary(results_root, optimizers=optimizers, seeds=seeds)\n", + "test_results = load_test_results(results_root, optimizers=optimizers, seeds=seeds)\n", + "print(\"rows:\", {\"metrics\": len(metrics), \"epoch\": len(epoch_metrics), \"layers\": len(layer_metrics)})\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "141d8115", + "metadata": {}, + "outputs": [], + "source": [ + "diagnostics = run_diagnostics_table(metrics, test_results)\n", + "diagnostic_summary = summarize_run_diagnostics(diagnostics)\n", + "summary = final_test_summary(test_results)\n", + "paired = paired_test_differences(test_results, optimizers=optimizers)\n", + "\n", + "for frame, name in (\n", + " (diagnostics, \"run_diagnostics.csv\"),\n", + " (diagnostic_summary, \"run_diagnostics_summary_95ci.csv\"),\n", + " (summary, \"final_and_validation_selected_95ci.csv\"),\n", + " (paired, \"paired_optimizer_differences_95ci.csv\"),\n", + "):\n", + " frame.to_csv(plot_root / name, index=False)\n", + "\n", + "display(diagnostics.sort_values([\"optimizer\", \"seed\"]))\n", + "display(diagnostic_summary.sort_values([\"metric\", \"optimizer\"]))\n", + "display(summary.sort_values([\"checkpoint\", \"metric\", \"optimizer\"]))\n", + "display(paired.sort_values([\"checkpoint\", \"metric\", \"contrast\"]))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9dbb7edc", + "metadata": {}, + "outputs": [], + "source": [ + "for metric in (\n", + " \"train_loss\", \"val_loss\", \"test_loss\",\n", + " \"train_accuracy\", \"val_accuracy\", \"test_accuracy\",\n", + " \"val_generalization_gap\", \"test_generalization_gap\",\n", + " \"weight_norm\", \"update_to_weight_ratio\",\n", + "):\n", + " if metric not in epoch_metrics.columns:\n", + " continue\n", + " plot_epoch_metric(\n", + " epoch_metrics,\n", + " metric=metric,\n", + " optimizers=optimizers,\n", + " title=f\"{run_slug(cfg)}: {metric} (95% Student-t CI)\",\n", + " output=plot_root / f\"{metric}.png\",\n", + " )\n", + " plt.show()\n", + "\n", + "# Late-horizon zooms make drift after the first half epoch explicit.\n", + "late = epoch_metrics[epoch_metrics[\"nominal_epoch\"].astype(float) >= 0.5]\n", + "for metric in (\"val_loss\", \"test_loss\", \"val_accuracy\", \"test_accuracy\"):\n", + " plot_epoch_metric(\n", + " late,\n", + " metric=metric,\n", + " optimizers=optimizers,\n", + " title=f\"{run_slug(cfg)}: {metric}, epoch >= 0.5\",\n", + " output=plot_root / f\"zoom_epoch_0p5_{metric}.png\",\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d97a0e5a", + "metadata": {}, + "outputs": [], + "source": [ + "for metric in (\"alpha_median\", \"ERG_gap_median\", \"num_traps_mean\"):\n", + " plot_spectral_optimizer_summary(\n", + " spectral_summary,\n", + " metric=metric,\n", + " optimizers=optimizers,\n", + " output=plot_root / f\"spectral_{metric}.png\",\n", + " )\n", + " if metric == \"alpha_median\":\n", + " plt.axhline(2.0, color=\"black\", linestyle=\"--\", linewidth=1.0)\n", + " if metric == \"ERG_gap_median\":\n", + " plt.axhline(0.0, color=\"black\", linestyle=\"--\", linewidth=1.0)\n", + " plt.show()\n", + "\n", + "for optimizer in optimizers:\n", + " for metric in (\"alpha\", \"ERG_gap\", \"num_traps\"):\n", + " plot_layer_metric(\n", + " layer_metrics,\n", + " optimizer=optimizer,\n", + " metric=metric,\n", + " title=f\"{run_slug(cfg)} {OPTIMIZER_LABELS[optimizer]}: block-resolved {metric}\",\n", + " output=plot_root / f\"{optimizer}_block_resolved_{metric}.png\",\n", + " )\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (active conda environment)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/ngb/notebooks/03_run_v4_small_4x4.ipynb b/baseline/ngb/notebooks/03_run_v4_small_4x4.ipynb new file mode 100644 index 0000000..2f16653 --- /dev/null +++ b/baseline/ngb/notebooks/03_run_v4_small_4x4.ipynb @@ -0,0 +1,144 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e0575451", + "metadata": {}, + "source": [ + "# NGB v4 small 4×4 tuned two-epoch baselines\n", + "\n", + "Runs the distinct four-block/four-head small language-model architecture.\n", + "\n", + "The test split is monitoring-only. Validation cross-entropy selects the best\n", + "checkpoint. Runtime data and results remain under explicit `/tmp` paths.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e10856af", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "CONFIG_PATH = \"configs/v4_small_4x4.yaml\"\n", + "OPTIMIZER = \"all\"\n", + "SEEDS = \"\"\n", + "DEVICE = \"auto\"\n", + "DATA_ROOT = \"/tmp/rg-nanogpt-one-head/data\"\n", + "NGB_STORAGE_ROOT = \"/tmp/rg-ngb\"\n", + "FORCE_DATA = False\n", + "OVERWRITE = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c31be70", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os\n", + "import sys\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "candidates = [cwd, cwd.parent, cwd / \"baseline\" / \"ngb\"]\n", + "NGB_ROOT_DIR = next(\n", + " (path for path in candidates if (path / \"configs\" / \"v4_one_head.yaml\").is_file()),\n", + " None,\n", + ")\n", + "if NGB_ROOT_DIR is None:\n", + " raise FileNotFoundError(\"Run from baseline/ngb or the repository root\")\n", + "RUNTIME_SRC = NGB_ROOT_DIR.parent / \"nanogpt_one_head\" / \"src\"\n", + "if str(RUNTIME_SRC) not in sys.path:\n", + " sys.path.insert(0, str(RUNTIME_SRC))\n", + "\n", + "from rg_nanogpt_one_head import (\n", + " SUPPORTED_OPTIMIZERS,\n", + " canonical_seeds,\n", + " load_config,\n", + " prepare_fineweb_edu,\n", + " run_all_replicates,\n", + " run_optimizer_replicates,\n", + " run_slug,\n", + " run_status_table,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17ac523b", + "metadata": {}, + "outputs": [], + "source": [ + "config_path = (NGB_ROOT_DIR / CONFIG_PATH).resolve()\n", + "cfg = load_config(config_path)\n", + "selected_seeds = (\n", + " tuple(int(value.strip()) for value in SEEDS.split(\",\") if value.strip())\n", + " if SEEDS.strip()\n", + " else canonical_seeds(cfg)\n", + ")\n", + "if not selected_seeds or len(set(selected_seeds)) != len(selected_seeds):\n", + " raise ValueError(\"SEEDS must contain unique integers\")\n", + "if OPTIMIZER != \"all\" and OPTIMIZER not in SUPPORTED_OPTIMIZERS:\n", + " raise ValueError(f\"OPTIMIZER must be all or one of {SUPPORTED_OPTIMIZERS}\")\n", + "\n", + "data_root = Path(DATA_ROOT)\n", + "results_root = Path(NGB_STORAGE_ROOT) / \"results\" / run_slug(cfg)\n", + "plots_root = Path(NGB_STORAGE_ROOT) / \"plots\" / run_slug(cfg)\n", + "for directory in (data_root, results_root, plots_root):\n", + " directory.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(\"config:\", config_path)\n", + "print(\"data:\", data_root)\n", + "print(\"results:\", results_root)\n", + "print(\"seeds:\", selected_seeds)\n", + "display(pd.DataFrame([cfg[\"model\"]]))\n", + "display(pd.DataFrame.from_dict(cfg[\"optimizer_profiles\"], orient=\"index\"))\n", + "\n", + "prepare_fineweb_edu(cfg, data_root, force=FORCE_DATA)\n", + "common = dict(\n", + " cfg=cfg,\n", + " config_path=config_path,\n", + " seeds=selected_seeds,\n", + " data_root=data_root,\n", + " results_root=results_root,\n", + " device=DEVICE,\n", + " resume=not OVERWRITE,\n", + " overwrite=OVERWRITE,\n", + " progress=True,\n", + ")\n", + "if OPTIMIZER == \"all\":\n", + " run_dirs = run_all_replicates(**common)\n", + "else:\n", + " run_dirs = run_optimizer_replicates(\n", + " optimizer_name=OPTIMIZER,\n", + " prepare_data=False,\n", + " **common,\n", + " )\n", + "print(\"run directories:\", len(run_dirs))\n", + "display(run_status_table(results_root, optimizers=SUPPORTED_OPTIMIZERS, seeds=selected_seeds))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (active conda environment)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/ngb/notebooks/04_compare_v4_small_4x4.ipynb b/baseline/ngb/notebooks/04_compare_v4_small_4x4.ipynb new file mode 100644 index 0000000..5c9cb9e --- /dev/null +++ b/baseline/ngb/notebooks/04_compare_v4_small_4x4.ipynb @@ -0,0 +1,244 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "05c6b327", + "metadata": {}, + "source": [ + "# NGB v4 small 4×4 comparison\n", + "\n", + "This notebook discovers the intersection of **completed matched seeds** across\n", + "SGD, AdamW, and Muon. It reports final and validation-selected metrics, paired\n", + "optimizer contrasts, late-horizon drift, corrected perplexity intervals, and\n", + "block-resolved WeightWatcher trajectories. Blocks are never statistical\n", + "replicates.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b874a5c", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "CONFIG_PATH = \"configs/v4_small_4x4.yaml\"\n", + "SEEDS = \"\"\n", + "NGB_STORAGE_ROOT = \"/tmp/rg-ngb\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7f868dd", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os\n", + "import sys\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "candidates = [cwd, cwd.parent, cwd / \"baseline\" / \"ngb\"]\n", + "NGB_ROOT_DIR = next(\n", + " (path for path in candidates if (path / \"configs\" / \"v4_one_head.yaml\").is_file()),\n", + " None,\n", + ")\n", + "if NGB_ROOT_DIR is None:\n", + " raise FileNotFoundError(\"Run from baseline/ngb or the repository root\")\n", + "RUNTIME_SRC = NGB_ROOT_DIR.parent / \"nanogpt_one_head\" / \"src\"\n", + "if str(RUNTIME_SRC) not in sys.path:\n", + " sys.path.insert(0, str(RUNTIME_SRC))\n", + "\n", + "import json\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "from rg_nanogpt_one_head import (\n", + " OPTIMIZER_COLORS,\n", + " OPTIMIZER_LABELS,\n", + " SUPPORTED_OPTIMIZERS,\n", + " discover_matched_complete_seeds,\n", + " final_test_summary,\n", + " load_config,\n", + " load_epoch_metrics,\n", + " load_layer_metrics,\n", + " load_metrics,\n", + " load_spectral_summary,\n", + " load_test_results,\n", + " paired_test_differences,\n", + " plot_epoch_metric,\n", + " plot_layer_metric,\n", + " plot_spectral_optimizer_summary,\n", + " run_diagnostics_table,\n", + " run_slug,\n", + " run_status_table,\n", + " summarize_run_diagnostics,\n", + ")\n", + "\n", + "pd.set_option(\"display.max_rows\", None)\n", + "pd.set_option(\"display.max_columns\", None)\n", + "pd.set_option(\"display.width\", None)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfa2cb61", + "metadata": {}, + "outputs": [], + "source": [ + "config_path = (NGB_ROOT_DIR / CONFIG_PATH).resolve()\n", + "cfg = load_config(config_path)\n", + "results_root = Path(NGB_STORAGE_ROOT) / \"results\" / run_slug(cfg)\n", + "plot_root = Path(NGB_STORAGE_ROOT) / \"plots\" / run_slug(cfg) / \"comparison\"\n", + "plot_root.mkdir(parents=True, exist_ok=True)\n", + "optimizers = tuple(SUPPORTED_OPTIMIZERS)\n", + "seeds = (\n", + " tuple(int(value.strip()) for value in SEEDS.split(\",\") if value.strip())\n", + " if SEEDS.strip()\n", + " else discover_matched_complete_seeds(results_root, optimizers=optimizers)\n", + ")\n", + "if not seeds:\n", + " raise RuntimeError(f\"No complete matched optimizer seeds under {results_root}\")\n", + "print(\"config:\", config_path)\n", + "print(\"results:\", results_root)\n", + "print(\"matched complete seeds:\", seeds)\n", + "display(run_status_table(results_root, optimizers=optimizers, seeds=seeds))\n", + "\n", + "config_rows = []\n", + "for optimizer in optimizers:\n", + " config_rows.append({\n", + " \"optimizer\": optimizer,\n", + " \"optimizer_label\": OPTIMIZER_LABELS[optimizer],\n", + " **cfg[\"optimizer_profiles\"][optimizer],\n", + " })\n", + "config_table = pd.DataFrame(config_rows)\n", + "display(pd.DataFrame([{\"run_slug\": run_slug(cfg), **cfg[\"model\"], **cfg[\"training\"]}]))\n", + "display(config_table)\n", + "config_table.to_csv(plot_root / \"optimizer_configurations.csv\", index=False)\n", + "\n", + "metrics = load_metrics(results_root, optimizers=optimizers, seeds=seeds)\n", + "epoch_metrics = load_epoch_metrics(results_root, optimizers=optimizers, seeds=seeds)\n", + "layer_metrics = load_layer_metrics(results_root, optimizers=optimizers, seeds=seeds)\n", + "spectral_summary = load_spectral_summary(results_root, optimizers=optimizers, seeds=seeds)\n", + "test_results = load_test_results(results_root, optimizers=optimizers, seeds=seeds)\n", + "print(\"rows:\", {\"metrics\": len(metrics), \"epoch\": len(epoch_metrics), \"layers\": len(layer_metrics)})\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06019c57", + "metadata": {}, + "outputs": [], + "source": [ + "diagnostics = run_diagnostics_table(metrics, test_results)\n", + "diagnostic_summary = summarize_run_diagnostics(diagnostics)\n", + "summary = final_test_summary(test_results)\n", + "paired = paired_test_differences(test_results, optimizers=optimizers)\n", + "\n", + "for frame, name in (\n", + " (diagnostics, \"run_diagnostics.csv\"),\n", + " (diagnostic_summary, \"run_diagnostics_summary_95ci.csv\"),\n", + " (summary, \"final_and_validation_selected_95ci.csv\"),\n", + " (paired, \"paired_optimizer_differences_95ci.csv\"),\n", + "):\n", + " frame.to_csv(plot_root / name, index=False)\n", + "\n", + "display(diagnostics.sort_values([\"optimizer\", \"seed\"]))\n", + "display(diagnostic_summary.sort_values([\"metric\", \"optimizer\"]))\n", + "display(summary.sort_values([\"checkpoint\", \"metric\", \"optimizer\"]))\n", + "display(paired.sort_values([\"checkpoint\", \"metric\", \"contrast\"]))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac067bb4", + "metadata": {}, + "outputs": [], + "source": [ + "for metric in (\n", + " \"train_loss\", \"val_loss\", \"test_loss\",\n", + " \"train_accuracy\", \"val_accuracy\", \"test_accuracy\",\n", + " \"val_generalization_gap\", \"test_generalization_gap\",\n", + " \"weight_norm\", \"update_to_weight_ratio\",\n", + "):\n", + " if metric not in epoch_metrics.columns:\n", + " continue\n", + " plot_epoch_metric(\n", + " epoch_metrics,\n", + " metric=metric,\n", + " optimizers=optimizers,\n", + " title=f\"{run_slug(cfg)}: {metric} (95% Student-t CI)\",\n", + " output=plot_root / f\"{metric}.png\",\n", + " )\n", + " plt.show()\n", + "\n", + "# Late-horizon zooms make drift after the first half epoch explicit.\n", + "late = epoch_metrics[epoch_metrics[\"nominal_epoch\"].astype(float) >= 0.5]\n", + "for metric in (\"val_loss\", \"test_loss\", \"val_accuracy\", \"test_accuracy\"):\n", + " plot_epoch_metric(\n", + " late,\n", + " metric=metric,\n", + " optimizers=optimizers,\n", + " title=f\"{run_slug(cfg)}: {metric}, epoch >= 0.5\",\n", + " output=plot_root / f\"zoom_epoch_0p5_{metric}.png\",\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6fb1b8c", + "metadata": {}, + "outputs": [], + "source": [ + "for metric in (\"alpha_median\", \"ERG_gap_median\", \"num_traps_mean\"):\n", + " plot_spectral_optimizer_summary(\n", + " spectral_summary,\n", + " metric=metric,\n", + " optimizers=optimizers,\n", + " output=plot_root / f\"spectral_{metric}.png\",\n", + " )\n", + " if metric == \"alpha_median\":\n", + " plt.axhline(2.0, color=\"black\", linestyle=\"--\", linewidth=1.0)\n", + " if metric == \"ERG_gap_median\":\n", + " plt.axhline(0.0, color=\"black\", linestyle=\"--\", linewidth=1.0)\n", + " plt.show()\n", + "\n", + "for optimizer in optimizers:\n", + " for metric in (\"alpha\", \"ERG_gap\", \"num_traps\"):\n", + " plot_layer_metric(\n", + " layer_metrics,\n", + " optimizer=optimizer,\n", + " metric=metric,\n", + " title=f\"{run_slug(cfg)} {OPTIMIZER_LABELS[optimizer]}: block-resolved {metric}\",\n", + " output=plot_root / f\"{optimizer}_block_resolved_{metric}.png\",\n", + " )\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (active conda environment)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/ngb/notebooks/05_compare_v4_architectures.ipynb b/baseline/ngb/notebooks/05_compare_v4_architectures.ipynb new file mode 100644 index 0000000..85c7441 --- /dev/null +++ b/baseline/ngb/notebooks/05_compare_v4_architectures.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "91ee97ee", + "metadata": {}, + "source": [ + "# NGB v4 architecture comparison\n", + "\n", + "Compares the one-block/one-head control with the distinct four-block/four-head\n", + "small language model using the same matched seeds and optimizer recipes. The\n", + "paired tables use seed-matched architecture differences.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e835f666", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "ONE_HEAD_CONFIG = \"configs/v4_one_head.yaml\"\n", + "SMALL_4X4_CONFIG = \"configs/v4_small_4x4.yaml\"\n", + "SEEDS = \"\"\n", + "NGB_STORAGE_ROOT = \"/tmp/rg-ngb\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6261d540", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os\n", + "import sys\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "candidates = [cwd, cwd.parent, cwd / \"baseline\" / \"ngb\"]\n", + "NGB_ROOT_DIR = next(\n", + " (path for path in candidates if (path / \"configs\" / \"v4_one_head.yaml\").is_file()),\n", + " None,\n", + ")\n", + "if NGB_ROOT_DIR is None:\n", + " raise FileNotFoundError(\"Run from baseline/ngb or the repository root\")\n", + "RUNTIME_SRC = NGB_ROOT_DIR.parent / \"nanogpt_one_head\" / \"src\"\n", + "if str(RUNTIME_SRC) not in sys.path:\n", + " sys.path.insert(0, str(RUNTIME_SRC))\n", + "\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "from rg_nanogpt_one_head import (\n", + " OPTIMIZER_COLORS,\n", + " OPTIMIZER_LABELS,\n", + " SUPPORTED_OPTIMIZERS,\n", + " discover_matched_complete_seeds,\n", + " final_test_summary,\n", + " load_config,\n", + " load_epoch_metrics,\n", + " load_spectral_summary,\n", + " load_test_results,\n", + " mean_ci95,\n", + " run_slug,\n", + ")\n", + "from rg_nanogpt_one_head.analysis import summarize_by_epoch\n", + "pd.set_option(\"display.max_rows\", None)\n", + "pd.set_option(\"display.max_columns\", None)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6500b2a9", + "metadata": {}, + "outputs": [], + "source": [ + "configs = {\n", + " \"one_head\": load_config(NGB_ROOT_DIR / ONE_HEAD_CONFIG),\n", + " \"small_4x4\": load_config(NGB_ROOT_DIR / SMALL_4X4_CONFIG),\n", + "}\n", + "optimizers = tuple(SUPPORTED_OPTIMIZERS)\n", + "results_roots = {\n", + " architecture: Path(NGB_STORAGE_ROOT) / \"results\" / run_slug(cfg)\n", + " for architecture, cfg in configs.items()\n", + "}\n", + "available = {\n", + " architecture: set(discover_matched_complete_seeds(root, optimizers=optimizers))\n", + " for architecture, root in results_roots.items()\n", + "}\n", + "seeds = (\n", + " tuple(int(value.strip()) for value in SEEDS.split(\",\") if value.strip())\n", + " if SEEDS.strip()\n", + " else tuple(sorted(set.intersection(*available.values())))\n", + ")\n", + "if not seeds:\n", + " raise RuntimeError(f\"No complete seed intersection across architectures: {available}\")\n", + "print(\"matched architecture seeds:\", seeds)\n", + "print(\"result roots:\", results_roots)\n", + "\n", + "all_epoch = []\n", + "all_spectral = []\n", + "all_test = []\n", + "for architecture, root in results_roots.items():\n", + " epoch = load_epoch_metrics(root, optimizers=optimizers, seeds=seeds)\n", + " spectral = load_spectral_summary(root, optimizers=optimizers, seeds=seeds)\n", + " test = load_test_results(root, optimizers=optimizers, seeds=seeds)\n", + " for frame in (epoch, spectral, test):\n", + " frame.insert(0, \"architecture\", architecture)\n", + " all_epoch.append(epoch)\n", + " all_spectral.append(spectral)\n", + " all_test.append(test)\n", + "epoch_metrics = pd.concat(all_epoch, ignore_index=True)\n", + "spectral_summary = pd.concat(all_spectral, ignore_index=True)\n", + "test_results = pd.concat(all_test, ignore_index=True)\n", + "plot_root = Path(NGB_STORAGE_ROOT) / \"plots\" / \"v4_architecture_comparison\"\n", + "plot_root.mkdir(parents=True, exist_ok=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70b82ab1", + "metadata": {}, + "outputs": [], + "source": [ + "summary_frames = []\n", + "for architecture, frame in test_results.groupby(\"architecture\"):\n", + " part = final_test_summary(frame)\n", + " part.insert(0, \"architecture\", architecture)\n", + " summary_frames.append(part)\n", + "architecture_summary = pd.concat(summary_frames, ignore_index=True)\n", + "architecture_summary.to_csv(plot_root / \"architecture_summary_95ci.csv\", index=False)\n", + "display(architecture_summary.sort_values([\"checkpoint\", \"metric\", \"optimizer\", \"architecture\"]))\n", + "\n", + "rows = []\n", + "for optimizer in optimizers:\n", + " for checkpoint in (\"final\", \"validation_selected\"):\n", + " selected = test_results[\n", + " (test_results[\"optimizer\"] == optimizer)\n", + " & (test_results[\"checkpoint\"] == checkpoint)\n", + " ]\n", + " for metric in (\"test_loss\", \"test_accuracy\", \"test_bleu\"):\n", + " left = selected[selected[\"architecture\"] == \"small_4x4\"][[\"seed\", metric]].rename(columns={metric: \"small_4x4\"})\n", + " right = selected[selected[\"architecture\"] == \"one_head\"][[\"seed\", metric]].rename(columns={metric: \"one_head\"})\n", + " paired = left.merge(right, on=\"seed\", validate=\"one_to_one\")\n", + " stats = mean_ci95(paired[\"small_4x4\"] - paired[\"one_head\"])\n", + " rows.append({\n", + " \"optimizer\": optimizer,\n", + " \"optimizer_label\": OPTIMIZER_LABELS[optimizer],\n", + " \"checkpoint\": checkpoint,\n", + " \"metric\": metric,\n", + " \"contrast\": \"small_4x4 - one_head\",\n", + " **stats,\n", + " })\n", + "architecture_paired = pd.DataFrame(rows)\n", + "architecture_paired.to_csv(plot_root / \"paired_architecture_differences_95ci.csv\", index=False)\n", + "display(architecture_paired.sort_values([\"checkpoint\", \"metric\", \"optimizer\"]))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5304e160", + "metadata": {}, + "outputs": [], + "source": [ + "styles = {\"one_head\": \":\", \"small_4x4\": \"-\"}\n", + "for optimizer in optimizers:\n", + " for metric in (\"val_loss\", \"val_accuracy\", \"test_loss\", \"test_accuracy\"):\n", + " figure, axis = plt.subplots(figsize=(9, 5))\n", + " for architecture in (\"one_head\", \"small_4x4\"):\n", + " subset = epoch_metrics[\n", + " (epoch_metrics[\"architecture\"] == architecture)\n", + " & (epoch_metrics[\"optimizer\"] == optimizer)\n", + " ]\n", + " summary = summarize_by_epoch(subset, metric, x=\"nominal_epoch\", group=(\"architecture\", \"optimizer\"))\n", + " axis.plot(\n", + " summary[\"nominal_epoch\"], summary[\"mean\"],\n", + " color=OPTIMIZER_COLORS[optimizer], linestyle=styles[architecture],\n", + " linewidth=2.2, label=architecture,\n", + " )\n", + " axis.fill_between(\n", + " summary[\"nominal_epoch\"], summary[\"ci95_lower\"], summary[\"ci95_upper\"],\n", + " color=OPTIMIZER_COLORS[optimizer], alpha=0.10,\n", + " )\n", + " axis.set(xlabel=\"Corpus-equivalent epoch\", ylabel=metric, title=f\"{OPTIMIZER_LABELS[optimizer]}: one-head vs small 4x4\")\n", + " axis.grid(alpha=0.25)\n", + " axis.legend(frameon=False)\n", + " figure.tight_layout()\n", + " figure.savefig(plot_root / f\"{optimizer}_{metric}.png\", dpi=170, bbox_inches=\"tight\")\n", + " plt.show()\n", + "\n", + "for optimizer in optimizers:\n", + " figure, axis = plt.subplots(figsize=(9, 5))\n", + " for architecture in (\"one_head\", \"small_4x4\"):\n", + " subset = spectral_summary[\n", + " (spectral_summary[\"architecture\"] == architecture)\n", + " & (spectral_summary[\"optimizer\"] == optimizer)\n", + " ]\n", + " summary = summarize_by_epoch(subset, \"alpha_median\", x=\"epoch\", group=(\"architecture\", \"optimizer\"))\n", + " axis.plot(summary[\"epoch\"], summary[\"mean\"], color=OPTIMIZER_COLORS[optimizer], linestyle=styles[architecture], linewidth=2.2, label=architecture)\n", + " axis.fill_between(summary[\"epoch\"], summary[\"ci95_lower\"], summary[\"ci95_upper\"], color=OPTIMIZER_COLORS[optimizer], alpha=0.10)\n", + " axis.axhline(2.0, color=\"black\", linestyle=\"--\", linewidth=1.0, label=\"alpha = 2\")\n", + " axis.set(xlabel=\"Corpus-equivalent epoch\", ylabel=\"Median alpha\", title=f\"{OPTIMIZER_LABELS[optimizer]}: architecture spectral comparison\")\n", + " axis.grid(alpha=0.25)\n", + " axis.legend(frameon=False)\n", + " figure.tight_layout()\n", + " figure.savefig(plot_root / f\"{optimizer}_alpha_median.png\", dpi=170, bbox_inches=\"tight\")\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (active conda environment)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/ngb/notebooks/06_compare_v3_v4_one_head.ipynb b/baseline/ngb/notebooks/06_compare_v3_v4_one_head.ipynb new file mode 100644 index 0000000..e131f46 --- /dev/null +++ b/baseline/ngb/notebooks/06_compare_v3_v4_one_head.ipynb @@ -0,0 +1,180 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d3823a8c", + "metadata": {}, + "source": [ + "# NGB v3 versus v4 one-head comparison\n", + "\n", + "This notebook compares the checked-in one-epoch v3 protocol with the separate\n", + "tuned two-epoch v4 protocol. Differences are paired by model seed. The result is\n", + "an overall protocol comparison—horizon and optimizer schedule both changed.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2a20c2b", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "V3_RESULTS_ROOT = \"/tmp/rg-nanogpt-one-head/results\"\n", + "V4_CONFIG = \"configs/v4_one_head.yaml\"\n", + "NGB_STORAGE_ROOT = \"/tmp/rg-ngb\"\n", + "SEEDS = \"\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b45c00e", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "candidates = [cwd, cwd.parent, cwd / \"baseline\" / \"ngb\"]\n", + "NGB_ROOT_DIR = next((p for p in candidates if (p / \"configs\" / \"v4_one_head.yaml\").is_file()), None)\n", + "if NGB_ROOT_DIR is None:\n", + " raise FileNotFoundError(\"Run from baseline/ngb or the repository root\")\n", + "RUNTIME_SRC = NGB_ROOT_DIR.parent / \"nanogpt_one_head\" / \"src\"\n", + "if str(RUNTIME_SRC) not in sys.path:\n", + " sys.path.insert(0, str(RUNTIME_SRC))\n", + "\n", + "from rg_nanogpt_one_head import (\n", + " OPTIMIZER_COLORS, OPTIMIZER_LABELS, SUPPORTED_OPTIMIZERS,\n", + " discover_matched_complete_seeds, final_test_summary, load_config,\n", + " load_epoch_metrics, load_spectral_summary, load_test_results, mean_ci95,\n", + " run_slug,\n", + ")\n", + "from rg_nanogpt_one_head.analysis import summarize_by_epoch\n", + "pd.set_option(\"display.max_rows\", None)\n", + "pd.set_option(\"display.max_columns\", None)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "941b7dc3", + "metadata": {}, + "outputs": [], + "source": [ + "v4_cfg = load_config(NGB_ROOT_DIR / V4_CONFIG)\n", + "roots = {\n", + " \"v3\": Path(V3_RESULTS_ROOT),\n", + " \"v4\": Path(NGB_STORAGE_ROOT) / \"results\" / run_slug(v4_cfg),\n", + "}\n", + "optimizers = tuple(SUPPORTED_OPTIMIZERS)\n", + "available = {name: set(discover_matched_complete_seeds(root, optimizers=optimizers)) for name, root in roots.items()}\n", + "seeds = (\n", + " tuple(int(v.strip()) for v in SEEDS.split(\",\") if v.strip())\n", + " if SEEDS.strip()\n", + " else tuple(sorted(set.intersection(*available.values())))\n", + ")\n", + "if not seeds:\n", + " raise RuntimeError(f\"No matched v3/v4 complete seeds: {available}\")\n", + "print(\"matched v3/v4 seeds:\", seeds)\n", + "\n", + "frames_epoch=[]; frames_spectral=[]; frames_test=[]\n", + "for protocol, root in roots.items():\n", + " epoch=load_epoch_metrics(root, optimizers=optimizers, seeds=seeds)\n", + " spectral=load_spectral_summary(root, optimizers=optimizers, seeds=seeds)\n", + " test=load_test_results(root, optimizers=optimizers, seeds=seeds)\n", + " for frame in (epoch, spectral, test): frame.insert(0, \"protocol\", protocol)\n", + " frames_epoch.append(epoch); frames_spectral.append(spectral); frames_test.append(test)\n", + "epoch_metrics=pd.concat(frames_epoch, ignore_index=True)\n", + "spectral_summary=pd.concat(frames_spectral, ignore_index=True)\n", + "test_results=pd.concat(frames_test, ignore_index=True)\n", + "plot_root=Path(NGB_STORAGE_ROOT)/\"plots\"/\"v3_vs_v4_one_head\"\n", + "plot_root.mkdir(parents=True, exist_ok=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f73fbc93", + "metadata": {}, + "outputs": [], + "source": [ + "parts=[]\n", + "for protocol, frame in test_results.groupby(\"protocol\"):\n", + " part=final_test_summary(frame); part.insert(0,\"protocol\",protocol); parts.append(part)\n", + "summary=pd.concat(parts, ignore_index=True)\n", + "summary.to_csv(plot_root/\"v3_v4_summary_95ci.csv\", index=False)\n", + "display(summary.sort_values([\"checkpoint\",\"metric\",\"optimizer\",\"protocol\"]))\n", + "\n", + "rows=[]\n", + "for optimizer in optimizers:\n", + " for checkpoint in (\"final\",\"validation_selected\"):\n", + " selected=test_results[(test_results[\"optimizer\"]==optimizer)&(test_results[\"checkpoint\"]==checkpoint)]\n", + " for metric in (\"test_loss\",\"test_accuracy\",\"test_bleu\"):\n", + " v4=selected[selected[\"protocol\"]==\"v4\"][[\"seed\",metric]].rename(columns={metric:\"v4\"})\n", + " v3=selected[selected[\"protocol\"]==\"v3\"][[\"seed\",metric]].rename(columns={metric:\"v3\"})\n", + " paired=v4.merge(v3,on=\"seed\",validate=\"one_to_one\")\n", + " stats=mean_ci95(paired[\"v4\"]-paired[\"v3\"])\n", + " rows.append({\"optimizer\":optimizer,\"optimizer_label\":OPTIMIZER_LABELS[optimizer],\"checkpoint\":checkpoint,\"metric\":metric,\"contrast\":\"v4 - v3\",**stats})\n", + "paired=pd.DataFrame(rows)\n", + "paired.to_csv(plot_root/\"paired_v4_minus_v3_95ci.csv\",index=False)\n", + "display(paired.sort_values([\"checkpoint\",\"metric\",\"optimizer\"]))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25032465", + "metadata": {}, + "outputs": [], + "source": [ + "styles={\"v3\":\":\",\"v4\":\"-\"}\n", + "for optimizer in optimizers:\n", + " for metric in (\"val_loss\",\"val_accuracy\",\"test_loss\",\"test_accuracy\"):\n", + " fig,ax=plt.subplots(figsize=(9,5))\n", + " for protocol in (\"v3\",\"v4\"):\n", + " subset=epoch_metrics[(epoch_metrics[\"protocol\"]==protocol)&(epoch_metrics[\"optimizer\"]==optimizer)]\n", + " summary_curve=summarize_by_epoch(subset,metric,x=\"nominal_epoch\",group=(\"protocol\",\"optimizer\"))\n", + " ax.plot(summary_curve[\"nominal_epoch\"],summary_curve[\"mean\"],color=OPTIMIZER_COLORS[optimizer],linestyle=styles[protocol],linewidth=2.2,label=protocol)\n", + " ax.fill_between(summary_curve[\"nominal_epoch\"],summary_curve[\"ci95_lower\"],summary_curve[\"ci95_upper\"],color=OPTIMIZER_COLORS[optimizer],alpha=0.10)\n", + " ax.set(xlabel=\"Corpus-equivalent epoch\",ylabel=metric,title=f\"{OPTIMIZER_LABELS[optimizer]}: v3 versus v4\")\n", + " ax.grid(alpha=0.25); ax.legend(frameon=False); fig.tight_layout()\n", + " fig.savefig(plot_root/f\"{optimizer}_{metric}.png\",dpi=170,bbox_inches=\"tight\")\n", + " plt.show()\n", + "\n", + "for optimizer in optimizers:\n", + " fig,ax=plt.subplots(figsize=(9,5))\n", + " for protocol in (\"v3\",\"v4\"):\n", + " subset=spectral_summary[(spectral_summary[\"protocol\"]==protocol)&(spectral_summary[\"optimizer\"]==optimizer)]\n", + " summary_curve=summarize_by_epoch(subset,\"alpha_median\",x=\"epoch\",group=(\"protocol\",\"optimizer\"))\n", + " ax.plot(summary_curve[\"epoch\"],summary_curve[\"mean\"],color=OPTIMIZER_COLORS[optimizer],linestyle=styles[protocol],linewidth=2.2,label=protocol)\n", + " ax.fill_between(summary_curve[\"epoch\"],summary_curve[\"ci95_lower\"],summary_curve[\"ci95_upper\"],color=OPTIMIZER_COLORS[optimizer],alpha=0.10)\n", + " ax.axhline(2.0,color=\"black\",linestyle=\"--\",linewidth=1.0,label=\"alpha = 2\")\n", + " ax.set(xlabel=\"Corpus-equivalent epoch\",ylabel=\"Median alpha\",title=f\"{OPTIMIZER_LABELS[optimizer]}: v3 versus v4 spectral flow\")\n", + " ax.grid(alpha=0.25); ax.legend(frameon=False); fig.tight_layout()\n", + " fig.savefig(plot_root/f\"{optimizer}_alpha_median.png\",dpi=170,bbox_inches=\"tight\")\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (active conda environment)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/ngb/tests/__init__.py b/baseline/ngb/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/baseline/ngb/tests/test_ngb.py b/baseline/ngb/tests/test_ngb.py new file mode 100644 index 0000000..42168a0 --- /dev/null +++ b/baseline/ngb/tests/test_ngb.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import ast +import json +import math +from pathlib import Path +import sys + +import nbformat +import pandas as pd +import pytest + +NGB_ROOT = Path(__file__).resolve().parents[1] +BASELINE_ROOT = NGB_ROOT.parent +RUNTIME_SRC = BASELINE_ROOT / "nanogpt_one_head" / "src" +if str(RUNTIME_SRC) not in sys.path: + sys.path.insert(0, str(RUNTIME_SRC)) + +from rg_nanogpt_one_head import ( # noqa: E402 + GPT, + GPTConfig, + expected_transformer_matrix_count, + final_test_summary, + load_config, + paired_test_differences, + roots, + run_slug, +) +from rg_nanogpt_one_head.model import transformer_matrix_items # noqa: E402 + + +def _config(name: str) -> dict: + return load_config(NGB_ROOT / "configs" / name) + + +def test_v4_protocols_are_separate_tuned_two_epoch_experiments() -> None: + one = _config("v4_one_head.yaml") + four = _config("v4_small_4x4.yaml") + + assert one["protocol"]["version"] == 4 + assert four["protocol"]["version"] == 4 + assert run_slug(one) == "v4_one_head" + assert run_slug(four) == "v4_small_4x4" + assert one["training"]["target_epochs"] == 2.0 + assert four["training"]["target_epochs"] == 2.0 + assert one["training"]["epoch_interval"] == 0.25 + assert one["training"]["eval_interval_steps"] == 500 + + assert one["optimizer_profiles"]["sgd_momentum"]["min_learning_rate"] == 5e-4 + assert one["optimizer_profiles"]["adamw"]["learning_rate"] == 3e-4 + assert one["optimizer_profiles"]["adamw"]["warmup_fraction"] == 0.025 + assert one["optimizer_profiles"]["muon"]["matrix_learning_rate"] == 0.01 + assert one["optimizer_profiles"]["muon"]["aux_learning_rate"] == 2e-4 + resolved = roots(one) + assert resolved["root"] == Path("/tmp/rg-ngb") + assert resolved["data"] == Path("/tmp/rg-nanogpt-one-head/data") + assert resolved["results"] == Path("/tmp/rg-ngb/results/v4_one_head") + + +def test_one_head_and_four_by_four_parameter_inventories() -> None: + one_cfg = _config("v4_one_head.yaml") + four_cfg = _config("v4_small_4x4.yaml") + one = GPT(GPTConfig(**one_cfg["model"])) + four = GPT(GPTConfig(**four_cfg["model"])) + + assert one.parameter_count() == 6_662_656 + assert four.parameter_count() == 7_253_248 + assert len(transformer_matrix_items(one)) == 6 + assert len(transformer_matrix_items(four)) == 24 + assert expected_transformer_matrix_count(one_cfg) == 6 + assert expected_transformer_matrix_count(four_cfg) == 24 + assert sum(item[3].numel() for item in transformer_matrix_items(four)) == 786_432 + + +def test_perplexity_interval_is_exponentiated_from_loss_space() -> None: + frame = pd.DataFrame( + { + "optimizer": ["adamw"] * 3, + "checkpoint": ["final"] * 3, + "test_loss": [5.0, 6.0, 7.0], + "test_perplexity": [math.exp(5.0), math.exp(6.0), math.exp(7.0)], + "test_accuracy": [0.1, 0.2, 0.3], + "test_bleu": [0.0, 0.1, 0.2], + } + ) + summary = final_test_summary(frame) + loss = summary[summary["metric"].eq("test_loss")].iloc[0] + perplexity = summary[summary["metric"].eq("test_perplexity")].iloc[0] + assert perplexity["interval_method"] == "exp_test_loss_student_t" + assert perplexity["mean"] == pytest.approx(math.exp(loss["mean"])) + assert perplexity["ci95_lower"] == pytest.approx(math.exp(loss["ci95_lower"])) + assert perplexity["ci95_lower"] > 0.0 + + +def test_paired_optimizer_contrasts_use_matched_seeds() -> None: + rows = [] + for optimizer, offset in (("sgd_momentum", 0.0), ("adamw", -0.2), ("muon", -0.1)): + for seed, base in ((11, 6.0), (13, 6.2), (17, 6.4)): + rows.append( + { + "optimizer": optimizer, + "checkpoint": "final", + "seed": seed, + "test_loss": base + offset, + "test_perplexity": math.exp(base + offset), + "test_accuracy": 0.1 - offset, + "test_bleu": 0.0, + } + ) + contrasts = paired_test_differences(pd.DataFrame(rows)) + row = contrasts[ + contrasts["metric"].eq("test_loss") + & contrasts["left_optimizer"].eq("sgd_momentum") + & contrasts["right_optimizer"].eq("adamw") + ].iloc[0] + assert row["n"] == 3 + assert row["mean"] == pytest.approx(0.2) + + +def test_ngb_notebooks_are_valid_python3_papermill_entrypoints() -> None: + paths = sorted( + path + for path in (NGB_ROOT / "notebooks").glob("*.ipynb") + if not path.name.endswith(".out.ipynb") + ) + assert [path.name for path in paths] == [ + "01_run_v4_one_head.ipynb", + "02_compare_v4_one_head.ipynb", + "03_run_v4_small_4x4.ipynb", + "04_compare_v4_small_4x4.ipynb", + "05_compare_v4_architectures.ipynb", + "06_compare_v3_v4_one_head.ipynb", + ] + for path in paths: + notebook = nbformat.read(path, as_version=4) + nbformat.validate(notebook) + assert notebook.metadata.kernelspec.name == "python3" + parameter_cells = [ + cell + for cell in notebook.cells + if "parameters" in cell.get("metadata", {}).get("tags", []) + ] + assert len(parameter_cells) == 1 + for cell in notebook.cells: + if cell.cell_type == "code": + ast.parse(cell.source, filename=str(path)) + + +def test_ngb_contains_no_home_or_wrapper_defaults() -> None: + forbidden = ( + "$HOME", + "${HOME}", + "Path.home(", + ".expanduser(", + "/home/", + "~/", + "scripts/setup_mac.sh", + ) + this_file = Path(__file__).resolve() + for path in NGB_ROOT.rglob("*"): + if ( + not path.is_file() + or path.suffix in {".pyc"} + or path.resolve() == this_file + ): + continue + text = path.read_text(encoding="utf-8") + for marker in forbidden: + assert marker not in text, f"{marker!r} found in {path}"