From 85defed792c86c815700200984746336b72e67a7 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:45:45 -0700 Subject: [PATCH 01/12] ci: stage MNIST trace-log robustness patch --- .github/workflows/harden-mnist-trace-log.yml | 435 +++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 .github/workflows/harden-mnist-trace-log.yml diff --git a/.github/workflows/harden-mnist-trace-log.yml b/.github/workflows/harden-mnist-trace-log.yml new file mode 100644 index 0000000..41acdd1 --- /dev/null +++ b/.github/workflows/harden-mnist-trace-log.yml @@ -0,0 +1,435 @@ +name: Harden MNIST trace-log diagnostics + +on: + push: + branches: + - "agent/harden-mnist-trace-log" + paths: + - ".github/workflows/harden-mnist-trace-log.yml" + +permissions: + contents: write + +jobs: + patch-test-and-publish: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + ref: agent/harden-mnist-trace-log + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + baseline/pyproject.toml + baseline/requirements.txt + + - name: Apply trace-log robustness patch + shell: bash + run: | + python - <<'PY' + from pathlib import Path + + diagnostics_path = Path("baseline/rg_baselines/diagnostics.py") + diagnostics = diagnostics_path.read_text(encoding="utf-8") + + old_clean = '''\ +def clean_positive_eigenvalues(values: Any) -> np.ndarray: + """Return finite positive eigenvalues in ascending order.""" + + evals = np.asarray(values, dtype=float).reshape(-1) + evals = evals[np.isfinite(evals) & (evals > 0.0)] + evals = np.sort(evals) + if evals.size < 2: + raise ValueError("fewer than two finite positive eigenvalues") + return evals +''' + new_clean = '''\ +def clean_positive_eigenvalues( + values: Any, + *, + expected_dimension: Optional[int] = None, +) -> np.ndarray: + """Return positive eigenvalues in ascending order. + + When ``expected_dimension`` is supplied, fail closed if the ESD is + incomplete, non-finite, or rank deficient. This preserves + WeightWatcher's full-M normalization instead of silently renormalizing a + filtered positive-rank spectrum. + """ + + evals = np.asarray(values, dtype=float).reshape(-1) + if expected_dimension is not None: + expected = int(expected_dimension) + if expected < 2: + raise ValueError("expected spectral dimension must be at least two") + if evals.size != expected: + raise ValueError( + "ESD dimension mismatch: " + f"expected {expected} eigenvalues, received {evals.size}" + ) + if not np.all(np.isfinite(evals)): + raise ValueError("full ESD contains non-finite eigenvalues") + if np.any(evals <= 0.0): + positive = int(np.count_nonzero(evals > 0.0)) + raise ValueError( + "rank-deficient ESD: " + f"expected {expected} positive eigenvalues, found {positive}" + ) + else: + evals = evals[np.isfinite(evals) & (evals > 0.0)] + + evals = np.sort(evals) + if evals.size < 2: + raise ValueError("fewer than two finite positive eigenvalues") + return evals +''' + if diagnostics.count(old_clean) != 1: + raise RuntimeError("unexpected clean_positive_eigenvalues source") + diagnostics = diagnostics.replace(old_clean, new_clean, 1) + + old_metrics = '''\ +def spectral_metrics_from_esd( + raw_evals_ascending: Any, + normalized_evals_ascending: Any, + *, + detx_num: int, + num_pl_spikes: int, + erg_gap: int, +) -> dict[str, float | int]: + """Compute transparent metrics from one WeightWatcher ESD. + + ``normalized_evals_ascending`` must be produced by WeightWatcher's own + ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not + recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and + ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. + """ + + raw = clean_positive_eigenvalues(raw_evals_ascending) + normalized = clean_positive_eigenvalues(normalized_evals_ascending) + if raw.size != normalized.size: + raise ValueError("raw and normalized ESDs have different sizes") + + count = int(raw.size) + m_detx = int(np.clip(int(detx_num), 1, count)) + m_pl = int(np.clip(int(num_pl_spikes), 1, count)) + expected_gap = m_detx - m_pl + if int(erg_gap) != expected_gap: + raise ValueError( + f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" + ) + m_midpoint = int(np.clip(math.floor((m_detx + m_pl) / 2.0), 1, count)) +''' + new_metrics = '''\ +def spectral_metrics_from_esd( + raw_evals_ascending: Any, + normalized_evals_ascending: Any, + *, + detx_num: int, + num_pl_spikes: int, + erg_gap: int, + expected_dimension: Optional[int] = None, +) -> dict[str, float | int]: + """Compute transparent metrics from one WeightWatcher ESD. + + ``normalized_evals_ascending`` must be produced by WeightWatcher's own + ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not + recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and + ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. + + ``expected_dimension`` is the full spectral dimension + ``min(weight.shape)``. Strict baseline measurements require all of those + eigenvalues to be finite and positive so WeightWatcher's normalization is + not silently changed by positive-eigenvalue filtering. + """ + + raw = clean_positive_eigenvalues( + raw_evals_ascending, + expected_dimension=expected_dimension, + ) + normalized = clean_positive_eigenvalues( + normalized_evals_ascending, + expected_dimension=expected_dimension, + ) + if raw.size != normalized.size: + raise ValueError("raw and normalized ESDs have different sizes") + + count = int(raw.size) + normalized_sum = float(np.sum(normalized)) + if not np.isclose( + normalized_sum, + float(count), + rtol=1e-10, + atol=1e-10 * max(count, 1), + ): + raise ValueError( + "WeightWatcher normalization audit failed: " + f"sum={normalized_sum:.17g}, expected={count}" + ) + + m_detx = int(detx_num) + m_pl = int(num_pl_spikes) + if not 1 <= m_detx <= count: + raise ValueError( + f"detX_num must lie in [1, {count}], received {m_detx}" + ) + if not 1 <= m_pl <= count: + raise ValueError( + f"num_pl_spikes must lie in [1, {count}], received {m_pl}" + ) + + expected_gap = m_detx - m_pl + if int(erg_gap) != expected_gap: + raise ValueError( + f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" + ) + m_midpoint = int(math.floor((m_detx + m_pl) / 2.0)) +''' + if diagnostics.count(old_metrics) != 1: + raise RuntimeError("unexpected spectral_metrics_from_esd source") + diagnostics = diagnostics.replace(old_metrics, new_metrics, 1) + + old_sum = '''\ + "rescaled_eigenvalue_sum": float(np.sum(normalized)), + "rescale_sum_minus_num_eigenvalues": float(np.sum(normalized) - count), +''' + new_sum = '''\ + "rescaled_eigenvalue_sum": normalized_sum, + "rescale_sum_minus_num_eigenvalues": float(normalized_sum - count), +''' + if diagnostics.count(old_sum) != 1: + raise RuntimeError("unexpected normalized-sum output source") + diagnostics = diagnostics.replace(old_sum, new_sum, 1) + + old_measure = '''\ + raw_esd = clean_positive_eigenvalues( + _get_esd_compat( + watcher, + model=model_cpu, + layer_id=int(layer_id), + params=get_esd_params, + ) + ) + normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) + computed = spectral_metrics_from_esd( + raw_esd, + normalized_esd, + detx_num=int(detx_num), + num_pl_spikes=int(num_pl_spikes), + erg_gap=erg_gap, + ) + + parameter = parameter_map.get(parameter_name) if parameter_name else None +''' + new_measure = '''\ + parameter = parameter_map.get(parameter_name) if parameter_name else None + if parameter is None: + raise ValueError( + "WeightWatcher layer could not be matched to a model matrix" + ) + expected_dimension = int(min(parameter.shape)) + raw_esd = clean_positive_eigenvalues( + _get_esd_compat( + watcher, + model=model_cpu, + layer_id=int(layer_id), + params=get_esd_params, + ), + expected_dimension=expected_dimension, + ) + normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) + computed = spectral_metrics_from_esd( + raw_esd, + normalized_esd, + detx_num=int(detx_num), + num_pl_spikes=int(num_pl_spikes), + erg_gap=erg_gap, + expected_dimension=expected_dimension, + ) + +''' + if diagnostics.count(old_measure) != 1: + raise RuntimeError("unexpected WeightWatcher measurement source") + diagnostics = diagnostics.replace(old_measure, new_measure, 1) + + old_shape = '''\ + "layer_rows": int(parameter.shape[0]) if parameter is not None else np.nan, + "layer_cols": int(parameter.shape[1]) if parameter is not None else np.nan, + "layer_parameter_count": int(parameter.numel()) if parameter is not None else np.nan, +''' + new_shape = '''\ + "layer_rows": int(parameter.shape[0]), + "layer_cols": int(parameter.shape[1]), + "layer_parameter_count": int(parameter.numel()), +''' + if diagnostics.count(old_shape) != 1: + raise RuntimeError("unexpected layer-shape source") + diagnostics = diagnostics.replace(old_shape, new_shape, 1) + diagnostics_path.write_text(diagnostics, encoding="utf-8") + + tests_path = Path("baseline/tests/test_diagnostics.py") + tests_path.write_text( + '''\ +import unittest + +import numpy as np + +from rg_baselines.diagnostics import ( + clean_positive_eigenvalues, + spectral_metrics_from_esd, +) + + +class SpectralMetricsTests(unittest.TestCase): + def test_original_boundaries_and_midpoint(self) -> None: + raw = np.arange(1.0, 11.0) + normalized = raw * (len(raw) / raw.sum()) + metrics = spectral_metrics_from_esd( + raw, + normalized, + detx_num=8, + num_pl_spikes=4, + erg_gap=4, + expected_dimension=len(raw), + ) + self.assertEqual(metrics["m_midpoint"], 6) + self.assertEqual(metrics["ERG_gap"], 4) + self.assertAlmostEqual(metrics["rescaled_eigenvalue_sum"], 10.0) + self.assertAlmostEqual( + metrics["rescale_sum_minus_num_eigenvalues"], + 0.0, + ) + self.assertGreater(metrics["midpoint_energy_fraction"], 0.5) + + def test_trace_log_matches_analytic_top_spectrum_value(self) -> None: + raw = np.asarray([1.0, 2.0, 4.0, 8.0]) + normalized = raw * (len(raw) / raw.sum()) + metrics = spectral_metrics_from_esd( + raw, + normalized, + detx_num=4, + num_pl_spikes=2, + erg_gap=2, + expected_dimension=4, + ) + + retained = normalized[::-1][:3] + expected_total = float(np.sum(np.log(retained))) + expected_per_eval = float(np.mean(np.log(retained))) + self.assertEqual(metrics["m_midpoint"], 3) + self.assertAlmostEqual( + metrics["trace_log_midpoint_total"], + expected_total, + ) + self.assertAlmostEqual( + metrics["trace_log_midpoint_per_eval"], + expected_per_eval, + ) + self.assertAlmostEqual( + metrics["geometric_mean_midpoint"], + float(np.exp(expected_per_eval)), + ) + self.assertAlmostEqual( + metrics["trace_log_midpoint_total"], + 3.0 * metrics["trace_log_midpoint_per_eval"], + ) + + def test_gap_mismatch_is_rejected(self) -> None: + raw = np.arange(1.0, 11.0) + normalized = raw * (len(raw) / raw.sum()) + with self.assertRaisesRegex(ValueError, "ERG_gap audit failed"): + spectral_metrics_from_esd( + raw, + normalized, + detx_num=8, + num_pl_spikes=4, + erg_gap=3, + expected_dimension=len(raw), + ) + + def test_out_of_range_boundaries_are_rejected(self) -> None: + raw = np.arange(1.0, 6.0) + normalized = raw * (len(raw) / raw.sum()) + for field, detx_num, num_pl_spikes in ( + ("detX_num", 6, 2), + ("num_pl_spikes", 4, 0), + ): + with self.subTest(field=field): + with self.assertRaisesRegex(ValueError, field): + spectral_metrics_from_esd( + raw, + normalized, + detx_num=detx_num, + num_pl_spikes=num_pl_spikes, + erg_gap=detx_num - num_pl_spikes, + expected_dimension=len(raw), + ) + + def test_rank_deficient_full_esd_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "rank-deficient ESD"): + clean_positive_eigenvalues( + [0.0, 1.0, 2.0], + expected_dimension=3, + ) + + def test_incomplete_full_esd_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "ESD dimension mismatch"): + clean_positive_eigenvalues( + [1.0, 2.0], + expected_dimension=3, + ) + + def test_incorrect_weightwatcher_normalization_is_rejected(self) -> None: + raw = np.asarray([1.0, 2.0, 3.0, 4.0]) + with self.assertRaisesRegex(ValueError, "normalization audit failed"): + spectral_metrics_from_esd( + raw, + raw, + detx_num=4, + num_pl_spikes=2, + erg_gap=2, + expected_dimension=4, + ) + + +if __name__ == "__main__": + unittest.main() +''', + encoding="utf-8", + ) + PY + + - name: Install complete baseline test environment + run: | + python -m pip install --upgrade pip + python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + python -m pip install -e './baseline[experiment]' + python -m pip install nbformat + python -m pip check + + - name: Compile and test baseline suite + env: + PYTHONPATH: baseline + MPLBACKEND: Agg + run: | + git diff --check + python -m compileall -q baseline/rg_baselines baseline/tests + python -m unittest baseline.tests.test_diagnostics -v + python -m unittest discover -s baseline/tests -v + + - name: Commit tested patch + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + git add \ + baseline/rg_baselines/diagnostics.py \ + baseline/tests/test_diagnostics.py + git commit -m "Harden MNIST trace-log diagnostics" + git push origin HEAD:agent/harden-mnist-trace-log From d359fe7d38a2ec89ee1569eb18c4116f79dd3d51 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:47:43 -0700 Subject: [PATCH 02/12] ci: run MNIST trace-log robustness patch --- .../harden-mnist-trace-log-runner.yml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/harden-mnist-trace-log-runner.yml diff --git a/.github/workflows/harden-mnist-trace-log-runner.yml b/.github/workflows/harden-mnist-trace-log-runner.yml new file mode 100644 index 0000000..e8f151a --- /dev/null +++ b/.github/workflows/harden-mnist-trace-log-runner.yml @@ -0,0 +1,81 @@ +name: Run MNIST trace-log robustness patch + +on: + push: + branches: + - "agent/harden-mnist-trace-log" + paths: + - ".github/workflows/harden-mnist-trace-log-runner.yml" + +permissions: + contents: write + +jobs: + patch-test-and-publish: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + ref: agent/harden-mnist-trace-log + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + baseline/pyproject.toml + baseline/requirements.txt + + - name: Recover and apply reviewed patch script + shell: bash + run: | + set -euo pipefail + awk ' + /- name: Apply trace-log robustness patch/ { + section=1 + next + } + section && /run: \|/ { + capture=1 + next + } + capture && /^ - name:/ { exit } + capture { + sub(/^ /, "") + print + } + ' .github/workflows/harden-mnist-trace-log.yml > /tmp/apply_patch.sh + bash /tmp/apply_patch.sh + + - name: Install complete baseline test environment + run: | + python -m pip install --upgrade pip + python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + python -m pip install -e './baseline[experiment]' + python -m pip install nbformat + python -m pip check + + - name: Compile and test baseline suite + env: + PYTHONPATH: baseline + MPLBACKEND: Agg + run: | + git diff --check + python -m compileall -q baseline/rg_baselines baseline/tests + python -m unittest baseline.tests.test_diagnostics -v + python -m unittest discover -s baseline/tests -v + + - name: Commit tested patch + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + git add \ + baseline/rg_baselines/diagnostics.py \ + baseline/tests/test_diagnostics.py + git commit -m "Harden MNIST trace-log diagnostics" + git push origin HEAD:agent/harden-mnist-trace-log From 489fab9f6a9d6033dc64e3b36e545cce47bf583a Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:50:06 -0700 Subject: [PATCH 03/12] ci: add robust trace-log patch script --- .github/scripts/apply_trace_log_robustness.py | 307 ++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 .github/scripts/apply_trace_log_robustness.py diff --git a/.github/scripts/apply_trace_log_robustness.py b/.github/scripts/apply_trace_log_robustness.py new file mode 100644 index 0000000..d3011df --- /dev/null +++ b/.github/scripts/apply_trace_log_robustness.py @@ -0,0 +1,307 @@ +from pathlib import Path + +diagnostics_path = Path("baseline/rg_baselines/diagnostics.py") +diagnostics = diagnostics_path.read_text(encoding="utf-8") + +new_clean = """\ +def clean_positive_eigenvalues( + values: Any, + *, + expected_dimension: Optional[int] = None, +) -> np.ndarray: + \"\"\"Return positive eigenvalues in ascending order. + + When ``expected_dimension`` is supplied, fail closed if the ESD is + incomplete, non-finite, or rank deficient. This preserves + WeightWatcher's full-M normalization instead of silently renormalizing a + filtered positive-rank spectrum. + \"\"\" + + evals = np.asarray(values, dtype=float).reshape(-1) + if expected_dimension is not None: + expected = int(expected_dimension) + if expected < 2: + raise ValueError("expected spectral dimension must be at least two") + if evals.size != expected: + raise ValueError( + "ESD dimension mismatch: " + f"expected {expected} eigenvalues, received {evals.size}" + ) + if not np.all(np.isfinite(evals)): + raise ValueError("full ESD contains non-finite eigenvalues") + if np.any(evals <= 0.0): + positive = int(np.count_nonzero(evals > 0.0)) + raise ValueError( + "rank-deficient ESD: " + f"expected {expected} positive eigenvalues, found {positive}" + ) + else: + evals = evals[np.isfinite(evals) & (evals > 0.0)] + + evals = np.sort(evals) + if evals.size < 2: + raise ValueError("fewer than two finite positive eigenvalues") + return evals +""" +start = diagnostics.index("def clean_positive_eigenvalues(") +end = diagnostics.index("\ndef _entropy_effective_rank", start) +diagnostics = diagnostics[:start] + new_clean + diagnostics[end + 1 :] + +new_metrics_prefix = """\ +def spectral_metrics_from_esd( + raw_evals_ascending: Any, + normalized_evals_ascending: Any, + *, + detx_num: int, + num_pl_spikes: int, + erg_gap: int, + expected_dimension: Optional[int] = None, +) -> dict[str, float | int]: + \"\"\"Compute transparent metrics from one WeightWatcher ESD. + + ``normalized_evals_ascending`` must be produced by WeightWatcher's own + ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not + recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and + ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. + + ``expected_dimension`` is the full spectral dimension + ``min(weight.shape)``. Strict baseline measurements require all of those + eigenvalues to be finite and positive so WeightWatcher's normalization is + not silently changed by positive-eigenvalue filtering. + \"\"\" + + raw = clean_positive_eigenvalues( + raw_evals_ascending, + expected_dimension=expected_dimension, + ) + normalized = clean_positive_eigenvalues( + normalized_evals_ascending, + expected_dimension=expected_dimension, + ) + if raw.size != normalized.size: + raise ValueError("raw and normalized ESDs have different sizes") + + count = int(raw.size) + normalized_sum = float(np.sum(normalized)) + if not np.isclose( + normalized_sum, + float(count), + rtol=1e-10, + atol=1e-10 * max(count, 1), + ): + raise ValueError( + "WeightWatcher normalization audit failed: " + f"sum={normalized_sum:.17g}, expected={count}" + ) + + m_detx = int(detx_num) + m_pl = int(num_pl_spikes) + if not 1 <= m_detx <= count: + raise ValueError( + f"detX_num must lie in [1, {count}], received {m_detx}" + ) + if not 1 <= m_pl <= count: + raise ValueError( + f"num_pl_spikes must lie in [1, {count}], received {m_pl}" + ) + + expected_gap = m_detx - m_pl + if int(erg_gap) != expected_gap: + raise ValueError( + f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" + ) + m_midpoint = int(math.floor((m_detx + m_pl) / 2.0)) +""" +start = diagnostics.index("def spectral_metrics_from_esd(") +body = diagnostics.index(" raw_desc = raw[::-1]", start) +diagnostics = diagnostics[:start] + new_metrics_prefix + "\n" + diagnostics[body:] + +old_sum = ( + ' "rescaled_eigenvalue_sum": float(np.sum(normalized)),\n' + ' "rescale_sum_minus_num_eigenvalues": ' + 'float(np.sum(normalized) - count),\n' +) +new_sum = ( + ' "rescaled_eigenvalue_sum": normalized_sum,\n' + ' "rescale_sum_minus_num_eigenvalues": ' + 'float(normalized_sum - count),\n' +) +if diagnostics.count(old_sum) != 1: + raise RuntimeError("unexpected normalized-sum output source") +diagnostics = diagnostics.replace(old_sum, new_sum, 1) + +new_measure = """\ + parameter = parameter_map.get(parameter_name) if parameter_name else None + if parameter is None: + raise ValueError( + "WeightWatcher layer could not be matched to a model matrix" + ) + expected_dimension = int(min(parameter.shape)) + raw_esd = clean_positive_eigenvalues( + _get_esd_compat( + watcher, + model=model_cpu, + layer_id=int(layer_id), + params=get_esd_params, + ), + expected_dimension=expected_dimension, + ) + normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) + computed = spectral_metrics_from_esd( + raw_esd, + normalized_esd, + detx_num=int(detx_num), + num_pl_spikes=int(num_pl_spikes), + erg_gap=erg_gap, + expected_dimension=expected_dimension, + ) + +""" +start = diagnostics.index(" raw_esd = clean_positive_eigenvalues(") +end = diagnostics.index(" record = {", start) +diagnostics = diagnostics[:start] + new_measure + diagnostics[end:] + +diagnostics = diagnostics.replace( + ' "layer_rows": int(parameter.shape[0]) ' + 'if parameter is not None else np.nan,\n' + ' "layer_cols": int(parameter.shape[1]) ' + 'if parameter is not None else np.nan,\n' + ' "layer_parameter_count": int(parameter.numel()) ' + 'if parameter is not None else np.nan,\n', + ' "layer_rows": int(parameter.shape[0]),\n' + ' "layer_cols": int(parameter.shape[1]),\n' + ' "layer_parameter_count": int(parameter.numel()),\n', + 1, +) +diagnostics_path.write_text(diagnostics, encoding="utf-8") + +tests_path = Path("baseline/tests/test_diagnostics.py") +tests_path.write_text( + """\ +import unittest + +import numpy as np + +from rg_baselines.diagnostics import ( + clean_positive_eigenvalues, + spectral_metrics_from_esd, +) + + +class SpectralMetricsTests(unittest.TestCase): + def test_original_boundaries_and_midpoint(self) -> None: + raw = np.arange(1.0, 11.0) + normalized = raw * (len(raw) / raw.sum()) + metrics = spectral_metrics_from_esd( + raw, + normalized, + detx_num=8, + num_pl_spikes=4, + erg_gap=4, + expected_dimension=len(raw), + ) + self.assertEqual(metrics["m_midpoint"], 6) + self.assertEqual(metrics["ERG_gap"], 4) + self.assertAlmostEqual(metrics["rescaled_eigenvalue_sum"], 10.0) + self.assertAlmostEqual( + metrics["rescale_sum_minus_num_eigenvalues"], + 0.0, + ) + self.assertGreater(metrics["midpoint_energy_fraction"], 0.5) + + def test_trace_log_matches_analytic_top_spectrum_value(self) -> None: + raw = np.asarray([1.0, 2.0, 4.0, 8.0]) + normalized = raw * (len(raw) / raw.sum()) + metrics = spectral_metrics_from_esd( + raw, + normalized, + detx_num=4, + num_pl_spikes=2, + erg_gap=2, + expected_dimension=4, + ) + + retained = normalized[::-1][:3] + expected_total = float(np.sum(np.log(retained))) + expected_per_eval = float(np.mean(np.log(retained))) + self.assertEqual(metrics["m_midpoint"], 3) + self.assertAlmostEqual( + metrics["trace_log_midpoint_total"], + expected_total, + ) + self.assertAlmostEqual( + metrics["trace_log_midpoint_per_eval"], + expected_per_eval, + ) + self.assertAlmostEqual( + metrics["geometric_mean_midpoint"], + float(np.exp(expected_per_eval)), + ) + self.assertAlmostEqual( + metrics["trace_log_midpoint_total"], + 3.0 * metrics["trace_log_midpoint_per_eval"], + ) + + def test_gap_mismatch_is_rejected(self) -> None: + raw = np.arange(1.0, 11.0) + normalized = raw * (len(raw) / raw.sum()) + with self.assertRaisesRegex(ValueError, "ERG_gap audit failed"): + spectral_metrics_from_esd( + raw, + normalized, + detx_num=8, + num_pl_spikes=4, + erg_gap=3, + expected_dimension=len(raw), + ) + + def test_out_of_range_boundaries_are_rejected(self) -> None: + raw = np.arange(1.0, 6.0) + normalized = raw * (len(raw) / raw.sum()) + for field, detx_num, num_pl_spikes in ( + ("detX_num", 6, 2), + ("num_pl_spikes", 4, 0), + ): + with self.subTest(field=field): + with self.assertRaisesRegex(ValueError, field): + spectral_metrics_from_esd( + raw, + normalized, + detx_num=detx_num, + num_pl_spikes=num_pl_spikes, + erg_gap=detx_num - num_pl_spikes, + expected_dimension=len(raw), + ) + + def test_rank_deficient_full_esd_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "rank-deficient ESD"): + clean_positive_eigenvalues( + [0.0, 1.0, 2.0], + expected_dimension=3, + ) + + def test_incomplete_full_esd_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "ESD dimension mismatch"): + clean_positive_eigenvalues( + [1.0, 2.0], + expected_dimension=3, + ) + + def test_incorrect_weightwatcher_normalization_is_rejected(self) -> None: + raw = np.asarray([1.0, 2.0, 3.0, 4.0]) + with self.assertRaisesRegex(ValueError, "normalization audit failed"): + spectral_metrics_from_esd( + raw, + raw, + detx_num=4, + num_pl_spikes=2, + erg_gap=2, + expected_dimension=4, + ) + + +if __name__ == "__main__": + unittest.main() +""", + encoding="utf-8", +) From 134c2dfd18c71f7acc5b941c86be9e21eb52eb00 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:51:05 -0700 Subject: [PATCH 04/12] ci: finalize MNIST trace-log robustness patch --- .../harden-mnist-trace-log-final.yml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/harden-mnist-trace-log-final.yml diff --git a/.github/workflows/harden-mnist-trace-log-final.yml b/.github/workflows/harden-mnist-trace-log-final.yml new file mode 100644 index 0000000..cabf463 --- /dev/null +++ b/.github/workflows/harden-mnist-trace-log-final.yml @@ -0,0 +1,63 @@ +name: Finalize MNIST trace-log robustness patch + +on: + push: + branches: + - "agent/harden-mnist-trace-log" + paths: + - ".github/workflows/harden-mnist-trace-log-final.yml" + +permissions: + contents: write + +jobs: + patch-test-and-publish: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + ref: agent/harden-mnist-trace-log + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + baseline/pyproject.toml + baseline/requirements.txt + + - name: Apply reviewed trace-log patch + run: python .github/scripts/apply_trace_log_robustness.py + + - name: Install complete baseline test environment + run: | + python -m pip install --upgrade pip + python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + python -m pip install -e './baseline[experiment]' + python -m pip install nbformat + python -m pip check + + - name: Compile and test baseline suite + env: + PYTHONPATH: baseline + MPLBACKEND: Agg + run: | + git diff --check + python -m compileall -q baseline/rg_baselines baseline/tests + python -m unittest baseline.tests.test_diagnostics -v + python -m unittest discover -s baseline/tests -v + + - name: Commit tested patch + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + git add \ + baseline/rg_baselines/diagnostics.py \ + baseline/tests/test_diagnostics.py + git commit -m "Harden MNIST trace-log diagnostics" + git push origin HEAD:agent/harden-mnist-trace-log From 251636e9a7279a9e5055dd9d65a90e53377fb98f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:54:11 +0000 Subject: [PATCH 05/12] Harden MNIST trace-log diagnostics --- baseline/rg_baselines/diagnostics.py | 103 ++++++++++++++++++++++----- baseline/tests/test_diagnostics.py | 91 ++++++++++++++++++++++- 2 files changed, 173 insertions(+), 21 deletions(-) diff --git a/baseline/rg_baselines/diagnostics.py b/baseline/rg_baselines/diagnostics.py index 156d581..80a1b48 100644 --- a/baseline/rg_baselines/diagnostics.py +++ b/baseline/rg_baselines/diagnostics.py @@ -58,17 +58,44 @@ def _sanitize_key(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) -def clean_positive_eigenvalues(values: Any) -> np.ndarray: - """Return finite positive eigenvalues in ascending order.""" +def clean_positive_eigenvalues( + values: Any, + *, + expected_dimension: Optional[int] = None, +) -> np.ndarray: + """Return positive eigenvalues in ascending order. + + When ``expected_dimension`` is supplied, fail closed if the ESD is + incomplete, non-finite, or rank deficient. This preserves + WeightWatcher's full-M normalization instead of silently renormalizing a + filtered positive-rank spectrum. + """ evals = np.asarray(values, dtype=float).reshape(-1) - evals = evals[np.isfinite(evals) & (evals > 0.0)] + if expected_dimension is not None: + expected = int(expected_dimension) + if expected < 2: + raise ValueError("expected spectral dimension must be at least two") + if evals.size != expected: + raise ValueError( + "ESD dimension mismatch: " + f"expected {expected} eigenvalues, received {evals.size}" + ) + if not np.all(np.isfinite(evals)): + raise ValueError("full ESD contains non-finite eigenvalues") + if np.any(evals <= 0.0): + positive = int(np.count_nonzero(evals > 0.0)) + raise ValueError( + "rank-deficient ESD: " + f"expected {expected} positive eigenvalues, found {positive}" + ) + else: + evals = evals[np.isfinite(evals) & (evals > 0.0)] + evals = np.sort(evals) if evals.size < 2: raise ValueError("fewer than two finite positive eigenvalues") return evals - - def _entropy_effective_rank(evals: np.ndarray) -> float: total = float(np.sum(evals)) if total <= 0.0: @@ -84,29 +111,62 @@ def spectral_metrics_from_esd( detx_num: int, num_pl_spikes: int, erg_gap: int, + expected_dimension: Optional[int] = None, ) -> dict[str, float | int]: """Compute transparent metrics from one WeightWatcher ESD. ``normalized_evals_ascending`` must be produced by WeightWatcher's own - ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not + ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. + + ``expected_dimension`` is the full spectral dimension + ``min(weight.shape)``. Strict baseline measurements require all of those + eigenvalues to be finite and positive so WeightWatcher's normalization is + not silently changed by positive-eigenvalue filtering. """ - raw = clean_positive_eigenvalues(raw_evals_ascending) - normalized = clean_positive_eigenvalues(normalized_evals_ascending) + raw = clean_positive_eigenvalues( + raw_evals_ascending, + expected_dimension=expected_dimension, + ) + normalized = clean_positive_eigenvalues( + normalized_evals_ascending, + expected_dimension=expected_dimension, + ) if raw.size != normalized.size: raise ValueError("raw and normalized ESDs have different sizes") count = int(raw.size) - m_detx = int(np.clip(int(detx_num), 1, count)) - m_pl = int(np.clip(int(num_pl_spikes), 1, count)) + normalized_sum = float(np.sum(normalized)) + if not np.isclose( + normalized_sum, + float(count), + rtol=1e-10, + atol=1e-10 * max(count, 1), + ): + raise ValueError( + "WeightWatcher normalization audit failed: " + f"sum={normalized_sum:.17g}, expected={count}" + ) + + m_detx = int(detx_num) + m_pl = int(num_pl_spikes) + if not 1 <= m_detx <= count: + raise ValueError( + f"detX_num must lie in [1, {count}], received {m_detx}" + ) + if not 1 <= m_pl <= count: + raise ValueError( + f"num_pl_spikes must lie in [1, {count}], received {m_pl}" + ) + expected_gap = m_detx - m_pl if int(erg_gap) != expected_gap: raise ValueError( f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" ) - m_midpoint = int(np.clip(math.floor((m_detx + m_pl) / 2.0), 1, count)) + m_midpoint = int(math.floor((m_detx + m_pl) / 2.0)) raw_desc = raw[::-1] normalized_desc = normalized[::-1] @@ -151,8 +211,8 @@ def energy_fraction(m: int) -> float: "pl_energy_fraction": energy_fraction(m_pl), "detx_energy_fraction": energy_fraction(m_detx), "midpoint_energy_fraction": energy_fraction(m_midpoint), - "rescaled_eigenvalue_sum": float(np.sum(normalized)), - "rescale_sum_minus_num_eigenvalues": float(np.sum(normalized) - count), + "rescaled_eigenvalue_sum": normalized_sum, + "rescale_sum_minus_num_eigenvalues": float(normalized_sum - count), "normalized_lambda_max": float(normalized_desc[0]), "normalized_lambda_midpoint_cut": float(normalized_desc[m_midpoint - 1]), } @@ -379,13 +439,20 @@ def measure_weightwatcher_checkpoint( raise ValueError("WeightWatcher did not return ERG_gap") erg_gap = int(round(erg_gap_value)) + parameter = parameter_map.get(parameter_name) if parameter_name else None + if parameter is None: + raise ValueError( + "WeightWatcher layer could not be matched to a model matrix" + ) + expected_dimension = int(min(parameter.shape)) raw_esd = clean_positive_eigenvalues( _get_esd_compat( watcher, model=model_cpu, layer_id=int(layer_id), params=get_esd_params, - ) + ), + expected_dimension=expected_dimension, ) normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) computed = spectral_metrics_from_esd( @@ -394,9 +461,9 @@ def measure_weightwatcher_checkpoint( detx_num=int(detx_num), num_pl_spikes=int(num_pl_spikes), erg_gap=erg_gap, + expected_dimension=expected_dimension, ) - parameter = parameter_map.get(parameter_name) if parameter_name else None record = { **base_record, "status": "ok", @@ -411,9 +478,9 @@ def measure_weightwatcher_checkpoint( "weight_scale": weight_scale, "xmin": _safe_float(_row_value(row, ("xmin",))), "xmax": _safe_float(_row_value(row, ("xmax",))), - "layer_rows": int(parameter.shape[0]) if parameter is not None else np.nan, - "layer_cols": int(parameter.shape[1]) if parameter is not None else np.nan, - "layer_parameter_count": int(parameter.numel()) if parameter is not None else np.nan, + "layer_rows": int(parameter.shape[0]), + "layer_cols": int(parameter.shape[1]), + "layer_parameter_count": int(parameter.numel()), **computed, } for column in ( diff --git a/baseline/tests/test_diagnostics.py b/baseline/tests/test_diagnostics.py index 63c1b1e..4487af7 100644 --- a/baseline/tests/test_diagnostics.py +++ b/baseline/tests/test_diagnostics.py @@ -2,7 +2,10 @@ import numpy as np -from rg_baselines.diagnostics import spectral_metrics_from_esd +from rg_baselines.diagnostics import ( + clean_positive_eigenvalues, + spectral_metrics_from_esd, +) class SpectralMetricsTests(unittest.TestCase): @@ -15,23 +18,105 @@ def test_original_boundaries_and_midpoint(self) -> None: detx_num=8, num_pl_spikes=4, erg_gap=4, + expected_dimension=len(raw), ) self.assertEqual(metrics["m_midpoint"], 6) self.assertEqual(metrics["ERG_gap"], 4) self.assertAlmostEqual(metrics["rescaled_eigenvalue_sum"], 10.0) - self.assertAlmostEqual(metrics["rescale_sum_minus_num_eigenvalues"], 0.0) + self.assertAlmostEqual( + metrics["rescale_sum_minus_num_eigenvalues"], + 0.0, + ) self.assertGreater(metrics["midpoint_energy_fraction"], 0.5) + def test_trace_log_matches_analytic_top_spectrum_value(self) -> None: + raw = np.asarray([1.0, 2.0, 4.0, 8.0]) + normalized = raw * (len(raw) / raw.sum()) + metrics = spectral_metrics_from_esd( + raw, + normalized, + detx_num=4, + num_pl_spikes=2, + erg_gap=2, + expected_dimension=4, + ) + + retained = normalized[::-1][:3] + expected_total = float(np.sum(np.log(retained))) + expected_per_eval = float(np.mean(np.log(retained))) + self.assertEqual(metrics["m_midpoint"], 3) + self.assertAlmostEqual( + metrics["trace_log_midpoint_total"], + expected_total, + ) + self.assertAlmostEqual( + metrics["trace_log_midpoint_per_eval"], + expected_per_eval, + ) + self.assertAlmostEqual( + metrics["geometric_mean_midpoint"], + float(np.exp(expected_per_eval)), + ) + self.assertAlmostEqual( + metrics["trace_log_midpoint_total"], + 3.0 * metrics["trace_log_midpoint_per_eval"], + ) + def test_gap_mismatch_is_rejected(self) -> None: raw = np.arange(1.0, 11.0) normalized = raw * (len(raw) / raw.sum()) - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, "ERG_gap audit failed"): spectral_metrics_from_esd( raw, normalized, detx_num=8, num_pl_spikes=4, erg_gap=3, + expected_dimension=len(raw), + ) + + def test_out_of_range_boundaries_are_rejected(self) -> None: + raw = np.arange(1.0, 6.0) + normalized = raw * (len(raw) / raw.sum()) + for field, detx_num, num_pl_spikes in ( + ("detX_num", 6, 2), + ("num_pl_spikes", 4, 0), + ): + with self.subTest(field=field): + with self.assertRaisesRegex(ValueError, field): + spectral_metrics_from_esd( + raw, + normalized, + detx_num=detx_num, + num_pl_spikes=num_pl_spikes, + erg_gap=detx_num - num_pl_spikes, + expected_dimension=len(raw), + ) + + def test_rank_deficient_full_esd_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "rank-deficient ESD"): + clean_positive_eigenvalues( + [0.0, 1.0, 2.0], + expected_dimension=3, + ) + + def test_incomplete_full_esd_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "ESD dimension mismatch"): + clean_positive_eigenvalues( + [1.0, 2.0], + expected_dimension=3, + ) + + def test_incorrect_weightwatcher_normalization_is_rejected(self) -> None: + raw = np.asarray([1.0, 2.0, 3.0, 4.0]) + with self.assertRaisesRegex(ValueError, "normalization audit failed"): + spectral_metrics_from_esd( + raw, + raw, + detx_num=4, + num_pl_spikes=2, + erg_gap=2, + expected_dimension=4, ) From e03e3448b7ad8ab3e45a3de25d34d877460f12bb Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:56:34 -0700 Subject: [PATCH 06/12] Remove temporary trace-log patch workflow --- .github/workflows/harden-mnist-trace-log.yml | 435 ------------------- 1 file changed, 435 deletions(-) delete mode 100644 .github/workflows/harden-mnist-trace-log.yml diff --git a/.github/workflows/harden-mnist-trace-log.yml b/.github/workflows/harden-mnist-trace-log.yml deleted file mode 100644 index 41acdd1..0000000 --- a/.github/workflows/harden-mnist-trace-log.yml +++ /dev/null @@ -1,435 +0,0 @@ -name: Harden MNIST trace-log diagnostics - -on: - push: - branches: - - "agent/harden-mnist-trace-log" - paths: - - ".github/workflows/harden-mnist-trace-log.yml" - -permissions: - contents: write - -jobs: - patch-test-and-publish: - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - with: - ref: agent/harden-mnist-trace-log - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: pip - cache-dependency-path: | - baseline/pyproject.toml - baseline/requirements.txt - - - name: Apply trace-log robustness patch - shell: bash - run: | - python - <<'PY' - from pathlib import Path - - diagnostics_path = Path("baseline/rg_baselines/diagnostics.py") - diagnostics = diagnostics_path.read_text(encoding="utf-8") - - old_clean = '''\ -def clean_positive_eigenvalues(values: Any) -> np.ndarray: - """Return finite positive eigenvalues in ascending order.""" - - evals = np.asarray(values, dtype=float).reshape(-1) - evals = evals[np.isfinite(evals) & (evals > 0.0)] - evals = np.sort(evals) - if evals.size < 2: - raise ValueError("fewer than two finite positive eigenvalues") - return evals -''' - new_clean = '''\ -def clean_positive_eigenvalues( - values: Any, - *, - expected_dimension: Optional[int] = None, -) -> np.ndarray: - """Return positive eigenvalues in ascending order. - - When ``expected_dimension`` is supplied, fail closed if the ESD is - incomplete, non-finite, or rank deficient. This preserves - WeightWatcher's full-M normalization instead of silently renormalizing a - filtered positive-rank spectrum. - """ - - evals = np.asarray(values, dtype=float).reshape(-1) - if expected_dimension is not None: - expected = int(expected_dimension) - if expected < 2: - raise ValueError("expected spectral dimension must be at least two") - if evals.size != expected: - raise ValueError( - "ESD dimension mismatch: " - f"expected {expected} eigenvalues, received {evals.size}" - ) - if not np.all(np.isfinite(evals)): - raise ValueError("full ESD contains non-finite eigenvalues") - if np.any(evals <= 0.0): - positive = int(np.count_nonzero(evals > 0.0)) - raise ValueError( - "rank-deficient ESD: " - f"expected {expected} positive eigenvalues, found {positive}" - ) - else: - evals = evals[np.isfinite(evals) & (evals > 0.0)] - - evals = np.sort(evals) - if evals.size < 2: - raise ValueError("fewer than two finite positive eigenvalues") - return evals -''' - if diagnostics.count(old_clean) != 1: - raise RuntimeError("unexpected clean_positive_eigenvalues source") - diagnostics = diagnostics.replace(old_clean, new_clean, 1) - - old_metrics = '''\ -def spectral_metrics_from_esd( - raw_evals_ascending: Any, - normalized_evals_ascending: Any, - *, - detx_num: int, - num_pl_spikes: int, - erg_gap: int, -) -> dict[str, float | int]: - """Compute transparent metrics from one WeightWatcher ESD. - - ``normalized_evals_ascending`` must be produced by WeightWatcher's own - ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not - recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and - ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. - """ - - raw = clean_positive_eigenvalues(raw_evals_ascending) - normalized = clean_positive_eigenvalues(normalized_evals_ascending) - if raw.size != normalized.size: - raise ValueError("raw and normalized ESDs have different sizes") - - count = int(raw.size) - m_detx = int(np.clip(int(detx_num), 1, count)) - m_pl = int(np.clip(int(num_pl_spikes), 1, count)) - expected_gap = m_detx - m_pl - if int(erg_gap) != expected_gap: - raise ValueError( - f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" - ) - m_midpoint = int(np.clip(math.floor((m_detx + m_pl) / 2.0), 1, count)) -''' - new_metrics = '''\ -def spectral_metrics_from_esd( - raw_evals_ascending: Any, - normalized_evals_ascending: Any, - *, - detx_num: int, - num_pl_spikes: int, - erg_gap: int, - expected_dimension: Optional[int] = None, -) -> dict[str, float | int]: - """Compute transparent metrics from one WeightWatcher ESD. - - ``normalized_evals_ascending`` must be produced by WeightWatcher's own - ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not - recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and - ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. - - ``expected_dimension`` is the full spectral dimension - ``min(weight.shape)``. Strict baseline measurements require all of those - eigenvalues to be finite and positive so WeightWatcher's normalization is - not silently changed by positive-eigenvalue filtering. - """ - - raw = clean_positive_eigenvalues( - raw_evals_ascending, - expected_dimension=expected_dimension, - ) - normalized = clean_positive_eigenvalues( - normalized_evals_ascending, - expected_dimension=expected_dimension, - ) - if raw.size != normalized.size: - raise ValueError("raw and normalized ESDs have different sizes") - - count = int(raw.size) - normalized_sum = float(np.sum(normalized)) - if not np.isclose( - normalized_sum, - float(count), - rtol=1e-10, - atol=1e-10 * max(count, 1), - ): - raise ValueError( - "WeightWatcher normalization audit failed: " - f"sum={normalized_sum:.17g}, expected={count}" - ) - - m_detx = int(detx_num) - m_pl = int(num_pl_spikes) - if not 1 <= m_detx <= count: - raise ValueError( - f"detX_num must lie in [1, {count}], received {m_detx}" - ) - if not 1 <= m_pl <= count: - raise ValueError( - f"num_pl_spikes must lie in [1, {count}], received {m_pl}" - ) - - expected_gap = m_detx - m_pl - if int(erg_gap) != expected_gap: - raise ValueError( - f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" - ) - m_midpoint = int(math.floor((m_detx + m_pl) / 2.0)) -''' - if diagnostics.count(old_metrics) != 1: - raise RuntimeError("unexpected spectral_metrics_from_esd source") - diagnostics = diagnostics.replace(old_metrics, new_metrics, 1) - - old_sum = '''\ - "rescaled_eigenvalue_sum": float(np.sum(normalized)), - "rescale_sum_minus_num_eigenvalues": float(np.sum(normalized) - count), -''' - new_sum = '''\ - "rescaled_eigenvalue_sum": normalized_sum, - "rescale_sum_minus_num_eigenvalues": float(normalized_sum - count), -''' - if diagnostics.count(old_sum) != 1: - raise RuntimeError("unexpected normalized-sum output source") - diagnostics = diagnostics.replace(old_sum, new_sum, 1) - - old_measure = '''\ - raw_esd = clean_positive_eigenvalues( - _get_esd_compat( - watcher, - model=model_cpu, - layer_id=int(layer_id), - params=get_esd_params, - ) - ) - normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) - computed = spectral_metrics_from_esd( - raw_esd, - normalized_esd, - detx_num=int(detx_num), - num_pl_spikes=int(num_pl_spikes), - erg_gap=erg_gap, - ) - - parameter = parameter_map.get(parameter_name) if parameter_name else None -''' - new_measure = '''\ - parameter = parameter_map.get(parameter_name) if parameter_name else None - if parameter is None: - raise ValueError( - "WeightWatcher layer could not be matched to a model matrix" - ) - expected_dimension = int(min(parameter.shape)) - raw_esd = clean_positive_eigenvalues( - _get_esd_compat( - watcher, - model=model_cpu, - layer_id=int(layer_id), - params=get_esd_params, - ), - expected_dimension=expected_dimension, - ) - normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) - computed = spectral_metrics_from_esd( - raw_esd, - normalized_esd, - detx_num=int(detx_num), - num_pl_spikes=int(num_pl_spikes), - erg_gap=erg_gap, - expected_dimension=expected_dimension, - ) - -''' - if diagnostics.count(old_measure) != 1: - raise RuntimeError("unexpected WeightWatcher measurement source") - diagnostics = diagnostics.replace(old_measure, new_measure, 1) - - old_shape = '''\ - "layer_rows": int(parameter.shape[0]) if parameter is not None else np.nan, - "layer_cols": int(parameter.shape[1]) if parameter is not None else np.nan, - "layer_parameter_count": int(parameter.numel()) if parameter is not None else np.nan, -''' - new_shape = '''\ - "layer_rows": int(parameter.shape[0]), - "layer_cols": int(parameter.shape[1]), - "layer_parameter_count": int(parameter.numel()), -''' - if diagnostics.count(old_shape) != 1: - raise RuntimeError("unexpected layer-shape source") - diagnostics = diagnostics.replace(old_shape, new_shape, 1) - diagnostics_path.write_text(diagnostics, encoding="utf-8") - - tests_path = Path("baseline/tests/test_diagnostics.py") - tests_path.write_text( - '''\ -import unittest - -import numpy as np - -from rg_baselines.diagnostics import ( - clean_positive_eigenvalues, - spectral_metrics_from_esd, -) - - -class SpectralMetricsTests(unittest.TestCase): - def test_original_boundaries_and_midpoint(self) -> None: - raw = np.arange(1.0, 11.0) - normalized = raw * (len(raw) / raw.sum()) - metrics = spectral_metrics_from_esd( - raw, - normalized, - detx_num=8, - num_pl_spikes=4, - erg_gap=4, - expected_dimension=len(raw), - ) - self.assertEqual(metrics["m_midpoint"], 6) - self.assertEqual(metrics["ERG_gap"], 4) - self.assertAlmostEqual(metrics["rescaled_eigenvalue_sum"], 10.0) - self.assertAlmostEqual( - metrics["rescale_sum_minus_num_eigenvalues"], - 0.0, - ) - self.assertGreater(metrics["midpoint_energy_fraction"], 0.5) - - def test_trace_log_matches_analytic_top_spectrum_value(self) -> None: - raw = np.asarray([1.0, 2.0, 4.0, 8.0]) - normalized = raw * (len(raw) / raw.sum()) - metrics = spectral_metrics_from_esd( - raw, - normalized, - detx_num=4, - num_pl_spikes=2, - erg_gap=2, - expected_dimension=4, - ) - - retained = normalized[::-1][:3] - expected_total = float(np.sum(np.log(retained))) - expected_per_eval = float(np.mean(np.log(retained))) - self.assertEqual(metrics["m_midpoint"], 3) - self.assertAlmostEqual( - metrics["trace_log_midpoint_total"], - expected_total, - ) - self.assertAlmostEqual( - metrics["trace_log_midpoint_per_eval"], - expected_per_eval, - ) - self.assertAlmostEqual( - metrics["geometric_mean_midpoint"], - float(np.exp(expected_per_eval)), - ) - self.assertAlmostEqual( - metrics["trace_log_midpoint_total"], - 3.0 * metrics["trace_log_midpoint_per_eval"], - ) - - def test_gap_mismatch_is_rejected(self) -> None: - raw = np.arange(1.0, 11.0) - normalized = raw * (len(raw) / raw.sum()) - with self.assertRaisesRegex(ValueError, "ERG_gap audit failed"): - spectral_metrics_from_esd( - raw, - normalized, - detx_num=8, - num_pl_spikes=4, - erg_gap=3, - expected_dimension=len(raw), - ) - - def test_out_of_range_boundaries_are_rejected(self) -> None: - raw = np.arange(1.0, 6.0) - normalized = raw * (len(raw) / raw.sum()) - for field, detx_num, num_pl_spikes in ( - ("detX_num", 6, 2), - ("num_pl_spikes", 4, 0), - ): - with self.subTest(field=field): - with self.assertRaisesRegex(ValueError, field): - spectral_metrics_from_esd( - raw, - normalized, - detx_num=detx_num, - num_pl_spikes=num_pl_spikes, - erg_gap=detx_num - num_pl_spikes, - expected_dimension=len(raw), - ) - - def test_rank_deficient_full_esd_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "rank-deficient ESD"): - clean_positive_eigenvalues( - [0.0, 1.0, 2.0], - expected_dimension=3, - ) - - def test_incomplete_full_esd_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "ESD dimension mismatch"): - clean_positive_eigenvalues( - [1.0, 2.0], - expected_dimension=3, - ) - - def test_incorrect_weightwatcher_normalization_is_rejected(self) -> None: - raw = np.asarray([1.0, 2.0, 3.0, 4.0]) - with self.assertRaisesRegex(ValueError, "normalization audit failed"): - spectral_metrics_from_esd( - raw, - raw, - detx_num=4, - num_pl_spikes=2, - erg_gap=2, - expected_dimension=4, - ) - - -if __name__ == "__main__": - unittest.main() -''', - encoding="utf-8", - ) - PY - - - name: Install complete baseline test environment - run: | - python -m pip install --upgrade pip - python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - python -m pip install -e './baseline[experiment]' - python -m pip install nbformat - python -m pip check - - - name: Compile and test baseline suite - env: - PYTHONPATH: baseline - MPLBACKEND: Agg - run: | - git diff --check - python -m compileall -q baseline/rg_baselines baseline/tests - python -m unittest baseline.tests.test_diagnostics -v - python -m unittest discover -s baseline/tests -v - - - name: Commit tested patch - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email \ - "41898282+github-actions[bot]@users.noreply.github.com" - git add \ - baseline/rg_baselines/diagnostics.py \ - baseline/tests/test_diagnostics.py - git commit -m "Harden MNIST trace-log diagnostics" - git push origin HEAD:agent/harden-mnist-trace-log From 80760796045b1967d3f4a7800caca53ed6e48606 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:56:41 -0700 Subject: [PATCH 07/12] Remove temporary trace-log runner workflow --- .../harden-mnist-trace-log-runner.yml | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 .github/workflows/harden-mnist-trace-log-runner.yml diff --git a/.github/workflows/harden-mnist-trace-log-runner.yml b/.github/workflows/harden-mnist-trace-log-runner.yml deleted file mode 100644 index e8f151a..0000000 --- a/.github/workflows/harden-mnist-trace-log-runner.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Run MNIST trace-log robustness patch - -on: - push: - branches: - - "agent/harden-mnist-trace-log" - paths: - - ".github/workflows/harden-mnist-trace-log-runner.yml" - -permissions: - contents: write - -jobs: - patch-test-and-publish: - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - with: - ref: agent/harden-mnist-trace-log - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: pip - cache-dependency-path: | - baseline/pyproject.toml - baseline/requirements.txt - - - name: Recover and apply reviewed patch script - shell: bash - run: | - set -euo pipefail - awk ' - /- name: Apply trace-log robustness patch/ { - section=1 - next - } - section && /run: \|/ { - capture=1 - next - } - capture && /^ - name:/ { exit } - capture { - sub(/^ /, "") - print - } - ' .github/workflows/harden-mnist-trace-log.yml > /tmp/apply_patch.sh - bash /tmp/apply_patch.sh - - - name: Install complete baseline test environment - run: | - python -m pip install --upgrade pip - python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - python -m pip install -e './baseline[experiment]' - python -m pip install nbformat - python -m pip check - - - name: Compile and test baseline suite - env: - PYTHONPATH: baseline - MPLBACKEND: Agg - run: | - git diff --check - python -m compileall -q baseline/rg_baselines baseline/tests - python -m unittest baseline.tests.test_diagnostics -v - python -m unittest discover -s baseline/tests -v - - - name: Commit tested patch - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email \ - "41898282+github-actions[bot]@users.noreply.github.com" - git add \ - baseline/rg_baselines/diagnostics.py \ - baseline/tests/test_diagnostics.py - git commit -m "Harden MNIST trace-log diagnostics" - git push origin HEAD:agent/harden-mnist-trace-log From bd0821f7bb55cce27df3c59283fb0278da0d7acf Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:56:47 -0700 Subject: [PATCH 08/12] Remove temporary trace-log final workflow --- .../harden-mnist-trace-log-final.yml | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100644 .github/workflows/harden-mnist-trace-log-final.yml diff --git a/.github/workflows/harden-mnist-trace-log-final.yml b/.github/workflows/harden-mnist-trace-log-final.yml deleted file mode 100644 index cabf463..0000000 --- a/.github/workflows/harden-mnist-trace-log-final.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Finalize MNIST trace-log robustness patch - -on: - push: - branches: - - "agent/harden-mnist-trace-log" - paths: - - ".github/workflows/harden-mnist-trace-log-final.yml" - -permissions: - contents: write - -jobs: - patch-test-and-publish: - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - with: - ref: agent/harden-mnist-trace-log - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: pip - cache-dependency-path: | - baseline/pyproject.toml - baseline/requirements.txt - - - name: Apply reviewed trace-log patch - run: python .github/scripts/apply_trace_log_robustness.py - - - name: Install complete baseline test environment - run: | - python -m pip install --upgrade pip - python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - python -m pip install -e './baseline[experiment]' - python -m pip install nbformat - python -m pip check - - - name: Compile and test baseline suite - env: - PYTHONPATH: baseline - MPLBACKEND: Agg - run: | - git diff --check - python -m compileall -q baseline/rg_baselines baseline/tests - python -m unittest baseline.tests.test_diagnostics -v - python -m unittest discover -s baseline/tests -v - - - name: Commit tested patch - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email \ - "41898282+github-actions[bot]@users.noreply.github.com" - git add \ - baseline/rg_baselines/diagnostics.py \ - baseline/tests/test_diagnostics.py - git commit -m "Harden MNIST trace-log diagnostics" - git push origin HEAD:agent/harden-mnist-trace-log From c745569376911ed497ca55ccbe0fec38b84da28b Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:56:55 -0700 Subject: [PATCH 09/12] Remove temporary trace-log patch script --- .github/scripts/apply_trace_log_robustness.py | 307 ------------------ 1 file changed, 307 deletions(-) delete mode 100644 .github/scripts/apply_trace_log_robustness.py diff --git a/.github/scripts/apply_trace_log_robustness.py b/.github/scripts/apply_trace_log_robustness.py deleted file mode 100644 index d3011df..0000000 --- a/.github/scripts/apply_trace_log_robustness.py +++ /dev/null @@ -1,307 +0,0 @@ -from pathlib import Path - -diagnostics_path = Path("baseline/rg_baselines/diagnostics.py") -diagnostics = diagnostics_path.read_text(encoding="utf-8") - -new_clean = """\ -def clean_positive_eigenvalues( - values: Any, - *, - expected_dimension: Optional[int] = None, -) -> np.ndarray: - \"\"\"Return positive eigenvalues in ascending order. - - When ``expected_dimension`` is supplied, fail closed if the ESD is - incomplete, non-finite, or rank deficient. This preserves - WeightWatcher's full-M normalization instead of silently renormalizing a - filtered positive-rank spectrum. - \"\"\" - - evals = np.asarray(values, dtype=float).reshape(-1) - if expected_dimension is not None: - expected = int(expected_dimension) - if expected < 2: - raise ValueError("expected spectral dimension must be at least two") - if evals.size != expected: - raise ValueError( - "ESD dimension mismatch: " - f"expected {expected} eigenvalues, received {evals.size}" - ) - if not np.all(np.isfinite(evals)): - raise ValueError("full ESD contains non-finite eigenvalues") - if np.any(evals <= 0.0): - positive = int(np.count_nonzero(evals > 0.0)) - raise ValueError( - "rank-deficient ESD: " - f"expected {expected} positive eigenvalues, found {positive}" - ) - else: - evals = evals[np.isfinite(evals) & (evals > 0.0)] - - evals = np.sort(evals) - if evals.size < 2: - raise ValueError("fewer than two finite positive eigenvalues") - return evals -""" -start = diagnostics.index("def clean_positive_eigenvalues(") -end = diagnostics.index("\ndef _entropy_effective_rank", start) -diagnostics = diagnostics[:start] + new_clean + diagnostics[end + 1 :] - -new_metrics_prefix = """\ -def spectral_metrics_from_esd( - raw_evals_ascending: Any, - normalized_evals_ascending: Any, - *, - detx_num: int, - num_pl_spikes: int, - erg_gap: int, - expected_dimension: Optional[int] = None, -) -> dict[str, float | int]: - \"\"\"Compute transparent metrics from one WeightWatcher ESD. - - ``normalized_evals_ascending`` must be produced by WeightWatcher's own - ``RMT_Util.rescale_eigenvalues``. The trace-log boundary and gap are not - recomputed here: the supplied ``detx_num``, ``num_pl_spikes``, and - ``erg_gap`` must come from ``watcher.analyze(ERG=True)``. - - ``expected_dimension`` is the full spectral dimension - ``min(weight.shape)``. Strict baseline measurements require all of those - eigenvalues to be finite and positive so WeightWatcher's normalization is - not silently changed by positive-eigenvalue filtering. - \"\"\" - - raw = clean_positive_eigenvalues( - raw_evals_ascending, - expected_dimension=expected_dimension, - ) - normalized = clean_positive_eigenvalues( - normalized_evals_ascending, - expected_dimension=expected_dimension, - ) - if raw.size != normalized.size: - raise ValueError("raw and normalized ESDs have different sizes") - - count = int(raw.size) - normalized_sum = float(np.sum(normalized)) - if not np.isclose( - normalized_sum, - float(count), - rtol=1e-10, - atol=1e-10 * max(count, 1), - ): - raise ValueError( - "WeightWatcher normalization audit failed: " - f"sum={normalized_sum:.17g}, expected={count}" - ) - - m_detx = int(detx_num) - m_pl = int(num_pl_spikes) - if not 1 <= m_detx <= count: - raise ValueError( - f"detX_num must lie in [1, {count}], received {m_detx}" - ) - if not 1 <= m_pl <= count: - raise ValueError( - f"num_pl_spikes must lie in [1, {count}], received {m_pl}" - ) - - expected_gap = m_detx - m_pl - if int(erg_gap) != expected_gap: - raise ValueError( - f"WeightWatcher ERG_gap audit failed: {erg_gap} != {m_detx} - {m_pl}" - ) - m_midpoint = int(math.floor((m_detx + m_pl) / 2.0)) -""" -start = diagnostics.index("def spectral_metrics_from_esd(") -body = diagnostics.index(" raw_desc = raw[::-1]", start) -diagnostics = diagnostics[:start] + new_metrics_prefix + "\n" + diagnostics[body:] - -old_sum = ( - ' "rescaled_eigenvalue_sum": float(np.sum(normalized)),\n' - ' "rescale_sum_minus_num_eigenvalues": ' - 'float(np.sum(normalized) - count),\n' -) -new_sum = ( - ' "rescaled_eigenvalue_sum": normalized_sum,\n' - ' "rescale_sum_minus_num_eigenvalues": ' - 'float(normalized_sum - count),\n' -) -if diagnostics.count(old_sum) != 1: - raise RuntimeError("unexpected normalized-sum output source") -diagnostics = diagnostics.replace(old_sum, new_sum, 1) - -new_measure = """\ - parameter = parameter_map.get(parameter_name) if parameter_name else None - if parameter is None: - raise ValueError( - "WeightWatcher layer could not be matched to a model matrix" - ) - expected_dimension = int(min(parameter.shape)) - raw_esd = clean_positive_eigenvalues( - _get_esd_compat( - watcher, - model=model_cpu, - layer_id=int(layer_id), - params=get_esd_params, - ), - expected_dimension=expected_dimension, - ) - normalized_esd, weight_scale = _rescale_with_weightwatcher(raw_esd) - computed = spectral_metrics_from_esd( - raw_esd, - normalized_esd, - detx_num=int(detx_num), - num_pl_spikes=int(num_pl_spikes), - erg_gap=erg_gap, - expected_dimension=expected_dimension, - ) - -""" -start = diagnostics.index(" raw_esd = clean_positive_eigenvalues(") -end = diagnostics.index(" record = {", start) -diagnostics = diagnostics[:start] + new_measure + diagnostics[end:] - -diagnostics = diagnostics.replace( - ' "layer_rows": int(parameter.shape[0]) ' - 'if parameter is not None else np.nan,\n' - ' "layer_cols": int(parameter.shape[1]) ' - 'if parameter is not None else np.nan,\n' - ' "layer_parameter_count": int(parameter.numel()) ' - 'if parameter is not None else np.nan,\n', - ' "layer_rows": int(parameter.shape[0]),\n' - ' "layer_cols": int(parameter.shape[1]),\n' - ' "layer_parameter_count": int(parameter.numel()),\n', - 1, -) -diagnostics_path.write_text(diagnostics, encoding="utf-8") - -tests_path = Path("baseline/tests/test_diagnostics.py") -tests_path.write_text( - """\ -import unittest - -import numpy as np - -from rg_baselines.diagnostics import ( - clean_positive_eigenvalues, - spectral_metrics_from_esd, -) - - -class SpectralMetricsTests(unittest.TestCase): - def test_original_boundaries_and_midpoint(self) -> None: - raw = np.arange(1.0, 11.0) - normalized = raw * (len(raw) / raw.sum()) - metrics = spectral_metrics_from_esd( - raw, - normalized, - detx_num=8, - num_pl_spikes=4, - erg_gap=4, - expected_dimension=len(raw), - ) - self.assertEqual(metrics["m_midpoint"], 6) - self.assertEqual(metrics["ERG_gap"], 4) - self.assertAlmostEqual(metrics["rescaled_eigenvalue_sum"], 10.0) - self.assertAlmostEqual( - metrics["rescale_sum_minus_num_eigenvalues"], - 0.0, - ) - self.assertGreater(metrics["midpoint_energy_fraction"], 0.5) - - def test_trace_log_matches_analytic_top_spectrum_value(self) -> None: - raw = np.asarray([1.0, 2.0, 4.0, 8.0]) - normalized = raw * (len(raw) / raw.sum()) - metrics = spectral_metrics_from_esd( - raw, - normalized, - detx_num=4, - num_pl_spikes=2, - erg_gap=2, - expected_dimension=4, - ) - - retained = normalized[::-1][:3] - expected_total = float(np.sum(np.log(retained))) - expected_per_eval = float(np.mean(np.log(retained))) - self.assertEqual(metrics["m_midpoint"], 3) - self.assertAlmostEqual( - metrics["trace_log_midpoint_total"], - expected_total, - ) - self.assertAlmostEqual( - metrics["trace_log_midpoint_per_eval"], - expected_per_eval, - ) - self.assertAlmostEqual( - metrics["geometric_mean_midpoint"], - float(np.exp(expected_per_eval)), - ) - self.assertAlmostEqual( - metrics["trace_log_midpoint_total"], - 3.0 * metrics["trace_log_midpoint_per_eval"], - ) - - def test_gap_mismatch_is_rejected(self) -> None: - raw = np.arange(1.0, 11.0) - normalized = raw * (len(raw) / raw.sum()) - with self.assertRaisesRegex(ValueError, "ERG_gap audit failed"): - spectral_metrics_from_esd( - raw, - normalized, - detx_num=8, - num_pl_spikes=4, - erg_gap=3, - expected_dimension=len(raw), - ) - - def test_out_of_range_boundaries_are_rejected(self) -> None: - raw = np.arange(1.0, 6.0) - normalized = raw * (len(raw) / raw.sum()) - for field, detx_num, num_pl_spikes in ( - ("detX_num", 6, 2), - ("num_pl_spikes", 4, 0), - ): - with self.subTest(field=field): - with self.assertRaisesRegex(ValueError, field): - spectral_metrics_from_esd( - raw, - normalized, - detx_num=detx_num, - num_pl_spikes=num_pl_spikes, - erg_gap=detx_num - num_pl_spikes, - expected_dimension=len(raw), - ) - - def test_rank_deficient_full_esd_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "rank-deficient ESD"): - clean_positive_eigenvalues( - [0.0, 1.0, 2.0], - expected_dimension=3, - ) - - def test_incomplete_full_esd_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "ESD dimension mismatch"): - clean_positive_eigenvalues( - [1.0, 2.0], - expected_dimension=3, - ) - - def test_incorrect_weightwatcher_normalization_is_rejected(self) -> None: - raw = np.asarray([1.0, 2.0, 3.0, 4.0]) - with self.assertRaisesRegex(ValueError, "normalization audit failed"): - spectral_metrics_from_esd( - raw, - raw, - detx_num=4, - num_pl_spikes=2, - erg_gap=2, - expected_dimension=4, - ) - - -if __name__ == "__main__": - unittest.main() -""", - encoding="utf-8", -) From 8fd623ca92d681ab88df81540d4e21e25a43010a Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:58:53 -0700 Subject: [PATCH 10/12] ci: format MNIST trace-log patch --- .github/workflows/format-mnist-trace-log.yml | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/format-mnist-trace-log.yml diff --git a/.github/workflows/format-mnist-trace-log.yml b/.github/workflows/format-mnist-trace-log.yml new file mode 100644 index 0000000..d8ef631 --- /dev/null +++ b/.github/workflows/format-mnist-trace-log.yml @@ -0,0 +1,50 @@ +name: Format MNIST trace-log patch + +on: + push: + branches: + - "agent/harden-mnist-trace-log" + paths: + - ".github/workflows/format-mnist-trace-log.yml" + +permissions: + contents: write + +jobs: + format-and-test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: agent/harden-mnist-trace-log + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Restore module-level spacing + run: | + python - <<'PY' + from pathlib import Path + + path = Path("baseline/rg_baselines/diagnostics.py") + text = path.read_text(encoding="utf-8") + old = " return evals\ndef _entropy_effective_rank" + new = " return evals\n\n\ndef _entropy_effective_rank" + if text.count(old) != 1: + raise RuntimeError("unexpected diagnostics function spacing") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + PY + git diff --check + python -m compileall -q baseline/rg_baselines baseline/tests + + - name: Commit formatting correction + run: | + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + git add baseline/rg_baselines/diagnostics.py + git commit -m "Format trace-log diagnostics" + git push origin HEAD:agent/harden-mnist-trace-log From 462bc53c2f0ae24779d3876971d4e4e8c87dc924 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:59:04 +0000 Subject: [PATCH 11/12] Format trace-log diagnostics --- baseline/rg_baselines/diagnostics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/baseline/rg_baselines/diagnostics.py b/baseline/rg_baselines/diagnostics.py index 80a1b48..cc705d9 100644 --- a/baseline/rg_baselines/diagnostics.py +++ b/baseline/rg_baselines/diagnostics.py @@ -96,6 +96,8 @@ def clean_positive_eigenvalues( if evals.size < 2: raise ValueError("fewer than two finite positive eigenvalues") return evals + + def _entropy_effective_rank(evals: np.ndarray) -> float: total = float(np.sum(evals)) if total <= 0.0: From 96ef24fd31185988f36317e3fe1c5913b8ac46d3 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Sun, 9 Aug 2026 16:59:34 -0700 Subject: [PATCH 12/12] Remove temporary trace-log formatting workflow --- .github/workflows/format-mnist-trace-log.yml | 50 -------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/format-mnist-trace-log.yml diff --git a/.github/workflows/format-mnist-trace-log.yml b/.github/workflows/format-mnist-trace-log.yml deleted file mode 100644 index d8ef631..0000000 --- a/.github/workflows/format-mnist-trace-log.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Format MNIST trace-log patch - -on: - push: - branches: - - "agent/harden-mnist-trace-log" - paths: - - ".github/workflows/format-mnist-trace-log.yml" - -permissions: - contents: write - -jobs: - format-and-test: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - with: - ref: agent/harden-mnist-trace-log - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Restore module-level spacing - run: | - python - <<'PY' - from pathlib import Path - - path = Path("baseline/rg_baselines/diagnostics.py") - text = path.read_text(encoding="utf-8") - old = " return evals\ndef _entropy_effective_rank" - new = " return evals\n\n\ndef _entropy_effective_rank" - if text.count(old) != 1: - raise RuntimeError("unexpected diagnostics function spacing") - path.write_text(text.replace(old, new, 1), encoding="utf-8") - PY - git diff --check - python -m compileall -q baseline/rg_baselines baseline/tests - - - name: Commit formatting correction - run: | - git config user.name "github-actions[bot]" - git config user.email \ - "41898282+github-actions[bot]@users.noreply.github.com" - git add baseline/rg_baselines/diagnostics.py - git commit -m "Format trace-log diagnostics" - git push origin HEAD:agent/harden-mnist-trace-log