From a4555bc15f05836b508ba87fdec0611c971a0cf0 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:25:18 -0700 Subject: [PATCH 01/49] bench: add ScoringBench integration --- .gitignore | 1 + benchmarks/scoringbench/README.md | 149 +++++++ benchmarks/scoringbench/__init__.py | 6 + benchmarks/scoringbench/openboost_wrapper.py | 166 ++++++++ benchmarks/scoringbench/requirements.txt | 11 + benchmarks/scoringbench/run.py | 381 ++++++++++++++++++ .../scoringbench/test_openboost_wrapper.py | 33 ++ 7 files changed, 747 insertions(+) create mode 100644 benchmarks/scoringbench/README.md create mode 100644 benchmarks/scoringbench/__init__.py create mode 100644 benchmarks/scoringbench/openboost_wrapper.py create mode 100644 benchmarks/scoringbench/requirements.txt create mode 100644 benchmarks/scoringbench/run.py create mode 100644 benchmarks/scoringbench/test_openboost_wrapper.py diff --git a/.gitignore b/.gitignore index 8294445..7b1b665 100644 --- a/.gitignore +++ b/.gitignore @@ -221,4 +221,5 @@ logs/ # Benchmark results (generated) benchmarks/results/*.json +benchmarks/results/scoringbench*/ tasks/ diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md new file mode 100644 index 0000000..d68bf19 --- /dev/null +++ b/benchmarks/scoringbench/README.md @@ -0,0 +1,149 @@ +# OpenBoost on ScoringBench + +[ScoringBench](https://github.com/jonaslandsgesell/ScoringBench) is an external +benchmark for probabilistic regression. It evaluates complete predictive +distributions with proper scoring rules and publishes accepted results at +[scoringbench.com](https://scoringbench.com/). This integration uses its dataset +loader, folds, metrics and Parquet schema without modifying the checkout. + +This is the primary third-party value benchmark for OpenBoost. It answers two +different questions with two deliberately separate protocols: + +1. **Official quality track**: ScoringBench's default 3,000-row cap and full + dataset suite. These results can be proposed for its public leaderboard. +2. **Scale extension**: selected ScoringBench datasets with a larger or removed + row cap. This measures OpenBoost's CPU/CUDA scaling but must not be presented + as an official ScoringBench leaderboard result. + +## Environment + +Use a separate Linux environment because ScoringBench currently constrains +NumPy to `>=2,<2.3` and imports PyTorch for its metrics. Intel macOS is not +supported by the complete launcher: the available PyTorch wheel uses the NumPy +1.x ABI and can crash with ScoringBench's NumPy 2.x requirement. Published CPU +and CUDA measurements should come from Linux in any case. The smaller wrapper +contract test remains useful for local adapter development. + +```bash +git clone https://github.com/jonaslandsgesell/ScoringBench .repos/ScoringBench + +uv venv .venv-scoringbench --python 3.12 +uv pip install --python .venv-scoringbench/bin/python \ + -r benchmarks/scoringbench/requirements.txt +uv pip install --python .venv-scoringbench/bin/python -e . +``` + +For CUDA, install OpenBoost's CUDA extra using the package versions appropriate +for the benchmark machine: + +```bash +uv pip install --python .venv-scoringbench/bin/python -e '.[cuda]' +``` + +Optional comparison models: + +```bash +uv pip install --python .venv-scoringbench/bin/python xgboostlss catboost +``` + +## Validate the adapter + +This uses one existing sklearn dataset and the complete ScoringBench metrics, +but is only an integration smoke test: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --smoke \ + --n-trees 20 \ + --output-dir /tmp/openboost-scoringbench-smoke +``` + +The smaller wrapper contract test can also be run directly: + +```bash +PYTHONPATH=.repos/ScoringBench \ + .venv-scoringbench/bin/python -m pytest \ + benchmarks/scoringbench/test_openboost_wrapper.py -q +``` + +## Official quality track + +Run the official default: five folds, one repeat, at most 3,000 rows per +dataset. Start with OpenBoost and the existing NGBoost wrapper: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --output-dir benchmarks/results/scoringbench-quality +``` + +Use `--dataset-index N` or `--dataset-name NAME` for resumable shards. Use +`--list-datasets` to display the validated list. Do not tune OpenBoost on the +test folds. If hyperparameters are changed, apply the same declared search +budget to every comparison model. + +After all shards complete, run ScoringBench's own aggregation and autoranking: + +```bash +cd .repos/ScoringBench +python aggregate_datasets.py \ + --raw_dir ../../benchmarks/results/scoringbench-quality/raw \ + --out_dir ../../benchmarks/results/scoringbench-quality +python autorank_leaderboard.py --output_dir ../../benchmarks/results/scoringbench-quality +``` + +For an upstream submission, copy `openboost_wrapper.py` into +`scoringbench/wrappers/`, register `OpenBoostWrapper` in the upstream wrapper +exports and add a zero-argument factory to its `MODELS` dictionary. Submit the +wrapper, raw/aggregated Parquet artifacts and leaderboard JSON for independent +review. + +## ScoringBench scale extension + +First identify large datasets from the official list, then run the same folds +and metrics without the 3,000-row cap. CPU and CUDA are separate model names so +their results cannot be confused: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,openboost_cuda,ngboost \ + --dataset-name '' \ + --sample-size 0 \ + --n-folds 5 \ + --output-dir benchmarks/results/scoringbench-scale +``` + +Run each CUDA measurement in a fresh process. Report cold and repeated runs +separately, and include failures/OOMs. The generated `openboost_manifest.json` +records both git commits, dirty state, arguments, package versions, platform and +GPU identity. Runs whose manifest says `scoringbench_scale_extension` are not +official leaderboard runs. + +Generated ScoringBench directories are gitignored. Publish accepted evidence in +ScoringBench's designated output/LFS repository or intentionally force-add a +frozen artifact; do not commit an arbitrary local smoke run. + +## Evidence gate + +OpenBoost should claim value only after all of the following are true: + +- the wrapper and results are accepted upstream by ScoringBench; +- quality is reported across the full suite, not a selected winning subset; +- paired fold-level CRPS/log-score/interval-score differences include + uncertainty intervals or the upstream statistical ranking; +- CPU and CUDA predictions pass a separate parity gate; +- a scale curve uses at least three real datasets and multiple data sizes; +- a speed claim is made only at matched predictive quality, with raw Parquet + files and `openboost_manifest.json` published. + +The benchmark is allowed to disprove the product hypothesis. If OpenBoost is +not competitive on proper scoring rules or does not accelerate at larger row +counts, the result should be published and the implementation fixed before the +README makes a performance claim. diff --git a/benchmarks/scoringbench/__init__.py b/benchmarks/scoringbench/__init__.py new file mode 100644 index 0000000..ee2d4bb --- /dev/null +++ b/benchmarks/scoringbench/__init__.py @@ -0,0 +1,6 @@ +"""OpenBoost integration for the external ScoringBench benchmark. + +The adapter is not imported here because ScoringBench is an optional external +checkout. Import ``benchmarks.scoringbench.openboost_wrapper`` explicitly after +putting that checkout on ``PYTHONPATH``. +""" diff --git a/benchmarks/scoringbench/openboost_wrapper.py b/benchmarks/scoringbench/openboost_wrapper.py new file mode 100644 index 0000000..afe8822 --- /dev/null +++ b/benchmarks/scoringbench/openboost_wrapper.py @@ -0,0 +1,166 @@ +"""ScoringBench wrapper for OpenBoost NaturalBoost. + +This module intentionally lives in OpenBoost's repository while the integration +is being validated. It is also shaped as an upstream-ready ScoringBench wrapper: +copy it to ``scoringbench/wrappers/openboost_wrapper.py`` and update the upstream +registry when submitting benchmark results. +""" + +from __future__ import annotations + +from contextlib import nullcontext + +import numpy as np + +try: + from scoringbench.wrappers.base import DistributionPrediction, ProbabilisticWrapper + from scoringbench.wrappers.quantile_based import quantiles_to_distribution +except ImportError as exc: # pragma: no cover - depends on the external checkout + raise ImportError( + "OpenBoostWrapper requires a ScoringBench checkout on PYTHONPATH. " + "See benchmarks/scoringbench/README.md." + ) from exc + + +class OpenBoostWrapper(ProbabilisticWrapper): + """OpenBoost NaturalBoost with a Gaussian predictive distribution. + + Parameters mirror ScoringBench's NGBoost Gaussian entry by default: 500 + boosting rounds, learning rate 0.01, depth-3 trees and 99 quantile levels. + The backend is explicit so CPU and CUDA results cannot be accidentally + conflated on a leaderboard. + + Parameters + ---------- + backend: + ``"cpu"``, ``"cuda"`` or ``"auto"``. ``"auto"`` uses OpenBoost's + normal backend detection; reproducible benchmark runs should use an + explicit backend. + n_trees: + Number of NaturalBoost rounds. + learning_rate: + Boosting shrinkage. + max_depth: + Maximum depth of each parameter tree. + n_bins: + Histogram bins. OpenBoost reserves bin 255 for missing values, so 254 + is the largest non-warning value. + n_quantiles: + Number of probability levels used to convert the analytic Normal + distribution into ScoringBench's common PMF representation. + model_params: + Additional keyword arguments forwarded to ``NaturalBoostNormal``. + """ + + _VALID_BACKENDS = {"auto", "cpu", "cuda"} + + def __init__( + self, + *, + backend: str = "cpu", + n_trees: int = 500, + learning_rate: float = 0.01, + max_depth: int = 3, + n_bins: int = 254, + n_quantiles: int = 99, + model_params: dict | None = None, + ) -> None: + backend = backend.lower() + if backend not in self._VALID_BACKENDS: + raise ValueError( + f"backend must be one of {sorted(self._VALID_BACKENDS)}, got {backend!r}" + ) + if n_quantiles < 2: + raise ValueError("n_quantiles must be at least 2") + + self.backend = backend + self.n_trees = n_trees + self.learning_rate = learning_rate + self.max_depth = max_depth + self.n_bins = n_bins + self.n_quantiles = n_quantiles + self.model_params = dict(model_params or {}) + + self._alphas = np.linspace( + 1 / (n_quantiles + 1), + n_quantiles / (n_quantiles + 1), + n_quantiles, + dtype=np.float64, + ) + self._model = None + self._resolved_backend: str | None = None + self._y_range = (0.0, 1.0) + + @staticmethod + def _sanitize_X(X) -> np.ndarray: + X = np.asarray(X, dtype=np.float32) + if X.ndim != 2: + raise ValueError(f"X must be 2-dimensional, got shape {X.shape}") + return np.nan_to_num(X, nan=0.0, posinf=1e7, neginf=-1e7) + + def _backend_context(self): + import openboost as ob + + if self._resolved_backend is None: + return nullcontext() + return ob.backend_context(self._resolved_backend) + + def _require_fitted(self) -> None: + if self._model is None: + raise RuntimeError("Model not fitted. Call fit() first.") + + def fit(self, X, y) -> OpenBoostWrapper: + import openboost as ob + + X = self._sanitize_X(X) + y = np.asarray(y, dtype=np.float32).reshape(-1) + valid = np.isfinite(y) + X, y = X[valid], y[valid] + if len(y) == 0: + raise ValueError("No valid finite training samples") + + lo, hi = float(y.min()), float(y.max()) + if lo == hi: + pad = max(abs(lo) * 1e-6, 1e-7) + lo, hi = lo - pad, hi + pad + self._y_range = (lo, hi) + + self._resolved_backend = ob.get_backend() if self.backend == "auto" else self.backend + params = { + "n_trees": self.n_trees, + "learning_rate": self.learning_rate, + "max_depth": self.max_depth, + "n_bins": self.n_bins, + **self.model_params, + } + self._model = ob.NaturalBoostNormal(**params) + with self._backend_context(): + self._model.fit(X, y) + return self + + def predict(self, X) -> np.ndarray: + self._require_fitted() + X = self._sanitize_X(X) + with self._backend_context(): + pred = self._model.predict(X) + return np.asarray(pred, dtype=np.float64).reshape(-1) + + def predict_distribution(self, X) -> DistributionPrediction: + self._require_fitted() + X = self._sanitize_X(X) + with self._backend_context(): + output = self._model.predict_distribution(X) + mean = np.asarray(output.mean(), dtype=np.float64).reshape(-1) + quantiles = np.column_stack( + [ + np.asarray(output.quantile(float(alpha)), dtype=np.float64) + for alpha in self._alphas + ] + ) + + return quantiles_to_distribution( + quantiles, + self._alphas, + mean=mean, + y_range=self._y_range, + ) diff --git a/benchmarks/scoringbench/requirements.txt b/benchmarks/scoringbench/requirements.txt new file mode 100644 index 0000000..44cf484 --- /dev/null +++ b/benchmarks/scoringbench/requirements.txt @@ -0,0 +1,11 @@ +# Base ScoringBench runtime, deliberately excluding its many heavyweight model +# extras. Install only the additional baselines you plan to run. +numpy>=2.0,<2.3 +scikit-learn>=1.3 +pandas>=2.0 +torch>=2.0 +pyarrow>=15 +autorank>=1.2 +openml>=0.15 +pytest>=7 +ngboost>=0.5 diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py new file mode 100644 index 0000000..8f3a552 --- /dev/null +++ b/benchmarks/scoringbench/run.py @@ -0,0 +1,381 @@ +"""Run OpenBoost through an unmodified ScoringBench checkout. + +The official suite owns datasets, folds, metrics and Parquet output. This +launcher only registers OpenBoost (plus selected existing baselines), records +provenance and exposes a small smoke mode for integration testing. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import os +import platform +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = PROJECT_ROOT / "src" + + +def _csv(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +def _git_state(path: Path) -> dict: + def run(*args: str) -> str | None: + try: + result = subprocess.run( + ["git", "-C", str(path), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + status = run("status", "--porcelain") + return { + "commit": run("rev-parse", "HEAD"), + "dirty": bool(status) if status is not None else None, + } + + +def _package_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _gpu_info() -> dict | None: + try: + from numba import cuda + + if not cuda.is_available(): + return None + device = cuda.get_current_device() + name = device.name.decode() if isinstance(device.name, bytes) else str(device.name) + return { + "name": name, + "compute_capability": list(device.compute_capability), + } + except Exception as exc: # provenance should never fail the benchmark + return {"error": f"{type(exc).__name__}: {exc}"} + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run OpenBoost on the external ScoringBench protocol" + ) + parser.add_argument( + "--scoringbench-dir", + default=os.environ.get("SCORINGBENCH_DIR", ".repos/ScoringBench"), + help="Path to a ScoringBench git checkout", + ) + parser.add_argument( + "--output-dir", + default="benchmarks/results/scoringbench", + help="ScoringBench Parquet and OpenBoost manifest output", + ) + parser.add_argument( + "--models", + type=_csv, + default=["openboost_cpu", "ngboost"], + help=( + "Comma-separated models: openboost_cpu, openboost_cuda, ngboost, " + "xgblss, catboost_quantile" + ), + ) + parser.add_argument("--n-trees", type=int, default=500) + parser.add_argument("--learning-rate", type=float, default=0.01) + parser.add_argument("--max-depth", type=int, default=3) + parser.add_argument("--n-quantiles", type=int, default=99) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--n-folds", type=int, default=5) + parser.add_argument("--n-repeats", type=int, default=1) + parser.add_argument( + "--sample-size", + type=int, + default=3000, + help="Official ScoringBench default is 3000; use 0 only for a scale extension", + ) + parser.add_argument( + "--dataset-index", + type=int, + action="append", + help="Run selected index from ScoringBench's validated dataset list (repeatable)", + ) + parser.add_argument( + "--dataset-name", + action="append", + help="Run exact case-insensitive dataset name from the validated list (repeatable)", + ) + parser.add_argument( + "--lite", + action="store_true", + help="Use two folds while retaining ScoringBench datasets and metrics", + ) + parser.add_argument( + "--smoke", + action="store_true", + help="Use sklearn diabetes with 2 folds; validates integration, not leaderboard evidence", + ) + parser.add_argument( + "--list-datasets", + action="store_true", + help="Print ScoringBench's validated dataset names and exit", + ) + return parser + + +def _select_datasets(all_datasets: list[dict], args) -> list[dict]: + if args.dataset_index: + invalid = [i for i in args.dataset_index if i < 0 or i >= len(all_datasets)] + if invalid: + raise ValueError( + f"dataset indices out of range: {invalid}; valid range is 0..{len(all_datasets) - 1}" + ) + return [all_datasets[i] for i in args.dataset_index] + + if args.dataset_name: + lookup = {dataset["name"].casefold(): dataset for dataset in all_datasets} + missing = [name for name in args.dataset_name if name.casefold() not in lookup] + if missing: + raise ValueError(f"unknown dataset names: {missing}; use --list-datasets") + return [lookup[name.casefold()] for name in args.dataset_name] + + return all_datasets + + +def _model_factories(args): + from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper + + common = { + "n_trees": args.n_trees, + "learning_rate": args.learning_rate, + "max_depth": args.max_depth, + "n_quantiles": args.n_quantiles, + } + + def openboost(backend: str): + return lambda: OpenBoostWrapper(backend=backend, **common) + + factories = { + "openboost_cpu": openboost("cpu"), + "openboost_cuda": openboost("cuda"), + } + + if "ngboost" in args.models: + from scoringbench.wrappers.ngboost_wrapper import NGBoostWrapper + + factories["ngboost"] = lambda: NGBoostWrapper( + dist="normal", + n_estimators=args.n_trees, + learning_rate=args.learning_rate, + n_quantiles=args.n_quantiles, + ngb_params={"random_state": args.seed}, + ) + + if "xgblss" in args.models: + from scoringbench.wrappers.xgblss_wrapper import XGBLSSWrapper + + factories["xgblss"] = lambda: XGBLSSWrapper( + n_quantiles=args.n_quantiles, + num_boost_round=args.n_trees, + distribution="Gaussian", + xgblss_params={"max_depth": args.max_depth, "eta": args.learning_rate}, + ) + + if "catboost_quantile" in args.models: + from scoringbench.wrappers.catboost_wrapper import CatBoostQuantileWrapper + + factories["catboost_quantile"] = lambda: CatBoostQuantileWrapper( + n_quantiles=args.n_quantiles, + iterations=args.n_trees, + catboost_params={ + "depth": args.max_depth, + "learning_rate": args.learning_rate, + "random_seed": args.seed, + }, + ) + + valid = set(factories) + unknown = [name for name in args.models if name not in valid] + if unknown: + allowed = [ + "openboost_cpu", + "openboost_cuda", + "ngboost", + "xgblss", + "catboost_quantile", + ] + raise ValueError(f"unknown models {unknown}; allowed values: {allowed}") + + return {name: factories[name] for name in args.models} + + +def _write_provenance( + output_dir: Path, + scoringbench_dir: Path, + args, + datasets: list[dict], + result_rows: int, +) -> Path: + import openboost as ob + + official_shape = ( + not args.smoke + and args.sample_size == 3000 + and args.n_folds == 5 + and args.n_repeats == 1 + ) + if args.smoke: + protocol_mode = "smoke" + elif args.sample_size != 3000: + protocol_mode = "scoringbench_scale_extension" + elif official_shape and (args.dataset_index or args.dataset_name): + protocol_mode = "official_quality_shard" + elif official_shape: + protocol_mode = "official_quality" + else: + protocol_mode = "scoringbench_protocol_deviation" + + manifest = { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "protocol": "ScoringBench", + "protocol_mode": protocol_mode, + "official_protocol_compatible": official_shape, + "warning": ( + None + if official_shape + else "This run is not directly comparable to the official 5-fold, sample_size=3000 leaderboard." + ), + "openboost_git": _git_state(PROJECT_ROOT), + "scoringbench_git": _git_state(scoringbench_dir), + "arguments": vars(args), + "datasets": [ + { + "name": dataset["name"], + "source": dataset.get("source", "openml"), + "id": dataset.get("id", dataset.get("loader")), + } + for dataset in datasets + ], + "result_rows": result_rows, + "platform": { + "python": platform.python_version(), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "processor": platform.processor(), + "cpu_count": os.cpu_count(), + "gpu": _gpu_info(), + }, + "versions": { + "openboost": ob.__version__, + "numpy": np.__version__, + **{ + name: _package_version(name) + for name in ( + "scipy", + "scikit-learn", + "pandas", + "pyarrow", + "torch", + "numba", + "numba-cuda", + "cupy-cuda12x", + "ngboost", + "xgboost", + "xgboostlss", + "catboost", + ) + }, + }, + } + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / "openboost_manifest.json" + path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + return path + + +def main() -> int: + args = _build_parser().parse_args() + if sys.platform == "darwin" and platform.machine() == "x86_64": + raise SystemExit( + "The complete ScoringBench runner is unsupported on Intel macOS: " + "the available PyTorch wheel uses the NumPy 1.x ABI while " + "ScoringBench requires NumPy 2.x. Run the benchmark on Linux " + "(the target environment for published CPU/CUDA results). The " + "wrapper contract test can still be run separately." + ) + scoringbench_dir = Path(args.scoringbench_dir).expanduser().resolve() + if not (scoringbench_dir / "scoringbench" / "runner.py").exists(): + raise SystemExit( + f"No ScoringBench checkout found at {scoringbench_dir}. " + "Clone https://github.com/jonaslandsgesell/ScoringBench first." + ) + + sys.path.insert(0, str(SRC_ROOT)) + sys.path.insert(0, str(PROJECT_ROOT)) + sys.path.insert(0, str(scoringbench_dir)) + + from scoringbench.datasets import get_DATASETS_CONFIG, validate_datasets + from scoringbench.runner import run_benchmark + from scoringbench.utils import set_seed + + set_seed(args.seed) + if args.smoke: + datasets = [ + { + "name": "diabetes_smoke", + "source": "sklearn", + "loader": "load_diabetes", + "abbr": "DBS", + "sample_size": min(args.sample_size or 442, 442), + } + ] + args.n_folds = 2 + else: + datasets = validate_datasets(get_DATASETS_CONFIG()) + if args.list_datasets: + for index, dataset in enumerate(datasets): + print(f"{index:3d} {dataset['name']}") + return 0 + datasets = _select_datasets(datasets, args) + + if args.lite: + args.n_folds = 2 + + model_factories = _model_factories(args) + output_dir = Path(args.output_dir).expanduser().resolve() + result = run_benchmark( + datasets_config=datasets, + model_factories=model_factories, + output_dir=output_dir, + n_folds=args.n_folds, + n_repeats_cv=args.n_repeats, + seed=args.seed, + sample_size=args.sample_size, + ) + manifest = _write_provenance( + output_dir, + scoringbench_dir, + args, + datasets, + result_rows=len(result), + ) + print(f"OpenBoost provenance: {manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/scoringbench/test_openboost_wrapper.py b/benchmarks/scoringbench/test_openboost_wrapper.py new file mode 100644 index 0000000..a9c7127 --- /dev/null +++ b/benchmarks/scoringbench/test_openboost_wrapper.py @@ -0,0 +1,33 @@ +"""Upstream-style smoke test for the OpenBoost ScoringBench wrapper.""" + +import numpy as np +from scoringbench.wrappers.base import DistributionPrediction + +from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper + + +def test_openboost_wrapper_distribution_contract(): + rng = np.random.default_rng(42) + X = rng.normal(size=(160, 4)).astype(np.float32) + sigma = 0.25 + np.abs(X[:, 1]) + y = (2 * X[:, 0] - X[:, 2] + rng.normal(scale=sigma)).astype(np.float32) + + model = OpenBoostWrapper( + backend="cpu", + n_trees=12, + learning_rate=0.05, + max_depth=2, + n_quantiles=15, + ) + returned = model.fit(X[:120], y[:120]) + distribution = model.predict_distribution(X[120:]) + + assert returned is model + assert isinstance(distribution, DistributionPrediction) + assert distribution.probas.shape == (40, 14) + assert distribution.bin_edges.shape == (40, 15) + assert distribution.mean.shape == (40,) + assert np.all(np.isfinite(distribution.probas)) + assert np.all(np.isfinite(distribution.bin_edges)) + assert np.allclose(distribution.probas.sum(axis=1), 1.0) + assert np.all(np.diff(distribution.bin_edges, axis=1) > 0) From cf6ae2e6e3e66c600baab83a61dd4b0679e015ff Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:28:03 -0700 Subject: [PATCH 02/49] docs: add agent guide and learning log --- AGENTS.md | 156 +++++++++++++++++ CLAUDE.md | 158 ++---------------- learnings/2026-08-15-repository-audit.md | 67 ++++++++ .../2026-08-15-scoringbench-integration.md | 59 +++++++ learnings/README.md | 40 +++++ learnings/TEMPLATE.md | 32 ++++ 6 files changed, 366 insertions(+), 146 deletions(-) create mode 100644 AGENTS.md create mode 100644 learnings/2026-08-15-repository-audit.md create mode 100644 learnings/2026-08-15-scoringbench-integration.md create mode 100644 learnings/README.md create mode 100644 learnings/TEMPLATE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5ca99ab --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,156 @@ +# OpenBoost Agent Guide + +This is the canonical repository guidance for coding agents and automated +contributors. Tool-specific instruction files should point here instead of +duplicating policy. + +## Mission + +OpenBoost is a readable Python gradient-boosting research platform. Its current +product focus is **calibration-first distributional boosting for tabular risk**: +NaturalBoost, proper scoring, calibration, exposure-aware targets, custom +distributions, and verified CPU/CUDA execution. + +Do not position the repository as a drop-in replacement for XGBoost, LightGBM, +or CatBoost. Standard GBDT, GAM, DART, linear leaves, Ray, multi-GPU, and +train-many are supporting or experimental capabilities unless a committed, +reproducible artifact proves otherwise. + +## Start Here + +Before a non-trivial change: + +1. Read this file and the relevant entries in `learnings/`. +2. Check `git status --short --branch`; preserve unrelated user changes. +3. Read the implementation, its tests, and the public documentation together. +4. Write a short plan for work spanning three or more meaningful steps. +5. Identify the smallest test that can fail before editing. + +Do not trust phase comments, docstrings, README claims, or green CI as proof by +themselves. Verify the actual call path and the tests that exercise it. + +## Current Priority Order + +1. Silent correctness and persistence bugs. +2. Deterministic CPU behavior and reference parity. +3. One verified single-GPU NaturalBoost path, including end-to-end quality. +4. Third-party evidence through ScoringBench and real domain case studies. +5. Stable packaging, versioned persistence, and a smaller public API. +6. New features only after the above gates are satisfied. + +Treat Ray, multi-GPU, out-of-core training, GOSS speedups, and fused train-many +as experimental. Do not expand or market them until exact correctness and +scaling artifacts exist. The repository audit in +`learnings/2026-08-15-repository-audit.md` records the current evidence gaps. + +## Architecture + +```text +Models (`src/openboost/_models/`) + -> tree core (`src/openboost/_core/`) + -> CPU/CUDA backends (`src/openboost/_backends/`) + -> distributions (`src/openboost/_distributions.py`) + -> validation and persistence +``` + +- `BinnedArray` is feature-major: `(n_features, n_samples)`. +- Bin 255 is reserved for missing values; use at most 254 regular bins. +- The backend is process-global, not thread-local. Use `backend_context` for a + scoped switch and do not run mixed-backend fits concurrently in one process. +- NaturalBoost fits one tree per distribution parameter per round. +- CUDA eligibility is narrower than the public model surface. A fallback must + be visible, tested, and represented honestly in benchmark provenance. + +## Correctness Rules + +- A serialization change requires prediction round trips for numeric, + categorical, missing-value, and specialized leaf/tree state that it touches. +- A CUDA change requires CPU/CUDA parity for gradients, splits, leaves, + predictions, and final task metrics—not just matching array shapes. +- Never silently ignore `sample_weight`, exposure, callbacks, evaluation sets, + constraints, or sampling parameters. Support them or reject them explicitly. +- Randomized behavior must be driven by the model's declared seed; do not use + unscoped global `numpy.random` state. +- Distributed child histograms must be derived from routed samples. Scaling a + parent histogram is not an exact substitute. +- Public examples are tests of product behavior. If an example cannot run, fix + it or label the feature experimental before documenting it. + +## Evidence and Benchmark Rules + +- Every performance or quality claim must link to a committed raw artifact. +- Record git SHA and dirty state, dataset/version/hash, split seed, package + versions, OS, CPU/RAM/thread count, GPU/driver/CUDA, and exact CLI arguments. +- Compare end-to-end fit and prediction, including distribution gradients, + transfers, compilation policy, and fallbacks. Kernel microbenchmarks cannot + support an end-to-end product claim. +- Use repeated folds/seeds and publish failures. Compare at matched predictive + quality; do not declare a speed win when CRPS/NLL/calibration regresses. +- Keep official ScoringBench results separate from OpenBoost's large-sample + extension. See `benchmarks/scoringbench/README.md`. +- Synthetic experiments generate hypotheses. Real third-party datasets and + upstream-accepted results generate evidence. + +## Commands + +Use `uv`; do not mutate the project environment with ad-hoc `pip` or Conda +commands. + +```bash +# Install +uv sync --extra dev +uv sync --extra cuda + +# Focused test while iterating +OPENBOOST_BACKEND=cpu uv run pytest tests/test_file.py -n 0 -q + +# CPU regression suite +OPENBOOST_BACKEND=cpu uv run pytest tests/ -m "not gpu and not benchmark" --tb=short + +# Lint production code and changed support files +uv run ruff check src/openboost/ path/to/changed_file.py + +# Documentation and packaging +uv run mkdocs build +uv build +``` + +Run CUDA tests only on real CUDA hardware. A skipped GPU job is not a passing +GPU validation. ScoringBench has a separate Linux environment documented under +`benchmarks/scoringbench/`. + +## Working and Commit Discipline + +- Keep changes small and cohesive. Prefer root-cause fixes over compatibility + shims that conceal invalid state. +- Commit after each independently verified slice: test/benchmark harness, + correctness fix, documentation/learning update, or infrastructure change. +- Do not bundle unrelated cleanup into a fix. Do not amend or rewrite existing + commits unless the user explicitly asks. +- Do not push, publish, create a release, or update an external leaderboard + unless the user asks for that external action. +- Before every commit: inspect the staged diff, run the narrowest meaningful + tests, and include the verification in the relevant learning entry. + +## Learning Log + +`learnings/` is the durable project memory for decisions, failed attempts, +experiments, and non-obvious operational facts. + +- Add or update an entry for every non-trivial change. +- Use `learnings/TEMPLATE.md`. +- Record evidence and falsified hypotheses, not a diary of shell commands. +- Link files and commits. State what was not verified. +- Never include credentials, tokens, private URLs, or user-specific secrets. +- Release notes describe user-facing changes; learning entries explain why the + implementation and evidence changed. + +## Definition of Done + +A change is done only when: + +1. The intended behavior is covered by a focused test or reproducible artifact. +2. Relevant regression tests and lint pass. +3. Documentation and capability claims match the implemented boundary. +4. A learning entry captures important decisions, failures, and follow-ups. +5. The change is committed as a cohesive unit and the remaining work is stated. diff --git a/CLAUDE.md b/CLAUDE.md index b8e57b9..b7405a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,148 +1,14 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -OpenBoost is a GPU-native, all-Python gradient boosting library (~20K lines). It uses Numba JIT for CPU kernels and CuPy/numba-cuda for GPU acceleration. Designed as a research-friendly alternative to XGBoost/LightGBM with full Python source. - -## Commands - -```bash -# Environment (always use uv, never pip/conda/poetry) -uv sync # Install/sync dependencies -uv sync --extra cuda # With GPU support -uv sync --extra dev # With dev tools (test + bench + sklearn + ruff) - -# Testing (parallelized with pytest-xdist, -n auto is in addopts) -uv run pytest tests/ -v --tb=short # All tests (CPU, parallel) -uv run pytest tests/test_core.py -v # Single test file -uv run pytest tests/test_core.py::test_name -v # Single test -uv run pytest tests/ -n 0 # Force serial (debugging) -OPENBOOST_BACKEND=cuda uv run pytest tests/ # GPU tests -OPENBOOST_BACKEND=cpu uv run pytest tests/ # Force CPU - -# Profiling -uv run python benchmarks/profile_loop.py # Profile training (50K samples default) -uv run python benchmarks/profile_loop.py --summarize # Machine-readable bottleneck summary -OPENBOOST_PROFILE=1 uv run python script.py # Profile any training run via env var - -# Linting -uv run ruff check src/openboost/ # Lint -uv run ruff check src/openboost/ --fix # Autofix - -# Docs -uv run mkdocs serve # Local docs server -uv run mkdocs build # Build docs - -# Build -uv build # Build wheel/sdist -``` - -## Architecture - -### Layer Overview - -``` -Models (_models/) → Core (_core/) → Backends (_backends/) - ↓ ↓ ↓ -GradientBoosting fit_tree() _cpu.py (Numba JIT) -NaturalBoost histograms _cuda.py (CuPy kernels) -OpenBoostGAM split finding -DART, LinearLeaf growth strategies -``` - -### Data Layer (`_array.py`) -`BinnedArray` is the fundamental data structure — quantile-bins continuous features into uint8 (max 255 bins). Missing values encode as `MISSING_BIN = 255`. Native categorical feature support with auto-detection of string/object columns. All tree-building operates on binned data. - -### Core (`_core/`) -- **`_tree.py`** — `fit_tree()`, `fit_tree_gpu_native()`, `fit_tree_symmetric()`: the main tree-fitting entry points -- **`_primitives.py`** — Low-level histogram building, split finding, sample partitioning -- **`_growth.py`** — Three growth strategies: `LevelWiseGrowth` (XGBoost-style), `LeafWiseGrowth` (LightGBM-style), `SymmetricGrowth` (CatBoost-style) - -### Backend Dispatch (`_backends/`) -`get_backend()` / `set_backend()` switch between CPU and CUDA implementations. Same interface, different kernels. Control via `OPENBOOST_BACKEND` env var or `set_backend('cuda')`. Use `backend_context('cpu')` context manager for temporary switches. - -### Models (`_models/`) -- **`_boosting.py`** — `GradientBoosting`, `MultiClassGradientBoosting`: main model classes with full callback/eval_set support -- **`_sklearn.py`** — sklearn-compatible wrappers (`OpenBoostRegressor`, `OpenBoostClassifier`, `OpenBoostDARTRegressor`, `OpenBoostGAMRegressor`, `OpenBoostDistributionalRegressor`, `OpenBoostLinearLeafRegressor`) -- **`_distributional.py`** — `NaturalBoost`: distributional GBDT with callbacks/eval_set support -- **`_dart.py`** — `DART`: dropout boosting with callbacks/eval_set support -- **`_linear_leaf.py`**, **`_gam.py`** — Specialized model variants (both support callbacks/eval_set/early stopping; GAM also supports pairwise interactions, smoothing, and monotone shape constraints) - -### Persistence (`_persistence.py`) -`PersistenceMixin` provides `save()`/`load()` on all models. Generic `ob.load(path)` auto-detects model class from saved state. - -### Profiling (`_profiler.py`) -`ProfilingCallback` instruments training by wrapping core primitives (`build_node_histograms`, `find_node_splits`, `partition_samples`, `compute_leaf_values`, `fit_tree`) with timers. Outputs JSON reports to `logs/` with per-phase breakdown, bottleneck identification, and run-over-run comparison. CLI runner: `benchmarks/profile_loop.py`. - -### Loss Functions (`_loss.py`) -9 objectives, each with CPU/GPU/dispatcher implementations returning `(gradient, hessian)`. Custom losses are callables with signature `fn(pred, y) -> (grad, hess)`; register by name via `register_loss` (optional `loss_value_fn` for true loss reporting), and mark GPU-capable losses with `@ob.device_loss`. `register_distribution` / `register_growth_strategy` extend the other factories. - -### Distributions (`_distributions.py`) -8 distributional families for NaturalBoost (Normal, LogNormal, Gamma, Poisson, StudentT, Tweedie, NegativeBinomial). Each implements `nll_grad_hess()` for natural gradient computation. - -## Key Conventions - -- **Python 3.10+** target. Ruff rules: E, F, I, UP, B, SIM (line length 100; E501, E402, F821 ignored). -- **uv only** for package management — never `pip install` or `conda`. -- All Numba-jitted functions use `@njit` or `@cuda.jit`. CPU kernels are in `_backends/_cpu.py`, CUDA in `_backends/_cuda.py`. -- Test environment variable `OPENBOOST_BACKEND=cpu` forces CPU backend in CI. -- Tests use `pytest-xdist` (`-n auto --dist loadfile`) for parallel execution. Shared fixtures are in `tests/conftest.py` (session-scoped datasets, function-scoped gradients). -- **GPU-native builder** (`fit_tree_gpu_native`) does not support missing values or categorical features. The training loop in `_boosting.py` auto-falls back to `fit_tree()` with a warning when the data has NaN or categorical columns. -- **Callbacks**: all models — `GradientBoosting`, `MultiClassGradientBoosting`, `DART`, `NaturalBoost`/`DistributionalGBDT`, `LinearLeafGBDT`, and `OpenBoostGAM` — support `callbacks` and `eval_set` in `fit()`. -- **`random_state`**: `GradientBoosting` and `DART` accept `random_state` for reproducibility. Sklearn wrappers pass it through. DART also accepts `seed` (alias). -- **`suggest_params()`**: Returns sklearn-style names by default (`n_estimators`). Pass `style='core'` to get core API names (`n_trees`). -- **Profiling**: `ProfilingCallback` wraps core primitives with timers. Enable via callback or `OPENBOOST_PROFILE=1` env var. Reports go to `logs/` as JSON. - -## Working Style - -### 1. Plan Mode Default -- Enter plan mode for ANY non-trivial task (3+ steps or architectural decisions) -- If something goes sideways, STOP and re-plan immediately -- don't keep pushing -- Use plan mode for verification steps, not just building -- Write detailed specs upfront to reduce ambiguity - -### 2. Subagent Strategy -- Use subagents liberally to keep main context window clean -- Offload research, exploration, and parallel analysis to subagents -- For complex problems, throw more compute at it via subagents -- One task per subagent for focused execution - -### 3. Self-Improvement Loop -- After ANY correction from the user: update `tasks/lessons.md` with the pattern -- Write rules for yourself that prevent the same mistake -- Ruthlessly iterate on these lessons until mistake rate drops -- Review lessons at session start for relevant project - -### 4. Verification Before Done -- Never mark a task complete without proving it works -- Diff behavior between main and your changes when relevant -- Ask yourself: "Would a staff engineer approve this?" -- Run tests, check logs, demonstrate correctness - -### 5. Demand Elegance (Balanced) -- For non-trivial changes: pause and ask "is there a more elegant way?" -- If a fix feels hacky: "Knowing everything I know now, implement the elegant solution" -- Skip this for simple, obvious fixes -- don't over-engineer -- Challenge your own work before presenting it - -### 6. Autonomous Bug Fixing -- When given a bug report: just fix it. Don't ask for hand-holding -- Point at logs, errors, failing tests -- then resolve them -- Zero context switching required from the user -- Go fix failing CI tests without being told how - -## Task Management - -1. **Plan First**: Write plan to `tasks/todo.md` with checkable items -2. **Verify Plan**: Check in before starting implementation -3. **Track Progress**: Mark items complete as you go -4. **Explain Changes**: High-level summary at each step -5. **Document Results**: Add review section to `tasks/todo.md` -6. **Capture Lessons**: Update `tasks/lessons.md` after corrections - -## Core Principles - -- **Simplicity First**: Make every change as simple as possible. Impact minimal code. -- **No Laziness**: Find root causes. No temporary fixes. Senior developer standards. +The canonical repository guidance is [`AGENTS.md`](./AGENTS.md). Read it in full +before changing code, benchmarks, documentation, CI, packaging, or release +state. + +Claude-specific compatibility notes: + +- Use `uv` for environments and commands. +- Store durable decisions, failed experiments, and user corrections in + `learnings/`, not the ignored `tasks/` directory. +- Keep commits small and verified. Do not push or publish unless requested. +- Treat GPU, distributed, out-of-core, and performance claims according to the + evidence gates in `AGENTS.md`; comments and green-but-skipped CI are not proof. diff --git a/learnings/2026-08-15-repository-audit.md b/learnings/2026-08-15-repository-audit.md new file mode 100644 index 0000000..6c6ad81 --- /dev/null +++ b/learnings/2026-08-15-repository-audit.md @@ -0,0 +1,67 @@ +# 2026-08-15: Repository Audit and Product Focus + +## Context + +A parallel code, benchmark, release, and ecosystem audit evaluated whether +OpenBoost had a credible path to a niche comparable in clarity—not impact—to +XGBoost. The audit was read-only and used local tests plus current primary +sources for competing libraries and publication routes. + +## Decision or Result + +OpenBoost has a substantive CPU tree core and a strong distributional subsystem, +but it is not yet a trustworthy general-purpose boosting library. The product +focus is now **calibration-first distributional boosting for tabular risk**: +NaturalBoost, exposure-aware count/severity models, proper scoring, calibration, +custom distributions, and verified single-GPU acceleration. + +Generic GBDT, GAM, DART, linear leaves, Ray, multi-GPU, out-of-core, GOSS, and +train-many must not share equal product priority. GPU remains strategically +important, but the next milestone is one correct and evidenced NaturalBoost CUDA +path—not broader unverified GPU surface area. + +## Evidence + +- CPU suite at audit time: 721 passed, 32 skipped, 3 deselected. +- Total coverage: 53%; `_models/_distributional.py` 95%, distributions 79%, + CUDA backend 0%, multi-GPU 16%, distributed tree 17%. +- The only committed third-party artifact was a three-dataset, one-seed CPU + NaturalBoost/NGBoost comparison showing approximate parity, not dominance. +- The strongest external validation opportunity was ScoringBench, which accepts + probabilistic model wrappers and publishes proper-scoring leaderboards. + +## Release-Blocking Findings + +- Categorical persistence used field names different from `TreeStructure`, so a + save/load round trip could change predictions. +- Categorical binning accepted up to 254 values while routing used one 64-bit + bitset. +- GPU GAM training dropped the base score after its first prediction update. +- Ray/multi-GPU workers initialized predictions inconsistently with final model + inference, and multi-GPU child histograms were approximate. +- The documented memmap out-of-core example passed a feature-major array to a + sample-major high-level API; `batch_size` was not connected to model training. +- The performance CI baseline was absent and regenerated on fresh runners, so + the check could succeed without detecting regressions. + +These findings must be re-verified against current code before fixing; this +entry records the audit state, not permanent truth. + +## Product and Evidence Gates + +1. Remove silent correctness failures. +2. Establish deterministic CPU reference behavior. +3. Verify end-to-end CPU/CUDA NaturalBoost parity. +4. Submit full-suite ScoringBench results with raw artifacts. +5. Add a real exposure-aware insurance case study such as freMTPL2. +6. Seek external users and contributions before JOSS/JMLR software submission. + +## Risks and Follow-ups + +- Distributional boosting mostly models aleatoric uncertainty; it does not by + itself solve epistemic/OOD uncertainty. +- PGBM, XGBoostLSS, LightGBMLSS, NGBoost, CatBoost uncertainty, and Py-Boost + already occupy adjacent positions. GPU probabilistic boosting alone is not a + unique claim. +- A benchmark is allowed to falsify the product hypothesis. Quality regressions + cannot be traded for speed without an explicit decision metric. diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md new file mode 100644 index 0000000..8d33563 --- /dev/null +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -0,0 +1,59 @@ +# 2026-08-15: ScoringBench Integration + +## Context + +OpenBoost needed an existing third-party benchmark or competition to demonstrate +value. ScoringBench was selected because it evaluates full probabilistic +regression distributions with proper scoring rules and accepts upstream model +wrappers and result artifacts. + +## Decision or Result + +The integration has two explicitly separated protocols: + +1. `official_quality`: ScoringBench's five-fold, 3,000-row protocol for an + upstream leaderboard submission. +2. `scoringbench_scale_extension`: the same datasets/folds/metrics with a larger + sample cap to compare OpenBoost CPU, OpenBoost CUDA, and existing baselines. + +Scale-extension results must never be represented as official leaderboard +results. ScoringBench proves general probabilistic quality; it does not exercise +OpenBoost's exposure-aware API, which still needs a domain benchmark. + +## Changes + +- `benchmarks/scoringbench/openboost_wrapper.py`: upstream-shaped NaturalBoost + Gaussian wrapper using ScoringBench's shared quantile-to-PMF conversion. +- `benchmarks/scoringbench/run.py`: launcher for an unmodified ScoringBench + checkout, baseline registration, protocol labeling, and provenance manifest. +- `benchmarks/scoringbench/README.md`: environment, official track, scale track, + upstream submission, and evidence gates. +- `.gitignore`: ignore arbitrary local ScoringBench result directories. + +## Verification + +- Wrapper contract: 1 passed against ScoringBench commit + `a938a667b7839b41e9272929010573410301c0b4`. +- OpenBoost distributional regression tests: 47 passed. +- `ruff check benchmarks/scoringbench`: passed. +- Python compilation and manifest protocol classification: passed. +- Integration commit: `a4555bc` (`bench: add ScoringBench integration`). + +## Failed Attempts + +- The composer-swarm Cursor scout repeatedly failed with macOS Keychain error + `SecItemCopyMatching failed -50`. Use local inspection until its CLI + authentication is repaired; do not repeatedly retry it during one task. +- The complete ScoringBench runner is not viable on Intel macOS. ScoringBench + requires NumPy 2.x, while the available PyTorch wheel uses the NumPy 1.x ABI. + One run crashed and later attempts entered an uninterruptible kernel exit + state. The launcher now refuses this platform before importing ScoringBench. + Published CPU/CUDA runs must use Linux. + +## Risks and Follow-ups + +- Run the official full suite on Linux and submit the wrapper/results upstream. +- Run a separate large-sample curve on at least three real ScoringBench datasets. +- Add CPU/CUDA prediction parity before interpreting a CUDA timing result. +- Add freMTPL2 or another real exposure-aware case study after the third-party + quality result exists. diff --git a/learnings/README.md b/learnings/README.md new file mode 100644 index 0000000..5b9f1db --- /dev/null +++ b/learnings/README.md @@ -0,0 +1,40 @@ +# OpenBoost Learnings + +This directory is the repository's durable engineering memory. It records why a +change was made, what evidence supports it, which attempts failed, and what +remains unknown. It is deliberately separate from release notes and generated +benchmark output. + +## When to Write an Entry + +Create or update an entry when work includes any of the following: + +- a non-obvious correctness fix; +- a benchmark, experiment, or falsified hypothesis; +- an architecture or product-scope decision; +- a dependency, platform, CI, packaging, or release incident; +- a user correction that should change future agent behavior. + +Use `YYYY-MM-DD-short-topic.md`. Prefer one entry per coherent investigation; +append follow-up evidence rather than creating many tiny diary files. + +## Required Content + +Start from `TEMPLATE.md` and include: + +- context and the question being answered; +- decision or result; +- files/behavior changed; +- verification and artifact locations; +- failed attempts and why they failed; +- remaining risks and next action. + +Keep entries factual and concise. Do not include credentials, secrets, private +URLs, or copied raw logs. Link the relevant commit after it exists. + +## Current Entries + +- `2026-08-15-repository-audit.md` — product focus, correctness risks, and + evidence gaps found in the deep audit. +- `2026-08-15-scoringbench-integration.md` — third-party benchmark integration, + validation, and Intel macOS runtime limitation. diff --git a/learnings/TEMPLATE.md b/learnings/TEMPLATE.md new file mode 100644 index 0000000..a8efe46 --- /dev/null +++ b/learnings/TEMPLATE.md @@ -0,0 +1,32 @@ +# YYYY-MM-DD: Topic + +## Context + +What question, bug, or decision triggered this work? + +## Decision or Result + +What did we decide or learn? Separate measured facts from hypotheses. + +## Changes + +- File or subsystem: behavioral change and reason. + +## Verification + +- Exact focused test or benchmark. +- Result and artifact location. +- Environment limitations that affect interpretation. + +## Failed Attempts + +- Attempt: failure mode and the rule learned from it. + +## Risks and Follow-ups + +- What remains unverified? +- What is the next concrete gate? + +## Commits + +- `SHA` — subject From aecf33f15e7d076f837387cca8026570f1ea2c69 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:31:02 -0700 Subject: [PATCH 03/49] fix: preserve categorical tree state on save --- src/openboost/_persistence.py | 58 ++++++++++++++++++++--------------- tests/test_persistence.py | 46 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/src/openboost/_persistence.py b/src/openboost/_persistence.py index 0287cd9..f028b6e 100644 --- a/src/openboost/_persistence.py +++ b/src/openboost/_persistence.py @@ -17,6 +17,8 @@ T = TypeVar("T", bound="PersistenceMixin") +_SERIALIZATION_VERSION = 2 + def _to_numpy(arr: Any) -> np.ndarray | None: """Convert array to numpy, handling GPU arrays. @@ -84,14 +86,14 @@ def _tree_to_dict(tree: TreeStructure) -> dict[str, Any]: data["level_thresholds"] = _to_numpy(tree.level_thresholds) # Phase 14: Missing value handling - if hasattr(tree, "missing_go_left") and tree.missing_go_left is not None: + if tree.missing_go_left is not None: data["missing_go_left"] = _to_numpy(tree.missing_go_left) # Phase 14.3: Categorical support - if hasattr(tree, "is_categorical") and tree.is_categorical is not None: - data["is_categorical"] = _to_numpy(tree.is_categorical) - if hasattr(tree, "category_masks") and tree.category_masks is not None: - data["category_masks"] = _to_numpy(tree.category_masks) + if tree.is_categorical_split is not None: + data["is_categorical_split"] = _to_numpy(tree.is_categorical_split) + if tree.cat_bitsets is not None: + data["cat_bitsets"] = _to_numpy(tree.cat_bitsets) return data @@ -157,7 +159,15 @@ def _dict_to_tree(data: dict[str, Any]) -> TreeStructure: else: values = values_arr - tree = TreeStructure( + # Accept the pre-Phase 14.3 draft key names for compatibility with any + # model states produced while those names were in use. + is_categorical_split = data.get( + "is_categorical_split", + data.get("is_categorical"), + ) + cat_bitsets = data.get("cat_bitsets", data.get("category_masks")) + + return TreeStructure( features=data["features"], thresholds=data["thresholds"], left_children=data["left_children"], @@ -169,20 +179,11 @@ def _dict_to_tree(data: dict[str, Any]) -> TreeStructure: is_symmetric=data.get("is_symmetric", False), level_features=data.get("level_features"), level_thresholds=data.get("level_thresholds"), + missing_go_left=data.get("missing_go_left"), + is_categorical_split=is_categorical_split, + cat_bitsets=cat_bitsets, ) - # Phase 14: Missing value handling - if "missing_go_left" in data: - tree.missing_go_left = data["missing_go_left"] - - # Phase 14.3: Categorical support - if "is_categorical" in data: - tree.is_categorical = data["is_categorical"] - if "category_masks" in data: - tree.category_masks = data["category_masks"] - - return tree - class PersistenceMixin: """Mixin class providing save/load functionality for models. @@ -224,7 +225,10 @@ def _to_state_dict(self) -> dict[str, Any]: Returns: Dictionary containing all model state """ - state = {"__class__": type(self).__name__, "_serialization_version": 1} + state = { + "__class__": type(self).__name__, + "_serialization_version": _SERIALIZATION_VERSION, + } for attr in self._get_persist_attrs(): value = getattr(self, attr, None) @@ -284,7 +288,6 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: """ import warnings - _CURRENT_SERIALIZATION_VERSION = 1 saved_version = state.get("_serialization_version") if saved_version is None: warnings.warn( @@ -293,14 +296,23 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: UserWarning, stacklevel=2, ) - elif saved_version > _CURRENT_SERIALIZATION_VERSION: + elif saved_version > _SERIALIZATION_VERSION: warnings.warn( f"Model was saved with serialization version {saved_version}, " - f"but current version is {_CURRENT_SERIALIZATION_VERSION}. " + f"but current version is {_SERIALIZATION_VERSION}. " "Some features may not load correctly.", UserWarning, stacklevel=2, ) + elif saved_version < 2 and np.any(state.get("_is_categorical", False)): + warnings.warn( + "This model uses categorical features and was saved with " + "serialization version 1, which did not reliably preserve " + "categorical tree routing. Retrain and resave the model before " + "using its predictions in production.", + UserWarning, + stacklevel=2, + ) trees_type = state.get("_trees_type", "list") @@ -341,8 +353,6 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: # Restore bin edges for transform if "_bin_edges" in state: - import numpy as np - from ._array import BinnedArray # Create a minimal BinnedArray with just bin edges for transform diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 900e50f..f9f2444 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -73,6 +73,52 @@ def test_save_load_basic(self, regression_data, tmp_path): # Predictions should match np.testing.assert_allclose(pred_before, pred_after, rtol=1e-5) + def test_save_load_preserves_categorical_tree_state(self, tmp_path): + """Categorical split routing survives a model round trip.""" + import openboost as ob + + categories = np.tile( + np.array([0.0, 1.0, 2.0, np.nan], dtype=np.float32), + 60, + ) + X = categories[:, None] + y = np.select( + [categories == 0.0, categories == 1.0, categories == 2.0], + [4.0, -3.0, 2.0], + default=7.0, + ).astype(np.float32) + + X_binned = ob.array(X, categorical_features=[0]) + model = ob.GradientBoosting(n_trees=8, max_depth=2, learning_rate=0.2) + model.fit(X_binned, y) + + assert any( + tree.is_categorical_split is not None + and np.any(tree.is_categorical_split[: tree.n_nodes]) + for tree in model.trees_ + ) + + state = model._to_state_dict() + assert state["_serialization_version"] == 2 + assert all("is_categorical_split" in tree for tree in state["trees_"]) + assert all("cat_bitsets" in tree for tree in state["trees_"]) + + pred_before = model.predict(X) + save_path = tmp_path / "categorical_model.joblib" + model.save(save_path) + loaded = ob.GradientBoosting.load(save_path) + pred_after = loaded.predict(X) + + for expected, actual in zip(model.trees_, loaded.trees_, strict=True): + np.testing.assert_array_equal( + expected.is_categorical_split, + actual.is_categorical_split, + ) + np.testing.assert_array_equal(expected.cat_bitsets, actual.cat_bitsets) + np.testing.assert_array_equal(expected.missing_go_left, actual.missing_go_left) + + np.testing.assert_allclose(pred_before, pred_after, rtol=0, atol=0) + def test_save_load_with_different_losses(self, regression_data, tmp_path): """Test save/load with various loss functions.""" import openboost as ob From e199e93322646328ad2abf157c84aaaf88475e0c Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:31:20 -0700 Subject: [PATCH 04/49] docs: record categorical persistence fix --- .../2026-08-15-categorical-persistence.md | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 learnings/2026-08-15-categorical-persistence.md diff --git a/learnings/2026-08-15-categorical-persistence.md b/learnings/2026-08-15-categorical-persistence.md new file mode 100644 index 0000000..36df21a --- /dev/null +++ b/learnings/2026-08-15-categorical-persistence.md @@ -0,0 +1,61 @@ +# 2026-08-15: Categorical Tree Persistence + +## Context + +`GradientBoosting.save()` serialized categorical trees with the draft field +names `is_categorical` and `category_masks`. `TreeStructure` actually stores +the routing state as `is_categorical_split` and `cat_bitsets`, so those arrays +were omitted and categorical predictions could change after loading a model. + +## Decision or Result + +Tree persistence now writes the canonical routing fields and reconstructs them +through the `TreeStructure` constructor. The loader also recognizes the draft +key names in case an external state contains them. + +Serialization version 2 identifies files written with the corrected schema. +When a version 1 file advertises categorical input metadata, loading emits a +warning because a file produced by the broken serializer cannot reconstruct +the category bitsets that were never written. Such a model must be retrained +and resaved before production use. + +## Changes + +- `src/openboost/_persistence.py`: persist categorical bitsets and split flags, + restore missing/categorical arrays in the constructor, retain draft-key read + compatibility, and bump the serialization version to 2. +- `tests/test_persistence.py`: add an exact prediction round trip containing + categorical group splits and missing values; assert schema version and tree + arrays. + +## Verification + +- Before the fix, the new test failed because loaded + `is_categorical_split` was `None`. +- `uv run pytest -q tests/test_persistence.py tests/test_categorical.py`: + 34 passed. +- Focused categorical round-trip test: 1 passed. +- `uv run ruff check src/openboost/_persistence.py`: passed. +- `git diff --check`: passed. + +## Failed Attempts + +- A combined sandboxed `uv` verification could not read the shared uv cache; + rerunning with the already approved uv cache permission completed normally. +- The first lint pass found a local `numpy` import that shadowed the module + import inside `_from_state_dict`; removing the redundant local import fixed + the scope error. + +## Risks and Follow-ups + +- Version 1 categorical files created by the broken serializer are not + repairable because their bitsets are absent; the warning is detection, not a + migration. +- Category routing uses one `uint64` bitset. Inputs with more than 64 category + codes are currently accepted elsewhere but cannot be represented correctly; + add a fail-fast guard or implement multiword bitsets before claiming support + above 64 categories. + +## Commits + +- `aecf33f` — `fix: preserve categorical tree state on save` From 356103bb86c00e02e6943a27672c208011cb9bd2 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:34:53 -0700 Subject: [PATCH 05/49] fix: reject unrepresentable categorical tree splits --- src/openboost/_array.py | 1 + src/openboost/_core/_split.py | 22 ++++++++++++++++++++-- tests/test_categorical.py | 10 ++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/openboost/_array.py b/src/openboost/_array.py index 24bea82..dab35a7 100644 --- a/src/openboost/_array.py +++ b/src/openboost/_array.py @@ -198,6 +198,7 @@ def array( categorical_features: List of column indices that are categorical. These use category encoding instead of quantile binning. Max 254 unique categories per feature (255 reserved for NaN). + Tree models currently support at most 64 categories. device: Target device ("cuda" or "cpu"). Auto-detected if None. Returns: diff --git a/src/openboost/_core/_split.py b/src/openboost/_core/_split.py index 029e0cd..b589e4f 100644 --- a/src/openboost/_core/_split.py +++ b/src/openboost/_core/_split.py @@ -16,6 +16,9 @@ from numpy.typing import NDArray +_MAX_CATEGORICAL_SPLIT_CARDINALITY = 64 + + class SplitInfo(NamedTuple): """Information about a split. @@ -257,7 +260,9 @@ def find_best_split_with_categorical( min_gain: Minimum gain to make a split has_missing: Boolean array (n_features,) for features with NaN is_categorical: Boolean array (n_features,) for categorical features - n_categories: Number of categories per feature (0 for numeric) + n_categories: Number of categories per feature (0 for numeric). Categorical + tree splits support at most 64 categories because the left + category set is represented by one uint64 bitset. Returns: SplitInfo with best feature, threshold/bitset, gain, etc. @@ -273,6 +278,20 @@ def find_best_split_with_categorical( # Check if we have categorical features any_categorical = is_categorical is not None and np.any(is_categorical) any_missing = has_missing is not None and np.any(has_missing) + + if any_categorical: + if n_categories is None: + raise ValueError( + "n_categories is required when is_categorical contains True" + ) + categorical_counts = np.asarray(n_categories)[np.asarray(is_categorical)] + if np.any(categorical_counts > _MAX_CATEGORICAL_SPLIT_CARDINALITY): + observed = int(np.max(categorical_counts)) + raise ValueError( + f"Categorical feature has {observed} categories; maximum " + f"supported is {_MAX_CATEGORICAL_SPLIT_CARDINALITY} because " + "categorical tree routing uses a single uint64 bitset" + ) # If no categorical and no missing, use standard split if not any_categorical and not any_missing: @@ -333,4 +352,3 @@ def find_best_split_with_categorical( cat_bitset=cat_bitset, cat_threshold=cat_threshold, ) - diff --git a/tests/test_categorical.py b/tests/test_categorical.py index 868a4f9..71eaab2 100644 --- a/tests/test_categorical.py +++ b/tests/test_categorical.py @@ -178,6 +178,16 @@ def test_categorical_split_found(self): class TestGradientBoostingWithCategorical: """Tests for GradientBoosting with categorical features.""" + + def test_fit_rejects_unrepresentable_categorical_split(self): + """Tree training fails before silently truncating a category bitset.""" + categories = np.tile(np.arange(65, dtype=np.float32), 4) + X_binned = array(categories[:, None], categorical_features=[0]) + y = (categories % 2).astype(np.float32) + + model = GradientBoosting(n_trees=1, max_depth=1) + with pytest.raises(ValueError, match="maximum supported is 64"): + model.fit(X_binned, y) def test_fit_with_categorical(self): """GradientBoosting fits with categorical features.""" From 52385f205c23476c1c568673bad5f8a80ff513b4 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:35:09 -0700 Subject: [PATCH 06/49] docs: record categorical cardinality limit --- .../2026-08-15-categorical-cardinality.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 learnings/2026-08-15-categorical-cardinality.md diff --git a/learnings/2026-08-15-categorical-cardinality.md b/learnings/2026-08-15-categorical-cardinality.md new file mode 100644 index 0000000..219af63 --- /dev/null +++ b/learnings/2026-08-15-categorical-cardinality.md @@ -0,0 +1,60 @@ +# 2026-08-15: Categorical Tree Cardinality + +## Context + +Categorical values are encoded into `uint8`, so `BinnedArray` can represent up +to 254 non-missing categories. Tree nodes, however, represent the categories +sent left with one `uint64` bitset. The split search previously evaluated all +categories but silently omitted category codes 64 and above from that set. +Reported gain and actual routing could therefore disagree. + +## Decision or Result + +Keep the encoding limit at 254 because non-tree consumers can use those bins, +but reject categorical tree training above 64 categories in the shared split +dispatch before either the CPU or CUDA implementation runs. This preserves the +useful `BinnedArray` capability while preventing a tree from silently learning +an unrepresentable split. + +Supporting more than 64 categories correctly requires a multiword bitset (or a +different category-set representation) across split search, partitioning, CPU +prediction, CUDA prediction, tree storage, and persistence. It is a feature, +not a safe one-line limit increase. + +## Changes + +- `src/openboost/_core/_split.py`: require category counts when categorical + features are present and fail before dispatch when any count exceeds 64. +- `src/openboost/_array.py`: document the distinction between the 254-category + encoding limit and the 64-category tree-split limit. +- `tests/test_categorical.py`: prove 65 categories can be binned but cannot be + passed into categorical tree training. + +## Verification + +- Before the guard, the new 65-category training case did not raise. +- `uv run pytest -q tests/test_categorical.py tests/test_growth.py`: 46 passed. +- A focused 64-category probe separated category 63 correctly. +- `uv run ruff check src/openboost/_array.py src/openboost/_core/_split.py`: + passed. +- `git diff --check`: passed. + +## Failed Attempts + +- The first design rejected more than 64 categories inside `ob.array()`. That + conflated safe bin encoding with tree routing and would unnecessarily block + consumers such as GAM. The guard was moved to the shared tree split path. +- Linting the entire legacy categorical test file surfaced pre-existing style + issues unrelated to this change. The source files were used as the scoped + lint gate; the complete behavior files were still executed with pytest. + +## Risks and Follow-ups + +- Multiword bitsets are required before advertising native high-cardinality + categorical tree support. +- Real GPU verification should include category code 63 and CPU/CUDA parity; + this machine has no CUDA environment. + +## Commits + +- `356103b` — `fix: reject unrepresentable categorical tree splits` From ee555cbaf95dba9ebd91c0dcd66dc264dd7565b9 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:36:36 -0700 Subject: [PATCH 07/49] fix: fail fast for unsupported model batching --- src/openboost/_models/_boosting.py | 17 ++++++++++++++++- tests/test_large_scale.py | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/openboost/_models/_boosting.py b/src/openboost/_models/_boosting.py index 12ab4cc..984aa01 100644 --- a/src/openboost/_models/_boosting.py +++ b/src/openboost/_models/_boosting.py @@ -70,6 +70,16 @@ def _is_levelwise_growth(growth) -> bool: ) +def _validate_batch_size(batch_size: int | None) -> None: + """Reject the reserved high-level mini-batch option until it is implemented.""" + if batch_size is not None: + raise NotImplementedError( + "batch_size is reserved but high-level mini-batch training is not " + "implemented. Leave batch_size=None; the low-level mini-batch " + "histogram helpers are not an end-to-end model.fit path." + ) + + def _compute_loss_value(loss, pred, y, **kwargs) -> float: """Compute scalar loss using the true loss formula for known objectives. @@ -141,7 +151,8 @@ class GradientBoosting(PersistenceMixin): - 'goss': Gradient-based One-Side Sampling (LightGBM-style) goss_top_rate: Fraction of top-gradient samples to keep (for GOSS). goss_other_rate: Fraction of remaining samples to sample (for GOSS). - batch_size: Mini-batch size for large datasets. If None, process all at once. + batch_size: Reserved for a future high-level mini-batch training path. + Any non-None value currently raises NotImplementedError. growth: Tree growth strategy: - 'levelwise': XGBoost-style level-wise growth (default) - 'leafwise': LightGBM-style best-first growth (see max_leaves) @@ -250,6 +261,8 @@ def fit( ) ``` """ + _validate_batch_size(self.batch_size) + # Clear any previous fit self.trees_ = [] @@ -1329,6 +1342,8 @@ def fit( """ from .._loss import softmax_gradient + _validate_batch_size(self.batch_size) + # Clear previous fit self.trees_ = [] diff --git a/tests/test_large_scale.py b/tests/test_large_scale.py index d071b12..d9dc040 100644 --- a/tests/test_large_scale.py +++ b/tests/test_large_scale.py @@ -317,6 +317,30 @@ def test_goss_config_validation(self): # Integration Tests with GradientBoosting # ============================================================================= +class TestUnsupportedHighLevelBatching: + """High-level models must not silently ignore ``batch_size``.""" + + @pytest.mark.parametrize( + "model,y", + [ + (ob.GradientBoosting(n_trees=1, batch_size=16), np.arange(64)), + ( + ob.MultiClassGradientBoosting( + n_classes=2, + n_trees=1, + batch_size=16, + ), + np.arange(64) % 2, + ), + ], + ) + def test_batch_size_fails_fast(self, model, y): + X = np.arange(128, dtype=np.float32).reshape(64, 2) + + with pytest.raises(NotImplementedError, match="batch_size"): + model.fit(X, y) + + class TestGOSSIntegration: """Integration tests for GOSS with GradientBoosting.""" From 6440e31143e4cbd56e9d523a25a8f48bca302670 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:38:15 -0700 Subject: [PATCH 08/49] docs: mark experimental scaling boundaries --- docs/getting-started/gpu-setup.md | 28 ++-- docs/getting-started/installation.md | 2 +- docs/user-guide/models/gradient-boosting.md | 6 + docs/user-guide/training/large-scale.md | 153 ++++++++------------ examples/gpu_training.py | 32 ++-- src/openboost/_models/_boosting.py | 16 +- src/openboost/_models/_sklearn.py | 6 +- 7 files changed, 107 insertions(+), 136 deletions(-) diff --git a/docs/getting-started/gpu-setup.md b/docs/getting-started/gpu-setup.md index b748347..da17210 100644 --- a/docs/getting-started/gpu-setup.md +++ b/docs/getting-started/gpu-setup.md @@ -28,19 +28,20 @@ ob.set_backend("cuda") ## GPU Performance -GPU acceleration provides significant speedups for larger datasets: +GPU benefit depends on dataset shape, tree parameters, distribution, CUDA +stack, transfer policy, and JIT warm-up. OpenBoost does not publish a universal +speedup table without a checked-in benchmark artifact. -| Dataset Size | Typical Speedup | -|--------------|-----------------| -| <5K samples | ~1x (CPU overhead dominates) | -| 5K-10K | 2-7x | -| 25K+ | 2-3x | -| 100K+ | 5-10x | +For a defensible comparison: -!!! tip "Best practices for GPU" - - Ensure data is `float32` (not `float64`) - - Use larger datasets (GPU overhead not worth it for <5K samples) - - GPU shows best speedup at 10K+ samples +1. force the backend with `ob.set_backend("cpu")` or `"cuda"`; +2. run a warm-up that is excluded from timed repetitions; +3. compare predictions and task metrics before comparing runtime; +4. report repeated fit/predict timings, peak memory, failures, and hardware; +5. save raw results with the OpenBoost commit and dependency versions. + +The ScoringBench integration under `benchmarks/scoringbench/` defines separate +official-quality and scale-extension protocols for probabilistic models. ## Multi-GPU Training @@ -67,8 +68,9 @@ model.fit(X, y) ### Training seems slow on GPU - Ensure data is `float32` (not `float64`) -- Use larger datasets (GPU overhead not worth it for <5K samples) -- GPU shows best speedup at 10K+ samples +- Exclude first-use JIT compilation only when the benchmark protocol says so +- Check that `ob.get_backend()` reports `"cuda"` +- Measure fit and prediction separately; do not assume a crossover dataset size ### Model trained on GPU, loading on CPU machine diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 6a81837..eb16016 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -43,7 +43,7 @@ For CUDA GPU acceleration: |-------|-----------------|---------| | `cuda` | CuPy for GPU acceleration | `pip install "openboost[cuda]"` | | `sklearn` | scikit-learn integration | `pip install "openboost[sklearn]"` | -| `distributed` | Ray for multi-GPU training | `pip install "openboost[distributed]"` | +| `distributed` | Ray for experimental multi-GPU work | `pip install "openboost[distributed]"` | | `all` | Everything | `pip install "openboost[all]"` | ## Requirements diff --git a/docs/user-guide/models/gradient-boosting.md b/docs/user-guide/models/gradient-boosting.md index aa9b280..d789def 100644 --- a/docs/user-guide/models/gradient-boosting.md +++ b/docs/user-guide/models/gradient-boosting.md @@ -44,6 +44,7 @@ predictions = model.predict(X_test) | `n_bins` | int | 254 | Number of histogram bins | | `growth` | str | `'levelwise'` | Tree growth strategy: `'levelwise'`, `'leafwise'`, or `'symmetric'` | | `max_leaves` | int/None | None | Max leaves per tree for `'leafwise'` growth (defaults to `2**max_depth`) | +| `batch_size` | int/None | None | Reserved; non-None values raise `NotImplementedError` | | `random_state` | int/None | None | Seed for reproducible training | ## Loss Functions @@ -92,6 +93,11 @@ training path. CUDA, distributed, and multi-GPU training raise `NotImplementedError` when weights are supplied, so weighted observations are never silently treated as unweighted. +High-level mini-batch and out-of-core fitting are not implemented. The +`batch_size` parameter fails fast when set; low-level memmap and mini-batch +histogram helpers are experimental building blocks rather than a `model.fit` +path. + ## Feature Importance ```python diff --git a/docs/user-guide/training/large-scale.md b/docs/user-guide/training/large-scale.md index 40fdab7..b87ddba 100644 --- a/docs/user-guide/training/large-scale.md +++ b/docs/user-guide/training/large-scale.md @@ -1,71 +1,69 @@ -# Large-Scale Training +# Scaling Training -Train on datasets that don't fit in memory or need faster training. +OpenBoost has a supported full-dataset CPU/CUDA path, a supported sampling +option, and several experimental scaling primitives. Keep those categories +separate when choosing a training path or reporting a benchmark. + +| Capability | Status | Important boundary | +|------------|--------|--------------------| +| Single-device full-data training | Supported | Dataset and histograms must fit in memory | +| GOSS sampling | Supported | Speed and quality are data-dependent | +| `fit_trees_batch` | Reference implementation | Shares bins; configurations fit sequentially | +| Mini-batch histogram helpers | Low-level primitive | Not integrated with `model.fit` | +| Memory-mapped binned arrays | Storage primitive | Not an out-of-core `model.fit` path | +| Distributed/multi-GPU | Experimental | No published parity or scaling artifact yet | ## GOSS Sampling -Gradient-based One-Side Sampling (from LightGBM) - train 3x faster with minimal accuracy loss. +Gradient-based One-Side Sampling keeps high-gradient observations and samples +from the remainder on every boosting round: ```python import openboost as ob model = ob.GradientBoosting( n_trees=100, - subsample_strategy='goss', - goss_top_rate=0.2, # Keep top 20% high-gradient samples - goss_other_rate=0.1, # Sample 10% of the rest + subsample_strategy="goss", + goss_top_rate=0.2, + goss_other_rate=0.1, + random_state=42, ) model.fit(X_train, y_train) ``` -### How GOSS Works - -1. Sort samples by gradient magnitude -2. Keep top `goss_top_rate` samples (most informative) -3. Randomly sample `goss_other_rate` from the rest -4. Weight the random samples to maintain unbiased gradients - -### GOSS Parameters - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `goss_top_rate` | 0.2 | Fraction of high-gradient samples to keep | -| `goss_other_rate` | 0.1 | Fraction of remaining samples to keep | +With these rates, approximately 28% of observations participate in a round. +That arithmetic is not a speed or accuracy guarantee: compare GOSS with the +full-data path on fixed folds and seeds before using it. -**Result**: Train on ~28% of samples with similar accuracy. +## Single-GPU Scaling -## Memory-Mapped Arrays - -For datasets larger than RAM: +Select CUDA explicitly so an unavailable GPU cannot silently turn a benchmark +into a CPU run: ```python import openboost as ob -# Create memory-mapped binned array (saves to disk) -X_mmap = ob.create_memmap_binned('large_data.npy', X_large) - -# Load for training (no copy, uses disk) -X_mmap = ob.load_memmap_binned('large_data.npy', n_features, n_samples) - -# Train as normal -model = ob.GradientBoosting(n_trees=100) -model.fit(X_mmap, y_train) +ob.set_backend("cuda") +model = ob.GradientBoosting(n_trees=100, random_state=42) +model.fit(X_train, y_train) ``` -## Mini-Batch Training +Report the OpenBoost commit, environment, GPU model, driver/CUDA versions, +warm-up policy, seeds, fit time, prediction time, peak memory, and CPU/CUDA +prediction parity. Use the scale-extension protocol in +`benchmarks/scoringbench/` for probabilistic benchmark work. -Accumulate histograms in batches: +## Mini-Batch and Memory-Mapped Primitives -```python -from openboost import MiniBatchIterator, accumulate_histograms_minibatch +`MiniBatchIterator`, `accumulate_histograms_minibatch`, +`create_memmap_binned`, and `load_memmap_binned` are low-level building blocks. +The memmap layout is feature-major `(n_features, n_samples)`; it is not a raw +sample-major matrix accepted by high-level `model.fit`. -# Process 100k samples at a time -hist_grad, hist_hess = accumulate_histograms_minibatch( - X_mmap, grad, hess, - batch_size=100_000, - n_features=n_features, -) -``` +The `batch_size` model parameter is reserved. Passing a non-`None` value raises +`NotImplementedError` rather than pretending to perform mini-batch training. +Do not claim datasets larger than memory are supported end to end until a +training loop integrates these primitives and has correctness tests. ## Train Many Configurations @@ -90,65 +88,32 @@ trees_by_config = ob.fit_trees_batch( ) ``` -The current implementation is a correctness reference: it shares binned input -data but fits configurations sequentially. GPU kernel fusion is planned without -changing this API's results. +The current implementation shares the binned input but fits configurations +sequentially. Treat it as a correctness reference, not fused GPU training. -## Multi-GPU Training +## Experimental Multi-GPU Path -Distribute training across multiple GPUs: +The Ray multi-GPU path is available for development experiments: ```python import openboost as ob -# Automatic multi-GPU with Ray -model = ob.GradientBoosting(n_trees=100, n_gpus=4) -model.fit(X, y) - -# Or specify exact devices -model = ob.GradientBoosting(n_trees=100, devices=[0, 2]) -model.fit(X, y) -``` - -### Requirements - -```bash -pip install "openboost[distributed]" # Installs Ray +model = ob.GradientBoosting(n_trees=100, devices=[0, 1]) +model.fit(X_train, y_train) ``` -## Scaling Guidelines +It does not support `sample_weight`, and the repository does not yet contain a +validated two-/four-GPU parity and scaling artifact. Do not use it for release +claims until exact single-device parity, repeated timings, peak memory, and +failure cases are published on real multi-GPU hardware. -| Dataset Size | Recommendation | -|--------------|----------------| -| <100K samples | Standard training | -| 100K-1M | GOSS sampling | -| 1M-10M | GOSS + memory-mapped | -| >10M | Multi-GPU + GOSS | +## Evidence Gate -## Example: Large Dataset +A scaling claim is ready only when the checked-in artifact records: -```python -import numpy as np -import openboost as ob - -# Simulate large dataset (10M samples) -n_samples = 10_000_000 -n_features = 100 - -# Create memory-mapped data -X_mmap = ob.create_memmap_binned( - 'large_X.npy', - np.random.randn(n_samples, n_features).astype(np.float32) -) - -y = np.random.randn(n_samples).astype(np.float32) - -# Train with GOSS -model = ob.GradientBoosting( - n_trees=100, - subsample_strategy='goss', - goss_top_rate=0.1, - goss_other_rate=0.05, -) -model.fit(X_mmap, y) -``` +1. a frozen OpenBoost commit and dependency lock; +2. real datasets plus at least one controlled synthetic scaling curve; +3. CPU/CUDA prediction parity and task-quality metrics; +4. repeated fit/predict timings after an explicit warm-up policy; +5. hardware, drivers, thread counts, peak memory, seeds, and failures; +6. comparisons against maintained baselines under the same protocol. diff --git a/examples/gpu_training.py b/examples/gpu_training.py index 949d32a..622318a 100644 --- a/examples/gpu_training.py +++ b/examples/gpu_training.py @@ -226,23 +226,17 @@ def main(): """ print(best_practices) - # --- Scaling Guide --- - print("\n8. Expected GPU speedups by dataset size...") + # --- Benchmark Evidence --- + print("\n8. How to measure GPU value...") scaling_info = """ - | Dataset Size | Features | Trees | Expected Speedup | - |--------------|----------|-------|------------------| - | 5K samples | 10 | 100 | ~1-2x | - | 10K samples | 20 | 100 | ~2-5x | - | 50K samples | 20 | 100 | ~3-7x | - | 100K samples | 50 | 200 | ~5-10x | - | 500K samples | 100 | 500 | ~10-20x | - - Factors affecting speedup: - - More features = better GPU utilization - - More bins = better GPU utilization - - GAM shows best speedups (parallel feature updates) - - First run includes JIT compilation overhead + GPU speedup is workload- and hardware-dependent. For a publishable result: + - Force CPU and CUDA backends explicitly + - Verify prediction and task-metric parity first + - State whether JIT warm-up is excluded + - Run repeated fit and predict timings + - Record peak memory, hardware, CUDA/driver, seeds, and failures + - Store raw results with the exact OpenBoost commit """ print(scaling_info) @@ -250,8 +244,7 @@ def main(): print("\n9. Multi-GPU training...") print(""" - For datasets that don't fit on a single GPU or to speed up training further, - OpenBoost supports multi-GPU training via Ray: + OpenBoost includes an experimental multi-GPU path via Ray: # Install Ray pip install ray[default] @@ -273,7 +266,10 @@ def main(): Multi-GPU training uses data parallelism: - Each GPU processes a subset of samples - Histograms are aggregated across GPUs - - Near-linear scaling with number of GPUs + - sample_weight is not supported + + Do not infer scaling from this example. The repository still needs a + checked-in two-/four-GPU parity and repeated-timing artifact. """) # --- Summary --- diff --git a/src/openboost/_models/_boosting.py b/src/openboost/_models/_boosting.py index 984aa01..4fe69dc 100644 --- a/src/openboost/_models/_boosting.py +++ b/src/openboost/_models/_boosting.py @@ -3,13 +3,12 @@ Provides a scikit-learn-like API for training gradient boosting models with both built-in and custom loss functions. -This module implements batched training that keeps computation on the GPU -without returning to Python between trees, achieving performance competitive -with XGBoost. +This module implements GPU-aware training paths and reusable tree-building +primitives. Performance claims require workload-specific benchmark artifacts. Phase 13: Added callback support for early stopping, logging, etc. -Phase 17: Added GOSS sampling and mini-batch training for large-scale datasets. -Phase 18: Added multi-GPU support via Ray for data-parallel training. +Phase 17: Added GOSS sampling and low-level mini-batch primitives. +Phase 18: Added an experimental multi-GPU path via Ray. """ from __future__ import annotations @@ -189,11 +188,11 @@ class GradientBoosting(PersistenceMixin): ) ``` - Multi-GPU training: + Experimental multi-GPU training: ```python model = ob.GradientBoosting(n_trees=100, n_gpus=4) - model.fit(X, y) # Data parallel across 4 GPUs + model.fit(X, y) # Requires independent parity/scaling validation ``` """ @@ -426,7 +425,8 @@ def _fit_multigpu( Each GPU holds a shard of the data and computes local histograms, which are aggregated on the driver to build global trees. - This approach provides near-linear scaling for large datasets. + This path is experimental until single-device parity and repeated + two-/four-GPU scaling results are checked into the repository. """ if MultiGPUContext is None: raise ImportError( diff --git a/src/openboost/_models/_sklearn.py b/src/openboost/_models/_sklearn.py index 37a1f8d..d464537 100644 --- a/src/openboost/_models/_sklearn.py +++ b/src/openboost/_models/_sklearn.py @@ -114,7 +114,8 @@ class OpenBoostRegressor(BaseEstimator, RegressorMixin): goss_other_rate : float, default=0.1 Fraction of remaining samples to sample (for GOSS). batch_size : int, optional - Mini-batch size for large datasets. If None, process all at once. + Reserved for future high-level mini-batch training. Non-None values + currently raise NotImplementedError. early_stopping_rounds : int, optional Stop training if validation score doesn't improve for this many rounds. Requires eval_set to be passed to fit(). @@ -357,7 +358,8 @@ class OpenBoostClassifier(BaseEstimator, ClassifierMixin): goss_other_rate : float, default=0.1 Fraction of remaining samples to sample (for GOSS). batch_size : int, optional - Mini-batch size for large datasets. + Reserved for future high-level mini-batch training. Non-None values + currently raise NotImplementedError. early_stopping_rounds : int, optional Stop if validation doesn't improve. verbose : int, default=0 From 077211066af1956f38e2a7183cd77bdd0ace140c Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:38:34 -0700 Subject: [PATCH 09/49] docs: record scaling capability boundaries --- learnings/2026-08-15-scaling-boundaries.md | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 learnings/2026-08-15-scaling-boundaries.md diff --git a/learnings/2026-08-15-scaling-boundaries.md b/learnings/2026-08-15-scaling-boundaries.md new file mode 100644 index 0000000..2b52a3c --- /dev/null +++ b/learnings/2026-08-15-scaling-boundaries.md @@ -0,0 +1,68 @@ +# 2026-08-15: Scaling Boundaries + +## Context + +The high-level `GradientBoosting` and `MultiClassGradientBoosting` APIs exposed +`batch_size`, but neither training loop read it. The large-scale guide also +passed a feature-major binned memmap directly to a high-level `fit` method that +validates sample-major input. GPU and multi-GPU pages quoted speedups without a +checked-in reproducible artifact. + +## Decision or Result + +Unsupported scale features now fail or read as experimental instead of looking +production-ready: + +- a non-`None` high-level `batch_size` raises `NotImplementedError` before fit; +- memmap and mini-batch histogram utilities remain available as low-level + building blocks, not an out-of-core model API; +- GOSS is described by its sampling behavior, without a universal speed/quality + promise; +- multi-GPU is explicitly experimental until parity and repeated two-/four-GPU + measurements exist; +- numeric GPU speedup tables were removed in favor of an evidence checklist. + +## Changes + +- `src/openboost/_models/_boosting.py`: validate the reserved batch parameter in + single-output and multiclass fits; remove unsupported performance wording. +- `tests/test_large_scale.py`: cover both high-level model families. +- `docs/user-guide/training/large-scale.md`: replace the broken out-of-core + recipe with a capability/status matrix and evidence gate. +- GPU setup, installation, sklearn docstrings, model guide, and GPU example: + align public wording with the actual support boundaries. + +## Verification + +- Before the fix, both new high-level tests failed because no exception was + raised and training proceeded while ignoring `batch_size`. +- Focused batch-size tests: 2 passed. +- `uv run pytest -q tests/test_large_scale.py tests/test_core.py + tests/test_losses.py`: 77 passed, 3 expected bin-count warnings. +- `uv run mkdocs build`: passed with the repository's 29 existing griffe + warnings. +- Focused source Ruff checks, example compilation, and `git diff --check`: + passed. + +## Failed Attempts + +- `uv run mkdocs build --strict` stopped on 29 existing griffe warnings in API + docstrings across callbacks, distributions, losses, models, arrays, trees, + and importance helpers. The current docs workflow is non-strict, so the + matching build was used for this change. Strict docs cleanliness remains a + separate maintenance task. + +## Risks and Follow-ups + +- The low-level memmap and mini-batch helpers are not proof of end-to-end + out-of-core training. Implement and test a real loop before reintroducing that + claim. +- Multi-GPU correctness is not established by documentation. Require exact + single-device parity and real two-/four-GPU artifacts before promotion. +- Run single-GPU scale-extension benchmarks on Linux/CUDA with full provenance; + this Intel macOS environment cannot provide those results. + +## Commits + +- `ee555cb` — `fix: fail fast for unsupported model batching` +- `6440e31` — `docs: mark experimental scaling boundaries` From 30b9ab5f331660078290fe57f7961e7e1e8a15f6 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:43:41 -0700 Subject: [PATCH 10/49] ci: compare performance across pushed revisions --- .github/workflows/unit-tests.yml | 26 +++++++- benchmarks/check_performance.py | 105 ++++++++++++++++++++++++++----- tests/test_performance_check.py | 58 +++++++++++++++++ 3 files changed, 170 insertions(+), 19 deletions(-) create mode 100644 tests/test_performance_check.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8b1a1e1..e9dc616 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: run: | python scripts/run_doc_examples.py - # Performance regression check: main branch only + # Compare the pushed main range against the previous remote main on one runner. performance-check: runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' @@ -84,6 +84,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -98,6 +100,26 @@ jobs: - name: Performance regression check env: + BASELINE_SHA: ${{ github.event.before }} OPENBOOST_BACKEND: "cpu" run: | - python benchmarks/check_performance.py + BASELINE_ROOT="${RUNNER_TEMP}/openboost-baseline" + git worktree add --detach "${BASELINE_ROOT}" "${BASELINE_SHA}" + NUMBA_CACHE_DIR="${RUNNER_TEMP}/numba-baseline" \ + python benchmarks/check_performance.py \ + --source-root "${BASELINE_ROOT}" \ + --benchmark-only \ + --output "${RUNNER_TEMP}/performance-baseline.json" + NUMBA_CACHE_DIR="${RUNNER_TEMP}/numba-current" \ + python benchmarks/check_performance.py \ + --baseline "${RUNNER_TEMP}/performance-baseline.json" \ + --output "${RUNNER_TEMP}/performance-current.json" + + - name: Upload performance results + if: always() + uses: actions/upload-artifact@v4 + with: + name: performance-check-${{ github.sha }} + path: | + ${{ runner.temp }}/performance-baseline.json + ${{ runner.temp }}/performance-current.json diff --git a/benchmarks/check_performance.py b/benchmarks/check_performance.py index bc8aa5c..7fd4e90 100644 --- a/benchmarks/check_performance.py +++ b/benchmarks/check_performance.py @@ -1,10 +1,11 @@ -"""Performance regression check for CI. +"""Performance regression check for CI or a local baseline. Runs a fixed, small benchmark and compares against stored baselines. Fails if any metric degrades by more than 20%. Usage: - uv run python benchmarks/check_performance.py + uv run python benchmarks/check_performance.py --baseline baseline.json + uv run python benchmarks/check_performance.py --benchmark-only --output result.json uv run python benchmarks/check_performance.py --update-baselines """ @@ -12,6 +13,9 @@ import argparse import json +import os +import platform +import subprocess import sys import time import tracemalloc @@ -20,7 +24,6 @@ import numpy as np PROJECT_ROOT = Path(__file__).parent.parent -sys.path.insert(0, str(PROJECT_ROOT / "src")) BASELINE_FILE = Path(__file__).parent / "results" / "performance_baselines.json" @@ -90,17 +93,43 @@ def run_fixed_benchmark(): } -def save_baselines(results): - """Save results as new baselines.""" - BASELINE_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(BASELINE_FILE, "w") as f: +def collect_provenance(source_root: Path) -> dict[str, str]: + """Collect enough environment data to interpret a raw CI result.""" + import openboost as ob + + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=source_root, + check=False, + capture_output=True, + text=True, + ) + git_commit = commit.stdout.strip() if commit.returncode == 0 else "unknown" + + return { + "git_commit": git_commit, + "openboost_version": ob.__version__, + "python_version": platform.python_version(), + "numpy_version": np.__version__, + "platform": platform.platform(), + "processor": platform.processor() or "unknown", + "openboost_backend": os.environ.get("OPENBOOST_BACKEND", "auto"), + "numba_num_threads": os.environ.get("NUMBA_NUM_THREADS", "default"), + "numba_cache_dir": os.environ.get("NUMBA_CACHE_DIR", "default"), + } + + +def save_results(results, path: Path): + """Save benchmark results.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: json.dump(results, f, indent=2) - print(f"Baselines saved to {BASELINE_FILE}") + print(f"Results saved to {path}") -def load_baselines(): +def load_baselines(path: Path): """Load stored baselines.""" - with open(BASELINE_FILE) as f: + with open(path) as f: return json.load(f) @@ -140,10 +169,52 @@ def main(): "--update-baselines", action="store_true", help="Update baselines with current results" ) + parser.add_argument( + "--baseline", + type=Path, + default=BASELINE_FILE, + help="Baseline JSON to compare against", + ) + parser.add_argument( + "--output", + type=Path, + help="Write the current raw benchmark result to this JSON file", + ) + parser.add_argument( + "--benchmark-only", + action="store_true", + help="Run and save the benchmark without comparing it", + ) + parser.add_argument( + "--source-root", + type=Path, + default=PROJECT_ROOT, + help="Repository root whose src/openboost implementation should run", + ) args = parser.parse_args() + if ( + not args.update_baselines + and not args.benchmark_only + and not args.baseline.exists() + ): + print(f"No baseline found at {args.baseline}", file=sys.stderr) + print( + "Pass --baseline, use --benchmark-only, or explicitly create a " + "local baseline with --update-baselines.", + file=sys.stderr, + ) + sys.exit(2) + + source_dir = args.source_root.resolve() / "src" + if not source_dir.is_dir(): + parser.error(f"source root has no src directory: {args.source_root}") + sys.path.insert(0, str(source_dir)) + print("Running fixed benchmark...") results = run_fixed_benchmark() + results["benchmark_schema_version"] = 1 + results["provenance"] = collect_provenance(args.source_root.resolve()) print(f" fit_time: {results['fit_time_median']:.4f}s") print(f" predict_time: {results['predict_time_median']:.4f}s") @@ -151,17 +222,17 @@ def main(): print(f" mse: {results['mse']:.6f}") print(f" r2: {results['r2']:.4f}") - if args.update_baselines: - save_baselines(results) + if args.output: + save_results(results, args.output) + + if args.benchmark_only: return - if not BASELINE_FILE.exists(): - print(f"\nNo baselines found at {BASELINE_FILE}") - print("Run with --update-baselines to create them.") - save_baselines(results) + if args.update_baselines: + save_results(results, args.baseline) return - baselines = load_baselines() + baselines = load_baselines(args.baseline) regressions = check_regression(results, baselines) if regressions: diff --git a/tests/test_performance_check.py b/tests/test_performance_check.py new file mode 100644 index 0000000..9e54bec --- /dev/null +++ b/tests/test_performance_check.py @@ -0,0 +1,58 @@ +"""Tests for the CI performance comparison harness.""" + +import json +from pathlib import Path + +from benchmarks.check_performance import ( + check_regression, + collect_provenance, + load_baselines, +) + + +def _result(**overrides): + result = { + "fit_time_median": 1.0, + "predict_time_median": 0.1, + "peak_memory_mb": 10.0, + "mse": 0.05, + "r2": 0.95, + "n_samples": 5000, + "n_features": 10, + "n_trees": 100, + "max_depth": 6, + } + result.update(overrides) + return result + + +def test_equal_results_have_no_regression(): + baseline = _result() + + assert check_regression(_result(), baseline) == [] + + +def test_runtime_and_quality_regressions_are_reported(): + baseline = _result() + current = _result(fit_time_median=1.21, mse=0.061) + + regressions = check_regression(current, baseline) + + assert any("fit_time_median" in item for item in regressions) + assert any("mse" in item for item in regressions) + + +def test_load_baselines_uses_explicit_path(tmp_path): + baseline_path = tmp_path / "parent.json" + baseline_path.write_text(json.dumps(_result())) + + assert load_baselines(baseline_path) == _result() + + +def test_provenance_records_source_commit_and_environment(): + provenance = collect_provenance(Path.cwd()) + + assert len(provenance["git_commit"]) == 40 + assert provenance["python_version"] + assert provenance["numpy_version"] + assert provenance["openboost_version"] From 05cd8bc800595a2f40c4d08f51afb697968b9b3e Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 18:43:59 -0700 Subject: [PATCH 11/49] docs: record performance gate design --- learnings/2026-08-15-performance-gate.md | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 learnings/2026-08-15-performance-gate.md diff --git a/learnings/2026-08-15-performance-gate.md b/learnings/2026-08-15-performance-gate.md new file mode 100644 index 0000000..5fcdf81 --- /dev/null +++ b/learnings/2026-08-15-performance-gate.md @@ -0,0 +1,67 @@ +# 2026-08-15: Performance Regression Gate + +## Context + +The performance CI expected `benchmarks/results/performance_baselines.json`, +but that file was ignored and absent from fresh checkouts. On a missing file, +the script benchmarked the current commit, saved that same result as its own +baseline, and exited successfully. The advertised regression gate was therefore +a no-op on every fresh GitHub runner. + +## Decision or Result + +Compare the code before a push with the code after the push on the same GitHub +runner instead of committing an absolute timing baseline from unrelated +hardware. The baseline revision is `github.event.before`, not `HEAD^`, so one +workflow covers every commit in a multi-commit push to main. + +Both revisions run through the current fixed harness with separate Numba cache +directories. Raw parent/current JSON files are uploaded and include commit, +runtime versions, platform, backend, and relevant Numba environment fields. +Missing baselines now exit with status 2 before doing expensive work; creating a +local baseline requires an explicit flag. + +## Changes + +- `benchmarks/check_performance.py`: add explicit baseline/output/source-root + options, benchmark-only mode, provenance, and fail-closed missing-baseline + behavior. +- `.github/workflows/unit-tests.yml`: fetch history, create a detached worktree + at the previous remote main, benchmark old/current code on one runner with + isolated caches, and upload both artifacts. +- `tests/test_performance_check.py`: cover equal results, runtime/quality + regressions, explicit baseline loading, and provenance. + +## Verification + +- Missing-baseline CLI check exited 2 in 0.2 seconds and created no baseline. +- Unit suite: 4 passed. +- Ruff and workflow YAML parsing: passed. +- End-to-end temporary-worktree simulation: + - baseline commit `6440e31143e4cbd56e9d523a25a8f48bca302670`; + - current commit `077211066af1956f38e2a7183cd77bdd0ace140c`; + - both artifacts contained provenance and comparison returned no regressions. +- The temporary worktree was removed after verification. + +## Failed Attempts + +- A committed baseline generated on this Intel macOS host was rejected as a CI + design because absolute timings are not portable to GitHub's Linux runners. +- Comparing only `HEAD^` was rejected because a push containing several commits + would test only the final commit. `github.event.before` represents the actual + remote-main baseline for the pushed range. + +## Risks and Follow-ups + +- Shared hosted runners remain noisy. The current median-of-three and 20% + threshold are a regression alarm, not publication-quality performance proof. +- Parent and current code use dependencies installed from the current checkout; + this isolates source regressions but does not detect dependency-only speed + changes. +- The workflow must run on GitHub once to validate hosted-runner behavior and + artifact upload. External ScoringBench results remain the value proof; this CI + microbenchmark is maintenance infrastructure. + +## Commits + +- `30b9ab5` — `ci: compare performance across pushed revisions` From 7e26d0ec8f7b25d71b3a525071dbb1f45725ce16 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:03:11 -0700 Subject: [PATCH 12/49] ci: isolate JAX tests from xdist --- .github/workflows/unit-tests.yml | 18 ++++++++++++++++-- pyproject.toml | 1 + tests/test_distribution_gradients.py | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e9dc616..2a907e3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -6,10 +6,15 @@ on: pull_request: branches: [main] +concurrency: + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: # Fast tests: <3 min, runs on every PR fast-tests: runs-on: ${{ matrix.os }} + timeout-minutes: 20 strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -43,11 +48,19 @@ jobs: env: OPENBOOST_BACKEND: "cpu" run: | - pytest tests/ -v --tb=short -m "not slow and not benchmark" + pytest tests/ -v --tb=short -m "not slow and not benchmark and not jax" + + - name: Run JAX distribution tests serially (Linux, Python 3.12) + if: runner.os == 'Linux' && matrix.python-version == '3.12' + env: + OPENBOOST_BACKEND: "cpu" + run: | + pytest tests/test_distribution_gradients.py -v --tb=short -n 0 -m jax # Full tests: includes slow tests, runs after fast tests pass full-tests: runs-on: ubuntu-latest + timeout-minutes: 30 needs: fast-tests steps: @@ -68,7 +81,7 @@ jobs: env: OPENBOOST_BACKEND: "cpu" run: | - pytest tests/ -v --tb=short + pytest tests/ -v --tb=short -m "not jax" - name: Run documentation examples (CPU backend) env: @@ -79,6 +92,7 @@ jobs: # Compare the pushed main range against the previous remote main on one runner. performance-check: runs-on: ubuntu-latest + timeout-minutes: 15 if: github.ref == 'refs/heads/main' needs: full-tests diff --git a/pyproject.toml b/pyproject.toml index ae350e9..27913aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ markers = [ "numerical: marks numerical agreement tests against reference implementations", "parity: marks CPU/GPU parity tests", "benchmark: marks performance benchmark tests (not run in CI)", + "jax: marks tests that import and compile JAX (run serially in Linux CI)", ] [dependency-groups] diff --git a/tests/test_distribution_gradients.py b/tests/test_distribution_gradients.py index 0e1eeb1..2506734 100644 --- a/tests/test_distribution_gradients.py +++ b/tests/test_distribution_gradients.py @@ -400,6 +400,7 @@ def nll_fn(y, p): expected_hess = 2.0 * resid ** 2 / params['scale'] ** 2 assert_allclose(grads['scale'][1], expected_hess, rtol=1e-2) + @pytest.mark.jax def test_jax_gradient_matches_numerical_path(self): """JAX path must differentiate through the link, like the numerical path. @@ -474,6 +475,7 @@ def fail_autodiff(y, params): assert_allclose(actual[name][0], expected[name][0]) assert_allclose(actual[name][1], expected[name][1]) + @pytest.mark.jax def test_numpy_nll_falls_back_when_jax_is_installed(self): """Plain numpy NLLs remain correct when optional JAX is installed.""" pytest.importorskip('jax') From f380c53632cd7e39669f8c20c88ad71ceb53f364 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:03:27 -0700 Subject: [PATCH 13/49] docs: record Linux JAX CI isolation --- learnings/2026-08-15-linux-jax-ci.md | 58 ++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 learnings/2026-08-15-linux-jax-ci.md diff --git a/learnings/2026-08-15-linux-jax-ci.md b/learnings/2026-08-15-linux-jax-ci.md new file mode 100644 index 0000000..c830c51 --- /dev/null +++ b/learnings/2026-08-15-linux-jax-ci.md @@ -0,0 +1,58 @@ +# 2026-08-15: Linux JAX CI Isolation + +## Context + +PR #19's fast-test matrix passed on macOS/Python 3.10 and 3.12, while both +Ubuntu jobs remained in the pytest step for more than 15 minutes. Lint and +dependency installation had already passed. The test extra installs JAX only +on Linux, and two JAX compilation tests were running inside the repository-wide +`pytest-xdist -n auto` process pool. + +## Decision or Result + +Mark the two JAX-dependent tests explicitly and run them once, serially, on +Linux/Python 3.12. The regular fast and full suites exclude that marker. This +separates JAX compilation/runtime behavior from xdist worker behavior and avoids +duplicating the same optional-backend test across two Python versions. + +Add job-level timeouts and workflow concurrency cancellation so a future hang +cannot consume the default six-hour GitHub Actions limit or leave superseded PR +runs executing indefinitely. + +## Changes + +- `tests/test_distribution_gradients.py`: mark the two true JAX tests. +- `pyproject.toml`: register the `jax` pytest marker. +- `.github/workflows/unit-tests.yml`: exclude JAX from parallel suites, add one + serial Linux/Python 3.12 step, add bounded job timeouts, and cancel superseded + runs on the same ref. + +## Verification + +- Non-JAX distribution gradient selection: 49 passed, 2 deselected. +- JAX selection on macOS Intel: 2 selected and correctly skipped because no JAX + wheel is installed on that platform. +- Workflow YAML parsing and `git diff --check`: passed. +- The decisive verification is the replacement PR workflow on Ubuntu; local + macOS cannot reproduce the Linux-only JAX installation. + +## Failed Attempts + +- GitHub does not publish downloadable logs for an in-progress job; the log + endpoint returned 404, so there was no responsible way to name a specific + test from partial output. +- Repeated status polling confirmed the Ubuntu pair was symmetric and isolated + to pytest but could not distinguish JAX compilation from another Linux-only + stall. The new split is designed to make the next run diagnostic as well as + faster. + +## Risks and Follow-ups + +- If the non-JAX Ubuntu suite still stalls, JAX was not the cause; use the job + timeout's completed logs to identify the last test/file and fix that path. +- If only the serial JAX step stalls, pin or revise the Linux JAX test/runtime + rather than weakening the core suite. + +## Commits + +- `7e26d0e` — `ci: isolate JAX tests from xdist` From 1b5bd8fc3eb9e6595a0ded9d9a068bed6dd244b3 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:20:35 -0700 Subject: [PATCH 14/49] bench: pin ScoringBench contract environment --- benchmarks/scoringbench/README.md | 14 ++++++++++++++ benchmarks/scoringbench/SCORINGBENCH_COMMIT | 1 + benchmarks/scoringbench/requirements.txt | 1 + learnings/2026-08-15-scoringbench-integration.md | 14 +++++++++++++- 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 benchmarks/scoringbench/SCORINGBENCH_COMMIT diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index d68bf19..8de08ba 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -26,6 +26,7 @@ contract test remains useful for local adapter development. ```bash git clone https://github.com/jonaslandsgesell/ScoringBench .repos/ScoringBench +git -C .repos/ScoringBench checkout "$(cat benchmarks/scoringbench/SCORINGBENCH_COMMIT)" uv venv .venv-scoringbench --python 3.12 uv pip install --python .venv-scoringbench/bin/python \ @@ -33,6 +34,19 @@ uv pip install --python .venv-scoringbench/bin/python \ uv pip install --python .venv-scoringbench/bin/python -e . ``` +`SCORINGBENCH_COMMIT` freezes the upstream protocol used for committed results. +Test newer upstream revisions separately before updating that file. The manifest +records the checked-out revision and dirty state. + +On Intel macOS, the latest Numba release may not publish a compatible wheel. +The full benchmark remains unsupported there, but the wrapper contract can be +checked with the last compatible wheel instead of compiling llvmlite locally: + +```bash +uv pip install --python .venv-scoringbench/bin/python 'numba==0.63.1' +uv pip install --python .venv-scoringbench/bin/python --no-deps -e . +``` + For CUDA, install OpenBoost's CUDA extra using the package versions appropriate for the benchmark machine: diff --git a/benchmarks/scoringbench/SCORINGBENCH_COMMIT b/benchmarks/scoringbench/SCORINGBENCH_COMMIT new file mode 100644 index 0000000..b823077 --- /dev/null +++ b/benchmarks/scoringbench/SCORINGBENCH_COMMIT @@ -0,0 +1 @@ +a938a667b7839b41e9272929010573410301c0b4 diff --git a/benchmarks/scoringbench/requirements.txt b/benchmarks/scoringbench/requirements.txt index 44cf484..9e7ac5a 100644 --- a/benchmarks/scoringbench/requirements.txt +++ b/benchmarks/scoringbench/requirements.txt @@ -8,4 +8,5 @@ pyarrow>=15 autorank>=1.2 openml>=0.15 pytest>=7 +pytest-xdist>=3 ngboost>=0.5 diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 8d33563..58cf1a3 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -34,6 +34,8 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - Wrapper contract: 1 passed against ScoringBench commit `a938a667b7839b41e9272929010573410301c0b4`. +- Fresh isolated-environment contract after adding xdist: 1 passed on Intel + macOS with Numba 0.63.1; this validates the adapter only, not benchmark scores. - OpenBoost distributional regression tests: 47 passed. - `ruff check benchmarks/scoringbench`: passed. - Python compilation and manifest protocol classification: passed. @@ -48,7 +50,17 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. requires NumPy 2.x, while the available PyTorch wheel uses the NumPy 1.x ABI. One run crashed and later attempts entered an uninterruptible kernel exit state. The launcher now refuses this platform before importing ScoringBench. - Published CPU/CUDA runs must use Linux. +- A fresh isolated environment could not collect the wrapper test because the + repository-wide pytest configuration enables xdist while the benchmark + requirements omitted `pytest-xdist`. The isolated requirements now include + it. +- Installing unconstrained `numba>=0.60` on Intel macOS selected Numba 0.67, + for which no compatible wheel was available; llvmlite then tried to build + against LLVM 20 although that release requires LLVM 22. Installing the last + available Intel wheel (`numba==0.63.1`) and the editable project with + `--no-deps` is sufficient for the wrapper-only contract. This workaround is + not a supported full benchmark environment. Published CPU/CUDA runs must use + Linux. ## Risks and Follow-ups From af22e8818ade59fb89e825c5d45b5c113302500a Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:22:27 -0700 Subject: [PATCH 15/49] ci: run pinned ScoringBench smoke --- .github/workflows/scoringbench.yml | 120 ++++++++++++++++++ benchmarks/scoringbench/README.md | 7 + .../2026-08-15-scoringbench-integration.md | 6 + 3 files changed, 133 insertions(+) create mode 100644 .github/workflows/scoringbench.yml diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml new file mode 100644 index 0000000..9b8ec38 --- /dev/null +++ b/.github/workflows/scoringbench.yml @@ -0,0 +1,120 @@ +name: ScoringBench + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/scoringbench.yml" + - "benchmarks/scoringbench/**" + - "src/openboost/**" + - "pyproject.toml" + workflow_dispatch: + inputs: + mode: + description: Benchmark scope + required: true + default: smoke + type: choice + options: + - smoke + - quality_shard + dataset_index: + description: ScoringBench dataset index for quality_shard + required: true + default: "0" + type: string + n_trees: + description: Boosting rounds for quality_shard + required: true + default: "500" + type: string + +concurrency: + group: scoringbench-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + scoringbench-cpu: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + OPENBOOST_BACKEND: cpu + PYTHONHASHSEED: "0" + NUMBA_NUM_THREADS: "2" + OMP_NUM_THREADS: "2" + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: benchmarks/scoringbench/requirements.txt + + - name: Check out pinned ScoringBench + run: | + git clone --filter=blob:none https://github.com/jonaslandsgesell/ScoringBench .repos/ScoringBench + git -C .repos/ScoringBench checkout --detach "$(cat benchmarks/scoringbench/SCORINGBENCH_COMMIT)" + + - name: Install benchmark environment + run: | + python -m pip install --upgrade pip + python -m pip install -r benchmarks/scoringbench/requirements.txt + python -m pip install -e . + + - name: Validate wrapper contract + env: + PYTHONPATH: .repos/ScoringBench + run: | + python -m pytest -o addopts="" \ + benchmarks/scoringbench/test_openboost_wrapper.py -q + + - name: Run smoke benchmark + if: github.event_name == 'pull_request' || inputs.mode == 'smoke' + run: | + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --smoke \ + --n-trees 20 \ + --output-dir "${RUNNER_TEMP}/scoringbench-output" + + - name: Run official quality shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'quality_shard' + env: + DATASET_INDEX: ${{ inputs.dataset_index }} + N_TREES: ${{ inputs.n_trees }} + run: | + if [[ ! "${DATASET_INDEX}" =~ ^[0-9]+$ ]]; then + echo "dataset_index must be a non-negative integer" >&2 + exit 2 + fi + if [[ ! "${N_TREES}" =~ ^[1-9][0-9]*$ ]]; then + echo "n_trees must be a positive integer" >&2 + exit 2 + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --dataset-index "${DATASET_INDEX}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees "${N_TREES}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-output" + + - name: Verify benchmark artifact + run: | + test -s "${RUNNER_TEMP}/scoringbench-output/openboost_manifest.json" + test -n "$(find "${RUNNER_TEMP}/scoringbench-output/raw" -name '*.parquet' -print -quit)" + + - name: Upload benchmark artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: scoringbench-${{ github.event_name }}-${{ github.sha }} + path: ${{ runner.temp }}/scoringbench-output + if-no-files-found: warn + retention-days: 30 diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 8de08ba..403cb2a 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -82,6 +82,13 @@ PYTHONPATH=.repos/ScoringBench \ benchmarks/scoringbench/test_openboost_wrapper.py -q ``` +The `ScoringBench` GitHub workflow runs this contract plus the two-fold smoke +benchmark on relevant pull requests and uploads the raw Parquet files and +manifest. Its manual `quality_shard` mode runs one indexed dataset with the +official five-fold, 3,000-row protocol. A green smoke job proves integration, +not model quality; only completed official shards belong in a leaderboard +submission. + ## Official quality track Run the official default: five folds, one repeat, at most 3,000 rows per diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 58cf1a3..a082815 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -29,6 +29,8 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - `benchmarks/scoringbench/README.md`: environment, official track, scale track, upstream submission, and evidence gates. - `.gitignore`: ignore arbitrary local ScoringBench result directories. +- `.github/workflows/scoringbench.yml`: pinned Linux contract/smoke validation, + artifact upload, and a manually dispatched official-quality shard. ## Verification @@ -39,6 +41,8 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - OpenBoost distributional regression tests: 47 passed. - `ruff check benchmarks/scoringbench`: passed. - Python compilation and manifest protocol classification: passed. +- GitHub workflow YAML parsed locally; the pinned wrapper contract passed before + the workflow was added. The complete Linux smoke remains a CI gate. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts @@ -67,5 +71,7 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - Run the official full suite on Linux and submit the wrapper/results upstream. - Run a separate large-sample curve on at least three real ScoringBench datasets. - Add CPU/CUDA prediction parity before interpreting a CUDA timing result. +- Run the new Linux smoke workflow and retain its artifact; workflow syntax and + wrapper-only validation do not prove that the complete runner succeeds. - Add freMTPL2 or another real exposure-aware case study after the third-party quality result exists. From 0d5a0e25553d38ceaf14def960d47495b1b84022 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:28:20 -0700 Subject: [PATCH 16/49] ci: avoid duplicate cold test suite --- .github/workflows/unit-tests.yml | 44 ++++++++++++++++++++-------- learnings/2026-08-15-linux-jax-ci.md | 27 +++++++++++++---- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 2a907e3..dbf0b3d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,23 +11,31 @@ concurrency: cancel-in-progress: true jobs: - # Fast tests: <3 min, runs on every PR + # Core tests: runs on every PR across the supported OS/Python matrix. fast-tests: runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 25 strategy: matrix: os: [ubuntu-latest, macos-latest] python-version: ["3.10", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + - name: Restore compiled Numba kernels + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/numba-cache + key: numba-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('src/openboost/**/*.py') }} + restore-keys: | + numba-${{ runner.os }}-py${{ matrix.python-version }}- + - name: Install OpenMP runtime (macOS) if: runner.os == 'macOS' run: | @@ -47,6 +55,7 @@ jobs: - name: Run fast tests (CPU backend) env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | pytest tests/ -v --tb=short -m "not slow and not benchmark and not jax" @@ -54,38 +63,49 @@ jobs: if: runner.os == 'Linux' && matrix.python-version == '3.12' env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | pytest tests/test_distribution_gradients.py -v --tb=short -n 0 -m jax - # Full tests: includes slow tests, runs after fast tests pass + # Slow tests and docs run after the core matrix; do not duplicate core tests. full-tests: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 needs: fast-tests steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" + - name: Restore compiled Numba kernels + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/numba-cache + key: numba-${{ runner.os }}-py3.12-${{ hashFiles('src/openboost/**/*.py') }} + restore-keys: | + numba-${{ runner.os }}-py3.12- + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -e ".[test,sklearn]" pip install "xgboost>=2.0" - - name: Run all tests (CPU backend) + - name: Run slow tests (CPU backend) env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | - pytest tests/ -v --tb=short -m "not jax" + pytest tests/ -v --tb=short -m "slow and not benchmark and not jax" - name: Run documentation examples (CPU backend) env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | python scripts/run_doc_examples.py @@ -97,12 +117,12 @@ jobs: needs: full-tests steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/learnings/2026-08-15-linux-jax-ci.md b/learnings/2026-08-15-linux-jax-ci.md index c830c51..7312cb5 100644 --- a/learnings/2026-08-15-linux-jax-ci.md +++ b/learnings/2026-08-15-linux-jax-ci.md @@ -19,13 +19,27 @@ Add job-level timeouts and workflow concurrency cancellation so a future hang cannot consume the default six-hour GitHub Actions limit or leave superseded PR runs executing indefinitely. +The replacement Ubuntu/Python 3.12 job falsified the original performance +hypothesis: the non-JAX core suite passed 729 tests in 937.02 seconds, while the +two isolated JAX tests passed in 3.59 seconds. JAX was not the source of the long +step. Keep the isolation because it makes the optional path explicit, but treat +the 15-minute cold core suite as the CI cost to optimize. + +The downstream `full-tests` job was also rerunning the entire core suite after +all four matrix jobs had passed. It now runs only tests marked `slow` plus the +documentation examples. Numba's on-disk cache is restored per OS and Python +version so later commits can reuse compiled kernels; the first run for a new +source hash remains a cold run. + ## Changes - `tests/test_distribution_gradients.py`: mark the two true JAX tests. - `pyproject.toml`: register the `jax` pytest marker. - `.github/workflows/unit-tests.yml`: exclude JAX from parallel suites, add one serial Linux/Python 3.12 step, add bounded job timeouts, and cancel superseded - runs on the same ref. + runs on the same ref. A follow-up avoids the duplicate core suite, restores a + versioned Numba cache, and moves JavaScript actions off deprecated Node 20 + releases. ## Verification @@ -35,6 +49,10 @@ runs executing indefinitely. - Workflow YAML parsing and `git diff --check`: passed. - The decisive verification is the replacement PR workflow on Ubuntu; local macOS cannot reproduce the Linux-only JAX installation. +- Replacement Ubuntu/Python 3.12 job: 729 passed, 32 skipped in 937.02 seconds; + isolated JAX step: 2 passed in 3.59 seconds. +- Revised local downstream selection: 2 slow tests passed, 762 deselected in + 10.91 seconds; all 6 executable documentation files passed. ## Failed Attempts @@ -48,10 +66,9 @@ runs executing indefinitely. ## Risks and Follow-ups -- If the non-JAX Ubuntu suite still stalls, JAX was not the cause; use the job - timeout's completed logs to identify the last test/file and fix that path. -- If only the serial JAX step stalls, pin or revise the Linux JAX test/runtime - rather than weakening the core suite. +- Measure one warm-cache PR run before claiming the cache improved latency. +- The two slow tests and documentation examples still need to pass in the + revised downstream job before the duplicate-suite removal is accepted. ## Commits From ac2ac72ad061b2019704326691ce2cc3612fdc10 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:30:20 -0700 Subject: [PATCH 17/49] bench: record CI source provenance --- .github/workflows/scoringbench.yml | 1 + benchmarks/scoringbench/README.md | 5 ++- benchmarks/scoringbench/run.py | 22 +++++++++++ .../2026-08-15-scoringbench-integration.md | 11 ++++-- tests/test_scoringbench_provenance.py | 37 +++++++++++++++++++ 5 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/test_scoringbench_provenance.py diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index 9b8ec38..d3579da 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -39,6 +39,7 @@ jobs: timeout-minutes: 45 env: OPENBOOST_BACKEND: cpu + OPENBOOST_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} PYTHONHASHSEED: "0" NUMBA_NUM_THREADS: "2" OMP_NUM_THREADS: "2" diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 403cb2a..6d19512 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -144,8 +144,9 @@ their results cannot be confused: Run each CUDA measurement in a fresh process. Report cold and repeated runs separately, and include failures/OOMs. The generated `openboost_manifest.json` records both git commits, dirty state, arguments, package versions, platform and -GPU identity. Runs whose manifest says `scoringbench_scale_extension` are not -official leaderboard runs. +GPU identity. In GitHub Actions it also distinguishes the tested merge commit +from the pull request's source-head commit. Runs whose manifest says +`scoringbench_scale_extension` are not official leaderboard runs. Generated ScoringBench directories are gitignored. Publish accepted evidence in ScoringBench's designated output/LFS repository or intentionally force-add a diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 8f3a552..70f1ed5 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -70,6 +70,27 @@ def _gpu_info() -> dict | None: return {"error": f"{type(exc).__name__}: {exc}"} +def _ci_state() -> dict | None: + """Return non-secret GitHub Actions identity for artifact provenance.""" + if os.environ.get("GITHUB_ACTIONS") != "true": + return None + + names = { + "event_name": "GITHUB_EVENT_NAME", + "repository": "GITHUB_REPOSITORY", + "ref": "GITHUB_REF", + "tested_sha": "GITHUB_SHA", + "source_sha": "OPENBOOST_SOURCE_SHA", + "head_ref": "GITHUB_HEAD_REF", + "run_id": "GITHUB_RUN_ID", + "run_attempt": "GITHUB_RUN_ATTEMPT", + } + return { + "provider": "github_actions", + **{key: os.environ.get(env_name) for key, env_name in names.items()}, + } + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Run OpenBoost on the external ScoringBench protocol" @@ -260,6 +281,7 @@ def _write_provenance( ), "openboost_git": _git_state(PROJECT_ROOT), "scoringbench_git": _git_state(scoringbench_dir), + "ci": _ci_state(), "arguments": vars(args), "datasets": [ { diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index a082815..31756ab 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -42,7 +42,12 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - `ruff check benchmarks/scoringbench`: passed. - Python compilation and manifest protocol classification: passed. - GitHub workflow YAML parsed locally; the pinned wrapper contract passed before - the workflow was added. The complete Linux smoke remains a CI gate. + the workflow was added, then the complete Linux smoke passed in run #1. +- Linux ScoringBench run #1 passed the contract, two-fold OpenBoost/NGBoost + smoke, artifact verification, and upload. Artifact `9256565927` contains the + manifest and both raw Parquet files with 4 result rows; its digest is + `sha256:3c98f640af58291f7cb648ac38bfa04ff0a44bd198698fb47a2b4f22dfb98862`. + This proves the integration path only, not comparative model value. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts @@ -71,7 +76,7 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - Run the official full suite on Linux and submit the wrapper/results upstream. - Run a separate large-sample curve on at least three real ScoringBench datasets. - Add CPU/CUDA prediction parity before interpreting a CUDA timing result. -- Run the new Linux smoke workflow and retain its artifact; workflow syntax and - wrapper-only validation do not prove that the complete runner succeeds. +- The first artifact identified the PR merge commit but not the source-head SHA. + The manifest now records both; confirm the mapping in the next CI artifact. - Add freMTPL2 or another real exposure-aware case study after the third-party quality result exists. diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py new file mode 100644 index 0000000..3cf4e7f --- /dev/null +++ b/tests/test_scoringbench_provenance.py @@ -0,0 +1,37 @@ +"""Tests for ScoringBench artifact provenance that need no external checkout.""" + +from benchmarks.scoringbench.run import _ci_state + + +def test_ci_state_is_none_outside_github_actions(monkeypatch): + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + assert _ci_state() is None + + +def test_ci_state_distinguishes_tested_merge_from_source_head(monkeypatch): + values = { + "GITHUB_ACTIONS": "true", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REPOSITORY": "jxucoder/openboost", + "GITHUB_REF": "refs/pull/19/merge", + "GITHUB_SHA": "merge-sha", + "OPENBOOST_SOURCE_SHA": "head-sha", + "GITHUB_HEAD_REF": "codex/scoringbench-release-hardening", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "2", + } + for name, value in values.items(): + monkeypatch.setenv(name, value) + + assert _ci_state() == { + "provider": "github_actions", + "event_name": "pull_request", + "repository": "jxucoder/openboost", + "ref": "refs/pull/19/merge", + "tested_sha": "merge-sha", + "source_sha": "head-sha", + "head_ref": "codex/scoringbench-release-hardening", + "run_id": "123", + "run_attempt": "2", + } From 1c41e4f4330c33af081749fd330b26a80bc96a7a Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:31:48 -0700 Subject: [PATCH 18/49] bench: isolate upstream dataset artifacts --- benchmarks/scoringbench/README.md | 3 ++ benchmarks/scoringbench/run.py | 29 +++++++++++++++---- .../2026-08-15-scoringbench-integration.md | 6 ++++ tests/test_scoringbench_provenance.py | 18 +++++++++++- 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 6d19512..52e753f 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -147,6 +147,9 @@ records both git commits, dirty state, arguments, package versions, platform and GPU identity. In GitHub Actions it also distinguishes the tested merge commit from the pull request's source-head commit. Runs whose manifest says `scoringbench_scale_extension` are not official leaderboard runs. +ScoringBench's resolved `datasets.json` is redirected into the output directory +so the exact registry travels with official artifacts without dirtying the +OpenBoost checkout. Generated ScoringBench directories are gitignored. Publish accepted evidence in ScoringBench's designated output/LFS repository or intentionally force-add a diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 70f1ed5..4814cc3 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -14,6 +14,7 @@ import platform import subprocess import sys +from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path @@ -91,6 +92,18 @@ def _ci_state() -> dict | None: } +@contextmanager +def _working_directory(path: Path): + """Temporarily direct upstream relative outputs into an artifact directory.""" + original = Path.cwd() + path.mkdir(parents=True, exist_ok=True) + os.chdir(path) + try: + yield + finally: + os.chdir(original) + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Run OpenBoost on the external ScoringBench protocol" @@ -355,6 +368,7 @@ def main() -> int: from scoringbench.utils import set_seed set_seed(args.seed) + output_dir = Path(args.output_dir).expanduser().resolve() if args.smoke: datasets = [ { @@ -367,18 +381,21 @@ def main() -> int: ] args.n_folds = 2 else: - datasets = validate_datasets(get_DATASETS_CONFIG()) - if args.list_datasets: - for index, dataset in enumerate(datasets): - print(f"{index:3d} {dataset['name']}") - return 0 + # ScoringBench exports its resolved dataset registry to Path.cwd(). Keep + # that reproducibility artifact with the benchmark instead of dirtying + # the OpenBoost checkout. + with _working_directory(output_dir): + datasets = validate_datasets(get_DATASETS_CONFIG()) + if args.list_datasets: + for index, dataset in enumerate(datasets): + print(f"{index:3d} {dataset['name']}") + return 0 datasets = _select_datasets(datasets, args) if args.lite: args.n_folds = 2 model_factories = _model_factories(args) - output_dir = Path(args.output_dir).expanduser().resolve() result = run_benchmark( datasets_config=datasets, model_factories=model_factories, diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 31756ab..e626639 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -48,6 +48,7 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. manifest and both raw Parquet files with 4 result rows; its digest is `sha256:3c98f640af58291f7cb648ac38bfa04ff0a44bd198698fb47a2b4f22dfb98862`. This proves the integration path only, not comparative model value. +- CI/source provenance and artifact-working-directory tests: 3 passed. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts @@ -70,6 +71,11 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. `--no-deps` is sufficient for the wrapper-only contract. This workaround is not a supported full benchmark environment. Published CPU/CUDA runs must use Linux. +- Building ScoringBench's official dataset registry writes `datasets.json` to + `Path.cwd()`. A dataset-list probe therefore polluted the OpenBoost root and + would make a later manifest report a dirty checkout. The launcher now builds + and validates that registry from inside the output directory and restores the + original working directory even after an exception. ## Risks and Follow-ups diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index 3cf4e7f..ee6f53b 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -1,6 +1,9 @@ """Tests for ScoringBench artifact provenance that need no external checkout.""" -from benchmarks.scoringbench.run import _ci_state +from pathlib import Path + +import pytest +from benchmarks.scoringbench.run import _ci_state, _working_directory def test_ci_state_is_none_outside_github_actions(monkeypatch): @@ -35,3 +38,16 @@ def test_ci_state_distinguishes_tested_merge_from_source_head(monkeypatch): "run_id": "123", "run_attempt": "2", } + + +def test_working_directory_contains_upstream_output_and_restores_on_error(tmp_path): + original = Path.cwd() + artifact_dir = tmp_path / "artifact" + + with pytest.raises(RuntimeError, match="stop"), _working_directory(artifact_dir): + assert Path.cwd() == artifact_dir + Path("datasets.json").write_text("[]\n") + raise RuntimeError("stop") + + assert Path.cwd() == original + assert (artifact_dir / "datasets.json").read_text() == "[]\n" From 6d8ae09736123aa6bc0c25bd5c1d29ae16a1040b Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:36:13 -0700 Subject: [PATCH 19/49] bench: add official quality sentinel --- .github/workflows/scoringbench.yml | 55 ++++++++++++++----- benchmarks/scoringbench/README.md | 13 +++-- benchmarks/scoringbench/run.py | 21 +++++-- .../2026-08-15-scoringbench-integration.md | 11 +++- tests/test_scoringbench_provenance.py | 27 ++++++++- 5 files changed, 101 insertions(+), 26 deletions(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index d3579da..c292d33 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -18,10 +18,10 @@ on: options: - smoke - quality_shard - dataset_index: - description: ScoringBench dataset index for quality_shard + dataset_name: + description: Exact ScoringBench dataset name for quality_shard required: true - default: "0" + default: Abalone type: string n_trees: description: Boosting rounds for quality_shard @@ -36,7 +36,7 @@ concurrency: jobs: scoringbench-cpu: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 90 env: OPENBOOST_BACKEND: cpu OPENBOOST_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -80,16 +80,29 @@ jobs: --models openboost_cpu,ngboost \ --smoke \ --n-trees 20 \ - --output-dir "${RUNNER_TEMP}/scoringbench-output" + --output-dir "${RUNNER_TEMP}/scoringbench-smoke" + + - name: Run official quality sentinel + if: github.event_name == 'pull_request' + run: | + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --dataset-name Abalone \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees 500 \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" - name: Run official quality shard if: github.event_name == 'workflow_dispatch' && inputs.mode == 'quality_shard' env: - DATASET_INDEX: ${{ inputs.dataset_index }} + DATASET_NAME: ${{ inputs.dataset_name }} N_TREES: ${{ inputs.n_trees }} run: | - if [[ ! "${DATASET_INDEX}" =~ ^[0-9]+$ ]]; then - echo "dataset_index must be a non-negative integer" >&2 + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 exit 2 fi if [[ ! "${N_TREES}" =~ ^[1-9][0-9]*$ ]]; then @@ -99,23 +112,37 @@ jobs: python benchmarks/scoringbench/run.py \ --scoringbench-dir .repos/ScoringBench \ --models openboost_cpu,ngboost \ - --dataset-index "${DATASET_INDEX}" \ + --dataset-name "${DATASET_NAME}" \ --sample-size 3000 \ --n-folds 5 \ --n-repeats 1 \ --n-trees "${N_TREES}" \ - --output-dir "${RUNNER_TEMP}/scoringbench-output" + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Verify smoke artifact + if: github.event_name == 'pull_request' || inputs.mode == 'smoke' + run: | + MANIFEST="${RUNNER_TEMP}/scoringbench-smoke/openboost_manifest.json" + test -s "${MANIFEST}" + test -n "$(find "${RUNNER_TEMP}/scoringbench-smoke/raw" -name '*.parquet' -print -quit)" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4' "${MANIFEST}" - - name: Verify benchmark artifact + - name: Verify quality-shard artifact + if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' run: | - test -s "${RUNNER_TEMP}/scoringbench-output/openboost_manifest.json" - test -n "$(find "${RUNNER_TEMP}/scoringbench-output/raw" -name '*.parquet' -print -quit)" + MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" + test -s "${MANIFEST}" + test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" + test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == 10' "${MANIFEST}" - name: Upload benchmark artifact if: always() uses: actions/upload-artifact@v4 with: name: scoringbench-${{ github.event_name }}-${{ github.sha }} - path: ${{ runner.temp }}/scoringbench-output + path: | + ${{ runner.temp }}/scoringbench-smoke + ${{ runner.temp }}/scoringbench-quality if-no-files-found: warn retention-days: 30 diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 52e753f..b35895d 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -82,12 +82,13 @@ PYTHONPATH=.repos/ScoringBench \ benchmarks/scoringbench/test_openboost_wrapper.py -q ``` -The `ScoringBench` GitHub workflow runs this contract plus the two-fold smoke -benchmark on relevant pull requests and uploads the raw Parquet files and -manifest. Its manual `quality_shard` mode runs one indexed dataset with the -official five-fold, 3,000-row protocol. A green smoke job proves integration, -not model quality; only completed official shards belong in a leaderboard -submission. +The `ScoringBench` GitHub workflow runs this contract, the two-fold smoke, and +an Abalone quality sentinel on relevant pull requests, then uploads the raw +Parquet files and manifests. The sentinel uses the official five-fold, +3,000-row protocol and default 500 rounds; one dataset is still not a quality +claim. Manual `quality_shard` mode accepts an exact dataset name so the suite can +be sharded without downloading every dataset just to resolve an index. Only a +completed full suite belongs in a leaderboard submission. ## Official quality track diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 4814cc3..d01e2f1 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -140,13 +140,14 @@ def _build_parser() -> argparse.ArgumentParser: default=3000, help="Official ScoringBench default is 3000; use 0 only for a scale extension", ) - parser.add_argument( + selection = parser.add_mutually_exclusive_group() + selection.add_argument( "--dataset-index", type=int, action="append", help="Run selected index from ScoringBench's validated dataset list (repeatable)", ) - parser.add_argument( + selection.add_argument( "--dataset-name", action="append", help="Run exact case-insensitive dataset name from the validated list (repeatable)", @@ -188,6 +189,13 @@ def _select_datasets(all_datasets: list[dict], args) -> list[dict]: return all_datasets +def _validate_selected_datasets(all_datasets: list[dict], args, validate) -> list[dict]: + """Validate only named shards; indexed shards retain validated-list semantics.""" + if args.dataset_name: + return validate(_select_datasets(all_datasets, args)) + return _select_datasets(validate(all_datasets), args) + + def _model_factories(args): from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper @@ -385,12 +393,17 @@ def main() -> int: # that reproducibility artifact with the benchmark instead of dirtying # the OpenBoost checkout. with _working_directory(output_dir): - datasets = validate_datasets(get_DATASETS_CONFIG()) + all_datasets = get_DATASETS_CONFIG() if args.list_datasets: + datasets = validate_datasets(all_datasets) for index, dataset in enumerate(datasets): print(f"{index:3d} {dataset['name']}") return 0 - datasets = _select_datasets(datasets, args) + datasets = _validate_selected_datasets( + all_datasets, + args, + validate_datasets, + ) if args.lite: args.n_folds = 2 diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index e626639..5d47e19 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -48,7 +48,10 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. manifest and both raw Parquet files with 4 result rows; its digest is `sha256:3c98f640af58291f7cb648ac38bfa04ff0a44bd198698fb47a2b4f22dfb98862`. This proves the integration path only, not comparative model value. -- CI/source provenance and artifact-working-directory tests: 3 passed. +- CI/source provenance, artifact-working-directory, and named-shard selection + tests: 4 passed. +- Linux ScoringBench run #4 confirmed `source_sha`, tested PR merge SHA, clean + checkouts, pinned upstream SHA, and 4 smoke rows in artifact `9256692529`. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts @@ -76,10 +79,16 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. would make a later manifest report a dirty checkout. The launcher now builds and validates that registry from inside the output directory and restores the original working directory even after an exception. +- ScoringBench validates datasets by loading them. Selecting a shard by index + therefore validates the entire registry before the index exists. Exact-name + shards now select first and validate only the requested datasets; index and + list modes retain their validated-list semantics. ## Risks and Follow-ups - Run the official full suite on Linux and submit the wrapper/results upstream. +- Use the Abalone sentinel only as a reproducibility/integration gate; inspect + its 5-fold artifact before deciding whether OpenBoost merits broader shards. - Run a separate large-sample curve on at least three real ScoringBench datasets. - Add CPU/CUDA prediction parity before interpreting a CUDA timing result. - The first artifact identified the PR merge commit but not the source-head SHA. diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index ee6f53b..f85910b 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -3,7 +3,11 @@ from pathlib import Path import pytest -from benchmarks.scoringbench.run import _ci_state, _working_directory +from benchmarks.scoringbench.run import ( + _ci_state, + _validate_selected_datasets, + _working_directory, +) def test_ci_state_is_none_outside_github_actions(monkeypatch): @@ -51,3 +55,24 @@ def test_working_directory_contains_upstream_output_and_restores_on_error(tmp_pa assert Path.cwd() == original assert (artifact_dir / "datasets.json").read_text() == "[]\n" + + +def test_named_quality_shard_validates_only_selected_dataset(): + class Args: + dataset_index = None + dataset_name = ["Abalone"] + + registry = [ + {"name": "Abalone", "source": "openml", "id": 183}, + {"name": "large_unused", "source": "openml", "id": 999}, + ] + validated = [] + + def validate(datasets): + validated.extend(datasets) + return datasets + + result = _validate_selected_datasets(registry, Args(), validate) + + assert result == [registry[0]] + assert validated == [registry[0]] From 921c024b23fc4549b251222f35ce64a4b6846505 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:47:04 -0700 Subject: [PATCH 20/49] bench: avoid OpenML sentinel outage --- .github/workflows/scoringbench.yml | 2 +- benchmarks/scoringbench/README.md | 13 +++++++------ learnings/2026-08-15-scoringbench-integration.md | 10 +++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index c292d33..ba29007 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -88,7 +88,7 @@ jobs: python benchmarks/scoringbench/run.py \ --scoringbench-dir .repos/ScoringBench \ --models openboost_cpu,ngboost \ - --dataset-name Abalone \ + --dataset-name 1027_ESL \ --sample-size 3000 \ --n-folds 5 \ --n-repeats 1 \ diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index b35895d..b67958a 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -83,12 +83,13 @@ PYTHONPATH=.repos/ScoringBench \ ``` The `ScoringBench` GitHub workflow runs this contract, the two-fold smoke, and -an Abalone quality sentinel on relevant pull requests, then uploads the raw -Parquet files and manifests. The sentinel uses the official five-fold, -3,000-row protocol and default 500 rounds; one dataset is still not a quality -claim. Manual `quality_shard` mode accepts an exact dataset name so the suite can -be sharded without downloading every dataset just to resolve an index. Only a -completed full suite belongs in a leaderboard submission. +a `1027_ESL` quality sentinel on relevant pull requests, then uploads the raw +Parquet files and manifests. The PMLB/GitHub-backed sentinel avoids making the +basic quality gate depend on OpenML dataset uptime. It uses the official +five-fold, 3,000-row protocol and default 500 rounds; one dataset is still not a +quality claim. Manual `quality_shard` mode accepts an exact dataset name so the +suite can be sharded without downloading every dataset just to resolve an +index. Only a completed full suite belongs in a leaderboard submission. ## Official quality track diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 5d47e19..730bc17 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -83,11 +83,19 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. therefore validates the entire registry before the index exists. Exact-name shards now select first and validate only the requested datasets; index and list modes retain their validated-list semantics. +- The first Abalone sentinel never reached model fitting: all three OpenML suite + requests returned server errors and dataset 183 then exhausted retries with + HTTP 504. ScoringBench caught the dataset exception and returned an empty + result while exiting successfully; OpenBoost's `result_rows == 10` artifact + gate correctly failed the job. The PR sentinel now uses ScoringBench's + `1027_ESL` PMLB/GitHub source so this small gate does not depend on the OpenML + data endpoint. Full-suite runs still must report OpenML failures rather than + silently treating them as model results. ## Risks and Follow-ups - Run the official full suite on Linux and submit the wrapper/results upstream. -- Use the Abalone sentinel only as a reproducibility/integration gate; inspect +- Use the `1027_ESL` sentinel only as a reproducibility/integration gate; inspect its 5-fold artifact before deciding whether OpenBoost merits broader shards. - Run a separate large-sample curve on at least three real ScoringBench datasets. - Add CPU/CUDA prediction parity before interpreting a CUDA timing result. From 2a73ef4d5f266d46890244c135ae3aeb21ecc9dd Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 19:56:47 -0700 Subject: [PATCH 21/49] bench: freeze first ScoringBench quality shard --- .../scoringbench/1027_esl_20260816/README.md | 48 ++ .../1027_esl_20260816/datasets.json | 482 ++++++++++++++++++ .../1027_esl_20260816/openboost_manifest.json | 83 +++ .../raw/ngboost/1027_ESL.parquet | Bin 0 -> 29953 bytes .../raw/openboost_cpu/1027_ESL.parquet | Bin 0 -> 29985 bytes .../1027_esl_20260816/summary.json | 62 +++ benchmarks/scoringbench/README.md | 5 + .../2026-08-15-scoringbench-integration.md | 9 + 8 files changed, 689 insertions(+) create mode 100644 benchmarks/evidence/scoringbench/1027_esl_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json create mode 100644 benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/1027_esl_20260816/raw/ngboost/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_20260816/raw/openboost_cpu/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/README.md b/benchmarks/evidence/scoringbench/1027_esl_20260816/README.md new file mode 100644 index 0000000..302c6dc --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/README.md @@ -0,0 +1,48 @@ +# ScoringBench `1027_ESL` sentinel — 2026-08-16 + +This directory freezes OpenBoost's first successful real-dataset shard under +ScoringBench's official five-fold, 3,000-row-cap protocol. It is an integration +and direction-finding result, not a full-suite leaderboard claim. + +## Provenance + +- GitHub Actions run: [31922702524](https://github.com/jxucoder/openboost/actions/runs/31922702524) +- Actions artifact: `9256929555` +- Actions artifact digest: + `sha256:0c5176c4a28cd44444f6a086643a01f636b0dcb5e165a4a30d3d992f3e96da97` +- OpenBoost source head: `921c024b23fc4549b251222f35ce64a4b6846505` +- Tested PR merge: `71dd86b7c95245593030ca488981bc2607f02eb5` +- ScoringBench: `a938a667b7839b41e9272929010573410301c0b4` +- Models: OpenBoost NaturalBoost Normal CPU and NGBoost Normal +- Shared parameters: 500 rounds, learning rate 0.01, depth 3, 99 quantiles, + seed 42 + +See `openboost_manifest.json` for the complete environment and arguments. +`datasets.json` is ScoringBench's resolved registry. The two Parquet files under +`raw/` contain all fold-level metrics. + +## Descriptive result + +| Metric (lower is better unless noted) | OpenBoost mean | NGBoost mean | OpenBoost relative | Fold count | +| --- | ---: | ---: | ---: | ---: | +| CRPS | 0.300527 | 0.307714 | -2.34% | 2/5 lower | +| Log score | 0.710078 | 0.675967 | +5.05% | 1/5 lower | +| RMSE | 0.545923 | 0.554927 | -1.62% | 3/5 lower | +| PIT KS statistic | 0.089366 | 0.105493 | -15.29% | 3/5 lower | +| 90% interval score | 2.436058 | 2.737215 | -11.00% | 4/5 lower | +| Fit time (seconds) | 2.111374 | 3.298695 | -35.99% | 5/5 lower | +| 90% coverage (closer to 0.90 is better) | 0.854723 | 0.813718 | — | 5/5 closer | + +Mean fit time is 1.56× lower for OpenBoost in this run. The first OpenBoost fold +includes cold Numba compilation, but this artifact does not separately report +cold and warm timing, so it cannot support a general speed claim. + +## Interpretation limits + +- This is one small real dataset and five correlated CV folds, with no repeated + seeds or uncertainty interval over the paired differences. +- OpenBoost improves several metrics here but loses log score. CRPS also improves + on only two individual folds despite its better mean. +- The result says nothing about CUDA or large-data scaling. +- A defensible value claim requires the complete ScoringBench suite, upstream + review, and a separate multi-size CPU/CUDA extension. diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json b/benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json new file mode 100644 index 0000000..a15f942 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json @@ -0,0 +1,83 @@ +{ + "arguments": { + "dataset_index": null, + "dataset_name": [ + "1027_ESL" + ], + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "models": [ + "openboost_cpu", + "ngboost" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "smoke": false + }, + "ci": { + "event_name": "pull_request", + "head_ref": "codex/scoringbench-release-hardening", + "provider": "github_actions", + "ref": "refs/pull/19/merge", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31922702524", + "source_sha": "921c024b23fc4549b251222f35ce64a4b6846505", + "tested_sha": "71dd86b7c95245593030ca488981bc2607f02eb5" + }, + "created_at": "2026-08-16T02:54:19+00:00", + "datasets": [ + { + "id": null, + "name": "1027_ESL", + "source": "pmlb" + } + ], + "official_protocol_compatible": true, + "openboost_git": { + "commit": "71dd86b7c95245593030ca488981bc2607f02eb5", + "dirty": false + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "official_quality_shard", + "result_rows": 10, + "schema_version": 1, + "scoringbench_git": { + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": null +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/ngboost/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/ngboost/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..89c2217f0bde77e6352f27fcaef6701f60cdc44a GIT binary patch literal 29953 zcmd5_34Bvk)=$g!C_7GTKvYC{!)8g+HR^EllD26|ThanuB7QYn*CdTe+O%a<B!aqsaBt9p=imSx6?YkQFDq0hel z%ZMXrO39vsPYyne?wbAB(i?Bvk5Xl+X{lqUgWbr~fvE$>$tgioP6;|`vdI$CQKMus z*^tS}@b8qQe*Kmd$ig`+Nd9(x;hw*vZx^j!`c=m#=;p>bb4=86^x)nnhn;-n6ZG}s z%+c!)_egLMH06|_gE^4W*c=YbEnR-|<}cC8$I?$5J5Qqbmlw=Czvm>HdWU@7xt%A` zF5|`u`U+&8awq4E>YR^Ijcel)kOWWeu! z--VPplZV~Z^PU6;K~qi%I+z0~jm?32d-=KxPkxT3^&I^nC-(&U;=*roR%t#*?a#NN zdAm-al7Hu~@RS^r;2>DaDM1HwAf>T6Z2zx^r_B8IN2vbwT~B>E;1Jr<{O0U0jvPc= z7UVB&n)4BQZs4=0#$_Cm;2@O9DZvJFAf>T6Jed5^T|4A2Ak~>A!;*fv75!`fLsg~) zFQci&Ppo=<(u-)wZ{J+4?|xZ=gJ2=2ggP(>QYy(old36BrP#4KejJRL)r%hAg_0<0 zqG4wer8rjGz7v7;*vUcfV&tEt%Ecp6D3Mdx5=;FML4$Y54c3 zY{xg~vuo~~GU}OcP}jY;dEb2QpJ@6M>nyo9{1Y+P88yqOlM>DgnsQ3e!4ycTr1Qqq z;=sg&o$q{S%0cY>szu!cD9YJ$0Eb}d^MB(*a6|&1bM^>gL%`|=Lm-w$z!y!7PuI>$ zI?bF#e|}^6(GicIMO#Ye?VqvYJJkHu`@7a|KZ{C0r121$+>~q!9>+g7A;~ z7Ysdp!+A7q?c_}loj8wLKhCfJin)OP@ZINo3cTmhYs;pfp0aNxI0%|@O3=X^NNEHP z!H$4~pt0Wb1@^ad)uPWK2$}&i22;~*V^3h&YV+1FCGdq3HU=83%g2D#4@x88i(+8@ ze#95IiC;c#I^|4y06) zgC=!mU^c`tIQAR(BE#RizVSc@82j=+@QbWw{#O{7oDA6#)_E{EDL5Y{Wr3qjhTH5j zSF8IC))d-LY;1XYw5IUoLmOuv9;K-plzqeheKA~fG$rfF>mEtckSip?LQV-f*mqJY z>3hg3ZF=6${x|J9jqY!KZR!ZiDYRh7i|z+!HfXGU4wNyJhSDc5J)uj-FGm+<0XBlHZNooBZVG+sQZx z8gfd|!5m0wYz|$Mc29obhn;Br$@Ay_)%FT{W|?Bos@<=neGrX$o4nOpO&tD zQ6dh4rJNFUFb7f^o5S*bFTXKn-Aa^keC@+6!@E)5kspUH+PxJ$RdS9#l>Z@myP|35 zdFpcs4nm2X66(MlNNH>i9}gNmA!+6obo#q{=b|4sqt5?*%lPSEx527?{%7-#zJg9n z8r=HQ*1t+{5H#hKpo2M((%2kMF8o$z|Klsjv0}%bt#e*QX*-ifPd~F6eY@@B9(whw z$UCj!A^SH{90UzHCFo!dq%<~%0Uw^-^}&FR$bV#OLr=>_wC}y)-Ls!~0Tp)N+5FJ9 zjVRyt#DGuwt(S;{ped&W9n680#^&(B8|u-Oz9VR}|H-B9-ycCAw@-W|?SGCSQ%l|| z&B`OF?v*od-TlYI5*!3gIVI>|4x}_Thcy#cK6vMAAECE8i+@u!;t;Z~_|>7cszYeU zr0EMs4u!uR6VMD2OdNN zUir-N-P&&Sef9|VGlg9e90W}{CFo!dq%<~%(fikpbw9Wty?OtE{g2*o09kzV%1`|H z0GjaekkL=|JAll$9{A$jV;@Lx5H#hKpo2M((%2k)H8*eg)CRk^>GuqIF?B!EXTST& z#nb!IiPn1`U38`k-8O2|uTJ0iz61wBQ%(swm;)(|&Eaa;Nv32SD$ag(AG&_`@S1P^ zU1;$ywf{oT?nhIu&Ab0}Sr=Ni)3rv*vW#RDW?P-90yVwI}Ulbm~Yyx*@xC# zpqdXVy3ikH%_uE@dq3J?I+ed?co$lF>k{g^E$>LQg`g>?1RczQl*Z<8_NHt6>g)F* z^|Y_o9LeZHzuLF4rl;@#x}|Gj$_LkXp$GCNe|n?)`-cwqlOG+hi z<*DO}QhE!S z`Nw3BikDeEZZG#Z*wI{9c%8xCdb7vvvU~aR6kCJWZTEN^nq8d15NmU*-D9q^ z*QgYnWK@I8XZN(49b7#Dc~CPmf~Fu3YGy~!6y!n8+z6V2JgBLTpee{jnpx5wsd;&S ztH;5|g|gc0HI8N>OheoazM3X)jn`-PadiyiC2j~z3m@DRKMs72!(PiP4dDcyhPrxQ zGL(~8?Kq}fO{sR5-BZ`jN7rKanQIj3$|z+SQObBvB@$1qzDcrTd@{_+1~{8-b_Xv>OBpk6%*AwCHZC<~OqOim$T94= zO-)WqNn4nlwj`Nuqp3^FT8)_w^NTDs8Bc;E;m_?;29BX+DF811O_4FluS?G*v77(}ne={_=wD+(5zE zQiQg0Fufi(ldwpNBbgLm^n!D-2yOKc`h^rgBNIf}H&Pn6Z=#uA1h;+|y?H3Wk+~oY zM@r*{3(mSK1&en&DeX)Wy=nyAIUMky$FdnC_9=Q^D868lUn!B7Yevz39SNvp(g%A< zN`HcvnVgp}t`{cLYp$VR9UbseoaDiHobO_3+<4($22o))c`lWH^=E)ergbnXDUBO7 z*p}G=FBc`#8?L2yrg2_|WpQ{f!=!QJg?kx91ur*^p?AvxmCV~aJ3SMrzp58kSP|3^;MkS?jqXxZHi@n@-Bi(fa?`3Ev zM({FJ8aH0JmqAqUa`#Pi*LXlBb1xW`l*WxJ@-j=9eZEPeU%r_>Jc08vEak#`877Sz zkMNR1<-YRUCeeE*0xFqh!KkD(Zd7)?Dw^hY|D5ik0f=8xCYc=#H z1jyv(10$2t-XV+o_69AzjR9mb7lDyUY44E5eS1?Ly;}#!WCj8wlhWQHi~IJr0(x&g zAd`6qj7&;fy9?KfdxxAW>S23}=)(p;CUXoJnUwYpSv;z`is|FC z0GZ4zU}RDnH*)a6MG-QX4wurWN&v9Wcidt?@yW$;BYk!@0FpTa>>(-rNgj$j@Tqe8 zVi^FE2?7jAN`C@i==^kcF1>gz0Fvu63`k1j1{9r;37r+ThUY5iWfcGiTitjw0@1hm z{rLkl_WVr%f))LVD*E9n03r|jv0tP#re8yOAmMb6#afk=wib2|Ci=(faoJF>aQGjR z1|S?rGma6T&IvVhXM}9Mk0qu3F^N8S2fgMFT(6J>5N89(S_=ZTraf#*OQET>&GX^B zPCn3WcG=8c`FH6HCN#Kg_O=>pv!li7@=myQ{2g~rA3uS6tzpaTVN&*W!33AtX$SQQ zkY1Po7PSp_hpi?|$idY3+TE;bt)toO%g)5=t`?`eU8=wdhuZkG6smT$I2`v(A0LFm z`3n+DkoGm0N&Rr;7lz{%`r5#S2@7&^>7DFTuz zeM}<|f!t^j;0pT~I?O#O0+K3yOd}8hb+ia@g?$Vi=AIM*NtHgP5sW}qlzBkS7WOf8 zxO)-=BvtyD2EHjE;W{tvYcT#Uf&`$bq>n*GHy%jbu)@m&&0VdC2p%l%WBA}L4iY=G z?6Lr}w?!f$2MR9>Gq8=swIFml!M}@XIkq0@X`S ztKp@6?0N7qDB0qnWqk~cFP)N*f;0>awm(wGu>oex| z|82H3+O0k;kA}!zm?iN7;@+d-qi?g%=V`FC_}F9b1#td6!RoP_ea#*?kzO#t(O~hI zJ?;45+ud&Vc$)o~5L|=D*W$5Mm6Z+`3MODXo{zXNJp;Z@_sHZ!ney`4mA7&?_2s8b zGw{z}*T67L8T=3bW-&}Aqpf2K7^aTV)v+?IHl*B-h53QBt#?OEq z{Kb{%@(Vx=?+jBo5A?M;2B)LNq_4=VQkAw^3M+gTS4qR{2Cb!Pj=#>}gNG2)^BUAt}0WN>I@j8#a5~G+bh%xdzIFKu?*P<#=ls1;J z^;;?{6xEgG^)`Jz<_-F7-df;pZ!c=8u2gb1t|FgR6_HQ7rK;3nXaIj*+IsM_-l~j( z=c@2o3mvt#LWiIC)oIbI>uq_}3bqcX(qYpVG*wrX*JHjGSB2M72=OWmiBEe;1M8pD zTv_2Y6{tmA4ebf$(r9&QJ2>tpx2dwNsY;bqZ-Ms1?NOakaIRS>RV`6yLLy zmMU#ll=k$QATH46ifY_X^q{YAQy1%SKMZ0*JY7{8rHz)1BCcJWBrifIac$RO))%x} z^;r&UMoFYTPUs}8uRF`#)(ow~>MC_xRGE?ZHJPhQ{idp-4s)ftCDHxOl3{XKodqsa zMQ(cuH_pj%0CSJcS(L{82kz5=(f1ZA%>=UOM$whTBUA}&ehRk(^s_HoCRK? z-(CUEFkS+2Rm2z9va2DkQMgLQHQ}7aaaF|^*XpYwu2Hy3#Wmra#c|DyFRpc0LtLY9 zm5OV^Ig8_(6<=JNu7$i~aR+I87_-^ zVk|S*J&;oEHC39pJznDW8Ul8^xzg6mx1&THc|S6W+$L51mEzcFs?1WDstl3(q~J@4 zUc%biS@auVX1Kx{PbucX@k@{zBn}m%ih%M}yY_PZH_Qh4IB7DwJTcXR1 zICsXhNnAg6SYR(_%5cixI`@gvcQ~6X-*?)I8<|LRO{BgPu(|SmC;unRI{}+3-*@Kxgn37>xhne3CC@w1 zcQ~7?q3>M!yp!K@)yz9{GBYoE-VtoBioVnI6XqSxCUJeI+z)w$;2J05Ijf{z;^vn^ zE#w;lYZ)&0A#vVslG00DUud#I4l9l|i=^I_X><5i#MR~jy@a*7t+1%xT$xpC(>rj! zFv8xj9M1k7Es#rc<6IQZB_=M8rdEqS;(pKxdC#WE=Q?nv637LT`7rSN(sGxOkA!u8 zhbi)2pUpK|RTYuOhbc*V&t|m6G2{~MNE^4)L>vvTk7lw1pjRqyGEn$6D1i49Zzk~b= zpI5`V(UOMDj^f6WhFUh?UJXfgEkA}zAl04;?|GRzC~IMwAOr6&f^sW^j6LX=z?}fF2CM?V9%Oa+J(rosFvWhj zE1)xIq5gUJU5nrInsKh*i!THe;=2NjAHU}^z^@pp!?e zWyBXC8dWo9IsbV374C=S4feC(K+VDFUcpNkS2lp}P{8@S{G``AmMxY+PQD z6R6MG+xjqWkta}}vX}Lh;!u`a0`=t^UA^*`ZA)!&mJq+Wy{xYU^UKc()Ys74`fz&~ zJc0V!dRre<$COwC^)W_IFXG2={e>yc7UE~@WqoY>=H&$Hv-h?>9KS+OpgvzO>toxu z*b=BOx3~3Se#T(@ijA!y?GZk|vhz`)aF$-`VDC<_GX>+E#?7ZXMxU3%wFf^R5%aqj zK()ddF?Zh0Xe$HuMX~Ig#rEu3JSC88@H9!UTS7 zee4sgB7n5`{!WvUnH99p4zq`a5&VMo`S?*UnVV$2$_m7Tsha_xUeJ!(9$`fB zs-kxGT=$KG~VCb+Rm0nM=<3-#)r4ZM-0ehVx%s%e+DQUzlr_=NHI;JEKhVCV`PkRZY zQw8jevFzFTC*)ooEQ*~0dtDyPgwgP^Yg+NWLF_;!7~%Non1Z6SQbOhzFgP&4&__VAbeakD+c z{6Z!l?>2kxZn67rvAW&2u$Q>U^Ov;8bJw@wKgA6Ho#w0LAK6ED%4A0PKR^8sYFPj9 literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/openboost_cpu/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/openboost_cpu/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ce8c2ce2f2fe432175c1428c58e110652e58c818 GIT binary patch literal 29985 zcmd5_3w)E+*-uM>SXvPzHC!fGhV zKi+o$S;uXB^tgE+TJ*!}zwJ8NgJcp}nrz$*u)9h&Og3zST*_<8rMym>WU7dC)F_EW zGGb~X{5v^e$dIK)l5h?y^^+$r{qYdW*|6cjoT5X>@$OacJlb&t9UeV=&Dwp3&}*wI z?eDkm6XC#X%B8#x=0HlLbGX~oqH!)ihOYio^ZUo_Cs5yt9mBTXcN|&oxw`6KH=ICe zs}tP+9{#xq2fjou0f;9v-h&0H8UtM7jJH3WbAsweJBkfyBe z`}Fk$siJRf=T0o0wC=y&6JfxY$fbM@m;ot`z(6oNR443v;J(AB(U5Y(#>XY6QTvnA z=g+(HE7aKeP5#5{PotIJ%sE%I9Q1)GOaR`U-@UgNB}kLr zo;FmP(znsMFX1XhUtN0_6Pabtek_6yR(-gHi2r`suSv_nkrKQ@`vhz4EjO2VPSy<#jLzQW}9nXit#L`gk12#kJE8i9Zy2y-TXI&Rl= zNq$bxq&J@Z@JICfoi8pPu{$x}sCwa7-{1EG(zM^xrQG|i2nSwMF6DJF2U04^K_Sc3 z!$%i?u;q)uoBZu>GydK-aoh=J2pnfWOk-qn;RhNt`)~>4A!Eb&9VTTkldoq#(1}Bq zB!+*V*SsXz{M$W4^6ycrms~kzNdCb$OmEf>mF9O3`{&YyA0_6e_I&YO*Drn)ahuna zOL-kkfs~56oi9`CWm3lNNq8>2?aqdijJq1k=g{hs+C`r;UH@;1r!X|>{c2poc+Z$} z-iJxUy&n?p{gv(aEc^MC;rZ>B{Ap8P9hSefB>B2k>qq7<@Xw(>vkuR%7~v~Wyq1tp zu8??5xs=zz6iBJ4_aUpa#Y2FJ;Ck`&LzPqw^_))r2A z@}-?GqE=03(yujJM0z4$0xKj=7jqz`(K#5$8w!dxe1x`tJLP)Yb^k&S-Munt_SAz& zF?{4Fh3k%>PgcFVZs)l@A{=-Pxs=zz97t(&4!``ja^m04zJi_^|K@}GWt-47nRRde zxOgkN{o$U}8|H38UsRjM)SY`#gafZBm-0H811XKpVfgKHO`E>kgp4x#wc|~jko)PE z+W)g*GkV3+v3$g?w(ADVi#>HdSig4gHkV7;0rbv#eOG7k z0W{_f-}m3II)J`kyPO&~=K#8EUKLv%Sdi(3_8cb?sjC-?F2xF5cXWj84azgm$r=B(Eu#@;aCUDUHsd@ZI<2ihJKj z54^f#vua-t`Y~a}`;A7}NVQu(!|6>oK;71v~ppOi1QZs#>&2gT`N4x}_Xhs$9n zxh;8)?6Ft7(a5(S?Oos1g%&JcJOA;dUZfmx>;L^`OgDPrvyCgJzWt6!Tkx84DX)X$ zKuV*>Av@oE=VTkSfw6qq_=R0)=bi({R~Po8Gh3GadENhZp}t*nx6oVO7U95a%B8#x z=0HlLbGRqDYvbaIZZyBwH}ZGWx{z+qO%p~Q??tO!e^_qb(uH~tzH9h;@-7h$yrx{r z>tGI~G&+Z~C(_l{-*lt13sV1)lG%j{_sy^#KJ@{*-B)qsmy&KY&b0Mk+7tg2;lOLk zrMwR2KuV)?Sa|rUf+a6^qYXz6|LOJ8E_BzaCqI`xvKPJ6vA#z!yBpnq>xkddO=28) zO}UiU!5m0wbPhY~_dj2GrW=h~ajl}x*M*+?%Z{|4!Tfk;%WL=T`CT`9YWNGMvcKOc z5(i#WF6DJF2T~fH!{u~IS*IS=dEkDRfd@9I^K>fC?eqvE&j8r4~U zXv9-1e;n0GuH|@5xs=zzaUiA9<4~UY%y(D+@#}3vj=y#4I>otdJtGI~G&+Y%wx$j4?H9n#MlxhfN87+)UN=0$yvI%-wBD1GUnSseF6{?)t!ufi@87)f-ZeO#ZPQFYYcw*|j zViLPmO-RNk$>}nRuW?juJoJ?W7<+zJmP9_xYiY1K zO|nH2S@M)b`4Fec#y%x^oL=rR(WByI7LVJ@K3?HyuCMi4nmwG(72XDu$L+Ftx$TR_u1uGe4clm6dR+EFRCm;`MW=7EDSa)D-+xJP-MJjvp5aB-1ZthQQ5GasfAZo99x$y@97nS5*=shq?Ofob7_ zo9xGduXWh!IHeJ+z+Y zIsMq>MJ!X!?t~^zf`OM**X*!LWGV1C+-mZfyf)tyi5{O3GqR!1W~6;#KhH=4r#m zQj%nV7osq8f&gam5Nbst0FikqJadv#n1o^jf*D5uvver+ND{!1sV59WN@IqBaq0F7 zfL08rR^tpHnP0*{q%>xr;A|;Cd*ljgEp8?44{$;9*jy#V@3^nnH}&F+OqQ+swa*0GAy0Lc^M{+886(+AS!tI&#}~AIiQjW z9E?gzV@3^nnH%u(?nG+Wc&cw4>t$HJhVwE^8Z%zFmqAqUvg>Em;R%3BW@s=fDUBI5 z=w%-3<-vrsqY2cOiPZ6HSuew~Gn|)U(wOnWz2s2Y?_cM2)O(Wvl}yNBR8krk)%fXM6$1|p>~0|i#Y0u$-*_i-G=0Y$WDUBIUxC;^Bom5ii6@W+P zJun_AjTtXwk0Ss)n?Wtf06;RyfdNTr%zz?06WBoskk4mP_h$hzncBd}q;z1&83N=b zxzs~BfJ`PdFfu6}7_xBIx&J0=RUROdX$*`^N(Y9VBk=j5o2e&$0mx+X0wa^sfg$G! zkXOy7p1TE*$&>{~CZz*GhCQMH`H6gL0|I1n^MR2`>A;YMefv2zwUq{BGChHjN$J3l zg?)QN0rk2DkjbP3Mkb{LLl*Y!twq%CLO>={5g3`24h&h?w_h)&dbEH{CLk~}DIFMc zuAqnQE}{180GUiXU}RD{Fl6DV>d{k2<^VF8Y{1B*G-l-Bfr}z!Fzqj+PL={7xwpoE zr1U2N3QsPM7^t&z0gy~1U_es(lK_Ps_+&YCejWglIRgwxN`C@i==^kc0kvcS0Fvu6 z3`k021{9o-@tqa6hTl|D_g4ZOZ0KXn5%hoJl*ApNG3ReQ(CGx~vubKZH2{&v|JW~5 z8r8294v0V9W3W~wq&<;9y<(&m8z*E#y`qT$!htlySmEg$Uo(3~$ke+fA#Fzjb#x*1 z#6nzeXexlr43G$SCj=nU))>>0DQVilMR0g0ALcf>tR}DgKQr%0x4W#iwpvTGqs8g+ zrr$Ja;jJ?!rL(WKOqnfA%Dmo@?lL)Tpq>u7hIFu~v)dfj+AtvtQ|oJYGpcotW|J>F z6RW#gobGn90wHZE zkcc5ggA5xu%Oqh#iUt`rn7$%$L(2vkSdbd~HC)bxu0iVBEz75kNu7 zAcG37KajX#g%<~!y=M^-JXk!)@WCq`Bz9=oMFD26jzmHZ6kZf)_Kr#<>|p62gNxpV zAz4IJivyjE*yu27aiAkI>xo6@VobndI61LJhf#||bn;@O!>Cp0MfX=K4=wm8v=X=K6uxj50ujjiu6YH?_+mz$k;hV1)=aov zkdEy*KElG88Sr(wQzB2H%gg6h-Natom!CAw!ask#ou=t|@E`n}L(`eGx}Gkg>3UjI z&&bs3ka9m376!`a_^a7>_%~AvQ?8?FKMi*97gwSwECMmS({ynK=&N&dPDhJTTbWs{ zDr+?tSNhDZQu|!H+FU)~U$1j=GDn`<>@0WKw3QvY#^(AerNXPzlr)-EZAx=h7Awo} z+Re@)pK-n}8_Ik&RTXxfoo~J6) z=rBf$wMyx?Rpu#d)oKUE);Wz{v#Ll@);N!;-&|FxsHrM%uxbl2Z_sb^)&X~0dr3=8 zm6Ek_mG~^GhgXL~?8>`xy zs#RGHW@ta$9yJ*yeoH&U)n8MU1%8$F<9n9UT&>RPr#*c}hzqp2q6YU9E$D08^7I&|kwB}477xXRpS zRc0i9O{VHHzp=Wc!&H^m67T+I&M-PG&LWqwGPk{y9p~gYfVs!&tZc8TvO20ebpAMV zbemn}5JO{wxhSurMwQpzKUYVKRa@C^bryN~etQWx!*~hARS{cU=UonQ?T4#aT;tAJ z7*|znajm%=;@S^avAD*avoNlivBkCia)@g`T*cxVch17NX2llQrpqC&{cshFYuq`9 z#I>4X>>-Qi>wW_A7O$qxXyq6=_ zcrLZuT9oI3{gxlEovhrRF>Y&RmzCN3KwCSEu86VBVD>=DJg>3J$nNptx7XmY+f7y0 zX09DY;>h`tQQ|hL8ZH&bMq^c$!dR_~)F*jgLiFO+*3QZn*zdUDEP-3k=h^XIw7P?C zDUH-Wqua!7OhIg6&t!$YHM=jaR+Zp$hPqNsX2iKOs!ja*vBL~|Ib(*S&I0F57H4H$ zjT6poDs%n%#)#)KQEe`VzEj4YjgVuqpT5J{Tn>FFE7G}7KYfR_x%7RfP2WgInrkBU zoq)}y?>mJ*VcrSYT>8E<|0m2lyv=3NcP@C|fxg4qTn>Hb!sngBmdj?|nV*??!SfDp zb6NDArk^nHur~4QJLP`JD+Jd#5zkpg_2M_b6ssZM5LnBwxet-^exsOP{Q5$Z1#(z1 ztXV|$E=`-mw<4xC59r0M&8@{H4W_EBI;+-!^Mw)ihUIYf?`VNsk{joua4s=^aWuA? zwGsD&PRM&UMLySoGnGItkj#gH-xrp!rBtnXC;uE6!tsFuW)%aoEt5*XLjftOYL<`zP$#L>S}Hb zl|rgL6W$9lHBi<#kf8- z^a8QEl0q_Rz224fc$5bqs3PY{Hk4?SUw74Ej;U9R{;P*19h}JN~u;;io z_+boMVKBpe26&iP0opW;i3YDOKrL9RW%~~rR)z@_PsZ`u4z#!*^l&S3r5(f)uxrK0{Yq1J|e zx62q!O*V(Gp$!YT`U(q}Ac52HV;-3&-0%?3(C-=>j+}r$bn|p*8?4C1Bf38t&L53F zllQ0E(8#q9>kkt|+>y9n5q{K0I-ebD+LL zgKI$kGHt2WXYuh{Fu?jsF~7o`Kz;Us)`#0e=Lyu;HqiQ@I=a*xsE;;y1`t1*?Ju-G zn~$Gifb}u$TaXi|&o$?V{sFW)BOqkoGLc z?|97)_BfXnIKQ6Dbb9q%&K~012%lbN{lb24#rwA~hn!hTtEUI-^(w>0VCXO*6<<#=<3-p$Wf0c| z0eg)u%s%G!DQ(0pr_plu8oIOqhVBKmPkSk?Q3dP`(d?P|C*)ooEQ*~0drbk%g#F=T z*VMv$gV=#8Fv9WE&_yxX!`)ab*GKV!7REnsuM4+lCTPKW5Dz*vlMjUl_&({u67$v3 z9=tvm^at2jz%MvnnXARcx^|#x3gDG3;OzPNUKEeHd&^W;3VQ|)4@d!dSfAJ9JrC2R zn8iM$NAvYDashul3e@=CF;>%Z_VAbav9mo*-y)Gua+^H2x7d6)THNj%nM>T0xJ%lT b*z4Qyzsn5&qs^DeKa!)jN+bsOKS%w4gQp3U literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json b/benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json new file mode 100644 index 0000000..af83cd4 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json @@ -0,0 +1,62 @@ +{ + "artifact_digest": "sha256:0c5176c4a28cd44444f6a086643a01f636b0dcb5e165a4a30d3d992f3e96da97", + "artifact_id": 9256929555, + "dataset": "1027_ESL", + "files": { + "openboost_manifest.json": "sha256:e1e825d99c35c8f41eabbacfad388923ae6b6271267f827814d1644d46b87121", + "raw/ngboost/1027_ESL.parquet": "sha256:d9cd14271f2a6471bd01976f66363550f14eaaf5273694c9de6fe80a70a0c679", + "raw/openboost_cpu/1027_ESL.parquet": "sha256:58c0f434c97b8578a3d283b5d37ecf9f7103cfdbd6cc85246cb4a713722eb3e0" + }, + "folds": 5, + "metrics": { + "coverage_90": { + "ngboost_abs_error": 0.0862823605537415, + "ngboost_mean": 0.8137176394462585, + "openboost_abs_error": 0.04527667760848997, + "openboost_closer_folds": 5, + "openboost_mean": 0.85472332239151 + }, + "crps": { + "ngboost_mean": 0.3077139963362946, + "openboost_fold_wins": 2, + "openboost_mean": 0.3005270428812241, + "openboost_relative_percent": -2.3355952412434267 + }, + "interval_score_90": { + "ngboost_mean": 2.737215196639375, + "openboost_fold_wins": 4, + "openboost_mean": 2.436057677703067, + "openboost_relative_percent": -11.002332564354278 + }, + "log_score": { + "ngboost_mean": 0.67596661214236, + "openboost_fold_wins": 1, + "openboost_mean": 0.7100775390578736, + "openboost_relative_percent": 5.04624434147789 + }, + "pit_ks_stat": { + "ngboost_mean": 0.10549263800016417, + "openboost_fold_wins": 3, + "openboost_mean": 0.08936585621575437, + "openboost_relative_percent": -15.287115850098198 + }, + "rmse": { + "ngboost_mean": 0.5549269313560135, + "openboost_fold_wins": 3, + "openboost_mean": 0.5459231513016759, + "openboost_relative_percent": -1.6225163252279073 + }, + "train_time": { + "ngboost_mean": 3.298694705963135, + "ngboost_over_openboost_speedup": 1.5623446940621515, + "openboost_fold_wins": 5, + "openboost_mean": 2.1113744735717774, + "openboost_relative_percent": -35.99363803643326 + } + }, + "openboost_source_sha": "921c024b23fc4549b251222f35ce64a4b6846505", + "protocol_mode": "official_quality_shard", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4", + "tested_merge_sha": "71dd86b7c95245593030ca488981bc2607f02eb5", + "workflow_run_id": 31922702524 +} diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index b67958a..1f99add 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -157,6 +157,11 @@ Generated ScoringBench directories are gitignored. Publish accepted evidence in ScoringBench's designated output/LFS repository or intentionally force-add a frozen artifact; do not commit an arbitrary local smoke run. +The first frozen official-protocol sentinel is under +`benchmarks/evidence/scoringbench/1027_esl_20260816/`. It preserves the raw +Parquet rows and documents both favorable and unfavorable metrics. Do not +generalize that single-dataset result into a library-level claim. + ## Evidence gate OpenBoost should claim value only after all of the following are true: diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 730bc17..402be2e 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -52,6 +52,15 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. tests: 4 passed. - Linux ScoringBench run #4 confirmed `source_sha`, tested PR merge SHA, clean checkouts, pinned upstream SHA, and 4 smoke rows in artifact `9256692529`. +- Linux ScoringBench run #6 completed the `1027_ESL` official-quality sentinel: + 10 fold/model rows, clean provenance, and artifact verification all passed. + Artifact `9256929555` has digest + `sha256:0c5176c4a28cd44444f6a086643a01f636b0dcb5e165a4a30d3d992f3e96da97`. +- The frozen evidence under + `benchmarks/evidence/scoringbench/1027_esl_20260816/` preserves the manifest, + resolved dataset registry, both raw Parquet files, and a descriptive summary. + OpenBoost's mean CRPS/RMSE/90% interval score and time were better on this + shard, but mean log score was worse and CRPS won only 2/5 folds. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts From e43e31b0438af7ae91493ee68ea7ae459feaafee Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 20:42:31 -0700 Subject: [PATCH 22/49] bench: audit ScoringBench completeness --- .github/workflows/scoringbench.yml | 6 +- benchmarks/scoringbench/README.md | 8 + benchmarks/scoringbench/run.py | 199 +++++++++++++++++- .../2026-08-15-scoringbench-integration.md | 10 + tests/test_scoringbench_provenance.py | 53 +++++ 5 files changed, 272 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index ba29007..eee9627 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -124,17 +124,19 @@ jobs: run: | MANIFEST="${RUNNER_TEMP}/scoringbench-smoke/openboost_manifest.json" test -s "${MANIFEST}" + test -s "${RUNNER_TEMP}/scoringbench-smoke/benchmark_outcome.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-smoke/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4' "${MANIFEST}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4 and m["outcome"]["status"] == "complete"' "${MANIFEST}" - name: Verify quality-shard artifact if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' run: | MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" test -s "${MANIFEST}" + test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == 10' "${MANIFEST}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == 10 and m["expected_result_rows"] == 10 and m["outcome"]["status"] == "complete"' "${MANIFEST}" - name: Upload benchmark artifact if: always() diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 1f99add..3a3d94b 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -91,6 +91,14 @@ quality claim. Manual `quality_shard` mode accepts an exact dataset name so the suite can be sharded without downloading every dataset just to resolve an index. Only a completed full suite belongs in a leaderboard submission. +Every launcher invocation also writes `benchmark_outcome.json`. ScoringBench's +upstream runner deliberately catches dataset/model exceptions so the remaining +campaign can continue; therefore a zero return from upstream is not proof of a +complete shard. OpenBoost audits every expected dataset/model/fold row, rejects +duplicates, captured model errors, missing rows, and non-finite core metrics, +then exits non-zero when that audit is incomplete. The failure report remains +in the uploaded artifact and is evidence, not disposable CI noise. + ## Official quality track Run the official default: five folds, one repeat, at most 3,000 rows per diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index d01e2f1..4f9583f 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -10,6 +10,7 @@ import argparse import importlib.metadata import json +import math import os import platform import subprocess @@ -104,6 +105,178 @@ def _working_directory(path: Path): os.chdir(original) +_REQUIRED_DISTRIBUTIONAL_METRICS = ( + "crps", + "log_score", + "rmse", + "coverage_90", + "interval_score_90", + "train_time", +) + + +def _is_present_finite(value) -> bool: + """Return whether a benchmark value is present and numerically finite.""" + if value is None or isinstance(value, bool): + return False + try: + return math.isfinite(float(value)) + except (TypeError, ValueError): + return False + + +def _is_present_text(value) -> bool: + return value is not None and bool(str(value).strip()) and str(value).lower() != "nan" + + +def _audit_records( + records: list[dict], + datasets: list[dict], + model_names: list[str], + *, + n_folds: int, + n_repeats: int, +) -> dict: + """Audit exact dataset/model/fold coverage after the upstream runner returns. + + ScoringBench intentionally catches dataset and model exceptions so a long + campaign can continue. That behavior is useful for throughput, but its + return code cannot be used as a completeness signal. This audit turns + missing, duplicate, error, and non-finite metric rows into explicit data. + """ + expected_keys = { + (dataset["name"], model_name, fold) + for dataset in datasets + for model_name in model_names + for fold in range(n_folds * n_repeats) + } + rows_by_key: dict[tuple[str, str, int], list[dict]] = {} + unexpected_rows = [] + for row in records: + try: + key = (str(row["dataset"]), str(row["model"]), int(row["fold"])) + except (KeyError, TypeError, ValueError): + unexpected_rows.append( + { + "reason": "invalid_identity", + "dataset": repr(row.get("dataset")), + "model": repr(row.get("model")), + "fold": repr(row.get("fold")), + } + ) + continue + if key not in expected_keys: + unexpected_rows.append( + { + "reason": "unexpected_identity", + "dataset": key[0], + "model": key[1], + "fold": key[2], + } + ) + continue + rows_by_key.setdefault(key, []).append(row) + + missing_rows = [ + {"dataset": dataset, "model": model, "fold": fold} + for dataset, model, fold in sorted(expected_keys - rows_by_key.keys()) + ] + duplicate_rows = [ + { + "dataset": key[0], + "model": key[1], + "fold": key[2], + "count": len(rows), + } + for key, rows in sorted(rows_by_key.items()) + if len(rows) != 1 + ] + error_rows = [] + invalid_metric_rows = [] + valid_keys = set() + for key, rows in rows_by_key.items(): + if len(rows) != 1: + continue + row = rows[0] + error = row.get("error") + if _is_present_text(error): + error_rows.append( + { + "dataset": key[0], + "model": key[1], + "fold": key[2], + "error_type": ( + str(row["error_type"]) + if _is_present_text(row.get("error_type")) + else None + ), + "error": str(error), + } + ) + continue + invalid_metrics = [ + metric + for metric in _REQUIRED_DISTRIBUTIONAL_METRICS + if not _is_present_finite(row.get(metric)) + ] + if invalid_metrics: + invalid_metric_rows.append( + { + "dataset": key[0], + "model": key[1], + "fold": key[2], + "metrics": invalid_metrics, + } + ) + continue + valid_keys.add(key) + + dataset_outcomes = [] + expected_per_dataset = len(model_names) * n_folds * n_repeats + for dataset in datasets: + name = dataset["name"] + observed = sum(key[0] == name for key in rows_by_key) + valid = sum(key[0] == name for key in valid_keys) + dataset_outcomes.append( + { + "dataset": name, + "expected_rows": expected_per_dataset, + "observed_rows": observed, + "valid_rows": valid, + "status": "complete" if valid == expected_per_dataset else "incomplete", + } + ) + + complete = ( + len(valid_keys) == len(expected_keys) + and not missing_rows + and not duplicate_rows + and not error_rows + and not invalid_metric_rows + and not unexpected_rows + ) + return { + "schema_version": 1, + "status": "complete" if complete else "incomplete", + "expected_rows": len(expected_keys), + "observed_rows": len(records), + "valid_rows": len(valid_keys), + "missing_rows": missing_rows, + "duplicate_rows": duplicate_rows, + "error_rows": error_rows, + "invalid_metric_rows": invalid_metric_rows, + "unexpected_rows": unexpected_rows, + "datasets": dataset_outcomes, + } + + +def _write_outcome(output_dir: Path, outcome: dict) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / "benchmark_outcome.json" + path.write_text(json.dumps(outcome, indent=2, sort_keys=True) + "\n") + return path + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Run OpenBoost on the external ScoringBench protocol" @@ -269,6 +442,7 @@ def _write_provenance( args, datasets: list[dict], result_rows: int, + outcome: dict, ) -> Path: import openboost as ob @@ -290,7 +464,7 @@ def _write_provenance( protocol_mode = "scoringbench_protocol_deviation" manifest = { - "schema_version": 1, + "schema_version": 2, "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "protocol": "ScoringBench", "protocol_mode": protocol_mode, @@ -313,6 +487,17 @@ def _write_provenance( for dataset in datasets ], "result_rows": result_rows, + "expected_result_rows": outcome["expected_rows"], + "outcome": { + "status": outcome["status"], + "valid_rows": outcome["valid_rows"], + "missing_rows": len(outcome["missing_rows"]), + "duplicate_rows": len(outcome["duplicate_rows"]), + "error_rows": len(outcome["error_rows"]), + "invalid_metric_rows": len(outcome["invalid_metric_rows"]), + "unexpected_rows": len(outcome["unexpected_rows"]), + "file": "benchmark_outcome.json", + }, "platform": { "python": platform.python_version(), "system": platform.system(), @@ -418,15 +603,25 @@ def main() -> int: seed=args.seed, sample_size=args.sample_size, ) + outcome = _audit_records( + result.to_dict(orient="records"), + datasets, + list(model_factories), + n_folds=args.n_folds, + n_repeats=args.n_repeats, + ) + outcome_path = _write_outcome(output_dir, outcome) manifest = _write_provenance( output_dir, scoringbench_dir, args, datasets, result_rows=len(result), + outcome=outcome, ) + print(f"OpenBoost outcome: {outcome_path} ({outcome['status']})") print(f"OpenBoost provenance: {manifest}") - return 0 + return 0 if outcome["status"] == "complete" else 1 if __name__ == "__main__": diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 402be2e..c960d11 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -31,6 +31,9 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - `.gitignore`: ignore arbitrary local ScoringBench result directories. - `.github/workflows/scoringbench.yml`: pinned Linux contract/smoke validation, artifact upload, and a manually dispatched official-quality shard. +- `benchmark_outcome.json`: an exact dataset/model/fold completeness audit that + makes upstream-captured failures and non-finite distributional metrics + machine-readable and changes the launcher exit status to failure. ## Verification @@ -39,6 +42,8 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - Fresh isolated-environment contract after adding xdist: 1 passed on Intel macOS with Numba 0.63.1; this validates the adapter only, not benchmark scores. - OpenBoost distributional regression tests: 47 passed. +- ScoringBench provenance/outcome tests: 6 passed after adding exact-completion + and mixed missing/error/non-finite cases. - `ruff check benchmarks/scoringbench`: passed. - Python compilation and manifest protocol classification: passed. - GitHub workflow YAML parsed locally; the pinned wrapper contract passed before @@ -100,6 +105,11 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. `1027_ESL` PMLB/GitHub source so this small gate does not depend on the OpenML data endpoint. Full-suite runs still must report OpenML failures rather than silently treating them as model results. +- ScoringBench's outer runner catches dataset exceptions and its fold runner + converts model exceptions into rows; neither condition necessarily produces + a failing process. The OpenBoost launcher now audits the returned records + after the entire shard finishes, preserves all omissions/errors, and only + then exits non-zero for an incomplete outcome. ## Risks and Follow-ups diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index f85910b..cb91cbe 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -4,6 +4,7 @@ import pytest from benchmarks.scoringbench.run import ( + _audit_records, _ci_state, _validate_selected_datasets, _working_directory, @@ -76,3 +77,55 @@ def validate(datasets): assert result == [registry[0]] assert validated == [registry[0]] + + +def _complete_record(dataset="example", model="openboost_cpu", fold=0): + return { + "dataset": dataset, + "model": model, + "fold": fold, + "crps": 0.2, + "log_score": 0.4, + "rmse": 0.5, + "coverage_90": 0.9, + "interval_score_90": 1.2, + "train_time": 2.0, + } + + +def test_outcome_audit_accepts_exact_complete_distributional_rows(): + outcome = _audit_records( + [_complete_record(fold=0), _complete_record(fold=1)], + [{"name": "example"}], + ["openboost_cpu"], + n_folds=2, + n_repeats=1, + ) + + assert outcome["status"] == "complete" + assert outcome["expected_rows"] == 2 + assert outcome["valid_rows"] == 2 + + +def test_outcome_audit_publishes_missing_error_and_invalid_metric_rows(): + invalid = _complete_record(model="ngboost", fold=0) + invalid["log_score"] = float("nan") + error = _complete_record(fold=1) + error.update(error="model exploded", error_type="RuntimeError") + + outcome = _audit_records( + [_complete_record(fold=0), error, invalid], + [{"name": "example"}], + ["openboost_cpu", "ngboost"], + n_folds=2, + n_repeats=1, + ) + + assert outcome["status"] == "incomplete" + assert outcome["expected_rows"] == 4 + assert outcome["valid_rows"] == 1 + assert outcome["missing_rows"] == [ + {"dataset": "example", "model": "ngboost", "fold": 1} + ] + assert outcome["error_rows"][0]["error"] == "model exploded" + assert outcome["invalid_metric_rows"][0]["metrics"] == ["log_score"] From 47a08cde539f916332a735e74c829bc8e2d936b3 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 20:44:29 -0700 Subject: [PATCH 23/49] bench: shard frozen ScoringBench registry --- benchmarks/scoringbench/README.md | 20 ++++ benchmarks/scoringbench/run.py | 91 ++++++++++++++++++- .../2026-08-15-scoringbench-integration.md | 9 +- tests/test_scoringbench_provenance.py | 31 +++++++ 4 files changed, 145 insertions(+), 6 deletions(-) diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 3a3d94b..e367e48 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -119,6 +119,26 @@ Use `--dataset-index N` or `--dataset-name NAME` for resumable shards. Use test folds. If hyperparameters are changed, apply the same declared search budget to every comparison model. +For a parallel campaign, freeze the resolved upstream registry and partition +that exact ordered list into stable strided shards. The first sentinel's +registry is committed as evidence and can seed the first full campaign: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --dataset-registry \ + benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json \ + --shard-index 0 \ + --shard-count 10 \ + --output-dir benchmarks/results/scoringbench-quality/shard-0 +``` + +Run every index from zero through `shard-count - 1`. The union covers each +registry entry exactly once. Both the source-registry and copied artifact +hashes are recorded, so a campaign cannot silently mix changing OpenML suite +membership across jobs. + After all shards complete, run ScoringBench's own aggregation and autoranking: ```bash diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 4f9583f..94ddeb2 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import hashlib import importlib.metadata import json import math @@ -29,6 +30,29 @@ def _csv(value: str) -> list[str]: return [item.strip() for item in value.split(",") if item.strip()] +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_dataset_registry(path: Path) -> list[dict]: + """Load and minimally validate a frozen ScoringBench dataset registry.""" + payload = json.loads(path.read_text()) + datasets = payload.get("datasets") if isinstance(payload, dict) else payload + if not isinstance(datasets, list) or not datasets: + raise ValueError(f"dataset registry must contain a non-empty list: {path}") + if any(not isinstance(dataset, dict) or not dataset.get("name") for dataset in datasets): + raise ValueError(f"every dataset registry entry must be an object with a name: {path}") + names = [dataset["name"].casefold() for dataset in datasets] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"duplicate case-insensitive dataset names in {path}: {duplicates}") + return datasets + + def _git_state(path: Path) -> dict: def run(*args: str) -> str | None: try: @@ -325,6 +349,23 @@ def _build_parser() -> argparse.ArgumentParser: action="append", help="Run exact case-insensitive dataset name from the validated list (repeatable)", ) + selection.add_argument( + "--shard-index", + type=int, + help="Run one zero-based strided shard from the dataset registry", + ) + parser.add_argument( + "--shard-count", + type=int, + help="Total number of stable registry shards; requires --shard-index", + ) + parser.add_argument( + "--dataset-registry", + help=( + "Frozen ScoringBench datasets.json to use instead of rebuilding the " + "dynamic upstream registry" + ), + ) parser.add_argument( "--lite", action="store_true", @@ -359,12 +400,34 @@ def _select_datasets(all_datasets: list[dict], args) -> list[dict]: raise ValueError(f"unknown dataset names: {missing}; use --list-datasets") return [lookup[name.casefold()] for name in args.dataset_name] + if args.shard_index is not None: + if args.shard_count is None or args.shard_count <= 0: + raise ValueError("--shard-count must be a positive integer with --shard-index") + if args.shard_index < 0 or args.shard_index >= args.shard_count: + raise ValueError( + f"--shard-index must be in 0..{args.shard_count - 1}, got {args.shard_index}" + ) + selected = [ + dataset + for position, dataset in enumerate(all_datasets) + if position % args.shard_count == args.shard_index + ] + if not selected: + raise ValueError( + f"shard {args.shard_index}/{args.shard_count} selects no datasets " + f"from a registry of size {len(all_datasets)}" + ) + return selected + + if args.shard_count is not None: + raise ValueError("--shard-count requires --shard-index") + return all_datasets def _validate_selected_datasets(all_datasets: list[dict], args, validate) -> list[dict]: """Validate only named shards; indexed shards retain validated-list semantics.""" - if args.dataset_name: + if args.dataset_name or args.shard_index is not None: return validate(_select_datasets(all_datasets, args)) return _select_datasets(validate(all_datasets), args) @@ -456,13 +519,16 @@ def _write_provenance( protocol_mode = "smoke" elif args.sample_size != 3000: protocol_mode = "scoringbench_scale_extension" - elif official_shape and (args.dataset_index or args.dataset_name): + elif official_shape and ( + args.dataset_index or args.dataset_name or args.shard_index is not None + ): protocol_mode = "official_quality_shard" elif official_shape: protocol_mode = "official_quality" else: protocol_mode = "scoringbench_protocol_deviation" + registry_path = output_dir / "datasets.json" manifest = { "schema_version": 2, "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), @@ -482,10 +548,20 @@ def _write_provenance( { "name": dataset["name"], "source": dataset.get("source", "openml"), - "id": dataset.get("id", dataset.get("loader")), + "id": dataset.get("id", dataset.get("url", dataset.get("loader"))), } for dataset in datasets ], + "dataset_registry": { + "mode": "frozen_file" if args.dataset_registry else "scoringbench_dynamic", + "resolved_sha256": _sha256(registry_path) if registry_path.exists() else None, + "source_sha256": ( + _sha256(Path(args.dataset_registry).expanduser().resolve()) + if args.dataset_registry + else None + ), + "file": "datasets.json" if registry_path.exists() else None, + }, "result_rows": result_rows, "expected_result_rows": outcome["expected_rows"], "outcome": { @@ -578,7 +654,14 @@ def main() -> int: # that reproducibility artifact with the benchmark instead of dirtying # the OpenBoost checkout. with _working_directory(output_dir): - all_datasets = get_DATASETS_CONFIG() + if args.dataset_registry: + registry_path = Path(args.dataset_registry).expanduser().resolve() + all_datasets = _load_dataset_registry(registry_path) + Path("datasets.json").write_text( + json.dumps(all_datasets, indent=2, ensure_ascii=False) + "\n" + ) + else: + all_datasets = get_DATASETS_CONFIG() if args.list_datasets: datasets = validate_datasets(all_datasets) for index, dataset in enumerate(datasets): diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index c960d11..5e3edc2 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -34,6 +34,10 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - `benchmark_outcome.json`: an exact dataset/model/fold completeness audit that makes upstream-captured failures and non-finite distributional metrics machine-readable and changes the launcher exit status to failure. +- Frozen-registry sharding: `--dataset-registry`, `--shard-index`, and + `--shard-count` split one ordered registry across workers without rebuilding + a potentially changing OpenML suite in each job; source and copied-registry + hashes are recorded in schema-v2 manifests. ## Verification @@ -42,8 +46,9 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - Fresh isolated-environment contract after adding xdist: 1 passed on Intel macOS with Numba 0.63.1; this validates the adapter only, not benchmark scores. - OpenBoost distributional regression tests: 47 passed. -- ScoringBench provenance/outcome tests: 6 passed after adding exact-completion - and mixed missing/error/non-finite cases. +- ScoringBench provenance/outcome/sharding tests: 8 passed after adding frozen + registry loading, exact-completion and mixed missing/error/non-finite cases, + and proof that strided shards cover each entry exactly once. - `ruff check benchmarks/scoringbench`: passed. - Python compilation and manifest protocol classification: passed. - GitHub workflow YAML parsed locally; the pinned wrapper contract passed before diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index cb91cbe..4808012 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -6,6 +6,8 @@ from benchmarks.scoringbench.run import ( _audit_records, _ci_state, + _load_dataset_registry, + _select_datasets, _validate_selected_datasets, _working_directory, ) @@ -129,3 +131,32 @@ def test_outcome_audit_publishes_missing_error_and_invalid_metric_rows(): ] assert outcome["error_rows"][0]["error"] == "model exploded" assert outcome["invalid_metric_rows"][0]["metrics"] == ["log_score"] + + +def test_load_dataset_registry_accepts_frozen_scoringbench_list(tmp_path): + path = tmp_path / "datasets.json" + path.write_text('[{"name": "alpha", "source": "pmlb", "url": "https://example"}]') + + assert _load_dataset_registry(path) == [ + {"name": "alpha", "source": "pmlb", "url": "https://example"} + ] + + +def test_stable_strided_shards_cover_registry_exactly_once(): + registry = [{"name": f"dataset_{index}"} for index in range(7)] + + class Args: + dataset_index = None + dataset_name = None + shard_count = 3 + shard_index = 0 + + selected = [] + for shard_index in range(Args.shard_count): + Args.shard_index = shard_index + selected.extend(_select_datasets(registry, Args())) + + assert sorted(dataset["name"] for dataset in selected) == sorted( + dataset["name"] for dataset in registry + ) + assert len(selected) == len({dataset["name"] for dataset in selected}) From 900cecf6e1a33f0af20153bb0a05a76b345f1ae6 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 20:51:06 -0700 Subject: [PATCH 24/49] bench: compare strong boosting baselines --- .github/workflows/scoringbench.yml | 42 ++++++++++++++-- benchmarks/scoringbench/README.md | 49 +++++++++++++++++++ .../requirements-strong-baselines.txt | 5 ++ benchmarks/scoringbench/run.py | 45 ++++++++++++++--- .../2026-08-15-scoringbench-integration.md | 13 +++++ tests/test_scoringbench_provenance.py | 20 ++++++++ 6 files changed, 163 insertions(+), 11 deletions(-) create mode 100644 benchmarks/scoringbench/requirements-strong-baselines.txt diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index eee9627..d08d17d 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -18,10 +18,11 @@ on: options: - smoke - quality_shard + - strong_shard dataset_name: - description: Exact ScoringBench dataset name for quality_shard + description: Exact ScoringBench dataset name for a quality/strong shard required: true - default: Abalone + default: 1027_ESL type: string n_trees: description: Boosting rounds for quality_shard @@ -52,7 +53,9 @@ jobs: with: python-version: "3.12" cache: pip - cache-dependency-path: benchmarks/scoringbench/requirements.txt + cache-dependency-path: | + benchmarks/scoringbench/requirements.txt + benchmarks/scoringbench/requirements-strong-baselines.txt - name: Check out pinned ScoringBench run: | @@ -65,6 +68,11 @@ jobs: python -m pip install -r benchmarks/scoringbench/requirements.txt python -m pip install -e . + - name: Install strong comparison models + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'strong_shard' + run: | + python -m pip install -r benchmarks/scoringbench/requirements-strong-baselines.txt + - name: Validate wrapper contract env: PYTHONPATH: .repos/ScoringBench @@ -119,6 +127,30 @@ jobs: --n-trees "${N_TREES}" \ --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Run strong-baseline diagnostic shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'strong_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + N_TREES: ${{ inputs.n_trees }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + if [[ ! "${N_TREES}" =~ ^[1-9][0-9]*$ ]]; then + echo "n_trees must be a positive integer" >&2 + exit 2 + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost,xgboost_quantile,xgblss,catboost_quantile \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees "${N_TREES}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Verify smoke artifact if: github.event_name == 'pull_request' || inputs.mode == 'smoke' run: | @@ -129,14 +161,14 @@ jobs: python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4 and m["outcome"]["status"] == "complete"' "${MANIFEST}" - name: Verify quality-shard artifact - if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' + if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' run: | MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" test -s "${MANIFEST}" test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == 10 and m["expected_result_rows"] == 10 and m["outcome"]["status"] == "complete"' "${MANIFEST}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); expected=25 if sys.argv[2] == "strong_shard" else 10; assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete"' "${MANIFEST}" "${{ inputs.mode }}" - name: Upload benchmark artifact if: always() diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index e367e48..2c1070d 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -15,6 +15,35 @@ different questions with two deliberately separate protocols: row cap. This measures OpenBoost's CPU/CUDA scaling but must not be presented as an official ScoringBench leaderboard result. +## What counts as a good model + +NGBoost is a canonical natural-gradient reference, not the product bar. The +quality target is the strongest practical boosting alternative in the same +ScoringBench protocol. The acceptance comparison set is: + +- ScoringBench's XGBoost multi-quantile wrapper (`xgboost_quantile`); +- Gaussian XGBoostLSS (`xgblss`), which is the closest full-distribution + XGBoost-family competitor; +- CatBoost MultiQuantile (`catboost_quantile`); +- NGBoost as the method/reference baseline. + +OpenBoost is a **good ScoringBench model** only when the completed full suite +places it first or statistically tied for first on primary proper scores, it +beats the strongest XGBoost-family baseline on a majority of paired datasets, +and the result is not purchased with a material regression in log score, +interval score, calibration, point RMSE, or failure coverage. CRPS is the first +optimization target; log score, interval score/coverage, RMSE, failures, and +training time remain explicit guardrails. A single shard decides what to debug, +not whether this target has been achieved. + +The diagnostic deliberately uses the budgets registered by ScoringBench +rather than forcing every implementation to share one arbitrary tree count: +100 rounds/50 quantiles for XGBoost quantile, 100 Gaussian rounds for +XGBoostLSS, 1,000 iterations/99 quantiles for CatBoost, and 500 rounds for +OpenBoost/NGBoost. These choices and resolved package versions are recorded in +the manifest. Its timing is descriptive; a later speed claim requires a +quality-matched compute sweep. + ## Environment Use a separate Linux environment because ScoringBench currently constrains @@ -60,6 +89,13 @@ Optional comparison models: uv pip install --python .venv-scoringbench/bin/python xgboostlss catboost ``` +For the frozen strong-baseline environment used by CI diagnostics: + +```bash +uv pip install --python .venv-scoringbench/bin/python \ + -r benchmarks/scoringbench/requirements-strong-baselines.txt +``` + ## Validate the adapter This uses one existing sklearn dataset and the complete ScoringBench metrics, @@ -114,6 +150,19 @@ dataset. Start with OpenBoost and the existing NGBoost wrapper: --output-dir benchmarks/results/scoringbench-quality ``` +Run one strong-baseline diagnostic before changing model behavior: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost,xgboost_quantile,xgblss,catboost_quantile \ + --dataset-name 1027_ESL \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --output-dir benchmarks/results/scoringbench-strong-diagnostic +``` + Use `--dataset-index N` or `--dataset-name NAME` for resumable shards. Use `--list-datasets` to display the validated list. Do not tune OpenBoost on the test folds. If hyperparameters are changed, apply the same declared search diff --git a/benchmarks/scoringbench/requirements-strong-baselines.txt b/benchmarks/scoringbench/requirements-strong-baselines.txt new file mode 100644 index 0000000..f29da26 --- /dev/null +++ b/benchmarks/scoringbench/requirements-strong-baselines.txt @@ -0,0 +1,5 @@ +# Strong practical baselines for the manually dispatched diagnostic/full suite. +# Keep these exact so a campaign cannot silently change model implementations. +xgboost==3.3.0 +xgboostlss==0.6.1 +catboost==1.2.10 diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 94ddeb2..63b3aa4 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -321,13 +321,37 @@ def _build_parser() -> argparse.ArgumentParser: default=["openboost_cpu", "ngboost"], help=( "Comma-separated models: openboost_cpu, openboost_cuda, ngboost, " - "xgblss, catboost_quantile" + "xgboost_quantile, xgblss, catboost_quantile" ), ) parser.add_argument("--n-trees", type=int, default=500) parser.add_argument("--learning-rate", type=float, default=0.01) parser.add_argument("--max-depth", type=int, default=3) parser.add_argument("--n-quantiles", type=int, default=99) + parser.add_argument( + "--xgboost-rounds", + type=int, + default=100, + help="Boosting rounds for the ScoringBench XGBoost quantile baseline", + ) + parser.add_argument( + "--xgboost-quantiles", + type=int, + default=50, + help="Quantile outputs for the ScoringBench XGBoost quantile baseline", + ) + parser.add_argument( + "--xgblss-rounds", + type=int, + default=100, + help="Boosting rounds for the ScoringBench Gaussian XGBoostLSS baseline", + ) + parser.add_argument( + "--catboost-rounds", + type=int, + default=1000, + help="Iterations for the ScoringBench CatBoost MultiQuantile baseline", + ) parser.add_argument("--seed", type=int, default=42) parser.add_argument("--n-folds", type=int, default=5) parser.add_argument("--n-repeats", type=int, default=1) @@ -461,14 +485,23 @@ def openboost(backend: str): ngb_params={"random_state": args.seed}, ) + if "xgboost_quantile" in args.models: + from scoringbench.wrappers.xgb_vector import XGBQuantileVectorWrapper + + factories["xgboost_quantile"] = lambda: XGBQuantileVectorWrapper( + n_bins=args.xgboost_quantiles, + num_boost_round=args.xgboost_rounds, + xgb_params={"device": "cpu", "seed": args.seed, "nthread": 2}, + ) + if "xgblss" in args.models: from scoringbench.wrappers.xgblss_wrapper import XGBLSSWrapper factories["xgblss"] = lambda: XGBLSSWrapper( n_quantiles=args.n_quantiles, - num_boost_round=args.n_trees, + num_boost_round=args.xgblss_rounds, distribution="Gaussian", - xgblss_params={"max_depth": args.max_depth, "eta": args.learning_rate}, + xgblss_params={"device": "cpu", "seed": args.seed, "nthread": 2}, ) if "catboost_quantile" in args.models: @@ -476,11 +509,10 @@ def openboost(backend: str): factories["catboost_quantile"] = lambda: CatBoostQuantileWrapper( n_quantiles=args.n_quantiles, - iterations=args.n_trees, + iterations=args.catboost_rounds, catboost_params={ - "depth": args.max_depth, - "learning_rate": args.learning_rate, "random_seed": args.seed, + "thread_count": 2, }, ) @@ -491,6 +523,7 @@ def openboost(backend: str): "openboost_cpu", "openboost_cuda", "ngboost", + "xgboost_quantile", "xgblss", "catboost_quantile", ] diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 5e3edc2..e957090 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -38,6 +38,10 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. `--shard-count` split one ordered registry across workers without rebuilding a potentially changing OpenML suite in each job; source and copied-registry hashes are recorded in schema-v2 manifests. +- Strong-baseline mode adds the ScoringBench native XGBoost quantile, Gaussian + XGBoostLSS, and CatBoost MultiQuantile wrappers with frozen package versions + and their registered model-specific budgets. This makes NGBoost a reference, + not the acceptance bar. ## Verification @@ -49,6 +53,9 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - ScoringBench provenance/outcome/sharding tests: 8 passed after adding frozen registry loading, exact-completion and mixed missing/error/non-finite cases, and proof that strided shards cover each entry exactly once. +- Strong-baseline parser/provenance suite: 9 passed. The frozen Linux target + dependency contract resolved 87 packages including NumPy 2.2.6, Pandas + 2.2.3, Torch 2.9.1, XGBoost 3.3.0, XGBoostLSS 0.6.1, and CatBoost 1.2.10. - `ruff check benchmarks/scoringbench`: passed. - Python compilation and manifest protocol classification: passed. - GitHub workflow YAML parsed locally; the pinned wrapper contract passed before @@ -75,6 +82,12 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. ## Failed Attempts +- A first local dependency resolution appeared to make XGBoostLSS 0.6.1 select + Torch 2.2.2 beside NumPy 2.2.6. PyPI metadata disproved the suspected hard + pin: XGBoostLSS declares `torch>=2.1,<2.10`. The old selection came from + resolving for Intel macOS, where 2.2.2 is the final available Torch wheel; + benchmark dependencies must be resolved for the Linux target platform. + - The composer-swarm Cursor scout repeatedly failed with macOS Keychain error `SecItemCopyMatching failed -50`. Use local inspection until its CLI authentication is repaired; do not repeatedly retry it during one task. diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index 4808012..fb95ec9 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -5,6 +5,7 @@ import pytest from benchmarks.scoringbench.run import ( _audit_records, + _build_parser, _ci_state, _load_dataset_registry, _select_datasets, @@ -160,3 +161,22 @@ class Args: dataset["name"] for dataset in registry ) assert len(selected) == len({dataset["name"] for dataset in selected}) + + +def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): + args = _build_parser().parse_args( + ["--models", "openboost_cpu,ngboost,xgboost_quantile,xgblss,catboost_quantile"] + ) + + assert args.models == [ + "openboost_cpu", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile", + ] + assert args.n_trees == 500 + assert args.xgboost_rounds == 100 + assert args.xgboost_quantiles == 50 + assert args.xgblss_rounds == 100 + assert args.catboost_rounds == 1000 From cea891a8cb964c655c30092ab0a223f4f140c03b Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:02:59 -0700 Subject: [PATCH 25/49] bench: keep strong baseline checkout clean --- .github/workflows/scoringbench.yml | 2 +- benchmarks/scoringbench/run.py | 79 ++++++++++++------- .../2026-08-15-scoringbench-integration.md | 5 ++ tests/test_scoringbench_provenance.py | 7 ++ 4 files changed, 62 insertions(+), 31 deletions(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index d08d17d..4fbe9c9 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -168,7 +168,7 @@ jobs: test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); expected=25 if sys.argv[2] == "strong_shard" else 10; assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete"' "${MANIFEST}" "${{ inputs.mode }}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); expected=25 if sys.argv[2] == "strong_shard" else 10; assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False' "${MANIFEST}" "${{ inputs.mode }}" - name: Upload benchmark artifact if: always() diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 63b3aa4..1e47b90 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -70,6 +70,7 @@ def run(*args: str) -> str | None: return { "commit": run("rev-parse", "HEAD"), "dirty": bool(status) if status is not None else None, + "changes": status.splitlines() if status else [], } @@ -456,64 +457,79 @@ def _validate_selected_datasets(all_datasets: list[dict], args, validate) -> lis return _select_datasets(validate(all_datasets), args) -def _model_factories(args): - from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper - - common = { +def _model_parameters(args) -> dict[str, dict]: + """Return the exact benchmark constructor parameters for every model.""" + openboost_common = { "n_trees": args.n_trees, "learning_rate": args.learning_rate, "max_depth": args.max_depth, "n_quantiles": args.n_quantiles, } + return { + "openboost_cpu": {"backend": "cpu", **openboost_common}, + "openboost_cuda": {"backend": "cuda", **openboost_common}, + "ngboost": { + "dist": "normal", + "n_estimators": args.n_trees, + "learning_rate": args.learning_rate, + "n_quantiles": args.n_quantiles, + "ngb_params": {"random_state": args.seed}, + }, + "xgboost_quantile": { + "n_bins": args.xgboost_quantiles, + "num_boost_round": args.xgboost_rounds, + "xgb_params": {"device": "cpu", "seed": args.seed, "nthread": 2}, + }, + "xgblss": { + "n_quantiles": args.n_quantiles, + "num_boost_round": args.xgblss_rounds, + "distribution": "Gaussian", + "xgblss_params": {"device": "cpu", "seed": args.seed, "nthread": 2}, + }, + "catboost_quantile": { + "n_quantiles": args.n_quantiles, + "iterations": args.catboost_rounds, + "catboost_params": { + "allow_writing_files": False, + "random_seed": args.seed, + "thread_count": 2, + }, + }, + } + + +def _model_factories(args): + from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper - def openboost(backend: str): - return lambda: OpenBoostWrapper(backend=backend, **common) + parameters = _model_parameters(args) factories = { - "openboost_cpu": openboost("cpu"), - "openboost_cuda": openboost("cuda"), + "openboost_cpu": lambda: OpenBoostWrapper(**parameters["openboost_cpu"]), + "openboost_cuda": lambda: OpenBoostWrapper(**parameters["openboost_cuda"]), } if "ngboost" in args.models: from scoringbench.wrappers.ngboost_wrapper import NGBoostWrapper - factories["ngboost"] = lambda: NGBoostWrapper( - dist="normal", - n_estimators=args.n_trees, - learning_rate=args.learning_rate, - n_quantiles=args.n_quantiles, - ngb_params={"random_state": args.seed}, - ) + factories["ngboost"] = lambda: NGBoostWrapper(**parameters["ngboost"]) if "xgboost_quantile" in args.models: from scoringbench.wrappers.xgb_vector import XGBQuantileVectorWrapper factories["xgboost_quantile"] = lambda: XGBQuantileVectorWrapper( - n_bins=args.xgboost_quantiles, - num_boost_round=args.xgboost_rounds, - xgb_params={"device": "cpu", "seed": args.seed, "nthread": 2}, + **parameters["xgboost_quantile"] ) if "xgblss" in args.models: from scoringbench.wrappers.xgblss_wrapper import XGBLSSWrapper - factories["xgblss"] = lambda: XGBLSSWrapper( - n_quantiles=args.n_quantiles, - num_boost_round=args.xgblss_rounds, - distribution="Gaussian", - xgblss_params={"device": "cpu", "seed": args.seed, "nthread": 2}, - ) + factories["xgblss"] = lambda: XGBLSSWrapper(**parameters["xgblss"]) if "catboost_quantile" in args.models: from scoringbench.wrappers.catboost_wrapper import CatBoostQuantileWrapper factories["catboost_quantile"] = lambda: CatBoostQuantileWrapper( - n_quantiles=args.n_quantiles, - iterations=args.catboost_rounds, - catboost_params={ - "random_seed": args.seed, - "thread_count": 2, - }, + **parameters["catboost_quantile"] ) valid = set(factories) @@ -577,6 +593,9 @@ def _write_provenance( "scoringbench_git": _git_state(scoringbench_dir), "ci": _ci_state(), "arguments": vars(args), + "model_parameters": { + name: _model_parameters(args)[name] for name in args.models + }, "datasets": [ { "name": dataset["name"], diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index e957090..4831b70 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -87,6 +87,11 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. pin: XGBoostLSS declares `torch>=2.1,<2.10`. The old selection came from resolving for Intel macOS, where 2.2.2 is the final available Torch wheel; benchmark dependencies must be resolved for the Linux target platform. +- Strong-baseline run `31925230473` completed all 25 expected rows, but its + OpenBoost checkout was dirty after training. CatBoost writes `catboost_info/` + in the process working directory unless configured otherwise. The factory + now sets `allow_writing_files=False`, manifests include porcelain change + paths, and the artifact gate rejects a dirty OpenBoost checkout. - The composer-swarm Cursor scout repeatedly failed with macOS Keychain error `SecItemCopyMatching failed -50`. Use local inspection until its CLI diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index fb95ec9..dc3f143 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -8,6 +8,7 @@ _build_parser, _ci_state, _load_dataset_registry, + _model_parameters, _select_datasets, _validate_selected_datasets, _working_directory, @@ -180,3 +181,9 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): assert args.xgboost_quantiles == 50 assert args.xgblss_rounds == 100 assert args.catboost_rounds == 1000 + + parameters = _model_parameters(args) + assert parameters["xgboost_quantile"]["num_boost_round"] == 100 + assert parameters["xgblss"]["num_boost_round"] == 100 + assert parameters["catboost_quantile"]["iterations"] == 1000 + assert parameters["catboost_quantile"]["catboost_params"]["allow_writing_files"] is False From 24e9873a0eff442a33ab0f970624bb8f9c4a6f36 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:16:25 -0700 Subject: [PATCH 26/49] bench: freeze strong ScoringBench diagnostic --- .../1027_esl_strong_20260816/README.md | 43 ++ .../benchmark_outcome.json | 21 + .../1027_esl_strong_20260816/datasets.json | 482 ++++++++++++++++++ .../openboost_manifest.json | 158 ++++++ .../raw/catboost_quantile/1027_ESL.parquet | Bin 0 -> 30017 bytes .../raw/ngboost/1027_ESL.parquet | Bin 0 -> 29953 bytes .../raw/openboost_cpu/1027_ESL.parquet | Bin 0 -> 29984 bytes .../raw/xgblss/1027_ESL.parquet | Bin 0 -> 29946 bytes .../raw/xgboost_quantile/1027_ESL.parquet | Bin 0 -> 30010 bytes .../1027_esl_strong_20260816/summary.json | 123 +++++ .../2026-08-15-scoringbench-integration.md | 29 +- 11 files changed, 854 insertions(+), 2 deletions(-) create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/catboost_quantile/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/ngboost/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/openboost_cpu/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgblss/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgboost_quantile/1027_ESL.parquet create mode 100644 benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md new file mode 100644 index 0000000..4a6e962 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md @@ -0,0 +1,43 @@ +# 1027_ESL strong-baseline diagnostic + +This is a clean, official-protocol-compatible five-fold ScoringBench shard. It +compares OpenBoost CPU against NGBoost, native XGBoost multi-quantile, +Gaussian XGBoostLSS, and CatBoost MultiQuantile. All 25 expected +dataset/model/fold rows completed; the OpenBoost and ScoringBench checkouts were +clean. The exact model constructors, package versions, platform, source SHAs, +and CI identity are in `openboost_manifest.json`. + +Lower is better for every displayed score except raw coverage, whose target is +0.90. `Coverage error` is the fold-level mean of `abs(coverage_90 - 0.90)`. + +| Model | CRPS | Log score | CRLS | RMSE | 90% coverage | Coverage error | 90% interval score | PIT KS | Train seconds | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| OpenBoost CPU | **0.3005** | 0.7101 | 0.9305 | 0.5459 | **0.8547** | **0.0510** | **2.4361** | **0.0894** | 1.3845 | +| XGBoostLSS | 0.3043 | 0.6375 | 0.8546 | **0.5384** | 0.7952 | 0.1048 | 2.8794 | 0.0991 | **0.1519** | +| NGBoost | 0.3077 | 0.6762 | 0.8920 | 0.5550 | 0.8137 | 0.0863 | 2.7371 | 0.1055 | 2.4431 | +| CatBoost quantile | 0.3249 | -1.3190 | 0.5825 | 0.5901 | 0.7172 | 0.1828 | 4.0657 | 0.1466 | 2.4481 | +| XGBoost quantile | 0.3257 | **-2.2201** | **0.5757** | 0.6178 | 0.6127 | 0.2873 | 3.9807 | 0.2204 | 0.3869 | + +On this shard, OpenBoost has the best mean CRPS, interval score, coverage +error, and PIT KS statistic. Relative to native XGBoost quantile, its CRPS is +7.7% lower, interval score is 38.8% lower, coverage error is 82.2% lower, and +RMSE is 11.6% lower. It wins four of five folds on CRPS and all five folds on +RMSE, interval score, coverage error, and PIT KS. + +This is not an overall win. XGBoost quantile and CatBoost have much lower +density-based log score/CRLS while also producing severely under-covering 90% +intervals. XGBoostLSS has 1.4% better RMSE, 11.4% better log score, and is about +9.1 times faster in these warm per-fold timings; OpenBoost has 1.2% better CRPS +and substantially better interval calibration. The timing is not a scale claim +and differed materially across otherwise equivalent Actions runs. + +The result is one small dataset, one seed, five correlated folds, unequal +model-specific default budgets, CPU only, and no paired confidence interval. +It is a diagnostic signal, not a library-level marketing claim. The next +quality decision must come from multiple untouched datasets and ultimately the +complete official suite. Hyperparameter changes prompted by this shard must be +developed elsewhere and not re-labelled as held-out evidence. + +Source artifact: [GitHub Actions run 31925701435](https://github.com/jxucoder/openboost/actions/runs/31925701435), artifact `9257853524`, digest +`sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765`. +`summary.json` records every frozen file hash and the unrounded means. diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json new file mode 100644 index 0000000..177475a --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1027_ESL", + "expected_rows": 25, + "observed_rows": 25, + "status": "complete", + "valid_rows": 25 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 25, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 25, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 25 +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json new file mode 100644 index 0000000..90d47df --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json @@ -0,0 +1,158 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1027_ESL" + ], + "dataset_registry": null, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "models": [ + "openboost_cpu", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31925701435", + "source_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b", + "tested_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b" + }, + "created_at": "2026-08-16T04:12:18+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "name": "1027_ESL", + "source": "pmlb" + } + ], + "expected_result_rows": 25, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "ngboost": { + "dist": "normal", + "learning_rate": 0.01, + "n_estimators": 500, + "n_quantiles": 99, + "ngb_params": { + "random_state": 42 + } + }, + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "n_quantiles": 99, + "n_trees": 500 + }, + "xgblss": { + "distribution": "Gaussian", + "n_quantiles": 99, + "num_boost_round": 100, + "xgblss_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": true, + "openboost_git": { + "changes": [], + "commit": "cea891a8cb964c655c30092ab0a223f4f140c03b", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 25 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "official_quality_shard", + "result_rows": 25, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": null +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/catboost_quantile/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/catboost_quantile/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..989af954c84bf4ba8eeecd805b37decd2e5b89c2 GIT binary patch literal 30017 zcmd5_4SZ8omQPE;SYQw&B_Ir?;zI;Vnl}AV@Z}|K)08xXLYp??lzg>KnxrvFo3`l4 zsv|fKyWqGiGK{dwU`JL4XGPrsv5GRvh=Y8zP;jg$2#(_*ilT!vd+z&4UUFZWU`uFz z3h$kJ&i((-xgYP|d+tvSRAr7NO=6HN(Mi%JqH!Y8tsBQLU-;+zrM=|u+ICuA@SP;n zkIj8y*z50;S+3__TvGKH^2z4mcjmuygp`OR8Inn}!R`jh2+4@aQZc6?6>~ZnqFV)| zW5HY`6y{_pE~>pv$xA}?I`&FfVaNsndRahOw0hh+-a4C(rdFB0Iy}DuNyhRh*&cAw0@o+=K zOBY_Xjqkkm`09(Vw(YP>?k&6gD&8MB3#pjXK^$;td=B}VZF85dJ4`Xs&k>v-o&TrEm7Kj69Ar*7FhyyN-&tb-MC+$0j?lU_U_K!7Weictkfm+lO#epniV0B>@5@Vh-H~;9mmk?ifvOrM$qFWQvSTemC%mTx~%)OeppIF zr%z|*rPN^$4U3+Zw*746)0YliAirL);Jc4@e?@ktDrcFFUL;>RbM!0esf*-C+rRvB z`z)a;fzyzRIUPg+mkN4skVJL?(dvsIB6l-)Z#)HE0G|4w3*3HN9};fvnjjQ@u7vg$ zS#Vu|)(=Z#5a4xzvfI!8@s8e0)^Sz;yNccm)F|8s5jPq&Xy&el(T{L#R5%K6`1ccarVLRl_; zZ42%Z4SA(xzN5_0BzA0njo+5jvcTCvUdWuZ_?O(R9`{~EzG~=ezZ_MZu=nb5P zRLtoh4!ATv2j@o9)01R5T{ov^y|l-CTi2!ye?4g#X6bri@uuT{d|uU6XJ0$^-F+2Z zcyHh=q+-qnalobVIsEDH%Z`)Zyhc8B-``ere()N3X8i7r5B%3&a>dMPv%9D7BR7tG zync?lOCS!MhE&YyAP%@RK8Ig_e(CKx&(q{M(b7Gc$}Obs;q^0LZGVi+-TB#fCoC=G zi1AOoU00*u_D&}+%2V5GT!)B-FqeBh5$gz)EvXu9{LP|IG{oJsji~M4aT=c$c zH+ga8r}tl2wL^ddrzsV4I*0={STIf=XR3QI=dg7*Sm{U-v3y(`ldgS;?py} z*!bBi)tp)9x67~d^6>J^2A*S@+=2V zl3PtH*LJ*nlDstG_}yF7y#gFK3#pjXK^$;td=44<-;daWY@ zAL;2n^7go6{TzrJwG{y1# z1Hm5hi4zrfE82R3L>?Uia*}INxhc>yju<&gf3o^*k`<_V4UIKmdA8+h;J_h^jFCRSc*U9e)v<0ULb2Y1jIN;Lw93Fb~ z0`+Lx5mHH<>fTo0O?r>K_w~mMkCUHl`R;>HZ#YJd|M>@teZfBoaNsnhVonEfz@_my zEM7BaXM^_$xpw6<&ph~0H~HqTdY$&Iz2tAdoi^i*w~vu+H}2av?_c``IB=S<$7gjA z2V5GT!=aWtwFh1}LVCAVe9*C}oBUw4Yf5A533BGy{6ja*KTeA0zW9s12ZcCrno=>R zgE-*Q_#B>)oL;sw^(dKld8V%R^=|S%59{9heA!7dd)kv#Yr2k;?|ymoSJR&r;=pN2 z#heb}fJ@_Z7z*btZ_kuDC;#E5uHL4;^Bey;sjI4S*YpY5le>--OzEBZ>∾pL>mZ zYTZ~Z$4!AUa9ESamBJ=o|30sd>LJ{$A0(zk1{g5_I#Z-f5uBU zbuH}K^X=3#(*!th7E&>%gE-*Q_#Cd;nl`-GoeVph&HE;#DO1GaC8lBUXD0as>~)uw zn1?B$Ec4*~!vyvgoE5#l*$jqC6a!ouBL?$A`%IB&bc!ftv{;lnT0CzURDgbI%wHmW zEzgzExU?3+#ifF{QpseUB$Ynq88sV`&5_9q>Pi;t0B4*eBYdWl4|Q^7($EuK=k=M) z`AteXI#s?+lFij9F`#2-t2tn4ZuST2?rAZ50uGm5pz5)hk`x!=U=uITRfwb`{MJUh z+bmfslBC~~DjnuF+nJ{{pWDwqruyWp%pWo5!VGKrEn_KNZbA!E3E@LEP9iD*Q*J^e#^@QYM&Ab?zoII?VA48Lqhcydh zXmawfW>E}HPR`R*2zyk-%G0bq7aJF`)n>1AHFIGa>2(C^n*4SCfH}a_F`AWlAuug$ zaMOb*@O3VGJ*za55%?SpjjUu8BeB|1OqrS_c8}fH(9TBJVh@<>WZ60Wl;!qQ#(E-H zXF)%9MKQ~Avf20w)S1%{&z$VoJae*R^UTSP%`+#vupgeR52I{eTOAulmO5X=juF0y zB@-5fd`=Je%nnaoz~P1_PJ@A$R^RNhi6o=o@j9D1VD{Ssw}^CTHl=05-OV<;ifJ5RJHDB0X*;sTt`RD^fF7r4ns~_{tJo6KBKhBa)<|De?yRYu?Ne z6A4i|!1Gb)Ie`bWVi>U|6@c)36`DnHDNI6%0l|#JgIP74Sf2(kcuESv;L?O)U|f3r zJfJn#5t~tF5YIayAY7U-P83=tRwgXP6KgE=?RQG#B&GHjgBBqyrkBOG0S4 zG+{K}OwWVs98J7D3gGax5rV^|3B!eFUAc_Idmxo~b_~(=Bh++QmWXaTOqwtr{*~fT zVVwPbEb;mcfQsjZ5GpQB7&Yu=9^>WbDH&g-5S=#?`^Pa}MkRw-FQcRh<3)SPqB66| zb_ua(JfPzFAB2ia6Gjbtna^*_9TSLs8H|@v=^ob0C~3lY(O!m8p)FsYNW3KlR6N0h zP;qI(s9`S)Ltd@`yqk!FlNc|f5<0AxQPPC*qP+~Gf|p%Wh=Y>>6;I(HR9u=cYS_!7 zke3go5^qc;j{TVNGAd!idKo267%$q(Fe-TY&NSj=CZOUe8ia~V6Gr8Eso>@T@bciz z#K&2Tmr)5B*2^eq!g!dMAyn}4*e%4_>41u-U=S)UO&FD4uX64IP~QyVJOMy>o&^En z(u9FRt6?74*_p(pTLBDDs30(0nlKoDt-14A?NatHy08+3XsoOYJ>I} zlMW1-KkIbfPP|+M$aq2nA>-14As6s`-Z78px&x5$GzLP(r2|7Q;vv6$C-K&NK*p07 z2pN|S1R3^-JmfAVagYRLeDi^jap}O2`F;B>6>*FLWIR!Uka6k2kokT4U@_6B24p-v zfsk?Oz>xWU`&cP)wgiyzqy$37r2|9e_wBwi;=Bfs@l*st#-#&8F68yFv*pAkEg<6w z2!xDF2ZqcaRp)iY3LPNhX$ORiOA|&8AGpXO2Gb=yu}Tks_}&@;;?f@k$UnJQv4~h> z06;vUfB^_2jKXAck{F8u+3k@M4zZ9bi4V4nKT7xaJPl*S&Q(dTa*(D@W%MJ>@;13-8x0QrSW#Qk zXsnGX8Cz0_gC=6VX>vZ)E0`p}9LO+C~biIdCGI~>phnEt&??UzB=>Xvj z0gm)+N(d_Bq$wkv5NG`R-EfE}9pN>5Y-YdoV)pVZhsS1btFtz{THGFg*6o>h-90-q zi+Qc1%j{87`gM7h$LzL)dKTmyvcRI=VRzZ;qJ#`gU7+1dtJb@k&4K(pr0!{Pd)tKy z+;Gy3j!>a$Pm9ZS-|WmV6iRB~Se$gQ!Q=u^v}}-p&@mh-G?tgc2?(kTGL4Xvc`}?pRN){)N4Y0NKu~3n zX@s22?=J#O;UGgtxhF(GP-T#5gd$MbUj&%KL57ZUPl$k^${^DSMWCp^2rz|%3?1d3 z5CK7zL8cLofTEvypolIUWaw!31PBPK3^EOLeL%oV>;(k@KwilpgYvFH;J8tR zR|lH8ZxItbTs+9|;j0}uc4XOA0j95z#6k`gUKMEOu1YNIaOog}3*L&sS;SOJ0-fB% z=+J6Opd&Er2}S2>Ou#}o1&Kw6R!c&3iV~wkt6gz2LMOT{_5~hi{1DEgo4m$iV0(Dh|m>qrhPMBXk@a68Co) z%ieRvv3Lbha4jB(6>h>JlgRVmW{cBq4Ip`ci1aO594{muI2tnQ1Qi1sfa`MJh}#gGg8s zDqj$+Vcy|Sx)!=zOHn}z?BH)yiMpf|#PCj0WmTZBD$u%JEhbHMUX5JeYALG@SUeSu zg$|XaW^u4V>t9T98JG9Q`24jxQuT_^jE%LS;i&4SIa{UgAyEI^0tj&kA zK&`RLp;J>1liTI8Xe#T4%B-4Fz%K2m$*ojaHC2F%^xIl%YZl4%YAwQOu^DrM_Ua;; zy+-9i*jl&AZ;_YE^v*?e{T5@jtkziBXw#G+-k{&+uLth-_VSimV-92EDGylXG5NGx zYVVrR`RY!ez~^h}Fk2ouu`3ccs^wt8!UAdap&E7mHt$xkeu})s%Ob zjYTcV?r)Y{lgsKZ^_Z#)+bfuHj*kPFdu;CN_FALORnwsjCYhtx;;Dofni?&oMIE*B zqW1o|x>{_S>UNvE)X(+XYrq-CODL|g#NxVWD8#iNu0nB5I%j@d<%z|$b|}QPAFe`i zO*&_OT=No(Yr{~8Yd>6t;+k~M{J1I-i)+(Rh-*Jwh2olY&Jl6dFOIQ(i}-Feu>$(Pxkt=R$l3v%Yh}o~a(zpdHEPX4SZ4HSz%Lne;N) zC&gOd1tPB(#^q*LWV4*H&8OgZ)+zt(|P_ zo-t`_Wsi;C`#@W}OrDsr%%%50IYoYx(ZuZWlDF63u-nZ>TQl2^0&!&h$SwDpaB3jWOY~9*Sg`{rn)ewbH+TEiEA?y z`i`DC8^On9KYfR_847(z5$oKipT5J`T>HM$rgKuU=9*Z2CuDQ&`%cLZn0G=p*S_y8 z{sHq2XEP-F&K1u)(03S{q0o1(eBLQ(88Y+E;=H^oo_9E#A<=i5e!#rL*d(v-R0bih z5MJZNJZBZuOWyoarhkCa*$YCY0W)al8Hf;{yiiFxcq?fcd zx0RJQnvIHjo5qFmg)#Pqm2md&YJpsm7v-W*E-`s=G__hZG53RR$a^-$KG%UWl~67a z&xe8ESC+ekd?c*%J4~_n`gE?*DzA>c@AFi*n2q{Y6XaM?93dweox^Ocb$ea4xmS== zb>~2R)&5xXxW!o1WHJV0wwukU@IqdtiEV#@Ja3W1?5=jQdtSl3A(Nvnl|g^<)_SU8 z?@8yb3sC+Sa@Xx9W2wxNt3`QX$S<~mPI;r%(?sVrA%_dk<*hbm{VosX!cdOcsm+14 zC9KcNAvekIcaUFU^J*wJTH(lR*EuU3^>n_y7Lw{Jb_`WOsyz?hi}Tb_)pIKj-Ilj_dLo9O2INzp9=Ez2J{XYd(f|dI{{z~SfxP? z$QsalAyq|Dx**&YP-|6Ce-(OHq4(lul zq7EV|Rz!uSRvbj8eq>tK0H*K{ysOc>9x9^LG%@5ksttY!gOV96aGwDl7FB^ZMIoZ$ zs|!#Il4_X#gG%HEbOFLCpF6h@-bza8N|57hedqzP0?SZeNA&?4TD9Cz&*EumJQTo|5_>-@~1-3A3D^U$nUnE)>P-S_-e|qgsrcnm<|#+ z4L|ykenJfo@r?YgG2ti(`9n3&g0?}5Y&_!oqh|e4>+(2%Y79=ceHee}Afk@M{z}oK zCf4~(Ny|c1URn^U&ppuk5N^3IRA0^j>(ir9>Mf!AN(`O>`AfH@N~hrBw`73zRUm#P z1)=&J1Fa9Wht?OWuWg|9L3LDxB~%|}@C_h-6w_ZQT|O5-!vO1}+qbwNRG)pI^`ZEc z`9k#t23Q~6zB)^&zQTdlhxi%7@zWVvBibW+ex>K5GVUy0@1pNc&@%<)p2f_kYD!aF zz_bTDA7S&mA3#;y88LI-O{t6_`|^19&3t?MtR7~1O~}3?${w2!Wyn69Hz|X=A7#}p z5K=9ztp5$gGPXW$K13fS{`{^(@8SFhoui@hL-thD+{pPf+MW(AFaLz9qcp7@O#B;E zRZ;fzr;(fY2@B(JVtxCA~Wt?+3?H8rRfMcKo`ETTOt(K}kRgFVV+h0d=R(w$zlkhO>S zI^okxuV0w&t#JS5=aBO%DAlZxy)Mc=@%E9^iwd2E#n)1W2G+lX+ov{2EezXh4KP$u zd|3O$+h-BoK4q<}eS@lYE({$yq{8bddc5%4M-Opb60%ooqwEuIp9&{xIkkqhS5pW2B~G z?cs0w$ISK=HD4sn^qPJ5wAcf4tX}UN`Vx00dr3Q!xxNkmeP;NNHxH42M4#U+5*grs Gp8Ef}kqAow literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/ngboost/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/ngboost/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2e781bc6c74f0b45d310724162d6636fd58d525f GIT binary patch literal 29953 zcmd5_3w%@68Bfc@P|8zV1EQ9SR}d^o+R|nyr#ESvrlfD$v_V($>N81ek~VEohl-Pl z2)Yf~h7KH46a;aIs7xLf<^!Edd4r-uR77OLm?+bI=RT5~oSO%>1oCtC-tTN|GccCyky2cEghfCJh`Tlk%D}DX)_(880FoF+w7d3?3g3 z|4xYO*Kc0FB$&hO_|q#3c6@?P-?wu9*Il2Y)TW9Gqx2|x?B&E8KmP5f=$pA2BUc_e zD8hl)lu3CV%z>0f=CEf*@xmK6eufcXY(t~!S z-v8yv1Mi7&;5B7ZUI%j^rI9&E-(L944^MoKsDmGWpOt+K{p*JZvzF$3j@JA!yL;-h~xATU?-SrweyL&;karOpu z&!Puj)Qw$-KDQ1`d*uBMA{=-NnUt>sb0DRn9CDI!^hr`?Y^IEX5wq;R$KOD4Qt3F| zwm7N$NL}YP1kxkNuXqij!3>lCdHOuf-o{EWsnobu_H3R-=coO1M;s!ze8nP zPoZO1{%XRAr%$1td#1YIeD2@qv3o|APDuGTTK!4o*2ksCMV#j~Wl~-TQy`_H&X*?X zeG?OQe#<)(_F?Cj-nVyvRO&dm2Zvz(s(*4JI4pwCI(rx~Az*a#Fde8dP>U4K-B09MB(Y!Gk-To8h!AqWV9@YTJu z6Hi`u4ozG!e*J=D=TQ4cc@1CFKcJB>E_h4gJ%>JA-2cyymYo*iz-!8+ybk6-N<(n) zcLW@S9LwEbV1KtPz3+1fLe7B6gQSzJqmN^qmL?UQ;IJbub4~ z8iIpA1{v7*2_0Wze{&uiH%KalfW!@#OJ3i2oQXkvt=N|d#{ieG-r^YYF$k1~z!$_o zk=%W7oU4CMZfo5470U+Y%+vg)(!66(PRZD>o~WHUFsE+lVmfi*k0LSPHDyv>2Xi2$ zq8xIPruk+=9D^ed!WS9-&N}6S4lsJdSNKIzd;8ZIn4Aom62^HjILudm zOv>wE4x}_P2T$$B&F^kJve`1@*tCR--h#MB>0(%A~vw=0Hj#b6B`*!}d|nEJ5iff3IyVy0b7>pYn$n zP)_|nwm-9FEh=ojXVl4lD@EeKYs#d&4(32gBXjs*yK1Dta~N&(J~6-b_lMEp&T)?> z|LHJVG1z|Yi6w{8->!Z%x%tUMA{=;4nUvST97t(o4$D)QJaot3_oKJA=pU>eb^w_c zJ#b(JtdPH`oP1>XkOSzg?j@_Ht=cQXf!CBtc^%Awlt$(-XWzHW^QY}a^!D<#+3WWr zTGHIEkbh;ZOFWl~-Tb0DRWIgH%>%;?sKcB41%-Lw05 z*X=|_0o|KL^$x8GAXaa#DTN-OCxh|*WR%DU)CN} zKI!hk>ymb(yeaQ~dj8~Y^upq+W&b?`ySR!;CA+VGUxWj%3A;E>2Xi2$kvUurJIRF1 z1Nte??n2jg53N1z?Lqh7rT#B^b~lndl7&gs<3XPA9V(8&U?DUp{WzUdFpU_4|-tN+S-E!drc3GJ?N`7fB*1TFYHA>PSZ^M&yF6H^7=Jfo;>iT z2nSwMCgpW72T~fD!-@aNm5b&=pX}K+?97in=+WEvzV%MkJ~VRYxLa#)*p0UCRn+bN zwHOCpQzqqgFb7f^nFD_!aw$((Tz9&M-(5api~QkNoaJeM4U6t=GP>tL|uH|@3SVwU>I1Z#VavYAeFF&~LxwD(g)84m?{^acDho&xGuzknb z&2_(h_@%=|XE#@LbQ~&4`+hUYf!B~pc^%Awlt$)o$=0-ioBW)V2NH7dzGh8#T%1(8 z?C5FO)$^TSra{(oc8MD7(ba4N{nSNQT5o;)X~i@k6Ay_Ja!W zuRr@oLI$5NVQ?8Oj7v&Iab-zk^hxo|@mAs_OjaRRX4U3Z=mBSBQnLRrYYNoKm&trj zOr3*L*n_LM1bmPTpANpp!Q-!h8lCm#mKL{1q{1t>A{qZcRyXSWSiAY*<0#s-If*?r!&aiU~;uOZEmhS!P@9< zwYl7lEl$>8u%)Hl<}%gWYL#+UGNRGxvANn!cD5d$+^?AtLX(&KHK&Bo=2r~ z+^?w$p~=eynwjDrsW^Fmi_6Z%MQX9yYV9q2mW2%*>DZ-SchbiMc5v?;T zjGZcEnIesguSlJWFgz7$p?NCOLi1Flh32V9%MQbn^C8jNYOUqM$Wa%HxG^FWab)}= zpU(*%m#NWN>uGdA6ZeOK*T1gCZj~e@!s8WIlgH$?d2W>G@yReF8{lZM+U%SpIbqb8 zQRmYr$(W>sQJIo~!$&dWHYq+iA$d-G^1OJegOXlY)@sxwm|rAG@puv(4u5W)FmMzl zNdR~u3Nt4NVCM9r7R3V)nS{dgBq@bSC^{gRaRe~)22e}-0}PpG!Z4&XY8V)ot!@F( zqARFnINe7klQ0k|jT*>5TME#Y45C)zX5tq~Q6!V1i=KZj7N9K~Oudi*Xk>y2`$kHm z_DwL;3*c4`p*AK09GMHkaHKS9IRC7xlrwlI?39y?Pa(l4%``N=l(A0wk5M(24!(LFN35};{|)^M+Gm}kD|I| zfJ)|VFe)jH8rAP*w$Dpw%k|e#J4dr#2IXluFN35};{|)^M+Gl8T}$m81E^$X2BVVF zs8Ri1s)Sx{x}NH}j`K1w6T^EMD2*B~*h@buc-cLc>PZ1qGWUW}Nomxm0xvW9+2>Ro zwc!TpP%7(XP|Ah#GDsRV9^oa6%6{cH-AL^m2dHG01*4MEs8N~qs$iPyzKQCg0EkSb zU?5T&HIQ#LECAa%fjTrEz{s2l1|y|Wg9+E#0=%Ay)X|#(kIau?JW?7pUf@bzfOzN@ z>ck{KB(otHk(5S_DA*?m;EqnF&ZGexnfSnPq%>+c;VwjgcS1p(mjfP|de*0%S6K zfssjR?~sMF&cnZ;mZ|`m%voS$QrbJ@EP>CzyOmlo75hwNDj1&$Y44C#0_3IBsMWUt zGMS&i$fUG4$goEgAg{=w)+0bBHy;?8l=co;*tb`!sZBH>leq|tOiFu)EbQCsbE$3( zAd?vgj7&;xEMR0(8a1;2z(pP~m<|_Bg1MDFw{aGFgJMf8e>UMv4e-Z3d9m3^&c z%4|VW=5=S1_@c1T2E&yqgrQgF?ptBV0CAkqqS44zyXKa z__P$NcDC8=cTY<3L*e`diN#C%8cZny#mM>?2%lJzP`tE{!F)L$5;35tk70eOBoa2D zsE=X&87mSuu&j@P1(`7tIjFFYp@|e6NkCMkk7@Xv%#f1=f(rW>I>piv!JG ztq2M3FYaS_|1Ay@JFx7c05i8mLLvJKFA6k!EhQATzqF6RMK8dREJCWqfKGaJbQrZ5 z&=HyS#G-RCCSWm~tmvY{sKp>Us_5u2Y8RajoWn#zhf|9I9Yu6>7_}JCiE3oQWire< z26UntS#WpP5E3>qt{fv!lq7z15+$&d_6aA_aMLQuWPfmU$o z#RC@^ECTS0F-CmVi%zS-rG4zV|1v1q;(=v-42&l~4Cp zv+wY4rWU4LN7G&!?BFl1M3a{fVtA+Nf=bX=XXzaFHlwyGqgq+qZZ4?un4Lw9C5>ux zb%nQH=ip>^Rjb)iZntTxx^zt~^#+C9tO zM!kk^G&<~dv$niWtjwa#2kiW=>h$tVi?$MQv3^H;O?8>FSfj%jZB~QAYpYVpZPjW! z#@0EEZnH99Uffj1)NeLa$!iSd4OVR)<_-EC?mFOZ>nv=mF(_CYXQ9WU49Ta{TwQF} zHG;oRbp!a>U{Qp@b5?mQ1@<~?f!)jb>M(0n4c6QmIa7y2VYh1Yn`^4e8!%t9v&wBQ zfOr)I#HX{Uk@3%AGE}*Z`6>ZdU1yBBG+CVLE|$Bo)oAEwu2yC?n4$e}d(@;CdM%v{ zS8t6W6Z|R;<9nvUT&>Ow)1Dq9#0A=1UW5CI7WB0pD!m5xLq8_O(^;Kf++-| zx!T*T+Nw^gBj3&U+e^S1#)~hm^627Pb~(f~3|FzZ#+QV#kKx&h-(7l_RU7PgoN$)Ft>;yZcrRMrMYk1&>YtHqVm78AwyK3c$>?j?==66d55)$UEeA9LSDhY z#tC`ODykQ|`K3S&`3B!whRuD5oc9~W^kUZ+nk|sSiek+os&{GH9KID%wYg6(W^HaQ zC~PnpGV82bJI)t|*c+C^*}uIFa!IW?7lm_)v5TXz-K-6{A9O(8vpMv+4xFj@a)D$% z4E(;Z+$H2AVV&P)487N9a*Y;cRp@=6v#QNxC~h}Gjupoda+1M0%=Q{btGy=u0&=Pj z1=LsN4mFRP4XS3N!5gyOtcJ{1$g4DS?Jtt&RW+I%RZZNUS2S** zWOCP8IR6W|>rSH~Uv5s<;k+>97dt?wu)*SNX7ZYl!-eO&Cozu8{G^UcyA$; zTk!jI{9alHIr&^#16oZuH=mDl^t1)PXV4Z<@)qFw)R3<)#qW@@2mK_rhHPjZO{qSK@aye$Q>exqdgk5Kw^c3eaBso=pQkJ%ED}Tn~UZcrV3N zG+s=_f~hdna=qBpjZF*c!4&?1cMX0ohKgtnLkxS4Yl9!gpyj1zxX%C&WtE^!)0n9L z>H^e)rCPTC;1Xpiz5vmroIE)j-tzL9O0eTxefR;h0!v?C$MpdlMzyfCj>FS2csP&& zc=%PzjA{IY9R-K$vG=Sh7DC9Nev1m2;0L+`zq5hIG2?!4FN3B7<}X03Z0GRnHAO-8 z5KUZ}?+3Uv9q=uy$Y30$b>?)|e;w`h`BNn74-;x_;CH*2(bP=g@HKSl46eSsTqa21 zH2j!H<_R}E#53@_#)c!y=MUX75!wbTa`A}lkB0L{qtD>|sV;5e+K2Ur2_o)D+%F$L zYD1mRVO_|52LeMOjG zUY4)E#@^P4+e7E_)z{J6`k*?x$n2|+E_L-Hel*)(X#EsEex<#vk7?iBEMI-L-qwfX zSK#v1=jmmAO#AB1zWTCzTOa0E>W`nkv^}6bg6CIeJ}ThP(u?iP-3ex$-YHs&z#l6Ot1CX7X{f9^PwEu`|~CRaQCC2 z$__%ht)26~K3C4w$IplOqa>W)_4wVN|6p=7^c0^x-8?yPJ`J{KLMzBW;p%8@dlwu3 zdUa)xJ@e@md|#OP9r6`!3-eV`YdL)lt;@|0h(BJ42)_qvY^oN!_mauXM+I4&y^d}x z<@l7b^EoU`;K$U*Ji#ggNDJ@pG%M)oe)}mw_OLL5pWi+YKWasDlZ;oHzIf2}li||~ z+F{!R)|q&H*I-mu)s@VEPf0s`8gWha>arkvSeOO0XE}byYj&{5xh&uLbqUky)g_!g z#J34Pz0CTB{oacAZ($BOqli{d^x5l!?4xfVC9|l|n>lRvB3h&L*_TGLXXc-Pdv&lVcKGZyxiAxk!^f_vh4%)r0|qd{@zc=xQQ5=Y zSS!~@@q!k{KX0!Kwr3`2!Fmu6IyI9Ig$MXP>A@27)zL1zJ{R-{*jT{NKVF%u#Ra-f zplR~q70=-8`T1THkGXrxR96Ih1`ZEM0eM)T*W*18)1{cjKBGtT^)Ye*e>@7*_}(#A z({lFkm-(@?Jx$*xk)^bnTz9tFJhxa{TW?`5ai?&Xv{TsY+wh-ahW}3UW%7??{~Z!Z JDg2+G{vXi1BDnwn literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/openboost_cpu/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/openboost_cpu/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..64267db77f5f1fd9bd5aa4b5898ba04273209a62 GIT binary patch literal 29984 zcmd5_3w%@6{ZC7USSSdR8juMVxxzq5(zjDiZ_+kRNlPg76>v4LwrP_zCTY{)ocxvl zL~+8LLpRw(rW?M%DS|NkQ}Mw>MAz1e@~|K>c_<7R_`v!9o%={`a&DSnOCTR-@BN+M z?|i?%-+A0~&hKn9&=pEqip(IJrsf$tz!)nbmXT-+Rzwb5D8(*L;HReRQt- z(>?o;b?k=6kDB+O1>djw^N!=)NG6e`%f?OvyQ^e_WrN4brM#wG%Il;{CW}Z%j+96w zLnkN0zf+S24O&txiR7?CKXKxc9}b}0_3QV|Dn5W5?_c%a;|mX=gQJG5UbE)_dShjk z{lmZ>5e~ejT*~WU4x}_ThkHz|8t1Yj=<46Me0aou4D~MGHh9YeN0Ig3tE>Nc<1v)J zD#`t?AzzAc;7jCEz7EWRl*Z`%2I2Zw&20ulrt@a>Uey zo+&+2`$gLOA{=;4xs=zz97t(w4##Y}J z_Pj`e+ut; ze)n(pq0>9pRxJH$zX%83LN4X&z#K?vYz|u2u+7)}>2GMk+iPxoSJ8ont$$dSR`Dv@ zBHy)t{v$7=Nr&%$!gu^N5e~eCT*}vhIgrvQ91Ov+nFk|g)jf~Cf#7d%&G;Qj(qXH5 zKYuexs_0$Qu^me%to`K&A`JKvxss;}+BO)AlO}UiUVOoN-4@skN&cu5Y3>aN51`*OI_<|VB-KKxymfOytzux}F z%tQB|Mt@j+LTed!7H$30=g%$Ka~hpb`*&~I@RK4Ocul#K*TEb}X%r6OJwY<#uf0s0 zAJcpWJwfvG@&nKlZeMZZ0W4iL-giiZ0bjyIfvJIu0;3<2Mqwa`LeYwo%XXdr8dXla zd)%y~uhA7x9B6EP=R36Px24aVd*K|~w07!y&%AzGgafZBm-0H811XKdp**Z76j<*1 z0(ye<;Tw)X5IUTD4nq)Dnf`tR2Vv1esmDbK@Fk49*aUGwVDv-MC}85_>;2q}Y^dIS4`P9C&4 zIr96w`en(c`*sZ~xL2)SJbcoig8grq-mbesTF^E4A4}$cl3bA1{nfXfKl?$%ZC+C@ z<#jLxQYz|pflRHJNg1~%;JNVjyBd!(?rJPwLaR&a7JSKc{XZw4z|f@kt8fY9J!8sw zA0dtOeo&qnQ{9k)t^A7>ErJ6=YEBj)}`B4OgV|V$8RyXKKK$fo|u?3 zHbZP4<~8L~UI%j^rLj3Yes_;lcCdGA&X8Y!{>Kpqx9qJ)qDfD` zy!|EArs+ueU(IHbp2(NTrMxcYKuTkCFkWjYEMET!+VIV!8*JDA6FqXzij1Nd?wD=b`0Ykyl-aMl*0d41pLsd( zpY@y2tCoeYuimo}J+$ZgE8M9si*VpI5&qS0$aIPjWsDX)V$kkZ&3c5T}6maSzUdT*V+v!i4m z8vT~!hzS6OL-m4ft1GPFwB!O@z&gr(D#p@ z@A~q&9^`uD?I%xNw;TPB?C@)gHuWH*)3G`!Ahwg_HRV!X2Xi2$u{jjI|DjxQ|A%P# zYuh%d_H?5kl9o>_*xiF>f1jGM?Q}Ofap1$|9YZ@s;=pUlrMwR2KuTkCsJ?mV?k-0+ z`c=bkKK|^9E;MI+d*$zD^q?~Nb!C1Oc!%lK* z>MYq4uXUl}?>^qMuCo)(TeN2GlPNt&IrO&w{%Uj=dhv@5%O}74o=98pnsOJJI%C`;M+E>OrSBFZsjT|La7(J7#aDH@_>wf!CBvc^%Aw zl*Z<8Z))d;MU`D>ZjW!oucve(-L9L*jX2ta{^a`YGV|t6)U*G6!`X>DL^$x8aw)Ha zIgrxW98Mq0P+Nc1h0e@N``fUrPE@pKn)Tp`kI)^y%0oYwbfGb(E&tRW`-cbzUQ;gR zbub4~8k@uXgMTSp{7M&Ef9T-v-z@7yi%vZKrR=fY=)HyOx)n3K(1W)Py^n4dX9A$9&|~bX&Bit;=KOu9_7f6z56~XdFSqt9SsMD z{$<4vBRj~o9Iq*t@;W#Uq%?LMDzcva_UhlA-8$&#J14GJoZH&HRQi)&G@adAbu3NW zYB;xb=51T@(<{E&N^;;el2NKj+U*xlXg@u&D*ZG<#R5N-)>t0j`OBuP@5RFa$~y=4$oi+?HX z9|;+JzJ$SLv~ciAsVJ^oHcl@~X0~+0rr{GQr9zckS2R}-IHP3gq1|f^)XA60gHKGI zS59Des!6H%AUQ)u@imStNP`+%+%}iFrN!&3v$$JDs!a=1kcYpL0AtS2$d0Eq8>Qwf@Q#n04Pv!LJJeAX<^HfgH>w_of!!WDcTE~Tvqb?M2V?-$8 z$b>{epHqDvligM4vpb=QQ()kw)VDaS64@|#9AY*3OkSIBl0=V>h#A=x&K9f9!Aa6n z$BY|uemW%?CrcfZEg5{(7-rnclG9Vu7bT}JNv7H<>4oL7#!Q15Mai4nAu~@HhLpw)1LM-| z6#y+CLao9HKr+3Afkyf`jOO|SK*)%*&imTgfwo{ke69OFBbvc)zr>Wtd|kl9?r`MY20{`UWQP? z%Plf$=V(AB^E()ol*Wx3@-iprCA4M7HB@&x>t#eXhx0N*8aG~~mmyT}@*iWU-Eu%B z^EVikl*Wx3@-i>z++f1Bwg9htDs||mfJbIbFdiw5 z8!vn%FF@RT6LowVAd>kJj7Uo3MilH51aOCDP-mtC9GMEiaHKSDIN>ftfOlL;omT)J znfAbVq%>~4usx0d@JuGPI1>QL90vv@rEvp_>`Y(>B|tu(O+A%Ba_npA!iDZ z7w1ur`WYaTxeJU;O8bYL zFF;;7lY0JEKqfO57@3sz2O0K=0_3L(sPzbt$;}5wCZ+vD7WVDu)zlUmkjd->Mkb~G zLl*Y!^@Y@%8bBs<5*V43_77Rux3?5iJBt9B%tT;hQrbUcVc&kUgzDA;GMRtC$fUG? z$a#VuwzHJls{>>*>wu9-Y5$OgqpDj^9hwEmWUc`tlhU}6LkBL3u)(yqoH||xfaKm9 z1Cr7o1t>hZIAoyC%mzR*i+}-1>5l>wcHrX`)cH98NTv)hASwM3fZ_AgnR(RWc>qYR z%P=4*jT=yKLdJJi*fgE3q8_XQIM~ofWCJi9DUAu2!X2P7=WjgF$t3D<4fRkB0FlT4 z7>Ja{1RBNx@uzzX*2<*xr(pMBr2bnwE(hurO$-o6X6c48!qYjvX7-GbsdsZy`nDwM z(0pq3d|YpMDuB!kkO=oA1tHQ`8`D!MY5Lv;aCj#l>^8ZqCa?TI)9=i%yR5eMI!lYA z)#>tP+&p3aZPO-Xu&;GYnJq%fyxy7NGC6Iao&mXr46vxT+Z@)q2q6no=L@(Q)p|#Z z$(NIb)m^PlcR;Mb2`Ak6=oG4UwK^PkO`8yc!pRI0i&tu2=~MYh^h=Qji8e`eMNvR9AM}O_rwT@sthoVU8P9bR@(fSIc!(U60M7X_NVqY@1}R64-m zqPJm47E#p_KqoUkI*eKZ=!ndEV$r!66R;RgZhX;U)DjS#{P^fFY8Raj9LmH)hf_-c z9c6rU7_|h@iECuR%`(h70d(RTS#X&xL3H98S#WpP5E0vao>fv%i>F$TKu z#vvJ$$kG9hg`j$o1Fgu?iw7<;SOnk~V~hl=7oAokO9$BV(9KY?#ly=67#QC`B_Vlf z1Q=|8#ExTw;=T@Jx%;gomY^U4uGM9?zztYz5`O;MVs5fod|2KWB6E9|#0!f1kA{!F zO+KH;Zf^B4$KH3s`E!QFV>9_$Ja8g?XNJRW_Lw{YeDLiKm^_{qKPCis;_)4MELCMs zhuZ}i*pA~PES#=_uhU%;`7pYoVs`b-?6rOQapMg9^EcROnw|sy!M|BFokgn~=wg~~ zpfwGQOsx(p_hVsEuzZ%khJA;BGqo_~I-2&=ULTl zSv9KiHgidp&+IC*&$g@0HFNz9IwvP{P*;(u}&ei2WnXk6G(yrIgcK9z%I?URNda*K#wivLB7uIA}WLvbAfQ$9p+iGj( zsLC}ujL~YXR{CvK`3hT&+JUikPNUbXDpr&?&0*>{S63-&t1B9<+9J#w^xM7lz}*%o zZLO_VvNo<#pG6gwPrzJL?$Fu6UzfTO{A{!+`@nNm`79-ldTWWp&-v;!Yx5heg|!N% z4yV##)fP9`)>JfNzGhdI*IWYeDhZ2Epv=zr=QLGUd5y*S00t{DwZFM&IRo;j1*-CSbI=hed^cf*8(B_I-+)uQiuWirQ zYj8gdVM07zHJRm2=FC#IU7RE@LML%;x6q_54p_9=4ohZPv_4MgB&@GHE8Lb$wZq~n zcbipN(fBo+YRdh_n$m@)>ipJ3_cwE<(P42GyNp$NfiiZSlj8v99;>q|P+M(v)GXBb z6U@j_VWGq5^#p`5{#=NzPQf09OBvsSFyMz zoU<^ls`%nsdpX3l53XWyO*m&^T(jbfYs2Lb*FLz4#Wmrag>lV}FRsm(LtOjdDi+s- za}JAZ`P?Y$x3KS4v+^?dZe>U3DjL^Bb8fOY+xuF7dK10=tW{MwYE=4R146g4Uz?3v0HuuqD%zVn-ED(X#{<@J_K*c-8Htc0!KLmbtr+Uj;C?33_b zj$Grp)NX5Wz6bVOe!OTHFvMi;G5^1g)WC9JKTRjsh!alu&vx1P_pGwkJznT~o3oHJRR zRrR$_IJc?F^Xr?Up3B6vxg7dVIeRujj>$gy4rg;Y^quTz=RSS(9oFX3_nmfq6CG`? ziPm?5HkZEd6#a;KCuno&`_9}SG4Jp;mqp*X;CTo74r_Bc^qmWzcZymsn|Wt$R@Md2 zJG{+h(RZ4E#Jt1WB(CpN_#v+lTH{1LXBE{;-276ahI~VCEyLzMM9%w-VtR?|3(XeD zVa2g#5!JgiZ4TdxxY|6Zm#{XsmXtP{s>>#oQuM_#KgtX z*k;y7-48k;@7WyvTnElng1JC49|nG3Snd+?k+9BRXpFwsXL5}eRaNwTpR20XR9)U? zgd8i5Bjh9_bC_+lPPd~r^8#|JP9@Y=<&8Fvo2&Dijn)3B?Pjgcc0*pJnQMQMJa4|; zY?SBdCQ-|}ykY8*Eozg~&tC`7b zLJk+6OWUmM`dt;wh2b1?lTHb1OIV+kLT*yn?;yXz<<)R*w9K9r&^MLY>zRCeEhN>| z+!!i@RC^Y@7iMXotd(wt47|S-$}RYP7JfI(ft-9HtpTkjoSQGkIeOZH-?L~7DEUip zeQL_NW_?gW4}U={ndAZx(yd2}UB>-}(7K%-Mb{gwD#jo%AfaIW8rF9ej} zy8^Tyzvt1wPY>Xr1lI%L4c-lyipGzrSTGfaTA?4Cda-Fq1DL`;@UFq{E^&Fm#!NY+J!^5vyW=!KJ>?k(w9cH#`mdw?L4V3b{b53_ z4gYSJGn$$l4qrnX=5h5E6*55rr{TvuGEcbSA)eviH8vc%L4W9$sn9l9k&8!ce>9vw z8hsY;PmQ68YaiAhCWyEralc~xsEu|$Q`9;emlx*->vQ(EK8#!H3D&3VXMN>3l;!4N zeMJUWzx-v|QmxPC<2SFL^_5|MMY+NH?ES3|w};LXtgpSl^+9!XnK@V=ZSeFXel*)( zXnhVJKSMw3W7@YcH&~yozxCnxm3V^n`TAKO)4qChu)e(h)`$5SLh;iZ+QQl+a(-pz zqZ0lsz1+dvonU4P+Bub-Pc^i*Fqdr)ZayOBcQ1gd`7>hnyqi{62klE^*|!MonX`JB z>9s-ovIu)(K2%`)P~M~j?tYY1IY3CawsHP96e_s-`1ufjl!WuU9>0h3A54yh&I#Jn z%`?L1(@1+Jw1WH-u8!8WEo9^0pstLtXFk1x?+Y`(L%zamVZJJAEvK)cb%l9h@y81h z;rBp|P1R!eUb30_s3e!O*U_y8j?Wx+K8J+~{FwTfCs;)QY2p2yW+goXMqE>adQOBrEX=~% zvjV^4H9OejTvqV>dN$MP)w4N!h;I{odYSbL`@I$K-@+VnRvE3H8no9(*vH>KDrQlk zH*@$pI?urQ7kB&A`svvrdz}G>Dvl3lAAkGIVcMspjk9l1*Uo^U!-Q0PJ;jU{Vf&Or zT;~PtHM$7so(t{=D ztD`-5eJ@7*_}(#A({lFkm-(@?Jx$*#kxy`&Ja@F(d^cI#?wgoP+!MG< d+7sC8+wi~34F99em&rep!?#H!2KfI*{eJ|m5a<8^ literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgblss/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgblss/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..583ad0f1a18cbc23d532d71b3af374a392e1cc26 GIT binary patch literal 29946 zcmd5_3w%@6{Z9*Yp@KYWyOe>{3KyJM+Vq*pJiSSqG^Gh`fu;?(npfMjN!lgp3x+aa z4joSBhv5VC4~QGvP|+cR0}99BQEur~1d++c3 ze&_rB{m$c_bAIRJO6l^9#KDQBiL*));}a#LC6dXjZ@=I6$ltPi(7O*`{_gL;?Lm1j z?_26Ra~yqoX4Rz0w;V^C-iW*B#Is!}QIeRHIA#jijZBP7j2kPH@|rR!uahL1EFv8> zN+OZmG&vUj9Un7j&{DM|m_x$Rld3nLIEl8_JvO)gmUq$rsvVOmFTIa8SB-yo=ezHr z#@Xs;rmO!e!hzS6NqHU2fs}^lpmR`~i4UGdlP*0Ro4@27dg&l_=FGvfXvc=)&wlg! zvuNkJA&VplAB%9{OJq{M4$Og+hUaj3*nd|3t)mkyzU}ZArGvZB<<*b0y)vR3C9PaF z!kus!t)FuAZ$CPJK!gLYDUshq}VuY-D&jKKh@}LZD-K< zaqD+o?0g?des_BO;;fS*9C%Hcl-I!=NNIQuo7aw6ccS4K$}B?XuI}tXqwl@1YRoG~ zQRadpzjW3fMZ2?fZ_q=IiE!X8WKv!ib0DSRIefads`KK(t!UqGlfJvW=5@5?qunoz z{OVQoGtH}O6CT-tHk6F{UHXZgA{=-NnUuG|97t&h4yC@anFk|g&7#L&NAOqTi9Zk{ z9k%A=>Af-1^piF1d$2U+h5O$UVZfI_OGqS)E@nVVLog7`4*y)a5Z&sKSi<=Z+R8Beu{dYKD6PV&wV7q!KW$Zbub4~8iIq~7lpCFd(DVP4x<>U zbkXC7APBqeXgdZ$STo++g@bV4m@Q&c17E@f0m66)aY118{n8Kw1VMQH7t8-MZqOwZ zx8N_I)I5C&U8&jQvcL8@`bT%i#h~ZDl65bgz)^TM_8le)xo^HN!T?7Bm#|#{*T6?1P#S`PAPSGU;%{C#@eB0W zNbd(99QYEo-MaSX&F_4TJ~4mVvi_^9sP5#2#c#ZJNrVHhDU6y zAFC8RAa<=@bcTt-A5WeJ_wPIE`T)P?583%4hQ1|HCj3V6C5-ziV8Xl4==-G%bo>PN z8x=cf`JmuWRMQV{m-bBl7InmbWn4ZyHs8H>!Mo?y56VwUc#LYBADe&jr}iT^_goe6 zp4XH~c^&LMDUHngJK8>C+|BAf1KuzExQ2O6kqTrL-*|`SKl$S94E9nbt@-t(jLZ98(tjs8_aej{~p>Hmf&y$?@;9A9XNFBi4N7plLwAxp6Vbw0dFCb z@;aCUDGkrz=!o6FI=x{l`mklK8`ql4AEG>Bi9=UoPK)_DwwhrS#ZQ z5e~eCOv>wG4x}_Zhwo0m*|lKtt7z$p2XC3G-ijnP&7qHDcc549O4+~c>sQgiA1~eX zx$|Wa4!ovJ%Ijbbq%=H-@%fX|_v*KxX)ie!{AtEE^weG7bsN@fLmO8;wT&LP9mQ=w zIPe-+`tYsjR$4(32g!*jUX@S6Ew6Q4(IZprI+SG|DdzBebgVB02i=T~Q(`XQT8 z>M~9%z>1K=YZPpe|UWH+o;(7&iXwsyp8Bg&C>_-Kodh6XH8X;=pUdE{@Z|97t(+4y#U3kIgAKgf#bcKUDbZF7#aeJ>6@R z-RPlXrs=cl4x^XP#SE2aiLKjtO_`L}VcLST_e;Zb7zjJb3wuttA1&)dKPz0;8MpEf z+MfDk`?FULqa77G-F-*8&_j>6HTL}9zeU=D*OWY1$@I(KxT_oQXFHy?XLgafZBlkz&411Sy9Vc|B_BYRR0 zqWN1Ee*CSv6J0SZqE}5lg4z^UM{J&O7>((iobtiSe~ED5HDyv>2Xi2$;W^Mbd6VA0 z^&nEOt@z+&MJM{?fj{3}{}&jWwF?ve0IV$ z>LA*`bnCY1Ih{zmYhlZlyN{xcZJ*EmuIn)R`SS-J$Y1wQkvQ<0GAXZvIgrxu90tM( z%L~JQ{rAPsj@mV6gkqj!;OIbIX` z1E+)IKuW{MVaw?X_3FQV(NQoj=HeXrm5u{*6LJq^Thr5SJ2cPO0DxDJl>TX8Vxn{RW zcjS$j-Hin6;1H0~5Fwc6gO^B#$4Fv^OC_cSPkQnQc+x4 z;@FbJSY|IjYziiukuJ}zE}U5cIHMDj{2TWisFN>~`JR|Mho-Qb^q6>jj11d-zQ*_o zaZqDht=Zx6h}0LymdLojWL?SMOwE?a;yji*o70pySCSY%F;+InX|l0TgWXOK_ZaJz zb25v&$-_Plbu`vidn}D^PG_j6&g5=#**si%yw&b$vbjC>Mi*;vlcll6<~G&Zs^#gd zWR%_IwYghN4z?bj+^?AxLX(&KHFH8}@^ZgsUIk6|7t#?@hx0N>8Zln5mwr_6a%UoS;1)n7^Eeolltzr|_cF)l zCA8(9A5z^(oR@)F8{W%6X~cNJUiwkN%YDgIj|@=BJPk%Ar4gh0z0701ybzOgIfmNx zBkH{|td~K_8P3ZfX~cNJUUI1Hm*F4dsP3_VO6FrQDk+T^)$gT3=;ikD)QMYJFN2aW zoR>k;i1C8G) z7?1FhMPV3q~cS5u-BeRlzj3cM{b_0T7u*!9b)mVj$mYSOB*F4(j-1 z03*{S7>txg3?^J_3-G$`q)z=9@W|8%#v`Q>;{~qd1&GJ*qRvkNL^2VA5lLyph=P5B z0PfUO>T(*ukvR_xM@l1x6YfF;c;_>yZ_)ve%x_>kQW`N{z#c~ccsY|=k_muhG6MsW z(ue^?b|$cc5+Hw*O)bj?WHN<;kx6OakTV6yOY*1{xqwV2E-*4F?HjUi)>-xwYPAB8 z$#exqCZ&Bt&K3B)Vj8vnZa^lJ6d0M5_6=DfKwfbs4fj4lZgh5OiKHPoG0jE`-`dLT0kb#3mBP{_6=D$s=7+3 zQ_}&NOe$bxQW`O`|G*_ZU@#rmQ|EO6NbapMASwM`fWniDQ>E19830J83osxl{a%2= z4t&0x`lbv3$=m=2B&FX2FmQglJd0W~3joP=83rV!5d#WN$oS6sZ4C8U1+}aK;9x_~ zfuTNB2W6UU!*jwU&A;c{&bJQS{;+LK8AYP zNPSf`HV5hz%>fV&B$Xx$Pv`iW*)u|>-sLe#PsC82bEs$L;Cjhi09ofWU!6%;s*>U< zY0?XG;j~T`*JN^8O&;0jY4cOqV2}j!atQ4wtH9H&& zrlk0xaQ1@4;-&o!CKrKXWc>_;&nrnNUfR!Kz6=kE7*N#Du)f1f5;mZypJDy^DiSxa zte=4ec`*_>sIZ@*i3A);KvbolY51JXN+$^f74|c9kb7bTL{<8khR?~I-Xg#j_A_*l zdtwAcRr;BRF9LbJMSv~rXXqgJ#0ZG0^fL`#1QfkRfGzB2=pgsR2#BimGYx+PvU`~a z6ii`1LkGJjLO@idpK0KG0wS(+(*6eHZX$>P3QGDJRB+#c#0@IEKG5vNijd&`;(mtr z-{By!1Iw-pFmqQV6tb`IxQ-ScHQZ~DNH1EIJGFy$%u>&qZS1^5sfUkN`_fSflfpt3vRDP ziB3c#3ogt>iB4W*eTPwtLSscV&@6%&F>@c2?Zmddl!;5tDnw&VB+3)AHAby^^i4WrA;XBd9M z-qV+zH%`UB)>=DF(`E2K{F_eGS+ug2R?~DXt*T{YN@YNKD;5^|%BQzhvhVP3rWU4L zOVh11*uh_1iKIY%+)#_%|=Z{R;66uVlJxinq4~k47<`?IkUA^>*Qn( zMU&ZC?yzYp=4tC2YYiFc9<8dl-Yjp+FdMR2S*FKscB;L`nc5sE^Hv#V+e=im-RN{U z%$o8Vu`-KB4cO{=m6_$)7R_wH#rkb6Rh4COy-JHQnyrS6R$GN4-Bzh|U~H|^=rPOH z>H7LIrhc=bBE8B`UT4)5V&0(N=BWYhw)W!YDnkZq<0|%A|V4VCijIy1B%ZjY+W;#NyL!?m@_kPUw6 zd+|Lx!(6G%?xj7wMu-cvd3qJ@CmPV#v?)qdxF7m4A)ct@?lOu(2CJiTp0+j09GlFpa)_a^&a76|%!nn#Ki)+YRmf&5kUt4Fe&ry>JzaYt%Uh#8p2t#QH7ZyVZ~}0KQw-I2xfk|K`gGVQg<9XWnkvg{ zOqu!`OD61%*fmzv*6)6fN_mx`Ed%yRcrQn;@m$I#t6JfP{Z=bpJ6X9sW7O8lE-SP5 zfwp!ST_Izc$?SnL6dt3&$nNo?x7XmY+f4>*BiD{1ape5SEN(K&>uwasdZQsb-B_s& z)hBsh0`#KR*3OD%*zdUDEP-3kE9`hLS~-ty)`jYy;ccQerXaSkXR^ZHn%x&y%8T(i zLyb{zjYZJYG>@dS#&Y0<_vA{W##aU5P<%DyaioDj6`jF=`VQmIN-_f&YBjlLu zrSEVy1EKF^hdTG^rSGsdH@@$*mDJOr=9*A_$7gfn`%dBan0I_OH@@%8{2ucTZ!;kJ z&Na_F(05pyfzWrZecmZ-9x(IH%&e?yo_Bbg0nv9FzQ?@7+C;DKl(#}&!N0}{dCn@T z7rptVND27{-&%&veTba*8^!dZ*B2Trki&{#%_6FIW7-_P6%n<$PcLe1ZY?UVGa0gL ztQrT-7lzmymc!Y^tle4srYh%WIhc1zP8*Y z=P z*zX{}!sXR)Zd7N_YA>nR*=v}5dle+rmE0K8L8?6q-V3r+P}WR0KnA|G7|Jd9eL8+G zErXnV0j&b9dYqeA;~YJ0!S7kL1(aHgaD7V1*O%gV$k>Cv4(n6D)1q21g@52(h2Qm15v^i~Vb5`G@WU8%dZ`)iGr&XHY|y4@Ow@mM0cycg z4cmWkiM$kFfT)*Gotg)4g=(e}>^N5+e!#52(%08@p&Rdnw!w;AJi_~<;`~vSWbyt~mezCa!}`Mn5qBi+r^b((Q0Fs+ z%`a+E=J{-Ryx34~L zAM0bIe9spJHXT{5p zIV?=z$JEC>!72hs3-9kVWYE+7_BlcJurPw3-@Xt(YD9CBj91ydc+j;|;nNG+y|xFe zGx7Sa&M2>_nK27KB`xr2#5L6_%Yy7-VHVJy<@g=1*})#?vV7;)Gnh`VoWa>ceCy%U z%dB76@2z^Y+?cduDdP$18KSxJcU$G)+D{ z{VdL&pYKKSn7g-3bvoEHaCkrp$iw=)7VmkOF2yYN89kb>kC6-b<58f*_l~iehO>vi z%#WSzY5J!USxS@1{qts<_by9Q(_PFZ?iB8lb_#oa8~#Jg@Ly>jApb~?+$)ik!vE{( F{{vrT`pN(R literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgboost_quantile/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgboost_quantile/1027_ESL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1420ed19cee7c49ff1de872221fa8e2562a0e866 GIT binary patch literal 30010 zcmd5_3wTpiwoXfdP{1ijIzg+J3P-#=nx@Zya(a?BX-Zp4fwT>JOI~f$CTU61rUg;J z`9@$y#Tg%U2DPXojN>3&m6wc02ar)2N&t zi+ARIO1?YSyz!HvpOU}R?VU2IxSN!SB*~JSCW2j(WRPS~id4*5O2wQ`vS_@3bi@de zNHlbOJp4N`Zoq(LYEdu;&(WjaTVDBsl#NV3@WLy{Nz;}F>y0PAAPxIpS=ZfpjLc8` zSG(uiPX#z|no=>RgE-*Q$Q+LCI=%hHKYvM{y6c8XC5}_%RCD#B*WWosuC(m9@7-}< zkw-^WL5O zpD#9y`uXjDT^Jp&ZWMX({=jKK&tU679B^r54yC=>Pwv^ek4*b(+@Si}kH}rCpGcG1 z_LEaqHhf)lct5#r{`*Hy7wi{^17{%>b9EpNxHK|{AGT(_q-;;_TJ$bIK8 z-jw&J?d1CV)~&s7`gYRl9$&kq_FVxEoP|`(=^zfcRFFfyB)>!=rpIP#9*h|8ooxq6 z_}jT*+D8M$6YuZ(5WO8)>+MCzqa;%Ph~)I8Vq}4I5i%~NkrT%RWO>KN z*QVX`9l0jqlZp}doh2umUrVyo{fqpraQf&8)6SCPE~(dEyz``h^PHtr%;_KsxKz;j za!I*wVnWV~KCJo-x$C{ybs$bW?8rv@p*XRu)9@x8k|SRY7Mc<`J30u+k_!S|n_n7& z0dH#j{&-LvSbyh(hYNRiC3Mh@RhYc%T@X^9TfCapHYPk#OFVcN`TU041~;?JhFpf)FGP zL4X&8FMqdf%3ZrIk#|4z*7C>PKaxGmmu6L+{*hedy|LltcQ2A3zw;Y)?y2ttIB=Fy zF{guqfJ;Mg2n<5!kuQ$ZK}cT%?u!@iyf{!So0t2+F(f=v+y1!#0Vtu(m>_^57la^b z2m-tyy#BN49{1l9^6MXcVQ$Z?LHVlNQ_hW9oREJ+wPH!vtpoEfbd>e{clb{N95@T9 znA1T)z@;HL1O_3$=bSoC*0;{B6Da@h#=5rSP+7wAqen-uy6jL)t!o@ zeD5;l!N*UI%+Go$>5X&nuKB~P31_O4@`*W1Mi=cJoR5DQI18zm(?LPNr6D*3P72;t zOHc5Ekb2YUV`x%fqd<2FoB?zN+763CkTe7XUKBRI|8i&V*p&P=73&7+E~MuFR=Y{| z>#uLlKYfWZyzt=7`M2(xAbT_SMu8}B7E&>%gQ9>-BXd|H>Kgd$j^m`~&F97}fhls0 zebJE-Umqiz6WX1_D~^*VHGA)zckr-4cjPRkVonEfz@?En44>6un~{COyX*deS$hp# z-mDY%>z2Qiod5ExYtB46?{n`zUK=rGMOT+V9Jmsxn5zSEz@?EnEFbf?2Q!m*l5O8i znKig^FPZ+-BX8fD@&QSGJNKubzWxU(ZaBSR+Kvwe;=q+i#atbT11^os;gvH>hBj<> zlf;C!N1pJWpSHBF_klGx$;iS%o4U>)Qj&v4yuW1Q?W?BYeUh`3ia8y`0hdPR@W9~3 zWvlmk$uIuCrRRC|Rx)Ay!ueD8zfE@D^5BJS`EQcze)Gi6{9mjWhy!OS6>~a>11^os zVfFTct@+l?SsSYd15R1=*GtTpBcANhy!aO6>~Z? z2TU57L-GSl=I^_A19^1I%Co9JzDhn``{kv%&#xzwe*CE|XXmTrzef#S_sxoz1UPV( zQZc84IN;LA9B%m`>*uc@J4B9aJ?K!l4v`Nee={j}&mrqTAer)_$ zfCHx~6>~at99Vn5G%|-Y=g$h+sa`TYeay4Z%mx+-EjxVjHO3U zlk+|j;J|4}#heb}fJ-BD*hD;hZqq+|NRjHH#5X?eCRe;Q_38Aq9&&Z!UpiKLddOAB z&%agnC!w*)SxUv64&s1IBXccunZx%#7TmH;)JryREC25OXS&IWyVh^JZA1@Aoq1!( z{71XVxru2v-2Ri$o{qDWia8y`0hdPR!0iLC<-T%^LQ?h<^ug~QdujWtH}sIV4u~6? zv9*Uhp_^)4)X`18Gj_b|P@GU4I7_LR(?M~-rIF)si2UON&h`W3v0ru{{%lJRS^90Q z=GjfXWap9p{mXy5yU8uPB~P!Z*dx#%I7_LR(?J|?X=DzEPdF)e+w*Uvu5;hgA4&s1IBXj7QcYdqqvyVyDszdW7 ze?35Md1l&kjjw%7rtbUaw623aRgE-*Q$Q%YQc<1(G?oY_0 z=YC(a|H47?(p&GF3_Cv|HzlPlIR9KPx&O&Oj9N5Chy$l76>~a>11^osfjcX?mgg-` zcmB^ex<|%)=loiq^GM2AZ~nfA2Mybp=AHd%++U^t9P540GqS7Ucc}trK3s`Z%+*1+ z1*`9uMvlYmKm0o7xsH3i?y;*j-`!p8J$kA~+p%Gq_k#VYB~1#QH^tky{L@n-1vqde zQZZKt;($vdbKusaF<;Xb`8ND1uu~DK$`6upVsYM%0|Uhg@7*yZY2uXDUV6{tx#gh1 z%)r?&yBnm7B7jRnMBpwTx=1uUP82s>EQ%j4zH0!g0DcL~9}&KS=sJg_GG|zEiResuyksI* zzwY!I4H^O3xNa2f0l37P~=mk4Ta@E?zpo zZm=*<2~N9|!2=*qZ8UT&5-`t24w^Z*Vp{EH1V@(QI`!TbwRylY=oB zYHDh=I1P1{8o7*-jIcV~7H6x$#?<4J`!%ydXmWDDW=;rAPVU#t3!%x${hEppnw*@c znJw&*f|VzjoHjNtVw2fYV{78VG_=|3u4#1DxZDOeQ^#;t(hPxVVS}6KL4mKaS!!9O zp^U(3t*d7x!x)Llf?~?lB(XRw&boFsx<-rJP$Qe15vD9NOd0EmV4b;P>=Yr(GA6U} z6{s^K49|?op?PLZ4$U)Ta%i3zlk>vxWPKQBZZ_AjVPvWEMeG>ii&!##kfpG9G~NR27;+aVbnf(E-7X!-H8ikmyJN7(5$=U~p;FFfcBgT|A(bgNaU*6y&xV zQ6!?G^T$71^3Xbl5NlC0;mIZB7A}q2E&p81L+cz$tWN|qJdK3VaB0+NyqTT{w{|$O zc^JUq86yOTOQVMK&$@CMjrUDl@^^8>s$US^8vqY_ES+tlpTcL4qH8+*E5-Bj`4Pnb zB>^g)8bV&;(kt*Xi}4c1^{?WImu@7sj%2(HN(3Q1@I6QxHD0inepHxE)=7xBMgc0G z`a!6;G-_0TTjuz@TpCZT|0S_Anej3x)5CfhB#jy`*h@buc)58ru}cc5cy0%w;?k&5 z{a)trz1(^;(RCB!Wl$=I^)g5rHD0inepK-C-CKx#DS(P6ZxAXjjT+VOrGoF}_S=Y$ zZ)Lm;%GI!521%pF3-;2F3SRCWLwu47sCae;q2ki0QF&fwbMpX<%dXpr<7teSLAe;# z%OGjgc$k+yRPge^IO613K*h5!2o;w`jY_XqIrjkQ$Q{I40)X%o3j)HWQ3Lr_!#uE) zcM|8v0~nr5L14HvYB2sIH0#UtZ zY*qj=p36YUxU_G`xjdiO-$i(*0y3V#K*+eXZ^#NB^5$v8uDbyl&s!j5T-q08*dy|g zz4=5J3CR7vB@yV`u+Bcv1eY>lGIHCe%JTrliacSR>`F;C< znmAbq$ap>iA>-1%A@lq8ks{)(29WVA1VYB8eM8RUwe!hh;=C4+@!SJK#-)8j=8vkg zCB)(qK*lo;2pN|~jqE>gkp&E<^JT=cG62N))(8-nUJ)Swv{(3G4wHeg4J)osA%0T7-CKz`xUh<*)Yfw8f#Tt@{4iA zZawk1J|ze0bnY=8TZ8cXwR!M{&gBdx*_*QN36;Krlam<8;RP zfRcOk$%%wG`B(SC5uS8Vv%z6DxTNPMFHEyK%$BwqQc$*ldd?ruv~!N(0B@r2P#h7l5K={S1T-F>xqP+RtFVv=5FLP}I+`zU&eX8&K5G zu>K?#jvH9k&%nH78IBxO*w4^dRt_g1sM60gd`@P`Z~{Sv{R|!Co)7^+m42q-b22Af z1en5ph7NL1h=8C1R;hB?ufhsPO7QGq){5 zg8Pg68Qy=f1IG?5yDGr+1(Hz6zQU^l&D>N8h3zlxXK=wgF*u8mYB8Xb866#3Ee3Q1 zW<82q!nX=+J60h>jvUI<(qVrvpbb(a>SlVn8P&Iy$sk4Cq8PvfzFhVjTlI zQH?CP(iS5+QH?CPMHeGFdC~P9S}g{R71cn`n0z$`I{)4w9+cqHevSpNdVvG2;L@uH zE-+Yl;8$ae_^KD2R)b6X*>nH>P`t$h%la7@-9yD8IcX3WY=4A~V}0Uqhq3G}R~(C1 z5Cqrau$tf=EHVi^|7|ihSWIpt4~IzKp~dlh;=ZGyqi=)T?X((O-1M>cLO6d;GdV2= zcasxNq!*^ytVXB7*^Umro7)XeXOjmJf}8Q^raY3$vnRtHgEVBv^5GXwmc!R+kw`j> zs;HPzIfc2rFFm7AM?X)Um7=Jb@IUmMPElEuvW`+yR2`+Nqh(5EK)DAA3w`C&Jypy* z{F|^I=9hLYMo(K8mne` z>a=!NW>Yj9?G-kQMz=uQ&{S8MA#-U}#SKPzTZXYRn~`O@tVX-qt)HdMfiib>xTU%>gRyZGyG`DEJBpv`5~sGn#+U(=>2QK5e5$Aow~sxr$OjG4tuyV!AF zm`?24Zh=9gZZ~PNZKll9P<~8P~ygD?2n- z!MMhnbA!p=7H<9NiuL-lT3%tRlDD;nS>H9lo~ahrpbKKn&7`r(s^o6iGnL6;pA>3+ z=P^`O)EY9&YE7B2H)7UUF&%c3N_b+>N`H0Yu|SYuVCKs* z!0*e;T|zz**7*zcq4)Z9uF)jdh2Hl$bS;L;vQ|ChSWz4yCmEc>Y^}C8+p04!Bd2Q5 zfckWH^e)q#CCGpb~jGx&YB2PfyQ-w?Z{t338mR4?Q4OVCn1Ys6Jprs}`5nvUpk=4+Sy+ z54~#WF^!&(qu_8I@}5yeLI@euZ&Cmg^gy+tcP8*CX6z5{Wl&VW`~`?}T3P%$RcVkt zL=#o!`vERR1$@h9WzmjOT4N^Tzn1d&{3#XmhYqzS@Vi|`YpQZsd=*tbo2{>~fDRHk z4L|ykenJfo@eKT~G2zJd`9n2LfVM%3Y&;_SqhkG0m1J@LRFyZd?Zfy(2N887_NPXV zno#F6g)K8sxjNTZpS`d3A>3l8ufB{v)>no?S!VRrS6J@olfQIZDoe7t_|5KPeWi$B zVXm(}YhUX_?V)x0>TBz3eNY`$YV_4dl{@1kbPZd{o4prI*?0yA$+GLD?rT^Qnr` z6y!4P!Oln6{O$r!C3i;5oOe^oN}qjkB>N`5J$+UWGrh)VUm9eO&4)5%@6VeQ!QGD{ zoehLkODpStU4e|PkDCwCM~Oebm!Nll{)5iZP&qz(sxduqJ`J{~L(9uQq3S42>jEbJ zb;>zG_VlNh_kE$~cgR<`O!QYpsbTe1l(rx*ApU3}!v7v9k*QMX-b*$;9~I@Y_FAf? zoaHlo20$U_Qiv$ONUP{ zXoqbNSZAX3UAVTC;;a%4PY^uV>JmUO9uc zhxj(Yr{%JXO&XQ2|oLhAp7XsM@}y)N{lSNmdYz<{foMNsy);UzrD5` zhAN5=Yaf04%%t0=sFk&^Q&y+L(4j*ryq==R3%`BJAg;50_9|_Vebnt!+JIV4rD5$= zRA~VW-OFm9_EJhE_t}?6vZv>tfO~bYD7O3TRRu5;hQmj$Df#yXkpq=rgyN^7)KS^P z-B>f*N6~^7#y@AT4YsEzXx@4d4LT*A4}}N#K9zta;;W^cXnoG>53sR-pMSj4SBs0Z z?LgDu!z-K3+H>>0ARc}9maeW8_6#f@kOK0sKCeT29=c1>i+x&;;_9R2JpO1DDAB!R zq^4o*;V=DTW_yadTO>_wHaO?ESlp9L&CQeOOWdjKCGAw^`ZoO6nc+X(e4YFwI&-f` KR1W{Iss9ghWhq<$ literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json new file mode 100644 index 0000000..2a024ba --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json @@ -0,0 +1,123 @@ +{ + "artifact": { + "digest": "sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765", + "id": 9257853524, + "workflow_run_id": 31925701435, + "workflow_url": "https://github.com/jxucoder/openboost/actions/runs/31925701435" + }, + "dataset": "1027_ESL", + "file_sha256": { + "benchmark_outcome.json": "d0640e3d74a7d14c615b61963121b873d8500d9664d07d78fb98234b7a14f3e4", + "datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "openboost_manifest.json": "70a1cc6c05fa5109d59bd319e3a0a3d8d76307664f40c7b0147f5283b5cae3c5", + "raw/catboost_quantile/1027_ESL.parquet": "461258ad37aa5d7fb4e3bb4a7659832d0bca7b6f2e714776746d8539466696b9", + "raw/ngboost/1027_ESL.parquet": "a87222737ce8bea41532f91af7f1fb0e826dae7aac0430f57078de199fc25194", + "raw/openboost_cpu/1027_ESL.parquet": "737edc08cab22cb3f28b1b6a237b718ce698f38aa03046f792b63ea11a1e42ae", + "raw/xgblss/1027_ESL.parquet": "fd24636edaa89d83217f26788da15e0d5a005b80cd1ba13940e7ea3af585d1c3", + "raw/xgboost_quantile/1027_ESL.parquet": "8e09f343e54ba328094954c6477c2b4a2e62a67466d3f071d0d5ba627ce50692" + }, + "folds": 5, + "means": { + "catboost_quantile": { + "coverage_90": 0.7172312140464783, + "coverage_90_abs_error": 0.18276878595352175, + "crls": 0.5825405189879156, + "crps": 0.32486018716094617, + "interval_score_90": 4.06573659420003, + "log_score": -1.3190115168420253, + "pit_ks_stat": 0.14662502054983878, + "rmse": 0.5901363119168295, + "train_time": 2.4480895519256594 + }, + "ngboost": { + "coverage_90": 0.8137176394462585, + "coverage_90_abs_error": 0.08628236055374147, + "crls": 0.8919766693054967, + "crps": 0.30774525268048586, + "interval_score_90": 2.7370853096534686, + "log_score": 0.676160510947884, + "pit_ks_stat": 0.10549263800016417, + "rmse": 0.5549747191688402, + "train_time": 2.443098306655884 + }, + "openboost_cpu": { + "coverage_90": 0.85472332239151, + "coverage_90_abs_error": 0.05104986906051636, + "crls": 0.9305498802189568, + "crps": 0.30052704288122417, + "interval_score_90": 2.436057677703067, + "log_score": 0.7100775390578737, + "pit_ks_stat": 0.08936585621575435, + "rmse": 0.545923151301676, + "train_time": 1.3844860553741456 + }, + "xgblss": { + "coverage_90": 0.7951819777488709, + "coverage_90_abs_error": 0.10481802225112917, + "crls": 0.8545918044143812, + "crps": 0.30426339893126575, + "interval_score_90": 2.879351361511471, + "log_score": 0.6375427933712055, + "pit_ks_stat": 0.09914356858963747, + "rmse": 0.538414204542853, + "train_time": 0.15186233520507814 + }, + "xgboost_quantile": { + "coverage_90": 0.6127077579498291, + "coverage_90_abs_error": 0.2872922420501709, + "crls": 0.5757246011831073, + "crps": 0.32567075673952595, + "interval_score_90": 3.980709384171024, + "log_score": -2.2200916983373404, + "pit_ks_stat": 0.22044577530811366, + "rmse": 0.6177832060042735, + "train_time": 0.38692564964294435 + } + }, + "openboost_source_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b", + "paired_openboost_relative_percent": { + "catboost_quantile": { + "coverage_90_abs_error": -72.06860635737965, + "crls": 59.73994080886593, + "crps": -7.490343612856022, + "interval_score_90": -40.08323900819789, + "log_score": 153.83406664696452, + "pit_ks_stat": -39.05142800278187, + "rmse": -7.492025100361004, + "train_time": -43.4462659143693 + }, + "ngboost": { + "coverage_90_abs_error": -40.83394481457232, + "crls": 4.324464107732061, + "crps": -2.3455145892229403, + "interval_score_90": -10.998109225485326, + "log_score": 5.016120811675712, + "pit_ks_stat": -15.287115850098193, + "rmse": -1.6309874224037508, + "train_time": -43.33072674143711 + }, + "xgblss": { + "coverage_90_abs_error": -51.296668297930594, + "crls": 8.888228907908463, + "crps": -1.2280004966636278, + "interval_score_90": -15.395609224144986, + "log_score": 11.377235605333752, + "pit_ks_stat": -9.862175139522938, + "rmse": 1.394641280907218, + "train_time": 811.6717805665942 + }, + "xgboost_quantile": { + "coverage_90_abs_error": -82.23068305074477, + "crls": 61.63107817638637, + "crps": -7.720593064612176, + "interval_score_90": -38.80342816810849, + "log_score": 131.9841536090451, + "pit_ks_stat": -59.46129786753722, + "rmse": -11.631921037053978, + "train_time": 257.81707846242597 + } + }, + "protocol_mode": "official_quality_shard", + "result_rows": 25, + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 4831b70..fda0779 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -78,6 +78,23 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. resolved dataset registry, both raw Parquet files, and a descriptive summary. OpenBoost's mean CRPS/RMSE/90% interval score and time were better on this shard, but mean log score was worse and CRPS won only 2/5 folds. +- Clean strong-baseline run `31925701435` completed all 25 expected rows for + `1027_ESL` at source commit `cea891a` and pinned ScoringBench commit + `a938a667`. Artifact `9257853524` has digest + `sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765`. + The source checkout had no porcelain changes, and every frozen input/result + file is checksummed in + `benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json`. +- On that one diagnostic shard, OpenBoost ranked first on mean CRPS, 90% + interval score, absolute 90% coverage error, and PIT KS. Against native + XGBoost quantile it reduced those metrics by 7.7%, 38.8%, 82.2%, and 59.5% + respectively and reduced RMSE by 11.6%. Against Gaussian XGBoostLSS it had + 1.2% lower CRPS and 51.3% lower coverage error, but 11.4% worse log score, + 1.4% worse RMSE, and about 9.1 times its warm per-fold training time. +- Those results define a useful hypothesis, not a win: the shard has one small + dataset, one seed, correlated folds, unequal model-specific budgets, CPU + only, and no confidence interval. Timing also varied materially between two + otherwise equivalent Actions runs, so it cannot support a speed claim. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts @@ -137,8 +154,16 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. ## Risks and Follow-ups - Run the official full suite on Linux and submit the wrapper/results upstream. -- Use the `1027_ESL` sentinel only as a reproducibility/integration gate; inspect - its 5-fold artifact before deciding whether OpenBoost merits broader shards. +- Treat `1027_ESL` as consumed diagnostic evidence. Do not tune on it and then + relabel the result as held out. Use separate development datasets to test + Normal scale/log-score hypotheses, preserve CRPS/calibration guardrails, and + make the final decision on untouched datasets or the complete suite. +- The acceptance bar is stronger than NGBoost parity: OpenBoost should rank + first or statistically tied on the primary proper scores, beat the strongest + XGBoost-family baseline on a majority of paired datasets, and avoid material + regressions in density score, interval score, calibration, RMSE, failure + rate, or resource use. Report per-dataset paired effects and uncertainty, not + only macro means. - Run a separate large-sample curve on at least three real ScoringBench datasets. - Add CPU/CUDA prediction parity before interpreting a CUDA timing result. - The first artifact identified the PR merge commit but not the source-head SHA. From db7d430649e779287d06539a0527848ffbf815b6 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:21:13 -0700 Subject: [PATCH 27/49] bench: isolate tuning from held-out evidence --- .github/workflows/scoringbench.yml | 50 ++++++++++++++++++- .../1027_esl_strong_20260816/README.md | 40 +++++++++------ .../1027_esl_strong_20260816/summary.json | 25 ++++++---- benchmarks/scoringbench/README.md | 48 +++++++++++++++--- benchmarks/scoringbench/run.py | 28 +++++++++-- .../2026-08-15-scoringbench-integration.md | 23 +++++++-- tests/test_scoringbench_provenance.py | 37 ++++++++++++++ 7 files changed, 211 insertions(+), 40 deletions(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index 4fbe9c9..c5e623a 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -19,6 +19,7 @@ on: - smoke - quality_shard - strong_shard + - development_shard dataset_name: description: Exact ScoringBench dataset name for a quality/strong shard required: true @@ -29,6 +30,26 @@ on: required: true default: "500" type: string + learning_rate: + description: OpenBoost learning rate for development_shard + required: true + default: "0.01" + type: string + max_depth: + description: OpenBoost tree depth for development_shard + required: true + default: "3" + type: string + reg_lambda: + description: OpenBoost L2 leaf regularization for development_shard + required: true + default: "1.0" + type: string + min_child_weight: + description: OpenBoost minimum child Hessian for development_shard + required: true + default: "1.0" + type: string concurrency: group: scoringbench-${{ github.workflow }}-${{ github.ref }} @@ -151,6 +172,31 @@ jobs: --n-trees "${N_TREES}" \ --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Run OpenBoost development shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'development_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + N_TREES: ${{ inputs.n_trees }} + LEARNING_RATE: ${{ inputs.learning_rate }} + MAX_DEPTH: ${{ inputs.max_depth }} + REG_LAMBDA: ${{ inputs.reg_lambda }} + MIN_CHILD_WEIGHT: ${{ inputs.min_child_weight }} + run: | + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees "${N_TREES}" \ + --learning-rate "${LEARNING_RATE}" \ + --max-depth "${MAX_DEPTH}" \ + --reg-lambda "${REG_LAMBDA}" \ + --min-child-weight "${MIN_CHILD_WEIGHT}" \ + --development-run \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Verify smoke artifact if: github.event_name == 'pull_request' || inputs.mode == 'smoke' run: | @@ -161,14 +207,14 @@ jobs: python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4 and m["outcome"]["status"] == "complete"' "${MANIFEST}" - name: Verify quality-shard artifact - if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' + if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' || inputs.mode == 'development_shard' run: | MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" test -s "${MANIFEST}" test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); expected=25 if sys.argv[2] == "strong_shard" else 10; assert m["protocol_mode"] == "official_quality_shard" and m["official_protocol_compatible"] and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False' "${MANIFEST}" "${{ inputs.mode }}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); mode=sys.argv[2]; expected=25 if mode == "strong_shard" else (5 if mode == "development_shard" else 10); protocol="development_tuning" if mode == "development_shard" else "official_quality_shard"; compatible=mode != "development_shard"; assert m["protocol_mode"] == protocol and m["official_protocol_compatible"] is compatible and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False' "${MANIFEST}" "${{ inputs.mode }}" - name: Upload benchmark artifact if: always() diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md index 4a6e962..c0968cb 100644 --- a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md @@ -1,6 +1,6 @@ # 1027_ESL strong-baseline diagnostic -This is a clean, official-protocol-compatible five-fold ScoringBench shard. It +This is a clean, ScoringBench-shaped five-fold diagnostic shard. It compares OpenBoost CPU against NGBoost, native XGBoost multi-quantile, Gaussian XGBoostLSS, and CatBoost MultiQuantile. All 25 expected dataset/model/fold rows completed; the OpenBoost and ScoringBench checkouts were @@ -9,14 +9,16 @@ and CI identity are in `openboost_manifest.json`. Lower is better for every displayed score except raw coverage, whose target is 0.90. `Coverage error` is the fold-level mean of `abs(coverage_90 - 0.90)`. +The CRPS values are ScoringBench's histogram-based scores after every model is +converted to its common `DistributionPrediction` representation. -| Model | CRPS | Log score | CRLS | RMSE | 90% coverage | Coverage error | 90% interval score | PIT KS | Train seconds | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| OpenBoost CPU | **0.3005** | 0.7101 | 0.9305 | 0.5459 | **0.8547** | **0.0510** | **2.4361** | **0.0894** | 1.3845 | -| XGBoostLSS | 0.3043 | 0.6375 | 0.8546 | **0.5384** | 0.7952 | 0.1048 | 2.8794 | 0.0991 | **0.1519** | -| NGBoost | 0.3077 | 0.6762 | 0.8920 | 0.5550 | 0.8137 | 0.0863 | 2.7371 | 0.1055 | 2.4431 | -| CatBoost quantile | 0.3249 | -1.3190 | 0.5825 | 0.5901 | 0.7172 | 0.1828 | 4.0657 | 0.1466 | 2.4481 | -| XGBoost quantile | 0.3257 | **-2.2201** | **0.5757** | 0.6178 | 0.6127 | 0.2873 | 3.9807 | 0.2204 | 0.3869 | +| Model | CRPS | RMSE | 90% coverage | Coverage error | 90% interval score | PIT KS | Train seconds | +|---|---:|---:|---:|---:|---:|---:|---:| +| OpenBoost CPU | **0.3005** | 0.5459 | **0.8547** | **0.0510** | **2.4361** | **0.0894** | 1.3845 | +| XGBoostLSS | 0.3043 | **0.5384** | 0.7952 | 0.1048 | 2.8794 | 0.0991 | **0.1519** | +| NGBoost | 0.3077 | 0.5550 | 0.8137 | 0.0863 | 2.7371 | 0.1055 | 2.4431 | +| CatBoost quantile | 0.3249 | 0.5901 | 0.7172 | 0.1828 | 4.0657 | 0.1466 | 2.4481 | +| XGBoost quantile | 0.3257 | 0.6178 | 0.6127 | 0.2873 | 3.9807 | 0.2204 | 0.3869 | On this shard, OpenBoost has the best mean CRPS, interval score, coverage error, and PIT KS statistic. Relative to native XGBoost quantile, its CRPS is @@ -24,12 +26,19 @@ error, and PIT KS statistic. Relative to native XGBoost quantile, its CRPS is RMSE is 11.6% lower. It wins four of five folds on CRPS and all five folds on RMSE, interval score, coverage error, and PIT KS. -This is not an overall win. XGBoost quantile and CatBoost have much lower -density-based log score/CRLS while also producing severely under-covering 90% -intervals. XGBoostLSS has 1.4% better RMSE, 11.4% better log score, and is about -9.1 times faster in these warm per-fold timings; OpenBoost has 1.2% better CRPS -and substantially better interval calibration. The timing is not a scale claim -and differed materially across otherwise equivalent Actions runs. +This is not an overall win. XGBoostLSS has 1.4% better RMSE and is about 9.1 +times faster in these fit-only timings; OpenBoost has 1.2% better CRPS and +substantially better interval calibration. The timing is a cold/warm mixture +from one persistent process, is not a scale claim, and differed materially +across otherwise equivalent Actions runs. + +The raw artifact also contains ScoringBench's reconstructed-density log score +and CRLS, but they are intentionally excluded from the comparison table. +Out-of-support targets are clamped to a quantile model's boundary density, and +CRLS is integrated over model-specific support; the upstream implementation +therefore does not make those two metrics comparable across these parametric +and finite-quantile predictions. Gaussian analytic NLL requires a separate +audit, and any cross-model density score requires a common support/tail rule. The result is one small dataset, one seed, five correlated folds, unequal model-specific default budgets, CPU only, and no paired confidence interval. @@ -40,4 +49,5 @@ developed elsewhere and not re-labelled as held-out evidence. Source artifact: [GitHub Actions run 31925701435](https://github.com/jxucoder/openboost/actions/runs/31925701435), artifact `9257853524`, digest `sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765`. -`summary.json` records every frozen file hash and the unrounded means. +`summary.json` hashes every frozen raw/result/provenance input except this +README and the summary itself, and records the unrounded diagnostic means. diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json index 2a024ba..7f6bb8b 100644 --- a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json @@ -17,6 +17,21 @@ "raw/xgboost_quantile/1027_ESL.parquet": "8e09f343e54ba328094954c6477c2b4a2e62a67466d3f071d0d5ba627ce50692" }, "folds": 5, + "metric_comparability": { + "cross_model_primary": [ + "crps", + "rmse", + "coverage_90", + "coverage_90_abs_error", + "interval_score_90", + "pit_ks_stat" + ], + "diagnostic_not_cross_model_comparable": { + "crls": "Integrated on model-specific support; upstream implementation documents that values are not directly comparable across differing bin grids.", + "log_score": "Finite quantile support clamps out-of-support targets to a boundary-bin density, creating a tail-boundary artifact." + }, + "timing": "Fit-only cold/warm mixture from one persistent process; not a scale or steady-state benchmark." + }, "means": { "catboost_quantile": { "coverage_90": 0.7172312140464783, @@ -75,43 +90,35 @@ } }, "openboost_source_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b", - "paired_openboost_relative_percent": { + "relative_percent_of_fold_means": { "catboost_quantile": { "coverage_90_abs_error": -72.06860635737965, - "crls": 59.73994080886593, "crps": -7.490343612856022, "interval_score_90": -40.08323900819789, - "log_score": 153.83406664696452, "pit_ks_stat": -39.05142800278187, "rmse": -7.492025100361004, "train_time": -43.4462659143693 }, "ngboost": { "coverage_90_abs_error": -40.83394481457232, - "crls": 4.324464107732061, "crps": -2.3455145892229403, "interval_score_90": -10.998109225485326, - "log_score": 5.016120811675712, "pit_ks_stat": -15.287115850098193, "rmse": -1.6309874224037508, "train_time": -43.33072674143711 }, "xgblss": { "coverage_90_abs_error": -51.296668297930594, - "crls": 8.888228907908463, "crps": -1.2280004966636278, "interval_score_90": -15.395609224144986, - "log_score": 11.377235605333752, "pit_ks_stat": -9.862175139522938, "rmse": 1.394641280907218, "train_time": 811.6717805665942 }, "xgboost_quantile": { "coverage_90_abs_error": -82.23068305074477, - "crls": 61.63107817638637, "crps": -7.720593064612176, "interval_score_90": -38.80342816810849, - "log_score": 131.9841536090451, "pit_ks_stat": -59.46129786753722, "rmse": -11.631921037053978, "train_time": 257.81707846242597 diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 2c1070d..3aa5023 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -30,11 +30,18 @@ ScoringBench protocol. The acceptance comparison set is: OpenBoost is a **good ScoringBench model** only when the completed full suite places it first or statistically tied for first on primary proper scores, it beats the strongest XGBoost-family baseline on a majority of paired datasets, -and the result is not purchased with a material regression in log score, -interval score, calibration, point RMSE, or failure coverage. CRPS is the first -optimization target; log score, interval score/coverage, RMSE, failures, and -training time remain explicit guardrails. A single shard decides what to debug, -not whether this target has been achieved. +and the result is not purchased with a material regression in interval score, +calibration, point RMSE, or failure coverage. CRPS is the cross-model +optimization target; interval score/coverage, RMSE, failures, and training time +remain explicit guardrails. A single shard decides what to debug, not whether +this target has been achieved. + +ScoringBench's current reconstructed log score and CRLS are not acceptance +metrics for this comparison. Finite quantile support clamps out-of-support +targets to a boundary-bin density, while CRLS is integrated over each model's +own support. Compare Gaussian models with a separately audited analytic NLL; +only restore a cross-model density score after validating one common grid, +resolution, support, and tail rule. The diagnostic deliberately uses the budgets registered by ScoringBench rather than forcing every implementation to share one arbitrary tree count: @@ -135,6 +142,11 @@ duplicates, captured model errors, missing rows, and non-finite core metrics, then exits non-zero when that audit is incomplete. The failure report remains in the uploaded artifact and is evidence, not disposable CI noise. +The manifest's `official_protocol_compatible` field means only that the run has +ScoringBench's 3,000-row, five-fold, one-repeat shape and was not marked as +development tuning. It does not certify leaderboard acceptance, immutable +dataset bytes, equal compute budgets, or statistical sufficiency. + ## Official quality track Run the official default: five folds, one repeat, at most 3,000 rows per @@ -163,6 +175,30 @@ Run one strong-baseline diagnostic before changing model behavior: --output-dir benchmarks/results/scoringbench-strong-diagnostic ``` +After freezing that baseline, test candidate OpenBoost settings on a different +development dataset. `--development-run` deliberately makes the manifest +ineligible for official evidence even though it retains the same five folds and +metrics: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu \ + --dataset-name 1028_SWD \ + --sample-size 3000 \ + --n-folds 5 \ + --n-trees 250 \ + --learning-rate 0.04 \ + --max-depth 2 \ + --reg-lambda 1.0 \ + --min-child-weight 1.0 \ + --development-run \ + --output-dir benchmarks/results/scoringbench-development +``` + +Record every tried configuration, including failures. Select one configuration +using only development datasets; do not repeatedly inspect the held-out suite. + Use `--dataset-index N` or `--dataset-name NAME` for resumable shards. Use `--list-datasets` to display the validated list. Do not tune OpenBoost on the test folds. If hyperparameters are changed, apply the same declared search @@ -245,7 +281,7 @@ OpenBoost should claim value only after all of the following are true: - the wrapper and results are accepted upstream by ScoringBench; - quality is reported across the full suite, not a selected winning subset; -- paired fold-level CRPS/log-score/interval-score differences include +- paired fold-level CRPS/interval-score differences include uncertainty intervals or the upstream statistical ranking; - CPU and CUDA predictions pass a separate parity gate; - a scale curve uses at least three real datasets and multiple data sizes; diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 1e47b90..61cb59c 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -328,6 +328,8 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--n-trees", type=int, default=500) parser.add_argument("--learning-rate", type=float, default=0.01) parser.add_argument("--max-depth", type=int, default=3) + parser.add_argument("--reg-lambda", type=float, default=1.0) + parser.add_argument("--min-child-weight", type=float, default=1.0) parser.add_argument("--n-quantiles", type=int, default=99) parser.add_argument( "--xgboost-rounds", @@ -401,6 +403,14 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Use sklearn diabetes with 2 folds; validates integration, not leaderboard evidence", ) + parser.add_argument( + "--development-run", + action="store_true", + help=( + "Mark this run as tuning-only evidence that must not be submitted or " + "reported as a held-out leaderboard result" + ), + ) parser.add_argument( "--list-datasets", action="store_true", @@ -464,6 +474,10 @@ def _model_parameters(args) -> dict[str, dict]: "learning_rate": args.learning_rate, "max_depth": args.max_depth, "n_quantiles": args.n_quantiles, + "model_params": { + "reg_lambda": args.reg_lambda, + "min_child_weight": args.min_child_weight, + }, } return { "openboost_cpu": {"backend": "cpu", **openboost_common}, @@ -564,8 +578,11 @@ def _write_provenance( and args.n_folds == 5 and args.n_repeats == 1 ) + official_protocol_compatible = official_shape and not args.development_run if args.smoke: protocol_mode = "smoke" + elif args.development_run: + protocol_mode = "development_tuning" elif args.sample_size != 3000: protocol_mode = "scoringbench_scale_extension" elif official_shape and ( @@ -583,11 +600,16 @@ def _write_provenance( "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "protocol": "ScoringBench", "protocol_mode": protocol_mode, - "official_protocol_compatible": official_shape, + "official_protocol_compatible": official_protocol_compatible, "warning": ( None - if official_shape - else "This run is not directly comparable to the official 5-fold, sample_size=3000 leaderboard." + if official_protocol_compatible + else ( + "This is a development/tuning run and must not be represented as " + "held-out leaderboard evidence." + if args.development_run + else "This run is not directly comparable to the official 5-fold, sample_size=3000 leaderboard." + ) ), "openboost_git": _git_state(PROJECT_ROOT), "scoringbench_git": _git_state(scoringbench_dir), diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index fda0779..99b5bc5 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -42,6 +42,10 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. XGBoostLSS, and CatBoost MultiQuantile wrappers with frozen package versions and their registered model-specific budgets. This makes NGBoost a reference, not the acceptance bar. +- Development mode exposes OpenBoost tree count, learning rate, depth, L2 leaf + regularization, and minimum child Hessian while forcing the manifest protocol + label to `development_tuning`. It preserves ScoringBench folds and metrics but + is intentionally ineligible for held-out or leaderboard evidence. ## Verification @@ -89,12 +93,20 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. interval score, absolute 90% coverage error, and PIT KS. Against native XGBoost quantile it reduced those metrics by 7.7%, 38.8%, 82.2%, and 59.5% respectively and reduced RMSE by 11.6%. Against Gaussian XGBoostLSS it had - 1.2% lower CRPS and 51.3% lower coverage error, but 11.4% worse log score, - 1.4% worse RMSE, and about 9.1 times its warm per-fold training time. + 1.2% lower CRPS and 51.3% lower coverage error, but 1.4% worse RMSE and about + 9.1 times its fit-only per-fold training time. - Those results define a useful hypothesis, not a win: the shard has one small dataset, one seed, correlated folds, unequal model-specific budgets, CPU only, and no confidence interval. Timing also varied materially between two otherwise equivalent Actions runs, so it cannot support a speed claim. +- A post-run metric audit found that the current ScoringBench reconstructed + log score clamps targets outside finite quantile support to the boundary-bin + density. CRLS is integrated over each model's own support, and the upstream + implementation explicitly warns that values are not comparable across + different bin grids. Both metrics remain in the raw artifact but are excluded + from cross-model conclusions. Use a separately audited analytic Gaussian NLL + for parametric-only density comparison; do not optimize OpenBoost against the + current quantile log-score artifact. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts @@ -161,9 +173,10 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. - The acceptance bar is stronger than NGBoost parity: OpenBoost should rank first or statistically tied on the primary proper scores, beat the strongest XGBoost-family baseline on a majority of paired datasets, and avoid material - regressions in density score, interval score, calibration, RMSE, failure - rate, or resource use. Report per-dataset paired effects and uncertainty, not - only macro means. + regressions in interval score, calibration, RMSE, failure rate, or resource + use. CRPS is the current cross-model primary metric. Density scoring becomes + a guardrail only after common-support handling is validated; report + per-dataset paired effects and uncertainty, not only macro means. - Run a separate large-sample curve on at least three real ScoringBench datasets. - Add CPU/CUDA prediction parity before interpreting a CUDA timing result. - The first artifact identified the PR merge commit but not the source-head SHA. diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index dc3f143..d0479da 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -181,9 +181,46 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): assert args.xgboost_quantiles == 50 assert args.xgblss_rounds == 100 assert args.catboost_rounds == 1000 + assert args.reg_lambda == 1.0 + assert args.min_child_weight == 1.0 + assert args.development_run is False parameters = _model_parameters(args) + assert parameters["openboost_cpu"]["model_params"] == { + "reg_lambda": 1.0, + "min_child_weight": 1.0, + } assert parameters["xgboost_quantile"]["num_boost_round"] == 100 assert parameters["xgblss"]["num_boost_round"] == 100 assert parameters["catboost_quantile"]["iterations"] == 1000 assert parameters["catboost_quantile"]["catboost_params"]["allow_writing_files"] is False + + +def test_development_parameters_are_explicit_in_manifest_constructor_contract(): + args = _build_parser().parse_args( + [ + "--models", + "openboost_cpu", + "--development-run", + "--n-trees", + "250", + "--learning-rate", + "0.04", + "--max-depth", + "2", + "--reg-lambda", + "3.0", + "--min-child-weight", + "5.0", + ] + ) + + assert args.development_run is True + assert _model_parameters(args)["openboost_cpu"] == { + "backend": "cpu", + "n_trees": 250, + "learning_rate": 0.04, + "max_depth": 2, + "n_quantiles": 99, + "model_params": {"reg_lambda": 3.0, "min_child_weight": 5.0}, + } From 56537d02e90a00af6a1c2c6a59c6d8604630b3bf Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:36:52 -0700 Subject: [PATCH 28/49] bench: record rejected learning-rate hypothesis --- .../1028_swd_lr_sweep_20260816/README.md | 42 ++ .../baseline_lr_001/benchmark_outcome.json | 21 + .../baseline_lr_001/datasets.json | 482 ++++++++++++++++++ .../baseline_lr_001/openboost_manifest.json | 158 ++++++ .../raw/catboost_quantile/1028_SWD.parquet | Bin 0 -> 30026 bytes .../raw/ngboost/1028_SWD.parquet | Bin 0 -> 29934 bytes .../raw/openboost_cpu/1028_SWD.parquet | Bin 0 -> 29993 bytes .../raw/xgblss/1028_SWD.parquet | Bin 0 -> 29945 bytes .../raw/xgboost_quantile/1028_SWD.parquet | Bin 0 -> 30011 bytes .../candidate_lr_003/benchmark_outcome.json | 21 + .../candidate_lr_003/datasets.json | 482 ++++++++++++++++++ .../candidate_lr_003/openboost_manifest.json | 124 +++++ .../raw/openboost_cpu/1028_SWD.parquet | Bin 0 -> 29979 bytes .../1028_swd_lr_sweep_20260816/summary.json | 79 +++ .../2026-08-15-scoringbench-integration.md | 19 + 15 files changed, 1428 insertions(+) create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/ngboost/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgblss/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md new file mode 100644 index 0000000..177bad3 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md @@ -0,0 +1,42 @@ +# 1028_SWD learning-rate development sweep + +This is tuning evidence, not held-out or leaderboard evidence. The clean +five-model baseline used 500 OpenBoost/NGBoost rounds at learning rate 0.01. +After inspecting that result, one OpenBoost-only candidate changed only the +learning rate to 0.03. Both runs used the same ScoringBench commit, dataset, +seed, five folds, sample cap, Normal distribution, depth, regularization, and +99-quantile representation. + +The baseline disproved a broad win: OpenBoost had the best mean 90% interval +score, absolute coverage error, and PIT KS, but its mean CRPS was 4.9% worse +than native XGBoost quantile, 1.7% worse than XGBoostLSS, and 2.0% worse than +NGBoost. Its CRPS lost to native XGBoost quantile on all five folds. + +| OpenBoost setting | CRPS | RMSE | 90% coverage | Coverage error | 90% interval score | PIT KS | Sharpness | +|---|---:|---:|---:|---:|---:|---:|---:| +| lr=0.01, 500 rounds | **0.3551** | **0.6260** | **0.8910** | **0.0310** | **2.5598** | **0.0873** | 0.5801 | +| lr=0.03, 500 rounds | 0.3566 | 0.6278 | 0.8680 | 0.0320 | 2.6811 | 0.0959 | **0.5490** | + +Increasing the learning rate sharpened the distribution by 5.4%, but mean +CRPS worsened by 0.44%, interval score by 4.74%, PIT KS by 9.82%, and coverage +moved farther below 90%. The candidate improved CRPS on only two of five +paired folds and interval score on one. This falsifies the hypothesis that the +current CRPS gap is primarily caused by an update budget that is too small; the +0.03 default used by an older NGBoost comparison should not be copied into the +ScoringBench wrapper. + +The apparent 11.6% fit-time change is from separate Actions processes and is +not a speed result. ScoringBench reconstructed log score and CRLS are retained +in the raw Parquet files but excluded from the decision because of the known +finite-support and model-specific-grid comparability problems. + +Baseline: [run 31926255124](https://github.com/jxucoder/openboost/actions/runs/31926255124), artifact `9258026048`, digest +`sha256:495ac3c8c02c0846423131c4636dfde05f0da872b637bca2be9493b3c02aa8b4`. + +Candidate: [run 31926664340](https://github.com/jxucoder/openboost/actions/runs/31926664340), artifact `9258115955`, digest +`sha256:b3b77cbf691bef574b0a27c897b8a92a22e90a4c158f0f7e3ae556400d60e599`. + +`summary.json` records exact unrounded effects and SHA-256 hashes for every +copied raw/result/provenance file. The next diagnostic must separate point-mean +error from scale calibration, then test post-fit scale calibration or a +different scale objective without changing the consumed `1027_ESL` shard. diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json new file mode 100644 index 0000000..27af05f --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 25, + "observed_rows": 25, + "status": "complete", + "valid_rows": 25 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 25, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 25, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 25 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json new file mode 100644 index 0000000..2b79b92 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json @@ -0,0 +1,158 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "models": [ + "openboost_cpu", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31926255124", + "source_sha": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36", + "tested_sha": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36" + }, + "created_at": "2026-08-16T04:26:25+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 25, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "ngboost": { + "dist": "normal", + "learning_rate": 0.01, + "n_estimators": 500, + "n_quantiles": 99, + "ngb_params": { + "random_state": 42 + } + }, + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "n_quantiles": 99, + "n_trees": 500 + }, + "xgblss": { + "distribution": "Gaussian", + "n_quantiles": 99, + "num_boost_round": 100, + "xgblss_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": true, + "openboost_git": { + "changes": [], + "commit": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 25 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "official_quality_shard", + "result_rows": 25, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": null +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..31d1266a1a7164401c36518ab4fbad9d1e35aa67 GIT binary patch literal 30026 zcmd5_4SZA8*-uNsPznP^yHrtYrxy?`N&2ne=}k(~l(w{mCT)b3e6>xRq%ldGwg_WW zH{2IYw}HyGifp1zhKdU4v^eL>6q)R$EmYJpKoNPB&4Gxh?{n@)a+7n@1X@D#Q@Hmz z&vX9&=Q$tuob#NYR?^jq)TGqP)Fox9$*Gd563NVs>VZd3{ZccCzI5F?w)|)BqrX1< z{>dlrA4K1|zGZjOe+JR1{*UfEFz#KHDoIUCy?PGVO->z`I&Qj5%4^7^yiS^ArigUP z6p2JKVP+!ynVc|Y%o>d(oWtYAum1V%HOG*(<;|8AZuooVy?5Q>I)zp5z{~Kkyn52hJ9AAf?ecG^Bj(T6O+Cboh&ZEM2O55B+?5`+>jY zzKP1%3Y zz|qeWew6Yb^i@N~?~mU68G0Z;m+C(GsR##NLnh^QFb7f^fkR~=1_i)7W2^QE_ILHG zX9g3bDL5nvQhCqby@##Vhg-Sp>ZR?4HBH!07EaNsp$QeFphAf*vFRD?ufuxiJ@m{z~; z-lGXpX~xpEV<*cQXTa5>BVETt2=FD0yNoUu1V%q7jX*#UgwGU*ns$8qEA;NhripET z`T~u4ec#sG##}(}oSJgX?!16bs^0kX;qQMT!hyGxNqHU2fs{tz5E_J{?ChgB2;C1p zcpQSTe8+WTp(mV~d5r4`$3+ORJ>fWF++`{V_5`6c0s%n~N*{m2RgjWcr0<-vP<>od z^w4&L=7o-L(7m(&aMkW?NzwMor9VA4C!vU(8+l8al()egNNEHPq2poow5@Cq-19$x z@c_dEVj+haf{DUl*R&7BqtFfdY!sLpxG024BQOv|!FK+Wmv2r=F516fak75OxT4*s ziYpG6B^C7@T9J6ZKdI=%dUQ(h=2aq5;4NiR-Uf3Zr4cxYM**e-9EHJ0-LXXh>yuFk zlSW`5h{AuLIp=@m(`iK~m#5tI!OqD=H){>Qn=YGFbmWzrKYnfPq@wjZE0%rPJXItL zyroRa+u$gW(&!vcsrOH+p*}@ISLd8WAM9E;`JSskMo*kB%=*V( zvEIOI$fUdu=0HlLb69rzf$qb1Z10&n{*{)`B|CcNUvpof_qkm?fB(VS&mNiio1WtG z?pnzYH;b$ncnz79x4|4pX><~ z-|!-uo?W-_n+rQp^0|+U_IIBb;lOLkq`VI1KuV)?7`P!*KJNS5(7%7PdHE+#Z9|Xk z)4$sP^)@vBC(pd_Ywy!2;e&G@rk~g z4-FKl7fI@;aCUDUHtIS%>5gudnPy z?^r)w`ZrxKIy`0VC(>W^qD2o+JaO!sUNp<~=B>_5F%G;2tRgub%z>0f=P(-fmv2ql zcWTGC`p|8+Cw%X^{k`b%!><+pcVZtZf9|18Sx@w#u0I|=SnU*x1Fs>I@;W#Uq%?XQ zu7A>$Uj2F>dVcm#`ZulaL#AJUzUrN4`_MmUJ)oU%cRyPD`SKkD{bC$=4VjeJ!5m0w zbPh+p{p1TXf7g$^-N`Rat?5VEtnmX2$ljJ{y>d!n)f3e~aD(-xu;*D$HLH{nwd!T=t7zbWM zCgpW72T~fH!)Q2nNlLrz!TAqSJ)y!Q|LZtB_d;@Nk0)!f*#(y-|G z-rFgWwH&V@lk&DW4x}`C98P3s-fg{6)${M)zUV9ZiLU4Lrhk0!Q=_UUdB5}6qA__r z33}~Q1%pV01Fr#X!P#OCq%=B*E4HQ$-R!QuQ!~8?cCYKM`uds#sdVs;>9FIU{N@x! zc-nU3bdk*ks555FW)~Y4;UJLG2tk+^IC79kCMHM{CQ2oV6Q%RUKn3^*$4|_UTrlt@ z3@)RCaY?Btt}JzWS!yD4Ae}M?lU2x-1@(&!#eg$4HEmAd&^Zt4-hINs6PSnsv8dN`f&-X@dB?Xr2f@?@*s z>$Z8k_Er~bFu~H=Zu6KLZS_hyE16<<`D~tclY^}%AP;KhM9}2rLCw4fn!G%ySr9>! zmj^WqBWUt+fo86_M}?d`$>MQvagkcAwt7b^AEpUzyRW{*TkrLmd~6*PIf)wr)5Zli z*^dKX@31v+N)uRt$KKe)Nm5ve#fD?b)|6^<**uM%Ty)JgpQ&D+r5K_tdx$d56VW>J zhp;P*SfbBN%VdSU_Mcfz>ia0Vs zQNZV9pT}f()%)yDXyPOocu5Vd4yzFF1;D9QBHY$_-m+4BM1GA4LHBkb8OoqSa%^H_Z zNs<9xh{DVX0+`ifsC9_|L?*8AEJ{jY5{eB7W*h;`nz7V|B!D4vQy7Mn#tZ}F((M%h zt-Ffq#wkKF0fm7`Y0N;u*;0VEVLY`3Hxrp^!f2#4cC^4;EI{j?Ky6C~G%~4#(MV~` zXo8tu0Jmi#wIc=K$ovt8Bc(CJ1!rBQoX1<8NNxER)iVk3pvN-#CH8|pWELs5u1J2R zcvKi?+on)GlL3{?5MeJ#=_PoX6Y%n$L~8rDsXbF!FT?Ud7!Q08lg5k}?qv`aynHE@ z+B*$U$qW!iC8aT=2HP?(;N=e!sh7V)9Y|xn3`_WMUWQ3y#tZi{hzef5o=)}20F_Mf zU{q2XGiuPw0-=|CuA$z$I?$HFj83pEh0>Vu!o3Wlf|qY!OASs3R5FKyQAugcs6j6a z171RpJ#amB;yTvLu#64oWtcQ(yl^jrsNm({8>mwmfJ)|QFe)jH8CBqAE z{+r2q8J3aZybP1Zj7NAGKm{*P+(ezb5m3n-3`QlTF{3i;Ro*=SIz5A0Jp+KqL<K0YbF-*5vjB|Ds9-Qs8Z(%1tu4S?J%?I18}P{N3C1I(G2?}<)W-PIb=(I5I7Q;YexBaKc@P0B?hm+M)nFG6jP1NNLP?A$uGF zV0R9+EgJyId`Y(>B|zSiNA1W3WHQTvkxA+Bkh2BI+X|?jd_X318yK0C z4i8y4>+HCd+EWO~WJUuclhWZK=L>x9nMb{S8z7T;42(=lhlgA!K;CmZb#Oi)li3T5 zOiG7?40}WY^4mq!AOd7^^MR2`>F|(+efyw_Iza<6nW@0Yq;z=5!oEFNOr2H(GMS&i z$fR_5$ilvTLPMQf0LWxk0wa^s;UNqA_URI8bqOGoxd@C*N{5GBAn0M|N~txafJ|l} zFfu6}9suDcfiP`G-l-Bfr~t3Fs)fcZKwc1a&L_RN$Dj43QsQ9RZ-oQ z07zyOFd!+tBtT&Y-msY3QVoD)`Tzrx(n|mgou9gEsBKFCkX)BxKvEhrpx}gz@2s#j z++?73=m8G4y0PX9hTi5UaR+G3`5O;(I)S>sj@nfVKx8Ta`$bBl`jx@~@uzzX*2aXi z%?Z@MjMNjx>3LAEXp(?%AgwZ8csj?|%$^Z4_5L#vS`!4USfmFE0nHyV{)YPO$lM#f%NevQTjW7^C#w4M5=?H@b5vfoAVp zLO+hw_->Z5!K>ACp$Jej9MJ% zh|GFo(YYKGuozB$Y|&xV;t-v}*yu27mz@qA&cs58Q;P!~MQn5!wK&j;X=K3-Gt4>; zbYdD=aH%a$bYdD=aE~rdbP8hYJB(T!8Y`xOuE@F^16_FYkPJ$A=?KR{P`${3R(R>< z0~Z-A0`SW*MgrA~POIUiBkXzbhA7$Mp=BcsjBlcnki0Yu47NXF$FTwNP=~SHJy#M- zP!I;!=CWJhCM-4yJ^yVrH`^>eEFTJyxkXFj1;oQg!$;pHpU-1AxA~Z3?-g+VoN4jc zOukkRoJg<8blA-vlcy6Oe7ieM9#5+u6N0<(_^v#bDs!{omO&=A*vy||4S}BpG z(ACw847ak^_hqMybMepLXs2nq3jW2vg*2T*s~TwyO*hi&Mn|Fh1{!Cmo}S~9SXA{mz8CE?PjOOXI!kygEC*8p~haO zrtLL$Hbk$~8=UTKifQ$7z+Usholoe_n#%Qw|6n>k&P;RSLIWV@) zY4n*P!wPKCp&)wI;rRySe3W|!V;E`fNJ zgv6(_+|KytG#T_>qoz>6Ro5A3F3lE~s*B}rbQ=vFEw#$rCNs1jZjZX`Qop5>;p(q5 z0lE?uS84h^MPIyQ0~gUCOqLljKF{ z#INnTOj=E+MVsreWS2+k!f&iC?J^k(+v45d z%-KeV#i?-_^#z^f>^LXK0n9yCr@ph!V0F}X>HKl#=r+5mA%?~#v!<}CPFdJFG*?HP zRjco`IyGLt-(CUEFkS+2mB$v>s?iYFA-IagHSU~+aaG0^*SgUV*CDuy#Wn7ng>lV^ zEv}8DA+AGk6^m=!ISb>O8(UmkMnhbO;3^i^xN{DPYsKOS>$i~aR*Pa3e7ABVa}|wi zyg4^poE<~0KfUo@f7U6h9kt4i_951H&9G-`fHi1Wyt!Gl4tcH82YaRpIqZ`nt?&G% z+Uf>Vc143F8}>%*8Y^z=_aH}=vd++Z%Ycv?yJzo6w8a#HV$zW~e+EFBqoFCbxZlkj4N^xv98gk{v zT3w_*$@>zb7q_-{>f2zya3D{iuzO(oe<{jQ< zRP>#To_C<{ur{Ni?_B)6v!HF%%sY#7axQw_;cZ4m-)Xsod55)$U*DE*}tO=a!GESi^93Y_{GuKZq`QJ4>}?5*%J9&2hLOixj-@>27X^$ z?h^8mu+Hx?M&9c)xkigpA9>&B(zlrm741gIvEn#FPBJ`)*qt?iwKe_8%df0n1x$AtK z|ApLjr_rF1o3nK|FAVv`4$vuWvbb89ye8yu;kmTk%C6s)fm|5QF*oZJu(pKtSt;Zu zh5ZilD_mX;=SIuzIh|$A<@N?9-(Ckvbrm;;$|2RB1MkH-YA9=?TOb4PFNJaoeqV^+ zE2|(UUreh(s~P9!H8@94Tkv}hZ2={J39e5C`T9!y4jFsUFNZq;U=3IrzZPVT_`QIx zq3JR|+!avkR8W5nepli5;#QpN_u>lyCHSrY?Z@v0H1I0}a8QEl0q_Rzm6(d!kEvKN z6^2@|ADeoyX-OlP!awk?#_ttS5v^v3Vb5`G@WU9iywVK!8Q`I+2DE7!6AfNnfLgFr z%l02!qO8OhAexnP=N7=*0u56McATpZKVVj18R+Y{K48PBmR2@!csd3T2QmZ?ziOE= zji0ci;BX`Mo>j#{2pQCGDFi0?f$qTXY~XRsxF6ihpy`nL3lMACIs8U-d6+#!6IT}a z0WM94e9IQ+FpknXb2jV0j`j!qDHrvJ3AHx#yIsL(s`EH}HC?%ct8YOu6C`jNe#|5D zgc}~>8Twsg!;v5Ghi;t>ZG#oLctrO{&H1A)%i;a0t!(DnhxLaEBJN1sPlF$|kJi6>B> zZ|2HQ%vn9m^xA-Zd6+#hAIh4iM69 z?VSIO#d5Abem=w>CE@&DhTntv4<<)L=LPKPmbszxX}CQTT0#B^S4V5xyV&?Qs%paQ znNP3a`@+ockgxDsn6HXT%jv6WU2#E3{P99W_&rczQUb#z-L$ES*& z&tYK#Kc+tB304t6T6lk_ML{nN+UJGY!@>xDLHh;xQ7f98WW34^#Di{}3!h%l9soDg#T*S4VsB`drW-U}FKl;CN-O7MJKcfu<>d zSFwb%=jVG-Jm&5#Q(Zai88|#31>|9U-iY@+OqXI7`-~pV*T={O{P8GI;d{qeP0QKC z-^`Dl?P+?xM3&(;dG2hp`EIee-M288xHGs*+8ONiZTRmq!+*Sal>8(4WSK-#3I7+> F{|By959a^? literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/ngboost/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/ngboost/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..52a1eb65315ee7ff8178137e39ef5e71b2729859 GIT binary patch literal 29934 zcmd5_4SZA8*-uM>P*6Tw0y3-}UQnRqqn`phxk+i7(vP%2(`LAuueM2(G&X6|7IDn! zhE6v(*brob%=aDgZqDHb43t(xn263Xr3FO3okMYeAS%4?-sjwpwNjy$J#miUgN!K>yzaO6kF(ADFQ z|M(HzaWwQx^J5==c^v(%Yh2dUhdx9pl9bexi8H`%RLYQ)A(LcMUPC73by6kMM5Lof zOC*xv(-Ps|$q9o7EiaISb9nXdLkt%Ur_pO$=DxDw^;77ryCyHa_{u4C=Ji!d!^YF- zlkYzCj(g@w5e~eDOv>wE4x}_XhhNoRxRhVhjk=E>dRzVbBWUul<;lOLiq`VI1KuV)?NEkBp*Xz&;RPyubSATToIQr`4?G=fp z6UZ@i*y?q)C(!*D=RSGgg%cti_!60vx5XStX><-2!*9QTd+TfHz|Wul>YZP8qA}+s z=QX=uN4m*Q<@btrBH#D!kzWnGCc=T&lu3CV%z>0f;7}SIo0DL~s8y;rP=Zu?qJ4LQ zbVTRnX|EL^<7wF+Pf0@1g z@gr%0IzDlHfD z-ZuFI?5{ex>mT6$`qiud&biNw$8jk#;k?8;`VnH>XLLjE3#ANn@)UMnN*wg`pzvww zj>B_SeDK37Xxmly?hCJciC%x zX-wWP+jaRkb~o)M-3jnseWIOtUG{1DX$(zz&y+CUGrFAjVbU<~A0@n>wEMYdwtsXD zsjP`x_Ia+NHy5w_V8!&SXrwGK=3whJwCByEC+=PNwTSn;rA*4}U<#x(ChynlPJE2L z@7%xt6W04r@auZ-+_U1|GbL=>V^hxiFlm_gFA&}jx^vFy3uhA4Pd>DC^~xy;>IGN+ zv-F*+1hsWk^RfKr6V$WsefW*7w@K9GXOXv*NqHTn?Ya6w(g@zGgFQbJ+*~#{`BUsJ z^!&k6dFSNEm^Vq+Z$1-uo+)9RXDm7A!=z!(uOys5{hfwa9vCxN{pzR7pVXaAQm-w( z|I(_%N$Qu3TmO3GRFZoAtFzli+%rhTdEQbc<#jLxQW~4{yDopmH9a^FlfXHy>`ApFALTQ-uQwZm~@{cWe`_fSLj^rOs39OT<$4r*I^Z37{>VF;CSo=ni zRIQg-)W2&PEaE(`DUU@ty*|G7O+|-y`rAH868{n$kxl| zKcan)(PP*A;g4vWw0zRXzRhUjl6A7LoBkjY2VMjEB&UNpkkaTJe)_@Q3A0ZeMW5Y2 zZb|c&qv+H(Z@%d6K8kAZnlVZ9*-=#X#@6m6>roL7yoOB5>tGI~G&+aH0oQ$Zo;i%% zKW;pGddXqre|y=5t~U;&r!vQ?9+n(IFRpb@H*7d0!hzS2NqHU2fs{t)aB=umPw9|D z=$iTV!tXV9qh~$;);_q7f4F8|+#WB(9o3tmGe<#liz zNNMyq)I4|etv@Kb(AYme^_1t1gXojBEB-Xpcn~?3q-htf?LtqzvgumgDlrbchD^%q zU=E}-I)|$l@9#YQY!?bVqSEbYIEZW&WwYv597M@0SCvmq>qZ@q-*e)!CwE z4x}_Xhrsv?spqzL!(6m~{V`~NbVucTSK#k!zs#As%6kZ*2UM~-BgMuAuOXB2I+z0~ zjm|;;(7BykW*zLmhmcA%Cf6tQ!6v|MriQ2TPOot{x_x zvG0XsERsB@O%~Z&KmlXKZ0#5w90F1rAp~=R@Dj;w36g}{q>{wjq;m#gK=>uGeji zlaP#0jo}2EuW|Uap-`iz&eGWE^NUnClq-_)4`g){uFcAl$cFf=^>(*8Ww9hB`R+v7 zAh+4hJ|#7|ecWSWlaiBJo4h{uahR*IuEu9=Y~pl=`RdJ0UXR_!l_%RAKCivW=V z2E(n5E%qjJoxMgWXC2Y*#q4713Ccs7nGrO3c}O!mf+jByY34@IjNUSEo8!VH*SBq@n_5*!78=G-%60wqZXcp(Zi zCkSAc4Wgb-1Ryfwgy%_83X@Q5KrrJ7V3rT2o=E~2GO2`NNNLP4FfP450npP!sr5Lw zM`n&N5GjorC^TCN(4HAaZN$w)=7=yFDUBU1I2Q}h)(@v%P6jkG8-&qFY0PMXnO*?5 z@iuDP2!JCKKNyab#tavlb(L}+?~z1m(@3iGcEE!k%jAWar{1$av2{iAE5)P2INLm$ z+BFJL$;=M+l9b+nmzk`WpC_bVN}x85rS^>pdMQljU_91$p)_W^a4$Jjb~bq_h1xj| zP{|AqMkS>&qlVftTj=Er~p)_VZ!b=vF^>WSK)TSwbO6FKFDk+T_m07O}rnwDMsm&Ask%<%xL`q`@ z3a*9)V4Lotwod~vGGl_lNNLPq!nL*lZ}W6&*S&y8W=Aj{DUBJg=Sp6HxP2zIZw4Te zxe$y=OiVyJk@bG60TDdtf+H8Z(@57b3vhr=SkY0gp^^U_4S9GhUB9jsWn0 ziaMzTKr)|!0ZD1hfFe5+*g*-94`)$lGXa^*Vqj!a+BamC0QqDNbuk-|$=n4-CZ&Bt z7S1|n^Qdof0h!ELU}RF-H{=|F&lhJ?kIV*SGEae#Non7Z^90D>%%z@~3&>=40wa^s zz97RMQGoo2n)(F-WODO?kx6OakcEBw2@SP|24pf5fssjR-;jlU`xp7t1}z|y`3H<#nanz1WK!BUWMSXlP)Kdo0Wz6uz{sSuZ^*fV9=557+O7v=GQ)t8 zNon7Zg`;Y7F|}(xjy923V4_V(V@3`gxX619rtPKFz7hZ=_tqGYl->}a@Z@5bfjUqI zfMmu11Cr7k0u*-OeHGNf{0dB-dpakd(#@C^#YGJL~@? zQ14YzXBPn+Y;|MJ2=uHnDaLt=zIdzSw($f1R(OTANxg0qxv<11L9Bj7_7Ak zsV^i@Pn)Rj>Pgv9uV@;8a3Iw%L3ldH*UX*~GWGr{A@#WgYV&ugYbIPTp9BzQ1IStn zg0-f0SEnXZ($rrphVwev5U<%|Gy7zhGnS+~JT`l4jkVF$?DqK5?@Rmc{WH?i+1DDT z%pN9XUYDeM%x*iVr$c%n9V}`cc9*RtOvu92_}jdUYOSl$?9a}`>Yiq|w@s|T4TswJ zv=plLG`n0&XQYLoaQ=eC;-&o!rWAo_;Pb^6&UfR!K!5j~X*rTYQVS~q)By5kO zeufQYtVrCRW&I2+$c&N5VTJt+O{CyR0-`GYOe5%Irko@YR@l$bVeW|$5LM}C8bK$s zdy4>D*w4^m?uii)Rq1CM!3gB`76G=fpP|Ft6C)t1($6%45yGoap4n)^`}SI5bvF16`4EGX}cwk|7zC@X~&cg`j$o1Fi7V zn+Gm3SOnlVV~hl=7oAqaOZ(aL&}C4v#e0_ZGcdk%N<#9|FfiEuh#kiU#l0QIa#vbO zEI~mST(ievg-fs4r04l>qs3{r`mww>MCQUQi5C?29St9SoBjSKho#xi9D6T;^XGJH zliloZY=RT%CFw4QrODjXh7Z2IZRVz?#sDS+*WmHBcq~AQeTu;*h8tmXNu0%Vp0L1W4(}jybUz4MEyP8e9%FHTdX^W+>(r@vUILaIv zOVxrvo!-sKTzOuLyTWDHRkrJ$jdeza+^5$TIW5Xog~gb~%2Ymw#a-YxEzoB}nZMe& z$Wg4N9VWNSWzkjCij`S)1%O@9UZtwYvg#HAF4k{tsjezlmTL7FquFLu1niZ0a(k7= zg|YQ+lh2|oke52knffipN_n-hqTZ&Phk1j3tFIQg+uMqotBnfQ##7|CDkJh~vs9J3 z^bYXXqp1f!>#d4j@H~}%YoV*wR_F?FzPc^Cyn0)HwVbKLt#H|N1r60z74?{}#Z&3C z6hgcTd&H-$#KHLIHXAE_rh+^HSAAQYxj3yJO*_lo*C9^%iJ9+#c1cqJXuH z;TotmW`SR&z4)G`uvBTXdTCF;3E~26F0aP@L zle`F>__bZTSy#|z)n&P?s**^3oY09|Uw2n{ttySn>M8YFl$nwEHJGbP1E#8?cC#_B zIo|!vqB6Ox?gEdgGPkXS9p~gYfVs!!u57C|+FVub`aqmHdM%y`h@q+8QjphPt;}od zovW+armJkTxeI)Jzr6*VVY~$6DvvF$JzaYuq^t{Di+tca~8%mE4H{c41~D$!c{D;ap&A4uB8hitlxTkw;B`!;JcL- znX71ALXEK`DJzo6w8a#HJ*=TFz+EFBqoFA$puSr>dt2jDM z#w@w1N*}3D^1k%Yi(6Z}E1O}z+cdmclnb$mE=A8wZnb$q<@HPXY?=;-Nyu;eWukTa@Ag>Tw z<3v1X71fL1{8Ffad_!<8!{$Cj&ihSbdhzQE4OYlu#js`()w?xq4&REH+B~Qiw>GyG z7S)@LS+zEu3+D?X>0fW17|A1Tp*bb z1HZ2?cM17OSm(E!BJcH?T%%Q48F}C5scbeIOIu8kW5scVoMdE!mjqIp9$M_nL?{^YIpRKnhq z$zA8*{4eCL+f2p+xkaVNd11&ewt`Mkz17peKbkgl|ZUJ6W;SPwNTbfH$VnHPz2>x z{5~JQ8_FRkpHFK+%ZYRI1vp1fTk(4)Z3U%3A+ApY`FaC>hm1Ywm%yC>um-GxfDUAJ z_&t|iMAO9qxGSL5YoPu`_+5kF^BZxl--j;*6ym!AbO68S(!j46z(EPF2f!P=8!#1Z z08_DIDh##!05j#{2pQCG%>yR*fo{d`Y~XRsxF6ihpy?j-7a%Tb;qdFUC1Lgu zOrd<6E{MlW~;RTU4z7dO8sFr$p2rCe*r~-|bRHQ=84x%*ll#w}_J)~D!WeWf^*rIuiQ^9-Io`OCDWrZ|g_-@-oDSAzM? z%L&%!=xcqrJ@ie%`da&1A5=$|Sc3J@hNeEmk7oM|U7XFw&(O#EnD))j3D#%tYkfF= zg-yZw{C%vCY2RW?u)f^B)`$5SLh&m$wDf3?@cEUQj|%y-^imgdcY>KIX!mq>KGo8? z{2aDDxcP{f-+ch8;m?TK^KM#W4B8h(vu_mIGiUWM)9ZrvC1Li&d??5Ep}a{U-2Eu5 zbb*j=ZsGi|%a?QY@$(`6C<*8HV*DP;e=s>3Iy-1jH_YlepN88rp%vtxaCNk0ATnU(D@O9iYoX_Id*h zRU9AAKKAx0XWFN*g|n~IRL_E;!-Q0PJ;jU{Vf&OqTo(rIwfZponA@ksiCa#quR625?ZSa+8d(TGxJZ6dv&lVb_eaX`7jgqhL2s-2=5JI2aI5ZsoDh5l;S5G(L^|_!wz{Uc8q4COGEiTlz z0ZnrduXG`2&(HUwc+A~frn(Z?GjMo73dqCyybkYqm@dUE_8C2zuaA)n_~TKa!S{}_ znvS!Fzs!%F?P+?hM3&|?H$B*F_s_I?y)&6h+-ck;?KJlKHvFfU;lI;7K>m>&x?dtO J!2i?f{{s(Er#b)t literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0b5adbc534a65debebca3d802a6a5ebe73652771 GIT binary patch literal 29993 zcmd5_3w%@6{ZGp)l!v@hgI26X?_gk?q;Ew}ZqhbQX-gl_v_Y+TwN0C(S(7$x8H$RG zA%dGSU#M*2V;c@-Oc@}pxH(|p-zk)bz!30(A|mo|3i|(@`$%qbZkk|AARlM%{hil$FV57RwT3k}(p=q~$xdZ(n@f{|+IEL~`hrKO915C*83+ z^SS?`bojkL^&_-y!h^%|u04Qc5?P|`nyFwnN;X(FI7u$$HRMuWCs8sLzX`E(u)&6Mu{7iK07=6C>ok`(}K~qBWTH28^$EcKStNyx?%X@ zio+rtcuTpI*TEb}X=DzE&5r5M%Xgvk#$(#5b9+(pEq}9IbKL>d{j9|M+YLQv&6wuy zuQ!UtfiICu`8qHMQW}}V0GZ*nX z@6^Ma?)+SY18*sp@;aCUDUHnGD#d$aZmvI!D*vPX?%?LbX!e57SKYtlF#2Tmebf>A zVRU}%FZRE&TC6|t7IGsWgaM8ME@Av-bonR*OT#b_L}Bg6NzU^3E}+Fr4!-c@xbtXY_rp)WaQ*ja z$+kCEzB2X#vX|Cu%Y6TTA{=;2xs=zz97w4s2enL7ER!-Ht7K+CjQh){I11p)F>oIS z#Ao;wqQT5b<79H-H;OM|+@FU{Irka;fRuqwxRL!v#Sd6HAoLTpV#l)7l3&HE2f9D{ zAbWM3`k@V3sn6_{sMF7neRJM~cy(oVe%j=dKZLkXNL%V=u>ZoNOqzzHen^`-n z=Rv8u{H@mpC_9FzpRm3=cf%&BdgZIepJ{$GNW^(wLoVfYu=Au;)cK&*+N!tSo^$<< zQ|PJvOMbWZk6)vO2XFi_vHK)iI4^z1;`>jbuOI*6!G|hNigW_rQZD6nFb7f^nZump zrS)BJ9_-qXe`EL6&mZhsbY?}-6Z;Q#UH|skr&XH{blp4jhX;22=1>>e33v^;l-I!= zNNHpa>W7^R9@V~%R>j?|IQIM3k-T#0ncJ7IMNd$V4!mdgdNgmZ`!_eV{#7InyoOxL z>o9G>*#)GLIjo>w_$rYL!$Q*1xy`-L=e@`q0VsXh7#@Z9`ALiEh`uJ$Uj} zVyhHhLoVfYFb7f^nS=6%0dGCt_$Jymp>d@7=8b6k(qA4}yLCM}_3hAtuaY*RX^#wh zZ0xCZB5~j?$fdjv=0Hj#b1159_`sX`Av&_Y@})%w_o4h{gQstPdLJrZI45J& zln>FDao_a(`Pg2OIPjKoDX)V$kkZH;w$Au!!;W9>MXw**_WNDu_8^n|-n%v|+JmU8 zpBXW$WiLu7%22NQ)ou|Eyro>q>tGI~G%|-jKEG-HuNUk=#}~;Ce)jL(XnWyp=Aw+< z$a8Sv^*Zk!bimT_K-bA#A{=;2xs=zz97t(o4p%)=vG3ih_n@t9m6JzNyU|BqjGtU% z?m@N_UnkxE!)|nR;l>$-pNZ|}cuTpI*TEb}X=D!k4)AjBD4(A6hmqsQ?m-QQJ03Xt zX%8Ac{k&3pW*2&6nn}{|&2Dtf|T)zjsxa_IH zi+=1u>Ge5le%!kYee>tkRWr1E(0vu}I^ULvao{cGQeFphAf=Hxlr|(g?j5!lJ=g9m z{I_{GT2ngZuA2w-ph4$W$&)|cgRYzWmk-{(@PSC5`d9U7cIW;lNCd9#p1wQ%B8#x=0Hj#b6E1Lj%(go zvJb`G^TV9Ub6|dW^YtYo-|RuR&Up5(#>Rc9`()yLqeF}XZz-4ZI+z0~jm+Vw;>ZGX z=7;D`qw_-I_j}N>msiZ+FW-$`n10tZ^Vkp3j;jAmdHEGF4!osY%Ijbbq%<-I{$%8G zp0wOH{=Mw-7e{sbetTigp&g^TpZnd~fBpRQsP4Izy44@QGOD}!>^;Vz<41RsYdPLh zF6DJ_97t*8IGo+GsQ%Q>(_M?5r|Jf0o#|R%vgNg&WoNo9!&E-|`=`53+@8DCt~%RA za^N-OQeFphAf=HxT(&iB@b>P|06Ia9H#T__hrsquz59QIVS9Jz#)(6)RFbxJh{)~& z3K%5pj~Z5mHI~2rlEmI;N?6EZ1oF>^F5?R7c@$vyqlZ|~E z*y8kXkMS)kPG)Iwd)UXJj^_GWkEOYV(;4b%FtxZ{HV;>xV6}VPwib`Q*~JyD;L}Z9}ax2!&b*B4Pyl@ z_WA}+GMtrIY&fQDO){Iy)>7ZWMb~WenraoP%3jLSdnw~Q5v?=37rWfBWy(}8z9Mxh zd*P`}4bM}V8lI;zH9SvcYECaaIUj~w-PT$zj2v~Lh#Mn95l1E<^81|NZ86zhwO+du zns^`#yn%Ji4y!~q93F>QOr6vqec~YnaRup(3Yp-sCOn(pQrfSQkaDaw53oQ zH6Gz5i^{g;_M52vHv%e|c)_ToG-_04y(*aIcK(byN&yg=UBN)4G-@FKYFGfae=>D^ z5`d9u6bwd6qXrYMwFP)br%FG-^b_ zK0yF?I*pp225@991jCWisNsaW5CPtKCAC-ycx2uKJqh)MM!YNG3Tj zASsO+P-JHUJ17D2;w)-;CLoik4U9}m`-Yq@Kz=NTTA2;VWI_WYlhVE+3um3>w@|Bc z0hvr=U}RF-H{@)A&nu@Ui`GMQ|^$fPuCam#sNUqB;ASsO+P;f%VcUIT{FEmigs{jsf^Kot= ziX1`j+x&st0UC4u#si&s6anu$g^@uSk z3+fe33=j?^mW>yl&ha&~XM{|>>*EqP$5CfzQ)_4AdV^B|WM+UwxIfMhk+{Z~m_SJr zPuu~Ack;n*lgn!I$j_wCNwK@Ew)R>}v!m7N@}%67Jp0b6$tmn>EmLL-kutAyQd}md z4b)R0*N_4hb#|M>S{ou{VQRe{Zbr4v(QNW&WngtztJB>fR^WsaZhUkKRl8apj=58l z15h}bL1OXJ{svQtKrym@2ExaeBor_0XE1-lheQl2>StL0Stbb^RMgL~f%FxL8(h}U zz=AXxi5ybc&(K6ZjwB$e($6&fPG%@b0wIO{3?1U07y(h0ex~7fGOM=;u!a2$9pauC z0a2BHrs0o3PHz!l3;P*5#62+rqALAN!ykd%-Xg#j_A_*ddtwAcRr;AmAOe}a%mcYh zVLwBMx+g+FRHdJ3;A;aSu5;4<2IDR!hyV&o`WaMk{ei>{DZDh$>^+OH;DO?Ph7Vln zAhCnXE(tJmbtD|JzwnYkvv*X&VFyb48C>)>49Oy_S`6roXy|ZiF`%Q2jt-+113FQSEVx;QS;v4*R3i&6v&D!` zR3i)S&&7yNPIP^TQHw!iMK#cssh47)3vV2fK?yDG=U51;7dg-hExmN$B7;Q$eksO? zzk1PWHMF#!JrCRrC0jhWte=7L4O9}6mxh4B_DAeE)-UeuFqXUDN@58LLf~3mb_?8q z#U{b$zs=@Go5hRey&*EUXGuK2xbJBA=-cG=w%E8d*Qjm`B2rNX1v7B!kx?MkyDla-}=>}F?y*I22~f--N7VV1pEOWTc3hr_I^ zs1qx*=n4S4ptCx?BGaOq1-Mwhy{)FYTve*oV~kd-LFuzq6JlT|VXw`t6=N;BMd@Q4Uzer<{A{o& zd%<&6c`b#GI%}cB$NB0s>v9{cc{K{A4yV##)fF_=R97@$zGhdI$6N^UDh!HGM~R*B z&uKDLd5i_Q0Wg;rKO~s!M&w>Y`4QA-6Tw{mq{irM--z)_>BF|;dTpM>{vAL&JIqpIO@acndiG8M*ZeYif! z`x2xVv$l3twZeYK1!oD|dOp{V_oCIEbZbet{u$XOW@8Fs3wtIj?5){-akZ)lpEJ~z zXfwjjoe^zf*N+`$*vlEy9d#BsXRS~;DZc~-xD{c&XE)&t_O6WVK?AZu8CVS~S zoXwTccQV7B`}ERxSewh=ciM{^>2PyRxW41Jx%_=6|0m2lew)kRcPf9vyu;gE5q;;P z=N;%ftj(3scP@V3$#1=4=AFunjEkOkc$+Jt?=<~{d55)$UEitjL0%!S#tD1QDykQ| z`K3?;`3C=5hRuD5oc9~W^kUZ+nk4>}?5*%ba<2hLRdxj-@>27X^$?h^8m zu+Hx^hTrQmxkihsD*V3BRn=-Tl(rcm$BN?!Imyr*W?PNZ?Wjq=h@7fZ3H4QZ!p-Am zLvEAN;0xPsRzs#6@+wVS`-|jxbL}Q)RU^0O70ny6IqCuh^e1Ozt`x=YJu0 z-C;BoD9q`4oEL`tVms&*HCS9tOkNXmxbR%mW@Xp!Dt|5v=a?JyN?2RM`m6|Ylfr%n z`4uj&hI6AO_Kc3=#u9rSlW(tqq`HP1LnV-E&w%&53@wzk(oK+o_Z3081;0nf1Nr(g{0hXIHJ&UG` zeQ;MmtJgsNv+%nHzvngMT)zij2q?sN1!y0B&!K@|F@S>-Tn~UZcrU|Lv_4G5f~hdn z@_g9TgG~$T!4&?1cP)M|g^Fk`LkxS4Yl9!gpcQ3ixX%C&<+DJWrZLgL)di>pOLc7j z!6m9Pd;y|Sm6nzRZ}|mGCD?JUKKy`Lfu+B%&@w`|9aZz_oqbEA12hg;O}-Rqp8i}@U?W=Os>BCJSIrsH2j!H<_R}E z#54H2#)c!??+@KP1=nf0+-VM_Tc6t zVt)4ksD?iyX3x86jlpkU6v@6>XwRJ0!%VO9+n0pc6Z4@0+XwO{g>d(yu*v~Ky0wk- zzdldF)yL0=_@gA8-;42kApgPSXy`1zJ>8TRJfDWzGocmapKx`wuC0@ef4yc_h&}V^ z6?|Wq`5p2V9t-nT(damREv?VX35q{nhzP$28f>Z&yZ4gG%twXUoV}iIE#vrhw!5PMh{!7pH+j~{iSxk<*WOn*G+`ZW0Tf_AU%0qab> zzH2b5s_JIUgilEud>V00^_ub!dsvtSwPyu>$7^=5$GI&3`SlE@(`#mM_7LAj`1CUC z7xsHA-oJ%82>jMX{N~*V8#=oPSZbPmPbB5wO>n!BEBV;q0St zpK_*s3fnmQdQD9l3>_w<;_E49ya?N;6yiG5Z?Dyd*hk$yC5^b{v^vgSOPA!q(7mYk z=_sMKD!+YMBztE53A$GYi(;qWUYiFqVQ=`@HI4AzAa=k2MmT<2x*#fhxEpKb`Y2w| z!uaRy^`Z961T9z(;z6fj@}ckm-=|`*#C-L13tpcK`U7k%;1?LL%+=yTeFxAq`SD6; za`yauFN(+9y=AH^fjt9<2c&>Jtk3K5o`>mD%wnI>qxt$6xqv?&1sZ(s7^~?xd-%)z z*x8<@ZTs+$HT~_WCybcbVZo+I)rlBl+Y`iKGnvPg4IM D@{rdZ literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgblss/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgblss/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d2ff166242b758d43ff23141124e4d8e8b8e76f2 GIT binary patch literal 29945 zcmd5_3w)E+*-uNsQ0_=dP^N{+JDgC`q*n(!eUr9nN?OuFn>Mi3+}fs1(wMYOTX7;6 zKgQ>j>4%_Wn@*c`wPEoHtFNB{n~2-}gDs zbN>J5IhXgG^PHbE(B3*Xq*XgP^~?J?Tk-G2hTDeX19{`Lv9qTxithL=vF-=;3z zT{-5s2nSw6F6DJF2T~fHgXHjzpM8B@FM9LUQ}17FJA}?ASAJqzbqLvtQjbg=(Th?_ zZhPn#n?DfYz?aCS0bSrgN~3f5$L2B1nrHQ(aevC&F>hu!I?`D2FXQvwXkzEg>&D*M zgW64Ls)usk5#hjV$fdjvjsq!;&f(;+J71aple1`K(uY&-z4-*X_t1vDiZ!Ru%XdG6 zTGpOLN3Ku);?+H;ML6&paw%_%Igrxm97Y^DV_Np&QS`~${l?<@qv(~+8;Am+RkZl^wa@Q7 zya)a3{x_2g|Jj9BPF(fXiW7T8IPey7DX)t;kkSYoN&{naGK`odbCO>}2~z3DWqT8( zBU3kOUrLZFK;DJ$ziwFZcM$@7iChX#_yu7C_yOw&r4a}SCWq{9YQcT${)6sXy?ssf zKhLArU)%8IsMi0WzbyG~;;QAJp#?iOPkE&GoCpVALoVfYFb7f^frB9sgdE_Vr&1ll z{z5Q%APPqx_q+>H0O{LU3O})~z?U%oGP+z882z9$0s}!5cCLD+arR?hpv5cRo$~3O z7ts1q|Gt(Q{RKMngTl2XBQByNt6yF+YVBttQQ$S?QeFphAf*vF1iJ!`f~4p1quAfu z?%#b3dc>05pAQ|Y=z8M~MhFg`5Fr5SObOfQxB@N+VbTZ$1VOl4b^oG|-uoJ@e70IT ze9k4*b;z*hi}p)sS`e!uoh5e~eiT*~WU4y06+L!L~nmr0p#RVog` zquOJqa1ip6d;ST1V9Cd2r}68f4?lbcLr;*&g`X(CgmIrS<=kiVgHi@M`6l)gl{jR> zknk_m)<6AvhRl|jH>_^awL47-dH2@vpS10vRS9{$-@G(@)7_Fh@+-+( z%B8#xra($Xy*J48fq{?#&cFcp7`qDtc_{RN2^)vPWN`B$Mg|f2SS0$O&6F_CGWL8A z!1_UH1bjgcSbxL$M;gx!&Fh)jGFtxB&^*VDaSWjUGk;+OoE2|+MuYK?vElp>uJx>i!&TE@9%yieaBl3Pb;&#t-O(XJh0?r?-igV_6;ctmc}=;L z*TED>si@l_i?kIF-ar4$&QH;%ju$p9zwI2laMxdR(_TA=p8wH-4-GTUqw<8~TTT`I zTcr7U4Y`!p!5m0wbPm&>bv(2G(6L?r(R{M|%oC=r#YR`!`ja-MhT* zB~*`odGOe1Fs>M@;aCUDUHrSGwRFa1ywtdA!TUAUAa3@>9jHHW^dhr zHtat>`g-dQ^hZhcmKBRdIdGP8DX+tDAf(YbwEgvOr#dEdBGV85r|OsbO=xx3Z*G5T z*Cw>*=d*Shmvy3tKD@5|ch+Y`;=pUjrMwR2KuV)?cyPs~?OWe^9}W4{^EZ2&-$(n; zZZtSfypL*^)$JI6?0q!nwXu6Ut?!F);5Fq^UI%j^rO`P&wW8(0zwAGVO47!>rbs)8 zuK)PyX-nTbh`Nd|O+I|?AhM56`oRNFzbC?h*N{tj9n680M(0pmQT61D7kklXTWi)G z%j`wV#%x>MaiSMJe*gGe?bF^v2mka{;{Mq^A{=-Pxs=zz97t(&4h4tSPOCZAgC?dr z)1Mg9gItB)-;DWV4|<~e&DvR$deMfR$v^$^WABP^;5Fn@UI%j^rO`P^CS?81VCX^n z*3Ulsb#*s#u6uR-u`}JMZKLiNTi5iUV;?=MuUI3tiQ_foQeFphAf?ec42F&5@KOJ} zVD7XYbZP2_qWdRyqkP53w`4rnjkcC1s^%>2K~wH|defiJzb(=hyoOxL>)<$$(&%xR zG48FThe~?TJ56QO^qafU`E&Q|>AbHSZCp~8Gj&G~>ORo%_+K6q0VjOr4xs=zz97t(& z4jrZbSMt5RXw$i=jk~Axpw=no3yGCIXxX+S+Iv!AY)XH7s%MuN2VMilCZ~frkkaTJ zCj5O?>@evmU1a?i#d?e=p6VpD8DX^+nTmGu&rZu_FZ)c@Rmk0lHNO1nk2b#xOB!| z`Rk0xt=cOxMPg-a#O`h+H}2*4^ND@X#C5fY?vxnen;g`hzk&vP1 zOE_H6!=WdoqPTL|WW6kr*~yKZfzPAT70R5N`Nn*}881r-Zrrn=PQFYYcw*`tp2}|0 z6O!>MGHmzx8k27t1~sUYSX0OL>_i^RPHiyq+_xc=7Zq{Ig zwW-zaHP_i|lnPcd&f#vhdt1#;ww{1IsF@K#la~iIvm$8n@}Oo;1WjHZ)Xa^b$;$gqYkNLFIC~bTPrKfT6 z6{$1551#31k$I-4Mdq2F7MW*yT23E4IUh#aJhmDxj2v~Lh#Mn95l1E{3izDd>@_>w zHO&qeG;tCPyrkMDr%fUo36Iy<%*|$>y?L5Mj}OEd*-%%L&F^x3a{PdDtXMk__-d6lP8k zz$_U;txg0WGW~=nNm2@vP;5Xj;|O3@4yD#50SuX8!Z4&XW*8Wk9-jbc^)=L^IMGL@ zkuVS`jTtC7TME$D4X2*M%|s@QFd8Y19W5{y3(y`NK|PlYXk=;#qmj~>(F8NS0Pd+# z)Yg#zM`nRA94U<%E;#Ed6+GV3MC$1=RM%+0gB}}}^uc(fG-f>VE5)P2IO`lo?H&uL zWSR$~lG2z_gI;C?yj-40ZN8S;H=gwpTsq3GhWbwR#5UliGArOEv}M;ssyl`CGAwJuwPlzzX1s7OgQ(!;-bqxi98k$T4MruUF{1|CGKcl@ zlZ2Gd5~vrir;c97dKs3S;k*o!#*7#4C5Os>8D9AT_0D8KCG#;Dm6XPe8uT(Z(3VRQ zsaI~K4&T7FWoZ6|Z_7|=%y{8m22r6c-^oN%ozz&mgYb^OPGN2W%wbEGt8=R#NV0>pze zsgGs=BAE!mh@>=TM8Q5m0C#*Abv_N?$eag;Bc(CJ33nj^ypPhUuM`-M$Zs%sgfwQn zkUfq7@VtszsscbVnSlXGY0Q8kI}_MJ36Q_aq*i4DGMU1_$fUG?$SMKy(j4l6Y(ORx z7Z{n8_77P&>#Vw!dL$Q+$#exqCZ+vD&KCImz-;OdKLKPiNr90`Y5$OO1;~&5lzMs& zAd{&Hj7&=VgA98_0rDU6s7?gPya;DL)GWH22pr9LVFKyq)50ZHi<0SZqp zjvJ`+^8k=c7hphAdPRW34*XF$^;H=FlDPp4NJ_5&Fm!%8zkpi0000MkwI(n`!R|Ng;2--(N~8Lh#2uh9=WjgFxdiHH6}7SofXI`63`9z!0*&N=_|rWG zYkfk>h6L(46LqnAau(DpngbwKw<(56!qYjvX7-Gbsdsfk%DM#V{})p07vg%!TmV_; z>_D9<7pqf}DQU_xi{P|QKFniw+sr=s=V^CLb+~Qzwi;`bv&H51O}#aB;m>BIPGw(f zm@<2qlzF{ls@v?cgZflRE=&cBT8G_fs|gdbFg4BX9!9m+*<^0c%E0RG7MG`8tiT0F z+W4##s&=#J%7zm$Nl2E*KfWZP89uhI6Xn{IGLd!34|36Fm#xEVgy802AD>` z$*jI2z!nZLbeMZ$1VmK^m_{H1IekTdEgWFzF!#g=h^h=QjX(r)`-%WtIKa?h?uii) zRT*F!!3bpbF%RT2g#!#7?w$w%QI!Fvf$s^3xXwuj8jQP%AOa{T8DLPseFqXZtnlhU zvllBOf(MHS7(RH1gTxLkyDGrUU6Dx0fx@c-&0b20gdHp$U~tjvFC>eIYH^^Wij5AV z76&>avz}OVuEqo`hLaszbQrZbL?<^kI*i&?rvs-jvC!ev;y@=oHad)29O%R}vfwHi zW*rAQF^w#^y%r}rF^w#^Fc&8}IkEK}MlBAF71KaZPrDieU3kTi3`%(E0LMa5y~u%9 zcs?0N7iDB0qnWdjV1ubh&Qyfh39wm)LWu>o;khq2s+ zRuW545C+%cc39!cD>ex||824~*saZ2-WMWsU6#ZPi2IL*kG{>#&0dG4rI|VQz5~vm zr&_&sb90jyPNeUc>U3DVW^X$_`1Z7$z1}82CIpw@@uheyRc5Bab%Lqbj^iUNOjE+w z=}w7!Bwb!U&v+|)PhWo4Gzs-zjleRLWN?F=!DXMI?xJw-K z9BNC|e1DzJ#mStx9*e8oY1dYE=o*^pjOhxWPE*`qQMRR9jG3%V<#SkEh0UhaygwAZF#L&nN?c|*o7Tcs`5;$wgPamep_pGRhhC>qr(_2He%q@@YkD7e?#gCsk+arT_3z>oH%8 zyV7SVf_N2$#HYQ)!T9Gg8!LUL!dwAYU3;9lG+5p04wk#gV=}fiRw*;HW=dC2UoGU#+|b;uFBZrT0I!z+6Py$ zxW=8cFs>P~#kFoQ#I+BuVsVW-XJK43V~cC!V2EoUT*cxVcg`VkEu9}>{TA}wYD^yl z->uBZTt(v=Z_W)?S6g4}PhY&(pVi88XO*(8wU6~(1MHbYZatstzENwMGjupoda+2XW%+_j`$62kq zjGU?~9qOy}MViMg#@t4e(I2thY{pCv$%0Mm*=a?IG z>9Dqh^;t3GCWZYD@+(|k4d+Hn92xEUh7w0DlW(tvq`I0LLnV-E&w%&*3=NdE(2bCR z_ZLIC6~E8L?}jqS$>-A=&}zWB`9hqdr>*!sgSLW_zX;c-hJ3vNzeC0z^h@AQ09XT7 zpP7~4h{VD01irUJpkU| z-GHfR{FsUrQ(>s(`?0AHn-v1&78UEij`j!qDG~LD3AHx#yIsm?YO*+d4Q*J!)mMLSX#2tzI z72-#2r1P1AmU+0mFgsA6tH1SO++uH_zVv?9SBgVfY6;X=U~u=#U#2b9`b<843;J1K z3FcRj9jMRI-}-QS=)8gY+WK1`R7aOs0`<`bZ$IKkv;BqEXYuhf^s_#uee<&e_1XJd zAC6y?I+?o@%uGSMZeizB z4Xw@3X4`|CkBIr*2cT;HjF>&|rq#xPeQ`AVCZRoZRu411Heg>8W>3tA3Tz+Dn-sy_ zkD^K^2(GAL5UaaDLb0_h9~m$xm z(+k>twg;>;@%pabq^zu+w*WpRt?+5YHPxxh!t7yT7Sf*O_#Lm=!5-(b0_WHBm`<;r z$Js-C8{pH+tY6sgt$6?3!A5ZxB0R1S1?j4P6+M zJ=~48aeWjoXkq;G_PTI;W`Y*12l1d&Gx<<>fbWwYEHPgl?ZxYJL4Saa1^j~JmAP76 zq-zJ7<^W#l0?wYF??v&LySGerC9r4U@PHJMhxK_K-t#bBidpP4dNf}jBNy<;qd<-C z9b+{uXAggwA3NLA^c;yi)noSF-ePZ_Y4vz!GMBhhxl7uq?DcK<4>7}krFoG2BkB2> LL}Gyd)6@R~E-b`k literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..62598b112bc3d432ff2bc14536b2f69d78eccd8e GIT binary patch literal 30011 zcmd5_4SZA8*-y*I((++(qy`18RW2|}+oYdCcyDjgHce?mTcAlJSo761ZIZ?$ZQ3$t zWz3;Ng`aMMuZ;RJ$89rApP;_axgW_*&P@|+3C+*h zd!O?>=l_46^Ks8P&-uA3szxRmE2)w!t&k*3L|2PMvmYyX^|r%X-abKY{+-j_OAL}P zeYW=>W19!b*`2RH(sAD)nX~H59nv@6AtfS7s^r=XuuGAQmyDk-6>}O=F{hI%nk^um zGDRd3O_-es|4vRAGiFVxD4fGPf70~} z&7`08k+1K0d&$k$9TDKbX-dVMF5-YoqjRty%}+b->Lowfe0cvo*Y}av7`31OS=&pV zdu;Lk^-X=`d&;LDf8qPX0vtFEshHD29B^rL4nKcUyn6O)2|ed*6Y8qZU3it)xIOEt z^P-+duez`Fk44|T`s{5jC#EXL^x*x0vyh589hw7MPf!}2!{&WAlwBA%NNU<2`fXO~ zAo=OLXIwQogXH$qIiFb{8YE}#F#hxW#S;Q?;54LSP6u(orO`QD+^O$fyKWy@v})@H zN9A6!>(RLrr+@c%a!JkZY0=yURm zTgBxcO#Xu0_sZ6krrdMnQb2^9vE{(t;GzeW&pF4s4 zOAi*X!p81qg5@bP#A=HVCwSP#S>%F9=^eGH~_A)nAbh zC*Hj_{p45V%5Sx`M{2$zn=%$ZuxZbhE&YyAP%@R0*9)Qo-p{N z`y|)s&=Zgw3F2(%0)t4}tv>m00R~(N?JwP3*eKBYL1_dAyeNp%E*e*)ULY5KnsMj* zo4z6Y$Na8;*`jYq*Sj~Zp0W7?dB1tJbjHWu2yoyuq+-qnaloa59Ev2$3W=EhR;8iw zK=X#bC>r2C^o;k|PH+lAPnAgdpD0j-IGrMCN9h0tI{{xxLnq(Ne4-M^tR55og?fJb z+s|t<5{t_7I=a=bCloc_zUk=`(?vxO)_#4>_SX`MHhpnZisXDk5k4z&mQpcigDBuq zLGP<1!GVwq&J-p0Gwv=-9w(mIbAI-zgp_Pi|C6WaF0gUq`vT!dK|>{U47wI3zsJo6 zVbTZ$cwJ!T^sU_sU;R;0rm^7r#=dby)GJj#?MfR{v^0Ci`ukrTU$pgS)6Ttmome0U zT!~c7*`Ofc(g+-e48s2XaSg)bh0esA_M{Yj@a@jTnu8OH>Zkna*^DL0MOV-HWX%6P ze^n9r$+&O-YMLSt1g=CX=4?<9aA|}f@EYRlk3ad>GrEsSVq)I++ohk9@2q%eps(T# z`N)O2r=EW94Ed-{Vfoutrv*9#XCW1HI*0=L3odG&+Zkbz9S>++Dh0wq*Xn zNz2yX@}+Omf%6{^EO`4RLUP-e)eElu^_S^y{&8R-J~MD7u##qN5eHlvox`<9-*8PH z*hPM`xm&hL@iKYl=(tHoPVXcac>aDaebsJKf7h%(y*PfiKpZ#?shG1t9B^rL4vxK< z$uFJVN&exl_uR1h#a-mRk5*?*IJJ{Jx3@8|>6TsOg@0@+n)}gT1UPUOQZc84IN;Lg z9IV7`H#^+h$lSHtZzao~Co?8LJ}_(7Hu4rC>-xs7=gHLfes_9&(l!AOoQ71)=^zfc zG&+YF4?S*pdGj%H`IEmtn>6+~d2+q-P3;rM$T#OaoVVocG4kbA{|)`*gLkDI)qmViF8%M;-~V!cKS?&;U4G!Lellt9&bc*<`UN;} z7E&=+2jYNBqjPYbTOr@OvX>N{l~1j$>mwg%{?AqE>wC$krd-%@r>2iQy-D$I_fx$B z95@T9n5zSEz@^bS6g;u%=08p8CGC$E+gAU(mwfI0@&EguANP`l_gr)GtIvALFMpGo zdVi7-2TnsO=5!DTTpFE2?$x$K?)kl>O0uDN{rO(9f8O8|hi3MYcT|3qwC5*%cuox^C@SMKiqFn99{z2t%ie=L>i`pAsb6N=ft>?J?1xzDim z7Y2^(&%wGVKlZF>-)%ki@rZl@EP>Ub07Vx7WUc4EsLLc z=-z(vWP+sk`c5GZoQ71)=^zfcG&+a7=N_6htDv7e_1IfSe);EqQnmg5wT6H9k<$0( z=kNQ~09l?%?RQ@g;=pN0#heb}fJ>uuxc}B&Z_5h@$iw#E?;YGSK$f(Yp8n`xFfSaQ zoYL~z5%Pl%oaZ&a?GqTAoQ71)=^zfcG&+Z(M-tEN-8Vq$Cf@SO+}tDNEhXA}_8JCA z>%CK2+ucXWk}sY;SaVT`1E(Psb2^9vE{)D%Y2uAvJZm{ZN}s!P&nN#lLRM}#{N-z( z4UoTBoOS)-dB@0d?V4w%nuX>`P6OJ4)j=F^X><;w;k>2(<-2a%H|fTnmy&*4UUqUu zPg0-0{K*|R_B?)Y^0?c7eM3*t$1f#N``DIBO=v8N3lp5^q4)K2oSP+00i6$k8 z5+;d7iIc<&#-Iw|HcGf_ImX|ys=V?9nU`C?PVU594)qb zuers;>Lhs^jUKnl>SfE5Eq1Tl>haoJT#Ug4b4#1mW3*Z8<=Kp6irwY2dfJQ*rk;R2 zsF@oofY8Iwr9aHw31Y4Q{d@1-{;4ZD5rqFanR=*2qdGG7_^D#gwT@Vs%+PwhlJB zCacd_pPeZiqAX{KGS(BpItzxdQ$#G2WwP-Vs8cosPg!PUp0doyJY|`YdCD>ihv3Qj zFwx?+)U#n^sq;nb7~zXpGC@(m=VYJ9Xm{27>`rLnu`uw)Hncb_BFRK}{E@}zGkUGQ znWBp6F#c)TIA@E+>R=_Q$Fcrc08i<%!0pOryuGBr!EPwJR}oO(M}wh%YU< zH7x_C9+4yw&5tSYXThBD(+E*A!1Gb)Ie`b$HHKK12tatQ3QeK76egkAfMCYq!K@ia zY#0kLcsdHf;L@03U|hPrJfL+yBDzsl5YIUwAY2+VP;j>7p>0SawxDLZgq%}s(F@MS zJhbi!#I|HW!}CbUH(VOCZ@ihF2e)Msv11~@;VB~ohf8CI3(mUoYz}W#BJs>sM9*Zv zgC0w#2jC?ic4cg4SJ~vv?Ywozu!O{zn*Q&7&ADrO>*I022r6c58Oz+ zn+B+Ob_RKgOMi%$dE7hzULL%O_#izlUItLX%Ym81hc^Q%o_;}I;?kJCq}QvQdjK>z zi#SUF5T0W}K)5t!pulRF2ln9{;_KM}h9^@H7%q(&jK9|A;hmjJbj<}kJaK~XaB0kV zp(}YF;@9(tHS+)wPm>@-TpBYXZ=b+}>&hn9WdR(X5kYXcG-f#dE`*1-Mow&y0UnfOx6{0pik_0R?s@u!G_uZ^$RM=GAtpQ{_`GAmd>F|*Gqv~u0(NzJ+c*+4G*6$LC2Xw@iT7ZMCZmh|Ip|=xb*#k8C{EY)Tn?QW0C!W>=5S|A>e&N!neobV7 zxYIov>#>B?rxJ(*2I4n{>G@EvV1@v5Ahl{5|8$P4nK>h*>)oG_dN_etwVc>>E2cU76&@A*yzw|ai9~^ z$b$Q2h;Mn2*z@51P`t%M%SIR&-9yD8IcXReY=4A~V*}!$ z4rAF{t~eI2APlb6WjDh;SY#4<{@Y?|vYLHJJ`^H-hZe^Rh=-4cj=qgPpT}-$_0h-P zE8+Y(-R!X%eJvh1kzSeZu$w$aPX{{qc6S&(o)$kM1UKW+O?f1h=VihjgLGua^5GX| z%HivDr${=Hs;OD5o6lU{m!2_Xp`YJorzol#{)c{xC@Pmy+Ne^BvQa7Ug=H4dw~wo}v8V$;d8y&6?{lS$q#GwJdeS&r9ka+dlGOEmdV z=Bv{!vsb7nyTR#jnA9~5LS<%kDPWg&>T_!H%<5%;i}c&u>h#retxAJ1S}i)6-&(84 zw(6A*gspKJye4^RwzjF7uHU4q&92kcG+NXph&SlBdmDhewWGYXPA6k*T;)EqJR+YC zlV0o4*uh_yvJw1jG|Ps-bJhCHWsU|*nZwWe>NKepjh5oNY`PAo%wbWNHrMHE8WCTU ztJZ5OgLsvN#HXXuPW$IH>T11)QU#BzrX$W=n#?X`C&S&~Ht5=$_42$%6SN;{kGh<4 zzqy0v>aWw~fnVAoe9x1a^vb*;+S6x%xImj{*P(u*27PtAqC$oGVGtAI>C)$Dn@l<7 zOuIO7UYJh&+OE^6F6}U@^Bm@!%1C`2(}`POchM>N3<8c2qLs93KZT_gI{@9d$a3L*J?K$C;zsTj#i7hw!`8q^>Y383UG$;5{PSdY;mm~4RIZUt596y&Y2%qd2Dg58x3(Cf~!zm zsGpilhdbtnwOxkSN zCq-J{`HlLT24jx4!JGqoBW8^ixAl9Fqf%a{YnQ=33GL4u-|dPSpvJBSJ=^BRNqOpRz~Wd(QV>3rXaSkXR^TF zn%NiY<>ly{p`lWh8*%Q8Y7@VH>@dMz&XD71FvB^M*;(6A=Y(^c+CqOtQ^a$bs5Yaa z?`WB`5qwMz(RWyz(a?ADBAxpT(RUb|E8ln8E1IZCb4{ea6R^4ReW&CH%sT;_E8lmP z{D66fvl$hA=aT0g=sS$fXy`kaKJS#Yj+%LANp9{X&pVvWsOUS*KVaTrY~t5%Bi`s$mg%S3KHE{OtXoXyo8|9)1LRmy93dwep2KXbbGjXMIhT-A zb;_W=T5qIz+@w=98+87N?Pk&CxgoF8%(lNko>yTvI%}KQJ+ENikjYV(W`duE#g`pgClST$> zOIV+kLvE7a?;yXz=G9Pcw9=m2QPEUsZ=mz-b&yn7vSX+cQti3$UYx6fvR0}YGVuO# zC^w_`Md-b%8glZ*lnS((P;R~y<>)Chde5cIpyV$@^(i4=UxnTwV-Nb3a3=t)0jt!n z2AK`L7gEb8s=^O<1ymX()V~b9E75y#3(EC-(S?9AbXS1#qxV7z_*DQnD8cmrc!T#U zL`CICRLqD9O|96EOufjo%m$|L54@|;yA~>v-P0|#0o3}eI3;YY-rW;ss>-+{vcL~;DJtY!wj`H!l+u`T82>etKj2TLpg(k|)uG>QEv>1_XYo~3)l#;;l43eY z;57W`NBRjhJj65fyT*j0Am9(xG8ftgDYEg1?vIM~M^%x_`J=CDV%vxDhYlj@NbIi^ zJ*p#}&y=(-M&+dif%=@otqzhFPB$g;Hw@)K^mF8kWCwTPiE^xcDs{W_^{2 zUr9lrKKpR%L+zpQ1nO%aZhcT4RcQ*;M^$--5kHFQFH}W77r&}u)u~Es z@hkHL>hle=KDvD?Oo93ehg%=wR~3w3MO9l!dxX!g^n6stouz9X^xX-1rl6d2nfX*j zsf!Dk_F(5DY<~9wsFFJ)X3o1Qr7mD!9?iamZ%?1q!%VLZ*jI+xWAkA)vJd7>%HZxt zS*-(vRBIdSzpXf%t&f`z(MO3tzgM95VE%*7(NOsTd#X7rbUqEYr$fujKcVUg*o3c909u{UH?OB7~(V89XQ7$WRe!ZCP^vcDo zJ;b*OKE3q%h56nJ_iugtD?6Q|G4^2kkXg zFjP@|So_%9rL_A+AdU_9{)7ea!7s*@Rk7rDp9_ zRAn&?-Aihpj!H@;57<{lv!~~ukb8BoD0T+yRmCt94uy|gQ}XW(A_sI}gyN^7N@KEz zyRjCwkD>)FjDOBv6K+pW(7g2^8gxoJ9|{lfeX0OU#8*Rk(E6O$A7Envzu>0ARc}9maeW6_6#f@kOK0sKDVJg58b8c#XhY^arM!19)C0nl<3|u zQd6_`@R$BEvpq#E6iL(EM$a9sR^L3c+dYrI#GS@o(oSQpZ^M6`8UEAFqvRjanH3^Y J75v|({y$0$=g$BD literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json new file mode 100644 index 0000000..598c1da --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 5, + "observed_rows": 5, + "status": "complete", + "valid_rows": 5 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 5, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 5, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 5 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json new file mode 100644 index 0000000..1f835b3 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json @@ -0,0 +1,124 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "development_run": true, + "learning_rate": 0.03, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_cpu" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31926664340", + "source_sha": "db7d430649e779287d06539a0527848ffbf815b6", + "tested_sha": "db7d430649e779287d06539a0527848ffbf815b6" + }, + "created_at": "2026-08-16T04:34:19+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 5, + "model_parameters": { + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.03, + "max_depth": 3, + "model_params": { + "min_child_weight": 1.0, + "reg_lambda": 1.0 + }, + "n_quantiles": 99, + "n_trees": 500 + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "db7d430649e779287d06539a0527848ffbf815b6", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 5 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 5, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8567250961b567230f3a45796f50bc038285e765 GIT binary patch literal 29979 zcmd5_3w)E+*-uM>P%eTqHDJ|p@f|3XB)x-p`X+7Dl(w`5nkM3EZoMXH%%y1yeku%Q z`{vZ?oFdzp;-@(008tSTN}Wvc3PTD7CZH$+IzR*kwtdfeFUgynH%*`=ke{>f`<&-F z|Nrxx%X`jw&d*iQRZ3ZstU@-YRF*80Tq}`GT(-hBLfQDM%i`eZ(MZxbM&tXPyZ_U_0Lh=-IXO@9{U2F{Og!w z`??N^aNsTFQeFphAf?ec{O3Z-W9z1Wiso&c|L5r^d(i(RPAnz{BNu7)=xz^@Ro8ZuY);|(&!wP6}FA2xZ^mww%`{_KCe59_FX@D!kfE~qOZTA z9(()e-=J9;kIy}tc~pc0uOXN6I+z0~jn3h@uQn|IcFIBYo6K!!>#GORB*U1gUvwNq z9r=fkq;EfnQq#-RCapXu!hzS2OZhr52T~fH!w3853y+@YLh3vBe6X0_f>LT<>N;83 zh1MyacW!*3r_tkYEvwr8&yynF^BQs~uY)O&(%8ITchjeg zyYn8|0p5cnpKz}|;@+ptd!z?3-ZQ$K_hHg7?;j<+|DyS%WJAXf=;J@#_rJQGXHeaL zUv?A@Ig3;yIv==C_XEm)@SP;|`1c~-^OkZcuY)O&(wMx5#y)`Eg+6cyd*A&W^9tTD zQz6;*ObO#XV;c1S5E~Ol8s_~3!uz&kKYsa(pZ*6OI5DAT`NQYYg4{P+hI!AUjuG>| z`tgMeXxjf74zGCboQU_lrCiGEU<#yE)cXoqY2d4r3SNUVU!nx5bn&sT;oosAOw?tbmr|DJ~)5h#*go5d4FL3Nk!AP z)%%n3pTB4Pdy@47^T$luVc-7Y0Fe;zmU1btgE^4W2podlAd~kK{@&m9{Xpq3=my^m z969+|<#8MX@b6pk7%(MF4DQFeTnxgb5%2{u*q2(q;GHc)^HbiPrvKw_lk;2B3tk>R zc1Zrg(+Tg6o;f7{ovlC5nXqNBNDO#Oxs=zz97t(&4vWm+D6{`?3a#F}q_A-KNz}3Z zUn8DKJ&EqR_k- zxeguiovc0f`_1S;`t$OZ(Hqf@9T96E7@UT)SR`1Fs>M@;aCUDUHq{ zVWIr=vk$$6PJA+X>zutC(27Nm^nB*sfQA_N4A0)U0Zo^V`tXBkZ;Ei>E#*>P2Xi2$ z(K$5yv7%D3aSfW7ZBYCEvKoENvb@z4H{p#EzQ`f#FEJ9$G7`mS#7Gk44PqH_{YM)`PVy%e^A z<1OVGem;rw4VUH8UrH0994Q}65AjRurXk$U+hM$l|PwUwxAoWZqGFDeV`k?Qd9B1ziKynZ28QO zrZ?;o;lNwUrMwR2KuV)?_(*Z&oomMJMujgFJZXA&7h3lB`%c~Q_Ab;t^M@ZF+1ZUe zYtBm)?P41^-cl~*bub4~8l3~b>ARAf$nF;!E|jM2M!&vwlK*E5cA+tw#;!eFzYC?_ zD|zqFo!#iU_YbyePwfbimlccGOl7tDS0 zu3bn!Yq>tGI~G&+a*N80cB{ewMd+lIv_Z$8kC zzHR^L%ny!kG*;1)p&inL*6V)N<(?0#hbjW)k#>mJmaAxkpaT3$V*g0U;PWL6E~ABUNvSBVT$WlYOJq*AhE2v~ zl?qjE?QDGk;9M(9nH)G-&W1YqGI`*MsdGpgdt{Z6jL(qA$tb?YQTfB6#uk^&X>Mup zcxx>#pGdXCLloqpuOz_e3sbWs@h=5&U58cc4N)8^sIldX*&m(A^IY;m#%LoF?>Hn*wXR;yC5l2MIL zug%?Rva|IBla~iIb0cW-@}Op31WjHp(99C|D36mTS=@FmE>ert zR%>tJ!!*>@=&fz`)Ox%oFI&fOPU3>V__*LE`*GlF?Y25jX(%gjH`X_Bl3}dGV#6_I zYm(WVHg|nH7hSW>YpPYGD|;!+=%tMFM6}MFUhMKBmMPP@_=?o2?1iT?Ju**adSsr; z^vFDw>AAh|{f|v z7(5QPn!F~D&3lWa6rU6_vVo2ktIf_yQj$lfj=qpiNm6CWqq8J~Mvi92tt>GmIpzMu zlqHE&8zsHC6xQg;FylyMiFjHZ34d;%G-xy>Nd|Z!3Nt4NVD2A4J(>tWWJU_loTL;c zq1b?6#u30Q8Av^q1TbXM3B!=mm|hDl?_BfnAtFISAB){X>JG696WB&9K< z2EEJ_db#u`)W&OBFChUCQ!XgB4jS$yhsw?-%VpH6>j0HZ^k6SZ=_PoXE%frq>#21q z0WXEw9gGJp9F_@+886&R4wdzC*=XuzIiQjW9gIp!V@3_OWiIO_;N71{ExUnQJBITz zG=al=87hq#FWk!@DtsAM+(@lW1ynL+gHcIo%&0*x^8#M}CxLqQW@_V2te0Uq8qUiw zY0P-xUItOY%a_Jd>(c<0%*tR?QW`U=z{@Ot9)PxdX*|_6j`cDu3&VLCCXE@7@G^i3 zUaq-?>Y4zkWZng%lG2z_ne{5~9ssTXDYcCPATqgvfkZW&tvp*}%x8v~S260_1Z!)Gu=YnapQkWK!BUWZ|r{a2oZSJU}M17#Nw9 z_6<2l;PWqUrgX&0B(n$@kd$5$ps)iUs-jL+ z0w9?(z<{Lm5&%Qzr=xSIbF%@ET$f=$QW`U$;Dn6ttP2yVB5HNTBxBP(QB$AoBPh`$bBl`ZbIL;!pP&tYrx)D-x)ejnrvFYBtm>niwD) zNU0btJe}ifX3q$jdeufzSc5jwlFF4I)9wgWtvA|iOOxS!#JS2{@S(6Y+{ z%v>Fbgd8ZmEYR#7l}OmZ(tZXPy$wUMh^Q6^IvKIiVbtP4M`YF$i_YbkfW>ffVv7!= z7KiBM#YTrwyX^dH(;?z==pDp zxyfenVtH?f%D+vIM? z2j8xCliS_m$AsWcJia53rK+rSxLq&~+i`q^h3Ogab-Gs~A4XSI&C*X}ukFi^8>ix* zzrK;C=}Pz?{>`N6Oj=z}7twS*t*K{ZYIR7t9}5cu^uCMsf8)m(X^iiJNSz$ z(G(Ve7~W~RcrNIxb94^7�}=tWlM>nv1KwW@lOBtVXrDX12dx=ip@aJeS!~Ww&Xo zJ9JGg^?Iekqtlc$nN@8{vp$QJWq2CRjv}vdwk{jWyaxT;#!?O4Xmr@^W^GlSSeZpz z1lUC#H5pY|7VTWX#rkcnhMG!MxkiUEd{({EZ>!Ez*lN^vjIDDRJ!VyrqP(e+so$)x zRv7eE4OVR-<_-F7o;u)eYcKH`^h(yoS>m;*BJycB*Oc3Jjo`0S-2i?zSd_isIjg;v zVtbvn*zV_ib(pnz4b}pKf~muyv|F`B&4!w)2F%y&toE3TAzsBH@o6t>Wc+iO^wl0? zQJ#RSu076Nnk-Ir2g}{)GV0r!YgAbcW@ta$9)^q(zonhw>Nn`Kz_0RNe9uyvYt&i2 zw5Qhyae+2h7;rz)g1)vbuT+EkVGtAI>8!~pZ!%|;uHXSvI)%8bOX*;G^RH`bJNnDlwRc=tDRhS6?u6giF6x$R}_ zI48#e%so~|b-O`twbyj${Bh>!GCQjvhQQ#x*mxxYl0{aqWexSX|@I zSs2%>*y7rJHN>?Su3~YGJLizNmd}o`ehc|-H7l=z?^af1uA*^`H|Hjcqpi2~rzhU) zPlKw;UZZMj?PYz}1be1BSc7)No0~;zSJbGyuxBb)z&^ zzkX~t!(PsqVXw2mIg`avU1xB>xlMJhzqBdhxlB}>tD*0dvu7jZnCzwRa5h&%-^q$} z?$b-(VQsE_-)Sptq9e^Uk@`-+=F0b-!b_NU0ybB^@65i0d55>TD*Dbv&pXg}SevV% z?_B)6Q|P;D=AGG@nHN3p@HSUP-)X*td55)$U*DC z*}vTfxg;0PMd4gx{NiYAHESd82OW_2Y>s@c17|9MTp*bb1HUgWcM17OSm$>bBk%Q@ zT%$!*9eLm9toE7o<*i1@vEn#FPBJ`)*=lgO?1qes$f-J%P+zqt(mZa~=QSJk{)p{n z)n~aNuhPu5zet`puhHbFZsPX5qIp9$M_r_V{^T+^t6}fSj* znAuUo=nebkasev*d-3%Fce+iUZ@cT^sUQr1-`2t!4T1_}N zUxah?v<1Is(iTwi7vuWWkgu=6?~t(v{W7={0M>w23?(b7{KN4|fGLIyKZk z7r(3VdqE4%^?UGzfMR@CfcE3}TpIY50yrqa^#FK-_XiHm*tEDF zOyM7R*WmYZsEF1u#IWbMHuzx-T2Wz!`wZ|W&_TVWAX33i;T4?kd5U>WG^xISRRsFqaJad_$4*8bN&SV^= zb>4cZ1Pa`A}nkB0L{Q<};9Q&Z8zwGZnL6GYsRxL*-|)J8g=DfG?4dp<2R>|^_5|Mg*k!x8v9xwZV#P1P+wbL z>x1g(GIO9ly29Ou_|a^Cp-Z#*_*L|=KBj#Oasu_)`dS~3U$Hw-pSO?oG3{Gw4%C<1 z*ZMHOieUUoD_TR^BYb{k=A&Z%EWO;$+?`-%3fgfSJD+N3Z9xv(9^8CH%)(7lMqS?0y?U}QBnCZ0v`?4^5Vm?%0`(WOr818-)SKC2I`&v2w>kAZIef)fg zKT5*+y%fI(^B+u(hRzPy)6G*u=hJX|CbWY56RwWdwsx@buUF3vvu8fNg6|76zeB#l zV`083YAvU)p>+khA@Rox5#jehjZM{J_g=D?`KUODv)9qS3XV@DJDD1c+C#>IF}VTzn;Z(di5;M9^%^spI&DD!hUbX`?oNM zoLNSzZwuI$hS|s7J}PEWQEKM!b#!h8=U>e2WAM|ng7&%!7^*lvoPF%=Q^~YXaVuwE zuQp7Dp~Hk!d_BdC7h(I9LtN(s>@~VD`WAM$6f2=&}MBx);?x?Pat^6|k>} zX3xw&A@}NFQS1oVYYJc{>Z*}PLMF4q+15_cMRNjr_bz779fX84abUnT!YzPw8!seu1m)c*&*AR5R3 literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json new file mode 100644 index 0000000..5cd1987 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json @@ -0,0 +1,79 @@ +{ + "artifacts": { + "baseline_lr_001": { + "digest": "sha256:495ac3c8c02c0846423131c4636dfde05f0da872b637bca2be9493b3c02aa8b4", + "id": 9258026048, + "source_sha": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36", + "workflow_run_id": 31926255124 + }, + "candidate_lr_003": { + "digest": "sha256:b3b77cbf691bef574b0a27c897b8a92a22e90a4c158f0f7e3ae556400d60e599", + "id": 9258115955, + "source_sha": "db7d430649e779287d06539a0527848ffbf815b6", + "workflow_run_id": 31926664340 + } + }, + "candidate_vs_baseline": { + "coverage_90_abs_error": { + "baseline_mean": 0.030999999046325687, + "candidate_lower_wins": 3, + "candidate_mean": 0.03200000524520876, + "candidate_minus_baseline": 0.0010000061988830744, + "relative_percent": 3.2258265472482375 + }, + "crps": { + "baseline_mean": 0.35507492380808053, + "candidate_lower_wins": 2, + "candidate_mean": 0.3566396681314354, + "candidate_minus_baseline": 0.0015647443233548497, + "relative_percent": 0.4406800419960381 + }, + "interval_score_90": { + "baseline_mean": 2.559808672169682, + "candidate_lower_wins": 1, + "candidate_mean": 2.6811327630567057, + "candidate_minus_baseline": 0.12132409088702367, + "relative_percent": 4.739576524060681 + }, + "pit_ks_stat": { + "baseline_mean": 0.08734850000489737, + "candidate_lower_wins": 3, + "candidate_mean": 0.09592410150802438, + "candidate_minus_baseline": 0.008575601503127014, + "relative_percent": 9.817686053734416 + }, + "rmse": { + "baseline_mean": 0.6260427399635591, + "candidate_lower_wins": 3, + "candidate_mean": 0.6278397715490043, + "candidate_minus_baseline": 0.0017970315854451968, + "relative_percent": 0.28704615048324 + }, + "sharpness": { + "baseline_mean": 0.5801309335762859, + "candidate_lower_wins": 5, + "candidate_mean": 0.5489976021750633, + "candidate_minus_baseline": -0.03113333140122254, + "relative_percent": -5.366604261092824 + } + }, + "dataset": "1028_SWD", + "decision": "reject_lr_003", + "file_sha256": { + "baseline_lr_001/benchmark_outcome.json": "55f203900d837ce575c17eb55e165590c9e4b7cc33a29e3f4b363bb006934048", + "baseline_lr_001/datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "baseline_lr_001/openboost_manifest.json": "43c1a91fb8b6c76ece5a9557389a8f1fc4f9bda403c6cd1c604a0ddb62db5c26", + "baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet": "1be2fab8e079e9203c23c7a50b23d83b7798bce3a35e7cf6ee8374a35ac18a63", + "baseline_lr_001/raw/ngboost/1028_SWD.parquet": "4cd7c3de7e764ab296b4c3712335626537dbb0c961a6ea448907ba68c9d3b1d4", + "baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet": "20f5e2fd14d086cf2f14f78447c0816d3155ca5e9b9fe0228366678fe7e6051e", + "baseline_lr_001/raw/xgblss/1028_SWD.parquet": "b525e10262bc178e961b8cb1761f7b5aa1b397430e8bb9be8fee69b03036a3d2", + "baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet": "fabfacadaef81cdf354f5fe7a571391c4d6168f73766a92a444b9adc7c38b21f", + "candidate_lr_003/benchmark_outcome.json": "cefba05ad79fdb6056f55cba5afda606d79984537856a41868b7e4a88c12e4a1", + "candidate_lr_003/datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "candidate_lr_003/openboost_manifest.json": "b0e660512c9762fd7e1109b1d9f77d8b0e552424863f0197db2aaafab39d0182", + "candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet": "7672d89594c4cad05f413a0181fdf15b838534aa60a5ba4eab4998be343c7296" + }, + "folds": 5, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 99b5bc5..29c35d0 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -89,6 +89,13 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. The source checkout had no porcelain changes, and every frozen input/result file is checksummed in `benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json`. +- `1028_SWD` clean baseline run `31926255124` completed 25/25 rows in artifact + `9258026048` (digest `sha256:495ac3c8c02c0846423131c4636dfde05f0da872b637bca2be9493b3c02aa8b4`). + Development run `31926664340` completed 5/5 rows with the expected + `development_tuning` label in artifact `9258115955` (digest + `sha256:b3b77cbf691bef574b0a27c897b8a92a22e90a4c158f0f7e3ae556400d60e599`). + Both source checkouts were clean; every copied artifact file is checksummed + under `benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/`. - On that one diagnostic shard, OpenBoost ranked first on mean CRPS, 90% interval score, absolute 90% coverage error, and PIT KS. Against native XGBoost quantile it reduced those metrics by 7.7%, 38.8%, 82.2%, and 59.5% @@ -107,6 +114,18 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. from cross-model conclusions. Use a separately audited analytic Gaussian NLL for parametric-only density comparison; do not optimize OpenBoost against the current quantile log-score artifact. +- Untouched development dataset `1028_SWD` showed the opposite CRPS ranking + from `1027_ESL`: OpenBoost had the best interval score, coverage error, and + PIT KS, but mean CRPS was 4.9% worse than native XGBoost quantile, 1.7% worse + than XGBoostLSS, and 2.0% worse than NGBoost. It lost CRPS to native XGBoost + on all five folds. This disproves a general quality-win claim and identifies + a reproducible sharpness/calibration trade-off. +- A development-only `0.03 × 500` OpenBoost run narrowed mean sharpness by 5.4% + relative to `0.01 × 500`, but worsened CRPS by 0.44%, interval score by 4.74%, + PIT KS by 9.82%, and absolute 90% coverage error by 3.23%. It improved paired + CRPS in only two of five folds. Reject the larger fixed learning rate; the + next experiment must separate mean accuracy from post-fit scale calibration + or test a different scale objective. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts From 793a2ad9631d6a435b818bbadb1f789f070a8f97 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:45:36 -0700 Subject: [PATCH 29/49] feat: add explicit Gaussian CRPS training --- .../2026-08-15-scoringbench-integration.md | 19 ++++++ src/openboost/_distributions.py | 68 +++++++++++++++++++ src/openboost/_models/_distributional.py | 44 ++++++++++-- src/openboost/_models/_sklearn.py | 8 +++ src/openboost/_persistence.py | 2 +- tests/test_distribution_gradients.py | 43 ++++++++++++ tests/test_distributional.py | 62 +++++++++++++++++ tests/test_persistence.py | 24 ++++++- tests/test_sklearn.py | 8 ++- 9 files changed, 269 insertions(+), 9 deletions(-) diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 29c35d0..ed86d7a 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -126,6 +126,25 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. CRPS in only two of five folds. Reject the larger fixed learning rate; the next experiment must separate mean accuracy from post-fit scale calibration or test a different scale objective. +- Gaussian CRPS training is not a new algorithmic claim: NGBoost already + publishes a Normal CRPS score and generalized natural-gradient metric. A + direct local prototype of that metric was unstable at larger learning rates + on heteroscedastic synthetic data (mean predicted scale exploded), so it was + rejected as OpenBoost's implementation path rather than copied blindly. +- The exact Gaussian CRPS Hessian is indefinite in the tails and cannot be fed + to OpenBoost's positive-Hessian tree solver. The implemented explicit + `training_objective='crps'` instead uses the strictly positive expected CRPS + curvature under the current Normal prediction. This keeps the default NLL + path unchanged, keeps training objective independent from `eval_metric`, and + fails early for non-Normal distributions. +- In a local heteroscedastic synthetic diagnostic, expected-curvature CRPS + training reduced held-out CRPS relative to NLL training at each tested fixed + learning rate (`0.003`, `0.01`, `0.03`, and `0.1`) for 300 depth-3 rounds. + This is a development hypothesis only, not benchmark evidence. The core + mathematical/API slice passed 191 tests (2 skipped), including finite + differences, default-NLL identity, objective logging, sklearn cloning, and + persistence. It must still win on the frozen `1028_SWD` development folds + before being promoted into the ScoringBench wrapper experiment. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts diff --git a/src/openboost/_distributions.py b/src/openboost/_distributions.py index 7b519f2..74d7656 100644 --- a/src/openboost/_distributions.py +++ b/src/openboost/_distributions.py @@ -431,6 +431,74 @@ def nll_gradient( 'loc': (grad_loc.astype(np.float32), hess_loc.astype(np.float32)), 'scale': (grad_scale.astype(np.float32), hess_scale.astype(np.float32)), } + + def crps( + self, + y: NDArray, + params: dict[str, NDArray], + ) -> NDArray: + """Per-sample Gaussian continuous ranked probability score. + + Lower is better. The implementation uses the closed-form Normal + score in float64 so the same objective can be used both for training + diagnostics and finite-difference correctness tests. + """ + from scipy.special import erf + + y = np.asarray(y, dtype=np.float64) + loc = np.asarray(params['loc'], dtype=np.float64) + scale = np.asarray(params['scale'], dtype=np.float64) + z = (y - loc) / scale + cdf_contrast = erf(z / np.sqrt(2.0)) # 2 * Phi(z) - 1 + pdf = np.exp(-0.5 * z ** 2) / np.sqrt(2.0 * np.pi) + return scale * ( + z * cdf_contrast + 2.0 * pdf - 1.0 / np.sqrt(np.pi) + ) + + def crps_gradient( + self, + y: NDArray, + params: dict[str, NDArray], + ) -> dict[str, GradHess]: + """Gaussian CRPS gradient with positive expected curvature. + + Raw parameters are ``loc`` and ``log(scale)``. The exact CRPS + Hessian is indefinite for sufficiently large standardized residuals, + so it is not suitable for OpenBoost's positive-Hessian tree solver. + Instead this method returns the expected CRPS Hessian under the + current Normal prediction: + + ``diag(1 / (sqrt(pi) * scale), scale / (2 * sqrt(pi)))``. + + This is a score-specific, strictly positive curvature surrogate. It + is deliberately distinct from the Fisher information used by the NLL + natural-gradient objective. + """ + from scipy.special import erf + + y = np.asarray(y, dtype=np.float64) + loc = np.asarray(params['loc'], dtype=np.float64) + scale = np.asarray(params['scale'], dtype=np.float64) + z = (y - loc) / scale + cdf_contrast = erf(z / np.sqrt(2.0)) # 2 * Phi(z) - 1 + pdf = np.exp(-0.5 * z ** 2) / np.sqrt(2.0 * np.pi) + sqrt_pi = np.sqrt(np.pi) + + grad_loc = -cdf_contrast + grad_scale = scale * (2.0 * pdf - 1.0 / sqrt_pi) + curvature_loc = 1.0 / (sqrt_pi * scale) + curvature_scale = scale / (2.0 * sqrt_pi) + + return { + 'loc': ( + grad_loc.astype(np.float32), + curvature_loc.astype(np.float32), + ), + 'scale': ( + grad_scale.astype(np.float32), + curvature_scale.astype(np.float32), + ), + } def fisher_information( self, diff --git a/src/openboost/_models/_distributional.py b/src/openboost/_models/_distributional.py index a4151e0..cda1d19 100644 --- a/src/openboost/_models/_distributional.py +++ b/src/openboost/_models/_distributional.py @@ -66,6 +66,9 @@ #: Metrics accepted by ``fit(eval_metric=...)``. EVAL_METRICS = ('nll', 'crps', 'pinball', 'interval_score') +#: Objectives accepted by ``training_objective``. +TRAINING_OBJECTIVES = ('nll', 'crps') + def _validate_exposure(exposure, n_samples: int, context: str = "fit") -> NDArray: """Validate an exposure vector: positive, finite, shape (n_samples,). @@ -138,6 +141,10 @@ class DistributionalGBDT(PersistenceMixin): subsample: Row sampling ratio (0.0-1.0) colsample_bytree: Column sampling ratio (0.0-1.0) n_bins: Number of bins for histogram building + training_objective: Objective used to fit distribution parameters. + ``'nll'`` preserves the likelihood/Fisher path. ``'crps'`` is + currently available for Normal predictions and uses Gaussian + CRPS with a positive score-specific expected curvature. Attributes: trees_: Dict mapping param_name -> list of trees @@ -174,6 +181,7 @@ class DistributionalGBDT(PersistenceMixin): subsample: float = 1.0 colsample_bytree: float = 1.0 n_bins: int = 254 + training_objective: Literal['nll', 'crps'] = 'nll' # Fitted attributes (not init) trees_: dict[str, list[TreeStructure]] = field(default_factory=dict, init=False, repr=False) @@ -243,6 +251,19 @@ def fit( # Get distribution instance self.distribution_ = get_distribution(self.distribution) + if self.training_objective not in TRAINING_OBJECTIVES: + raise ValueError( + f"Unknown training_objective '{self.training_objective}'. " + f"Available: {', '.join(TRAINING_OBJECTIVES)}." + ) + if self.training_objective == 'crps' and not isinstance( + self.distribution_, Normal + ): + raise ValueError( + "training_objective='crps' currently supports only the " + "Normal distribution." + ) + y = np.asarray(y, dtype=np.float32).ravel() n_samples = len(y) @@ -425,11 +446,12 @@ def fit( params_report = self._constrained_params( raw_preds, exposure_param, train_log_offset ) + if self.training_objective == 'crps': + train_values = self.distribution_.crps(y, params_report) + else: + train_values = self.distribution_.nll(y, params_report) state.train_loss = float( - np.average( - self.distribution_.nll(y, params_report), - weights=sample_weight, - ) + np.average(train_values, weights=sample_weight) ) if last_metric is not None: # Early stopping monitors the LAST eval set's metric @@ -539,6 +561,8 @@ def _compute_gradients( Subclasses can override for different gradient computation. """ + if self.training_objective == 'crps': + return self.distribution_.crps_gradient(y, params) return self.distribution_.nll_gradient(y, params) def _predict_raw(self, X: NDArray | BinnedArray) -> dict[str, NDArray]: @@ -749,8 +773,10 @@ class NaturalBoost(DistributionalGBDT): Uses natural gradient instead of ordinary gradient, leading to faster convergence by accounting for the geometry of the parameter space. - Natural gradient: F^{-1} @ ordinary_gradient - where F is the Fisher information matrix. + With the default NLL objective, the natural gradient is + ``F^{-1} @ ordinary_gradient`` where F is the Fisher information matrix. + Normal CRPS training instead uses its score-specific expected curvature; + it does not reuse the NLL Fisher information. Key advantages over standard GBDT: - Full probability distributions, not just point estimates @@ -803,6 +829,12 @@ def _compute_gradients( Natural gradient = F^{-1} @ ordinary_gradient where F is the Fisher information matrix. """ + if self.training_objective == 'crps': + # Gaussian CRPS uses its own positive expected curvature rather + # than the NLL Fisher information. Returning gradient/curvature + # directly lets the tree solver perform the corresponding + # second-order leaf updates. + return self.distribution_.crps_gradient(y, params) return self.distribution_.natural_gradient(y, params) diff --git a/src/openboost/_models/_sklearn.py b/src/openboost/_models/_sklearn.py index d464537..bf39da2 100644 --- a/src/openboost/_models/_sklearn.py +++ b/src/openboost/_models/_sklearn.py @@ -644,6 +644,11 @@ class OpenBoostDistributionalRegressor(BaseEstimator, RegressorMixin): use_natural_gradient : bool, default=True If True, use NGBoost (natural gradient). Recommended for faster convergence and better uncertainty calibration. + training_objective : {'nll', 'crps'}, default='nll' + Objective used to fit the distribution parameters. CRPS training is + currently supported only for the Normal distribution and uses a + positive score-specific expected curvature. This is independent of + ``eval_metric``. early_stopping_rounds : int, optional Stop training if the validation metric (``eval_metric``) doesn't improve for this many rounds. Requires eval_set to be passed to @@ -706,6 +711,7 @@ def __init__( reg_lambda: float = 1.0, n_bins: int = 254, use_natural_gradient: bool = True, + training_objective: Literal['nll', 'crps'] = 'nll', early_stopping_rounds: int | None = None, verbose: int = 0, eval_metric: Literal['nll', 'crps', 'pinball', 'interval_score'] = 'nll', @@ -720,6 +726,7 @@ def __init__( self.reg_lambda = reg_lambda self.n_bins = n_bins self.use_natural_gradient = use_natural_gradient + self.training_objective = training_objective self.early_stopping_rounds = early_stopping_rounds self.verbose = verbose self.eval_metric = eval_metric @@ -782,6 +789,7 @@ def fit( min_child_weight=self.min_child_weight, reg_lambda=self.reg_lambda, n_bins=self.n_bins, + training_objective=self.training_objective, ) all_callbacks = list(callbacks) if callbacks else [] diff --git a/src/openboost/_persistence.py b/src/openboost/_persistence.py index f028b6e..b45a846 100644 --- a/src/openboost/_persistence.py +++ b/src/openboost/_persistence.py @@ -17,7 +17,7 @@ T = TypeVar("T", bound="PersistenceMixin") -_SERIALIZATION_VERSION = 2 +_SERIALIZATION_VERSION = 3 def _to_numpy(arr: Any) -> np.ndarray | None: diff --git a/tests/test_distribution_gradients.py b/tests/test_distribution_gradients.py index 2506734..ece53c4 100644 --- a/tests/test_distribution_gradients.py +++ b/tests/test_distribution_gradients.py @@ -118,6 +118,49 @@ def test_normal(self): } _check_family_gradients(Normal(), y, raw) + def test_normal_crps_raw_gradients_and_expected_curvature(self): + """Gaussian CRPS derivatives match FD; training curvature stays positive.""" + dist = Normal() + z_values = np.array([-6.0, -2.0, -0.5, 0.0, 0.5, 2.0, 6.0]) + scales = np.array([0.05, 1.0, 20.0]) + z = np.tile(z_values, len(scales)) + scale = np.repeat(scales, len(z_values)) + loc = np.linspace(-1.0, 1.0, len(z)) + y = loc + z * scale + raw = {'loc': loc, 'scale': np.log(scale)} + params = _params_from_raw(dist, raw) + grads = dist.crps_gradient(y, params) + + for name in dist.param_names: + fd = _fd_grad_raw( + dist, + y, + raw, + name, + objective=dist.crps, + eps=1e-5, + ) + assert_allclose(grads[name][0], fd, rtol=5e-4, atol=5e-5) + assert np.all(np.isfinite(grads[name][1])) + assert np.all(grads[name][1] > 0) + + sqrt_pi = np.sqrt(np.pi) + assert_allclose(grads['loc'][1], 1 / (sqrt_pi * scale), rtol=1e-6) + assert_allclose(grads['scale'][1], scale / (2 * sqrt_pi), rtol=1e-6) + + def test_normal_crps_loss_matches_public_metric(self): + from openboost import crps_gaussian + + dist = Normal() + y = np.array([-2.0, 0.2, 3.0]) + params = { + 'loc': np.array([-1.5, 0.0, 2.0]), + 'scale': np.array([0.2, 1.0, 4.0]), + } + assert np.mean(dist.crps(y, params)) == pytest.approx( + crps_gaussian(y, params['loc'], params['scale']), rel=1e-12 + ) + def test_lognormal(self): y = np.array([0.05, 1.0, 4.0, 50.0]) raw = { diff --git a/tests/test_distributional.py b/tests/test_distributional.py index 7ae233b..29347b5 100644 --- a/tests/test_distributional.py +++ b/tests/test_distributional.py @@ -960,6 +960,68 @@ def test_crps_metric_decreases(self): assert vals[-1] < vals[0] assert np.mean(vals[-5:]) < np.mean(vals[:5]) + def test_crps_training_reduces_crps_and_reports_objective(self): + """Explicit CRPS training improves its objective and logs CRPS, not NLL.""" + from openboost import HistoryCallback, NaturalBoost, crps_gaussian + + X, y = self._make_data(seed=41, n=400, noise=0.5) + initial = crps_gaussian( + y, + np.full_like(y, np.mean(y)), + np.full_like(y, np.std(y) + 1e-6), + ) + history = HistoryCallback() + model = NaturalBoost( + distribution='normal', + training_objective='crps', + n_trees=50, + max_depth=3, + learning_rate=0.1, + ) + model.fit(X, y, callbacks=[history]) + + output = model.predict_distribution(X) + final = crps_gaussian(y, output.params['loc'], output.params['scale']) + final_nll = model.nll(X, y) + assert final < initial + assert history.history['train_loss'][-1] == pytest.approx(final, rel=1e-5) + assert history.history['train_loss'][-1] != pytest.approx(final_nll, rel=1e-3) + + def test_training_objective_is_explicit_and_validated(self): + """CRPS is opt-in, independent of eval_metric, and Normal-only.""" + from openboost import NaturalBoost + + X, y = self._make_data(seed=42, n=100) + default = NaturalBoost( + distribution='normal', n_trees=5, max_depth=2, learning_rate=0.05 + ) + explicit_nll = NaturalBoost( + distribution='normal', + training_objective='nll', + n_trees=5, + max_depth=2, + learning_rate=0.05, + ) + default.fit(X, y, eval_set=[(X, y)], eval_metric='crps') + explicit_nll.fit(X, y) + for name in ('loc', 'scale'): + assert_allclose( + default.predict_params(X)[name], + explicit_nll.predict_params(X)[name], + rtol=0, + atol=0, + ) + + with pytest.raises(ValueError, match="supports only the Normal"): + NaturalBoost( + distribution='poisson', training_objective='crps', n_trees=1 + ).fit(X, np.maximum(np.rint(y - y.min()), 0)) + + with pytest.raises(ValueError, match="Unknown training_objective"): + NaturalBoost( + distribution='normal', training_objective='mystery', n_trees=1 + ).fit(X, y) + def test_pinball_and_interval_metrics(self): """pinball (with quantiles) and interval_score (with level) run.""" from openboost import NaturalBoost diff --git a/tests/test_persistence.py b/tests/test_persistence.py index f9f2444..8c4cdd4 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -99,7 +99,7 @@ def test_save_load_preserves_categorical_tree_state(self, tmp_path): ) state = model._to_state_dict() - assert state["_serialization_version"] == 2 + assert state["_serialization_version"] == 3 assert all("is_categorical_split" in tree for tree in state["trees_"]) assert all("cat_bitsets" in tree for tree in state["trees_"]) @@ -326,6 +326,28 @@ def test_save_load_normal(self, regression_data, tmp_path): np.testing.assert_allclose(interval_before[0], interval_after[0], rtol=1e-5) np.testing.assert_allclose(interval_before[1], interval_after[1], rtol=1e-5) + def test_save_load_crps_objective(self, regression_data, tmp_path): + """CRPS training semantics survive a persistence round trip.""" + import openboost as ob + + X, y = regression_data + model = ob.NaturalBoostNormal( + training_objective='crps', n_trees=10, max_depth=3 + ) + model.fit(X[:400], y[:400]) + pred_before = model.predict_distribution(X[400:]) + + save_path = tmp_path / "crps-model.joblib" + model.save(save_path) + loaded = ob.NaturalBoost.load(save_path) + pred_after = loaded.predict_distribution(X[400:]) + + assert loaded.training_objective == 'crps' + for name in ('loc', 'scale'): + np.testing.assert_allclose( + pred_before.params[name], pred_after.params[name], rtol=1e-6 + ) + def test_save_load_poisson(self, tmp_path): """Test save/load for NaturalBoost with Poisson distribution.""" import openboost as ob diff --git a/tests/test_sklearn.py b/tests/test_sklearn.py index 1525098..a9a746e 100644 --- a/tests/test_sklearn.py +++ b/tests/test_sklearn.py @@ -579,22 +579,28 @@ def test_clone_get_params_with_new_params(self): reg = OpenBoostDistributionalRegressor( distribution='gamma', n_estimators=15, + training_objective='nll', eval_metric='interval_score', quantiles=[0.1, 0.9], interval_alpha=0.2, ) params = reg.get_params() assert params['eval_metric'] == 'interval_score' + assert params['training_objective'] == 'nll' assert params['quantiles'] == [0.1, 0.9] assert params['interval_alpha'] == 0.2 reg_clone = clone(reg) assert reg_clone is not reg assert reg_clone.eval_metric == 'interval_score' + assert reg_clone.training_objective == 'nll' assert reg_clone.quantiles == [0.1, 0.9] assert reg_clone.interval_alpha == 0.2 - reg.set_params(eval_metric='nll', quantiles=None) + reg.set_params( + training_objective='crps', eval_metric='nll', quantiles=None + ) + assert reg.training_objective == 'crps' assert reg.eval_metric == 'nll' assert reg.quantiles is None From f092613f5d3b77c54b100c8a07be865ec3173fe3 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:46:24 -0700 Subject: [PATCH 30/49] bench: evaluate CRPS training objective --- .github/workflows/scoringbench.yml | 10 ++++++++++ benchmarks/scoringbench/README.md | 5 +++++ benchmarks/scoringbench/run.py | 10 ++++++++++ .../scoringbench/test_openboost_wrapper.py | 19 +++++++++++++++++++ .../2026-08-15-scoringbench-integration.md | 5 +++++ tests/test_scoringbench_provenance.py | 10 +++++++++- 6 files changed, 58 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index c5e623a..4a0f267 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -35,6 +35,14 @@ on: required: true default: "0.01" type: string + training_objective: + description: OpenBoost objective for development_shard + required: true + default: nll + type: choice + options: + - nll + - crps max_depth: description: OpenBoost tree depth for development_shard required: true @@ -178,6 +186,7 @@ jobs: DATASET_NAME: ${{ inputs.dataset_name }} N_TREES: ${{ inputs.n_trees }} LEARNING_RATE: ${{ inputs.learning_rate }} + TRAINING_OBJECTIVE: ${{ inputs.training_objective }} MAX_DEPTH: ${{ inputs.max_depth }} REG_LAMBDA: ${{ inputs.reg_lambda }} MIN_CHILD_WEIGHT: ${{ inputs.min_child_weight }} @@ -191,6 +200,7 @@ jobs: --n-repeats 1 \ --n-trees "${N_TREES}" \ --learning-rate "${LEARNING_RATE}" \ + --training-objective "${TRAINING_OBJECTIVE}" \ --max-depth "${MAX_DEPTH}" \ --reg-lambda "${REG_LAMBDA}" \ --min-child-weight "${MIN_CHILD_WEIGHT}" \ diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index 3aa5023..e2a9d96 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -189,6 +189,7 @@ metrics: --n-folds 5 \ --n-trees 250 \ --learning-rate 0.04 \ + --training-objective crps \ --max-depth 2 \ --reg-lambda 1.0 \ --min-child-weight 1.0 \ @@ -198,6 +199,10 @@ metrics: Record every tried configuration, including failures. Select one configuration using only development datasets; do not repeatedly inspect the held-out suite. +`--training-objective crps` is an explicit development candidate for Gaussian +CRPS. It does not change `eval_metric`, is not enabled by default, and must not +be described as an official result until it is frozen and rerun on untouched +data. Use `--dataset-index N` or `--dataset-name NAME` for resumable shards. Use `--list-datasets` to display the validated list. Do not tune OpenBoost on the diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 61cb59c..30ad727 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -328,6 +328,15 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--n-trees", type=int, default=500) parser.add_argument("--learning-rate", type=float, default=0.01) parser.add_argument("--max-depth", type=int, default=3) + parser.add_argument( + "--training-objective", + choices=("nll", "crps"), + default="nll", + help=( + "OpenBoost training objective. CRPS is a development candidate; " + "use --development-run while evaluating it." + ), + ) parser.add_argument("--reg-lambda", type=float, default=1.0) parser.add_argument("--min-child-weight", type=float, default=1.0) parser.add_argument("--n-quantiles", type=int, default=99) @@ -477,6 +486,7 @@ def _model_parameters(args) -> dict[str, dict]: "model_params": { "reg_lambda": args.reg_lambda, "min_child_weight": args.min_child_weight, + "training_objective": args.training_objective, }, } return { diff --git a/benchmarks/scoringbench/test_openboost_wrapper.py b/benchmarks/scoringbench/test_openboost_wrapper.py index a9c7127..a01b22e 100644 --- a/benchmarks/scoringbench/test_openboost_wrapper.py +++ b/benchmarks/scoringbench/test_openboost_wrapper.py @@ -31,3 +31,22 @@ def test_openboost_wrapper_distribution_contract(): assert np.all(np.isfinite(distribution.bin_edges)) assert np.allclose(distribution.probas.sum(axis=1), 1.0) assert np.all(np.diff(distribution.bin_edges, axis=1) > 0) + + +def test_openboost_wrapper_forwards_crps_training_objective(): + rng = np.random.default_rng(7) + X = rng.normal(size=(100, 3)).astype(np.float32) + y = (X[:, 0] + rng.normal(scale=0.4, size=100)).astype(np.float32) + model = OpenBoostWrapper( + backend="cpu", + n_trees=5, + learning_rate=0.05, + max_depth=2, + n_quantiles=9, + model_params={"training_objective": "crps"}, + ) + + model.fit(X[:80], y[:80]) + + assert model._model.training_objective == "crps" + assert np.all(np.isfinite(model.predict(X[80:]))) diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index ed86d7a..456b153 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -145,6 +145,11 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. differences, default-NLL identity, objective logging, sklearn cloning, and persistence. It must still win on the frozen `1028_SWD` development folds before being promoted into the ScoringBench wrapper experiment. +- The benchmark launcher and manual Actions workflow now record and forward + `training_objective`; the default remains `nll`, while CRPS candidates must + carry the `development_tuning` protocol label. The provenance suite passed + 10 tests and both NLL/CRPS wrapper contracts passed against the pinned + ScoringBench checkout. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index d0479da..816c636 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -183,12 +183,14 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): assert args.catboost_rounds == 1000 assert args.reg_lambda == 1.0 assert args.min_child_weight == 1.0 + assert args.training_objective == "nll" assert args.development_run is False parameters = _model_parameters(args) assert parameters["openboost_cpu"]["model_params"] == { "reg_lambda": 1.0, "min_child_weight": 1.0, + "training_objective": "nll", } assert parameters["xgboost_quantile"]["num_boost_round"] == 100 assert parameters["xgblss"]["num_boost_round"] == 100 @@ -206,6 +208,8 @@ def test_development_parameters_are_explicit_in_manifest_constructor_contract(): "250", "--learning-rate", "0.04", + "--training-objective", + "crps", "--max-depth", "2", "--reg-lambda", @@ -222,5 +226,9 @@ def test_development_parameters_are_explicit_in_manifest_constructor_contract(): "learning_rate": 0.04, "max_depth": 2, "n_quantiles": 99, - "model_params": {"reg_lambda": 3.0, "min_child_weight": 5.0}, + "model_params": { + "reg_lambda": 3.0, + "min_child_weight": 5.0, + "training_objective": "crps", + }, } From 692b9ad8b0ca5b2002ab458570b1545421744379 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 21:56:06 -0700 Subject: [PATCH 31/49] bench: freeze CRPS objective diagnostic --- .../1028_swd_crps_20260816/README.md | 37 ++ .../benchmark_outcome.json | 21 + .../1028_swd_crps_20260816/datasets.json | 482 ++++++++++++++++++ .../openboost_manifest.json | 126 +++++ .../raw/openboost_cpu/1028_SWD.parquet | Bin 0 -> 29982 bytes .../1028_swd_crps_20260816/summary.json | 57 +++ .../2026-08-15-scoringbench-integration.md | 9 + 7 files changed, 732 insertions(+) create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/raw/openboost_cpu/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md new file mode 100644 index 0000000..9ed3969 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md @@ -0,0 +1,37 @@ +# 1028_SWD Gaussian CRPS-objective diagnostic + +This is development/tuning evidence, not held-out or leaderboard evidence. It +compares the frozen NLL-trained OpenBoost baseline in +`../1028_swd_lr_sweep_20260816/baseline_lr_001/` with one candidate that +changed only `training_objective` from `nll` to `crps`. Both used 500 rounds, +learning rate 0.01, depth 3, identical regularization, seed 42, five folds, +sample cap 3000, 99 quantiles, and pinned ScoringBench commit `a938a667`. + +| OpenBoost objective | CRPS | RMSE | 90% coverage error | 90% interval score | PIT KS | Sharpness | +|---|---:|---:|---:|---:|---:|---:| +| NLL | 0.355075 | 0.626043 | 0.0310 | **2.559809** | 0.087349 | **0.580131** | +| Gaussian CRPS | **0.353996** | **0.625552** | **0.0280** | 2.660561 | **0.080873** | 0.595394 | + +The CRPS objective improved mean CRPS by 0.30% and won four of five paired +folds. It also improved PIT KS by 7.41%, absolute 90% coverage error by 9.68%, +and RMSE by 0.08%. However, it widened the predictive distribution by 2.63% +and worsened 90% interval score by 3.94% on every fold. This is a real but small +primary-metric improvement with a material guardrail regression. + +It does not meet the stated benchmark goal. Candidate CRPS remains 4.62% worse +than native XGBoost quantile, 1.35% worse than Gaussian XGBoostLSS, 1.67% worse +than NGBoost, and 5.24% worse than CatBoost quantile on the already-frozen +five-model baseline. Therefore the objective is retained as a mathematically +tested development capability, not selected as the final ScoringBench model. +The next diagnostic should tune scale without changing the mean model, using +training-fold-only calibration; test-fold scale selection would be leakage. + +Run: [31927426636](https://github.com/jxucoder/openboost/actions/runs/31927426636), +artifact `9258329504`, digest +`sha256:02f3827ee1787a6559f841cd277407d7faa1f2af2dd8031caf39794d8173594e`. +The run completed 5/5 expected rows from clean OpenBoost source `f092613` and a +clean pinned ScoringBench checkout. `summary.json` records unrounded effects +and SHA-256 hashes for every copied file. Fit-time differences from separate +Actions processes are not treated as speed evidence; reconstructed log score +and CRLS remain excluded from the decision for the documented evaluator +comparability reasons. diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json new file mode 100644 index 0000000..598c1da --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 5, + "observed_rows": 5, + "status": "complete", + "valid_rows": 5 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 5, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 5, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 5 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json new file mode 100644 index 0000000..f887dee --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json @@ -0,0 +1,126 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "development_run": true, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_cpu" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "crps", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31927426636", + "source_sha": "f092613f5d3b77c54b100c8a07be865ec3173fe3", + "tested_sha": "f092613f5d3b77c54b100c8a07be865ec3173fe3" + }, + "created_at": "2026-08-16T04:53:35+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 5, + "model_parameters": { + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "model_params": { + "min_child_weight": 1.0, + "reg_lambda": 1.0, + "training_objective": "crps" + }, + "n_quantiles": 99, + "n_trees": 500 + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "f092613f5d3b77c54b100c8a07be865ec3173fe3", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 5 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 5, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..964f849e89f8dea77340dc85ca432b54010e6e37 GIT binary patch literal 29982 zcmd5_4}4SA^-l|wM=4s+)YPFAh7TB6nzW%ULvLTwHce?uTWFI;u*<);X_GW2ZPQkm z%zqOV1$8Rw7!!0(`8n~=Kv4?QIX2yJbJFssQpG6~5D-B&(f!VSf0CEnmnPT}$j90D z&OPUTzvtXP@7{av$5k@bO8H26rM$XKo-CJ5kjbVSlY7@!_G}(NB_DNfs9iCDWHQ-+ z#d{zjHS4V>J{>^4>pkCo_{aT7E|aIpubc&Dsq!Rw(nJLX2CRmH5_HmJ(HnlsqfUUTD@aGYW{MZ_2<@J zWZc+3ca^_if`d?^poBUw2T~fH!&Aq!iDy6SMXfLGziQ1FyU~K=rv4Sz?nb*dUvYHm zYrSY?%D6|bSn*E@4uYkE5_B*JQW~AZLw9!Ab`~5%w({kM(fHCX`}T$-=+es? z{x!pW6y>blqdbg0m*5~+Dkwn*b0DSBIlOqFYijS=kCEf@GdCao@j*0k#OW_{ryfN2 zu0C6Cc@F2O<2P*6f0m;))5WF{si4iAJ_6^^*JX`Y>a^5&lSJHSw!fnA)K?ypT0x6YryHZ{em<}ca?TkAP??DOF z$meyVs7hJy`abOY^6Qd&+39J$M`|2`#sDs1W5DY2F$j}JARwBL_S+|)`r!66XyKDb zir<>{HJb3)SKlsq{4~l?uQVS>J%eUHop{N6V^2v0L9kR%f)3_DN+WOx1|bK$Dag(~ zi2aq#SUieKfgbP)Ov>OGD_nlK>Jte9*bEc{K{$x`AcRRH5D*2yk-lwJ{nl^M6M3e2 z-;F+t{`%B-pRw^9bX{A^+t(jCi=H{`zux|r|448UEESZXgE^4W2plQ{Js}T!lid3m z4noHBx_>87DbN=VB~V%G)~xwA+ZC#|NKHkcKY&ZP;ecxpxP^R3um`C_x8vAf*vF1iJ!` z0<`*J>@T$9kpzm`{_eY@s4VCgtnerI|KX@aGk`i*Y#`y*_lulF1sMa}BI{0scH&U4)t*n4tDWJ_4@SxdePgh^TC;EXtRMEH#O z?^oqtmrzu}b<^JX$Ip&X1slp!XFv1k=z=#I=l%DdhsG8xb*C@dF+WMdc|k)#2|Abp zDV21-QZ8zMp0Pjp_{ML(L}vQLr1$$zp!+G)ikHy|bkieEvW<0LqB}?aI(OUB;}V@f zuvAcj4(32gqjQ*b^+&#iOZ$5^UpXz~^U@D{I?}HG=6>ct&!69XZB4?G13j0$*^{_& z#Ya8lNDwR)l%Rt-kkaTJjz6-o>$x9nLMd;bZJ2cA6?Dmg~<9-xBe5Ydvj%H)uva`hmWkSTfF=Ow5tE$QrlA- z(ZKD;=ic|b-$`%~G!>Mfi#d?e=p0@;sQGyM&ezdLGw1j3ShE%7p89it`XyV@56`BL zJFw>sG^6Z6`=X>*B{&Eg3QEwy97t(&4rM1d6h7;C1G(@`$# z?CwAP>-bi5!1!RZMv{YIsh|WM%z>0f=Wtp@*Nl4kMbxn78g; z^xN*VuAbxl=*QOkXHV>uTCfO~3QEwy97t(&4*R;AR`gEYho+iVKYnBHUNrI9nh^yL z??n~6_UyQI!aj7@=QhpZf9{cpgJ7wk1RczQlt$;UVQ$%{o|Zkxd}@^cmq+`MeM!Ot zZ~VLu+4ArIW?a=CRG?HY+vV+*;2>BkC_x8vAf?ecWIy<=eYCp|&G=>F=SO;aQOBm+ zzVpxNMYSb||2Mg;5B+83^Q)g5_&|b#V5y)49n680M(42orAf*O{{ifGH@{Pt-ixZX zj%c#B>_%HYvYHm}>P02@KC%1m15%qf!4kICybk6-N~3cSHhvd!BRTI+Rnt?C^&-Qv z(p!r5?M9OupZZ&M(r&c=WPj4p*L%@R%h%t8N~Pi;SSlz%2giYwMvudfdh^bny1oxh z-TKAkjVE`b)tkquj-UJh?S198&)*o)hjtr&J42Q%#X+!CP=XHTKuV)?xTW={FVr9C zLvOBndP9M+7j0bTc>n*V??x%kxijv34EpLrxAZ@mD#bytgt?m6!5m0wbPf-#FTFM4 z-96~^Q}0jyJNUh5(&lZKuHB8a&)N27F580^Tsh_Kr8QC<1WN@a=wJ?{G&+an_w4w` zBRB3vn{HW`{wrS}TA6mW?CWE@(SH}!7`G+wMLX!HP$#{pm&F<|mJGpuS@}Oo;1WiF6)SMGRQ;-KW^CD;p@}OpZ1WiFM(o{=( zl+VjYTD?v_E|k?~uXnZzVH)dkwAHux>V0kIHm;6wyu<^6Y3GBR?8kwxciJ0xrLmmA z>u7A^B`KW5YR56a@w^DeySjW^Ob4>}}V`%J6|PD;wo%wb`A#BrQ38V*1%kS~gLhoUWE7 zrKYpvR-TxaoVGkM?aoBHgQm_cjg>wNW*nJ35l@S$@MqqPq;y)A4DezUc1{pr?n{5(qfw@?OwstK2d@`Vs86zA-QW|p*MKiq!?&)## z?^6JdOb%f7w%;c6^8j* zIsN?QfJ!ENFe)jH88ztT9I=<{uAqOP#(5c*+u^+olg5k}?qv`aynH5|-l70hGMj@@ zNomZeK`-+HUf!NaZ~Oth{Yt(qLvuGmTZT$w#tZi{hzef5d=>rHL_j68H5iqY#*7;D zGC$xY^w{4|qTjum^D-0#t179Sh zeVIVN_(OX46wb@ABn*;5%17tE|fssk+P>^AdC_-LWK)-+hdDyok5`7zX-6G^?H1rk*kjd->_L-Cp&1Z4n zexZ=wt_5T=CxMYk>ClkHeS1qWy|W0A$xH-BCZ$6|7WeJ#CG>kbKqm7K7@3p~4LMKL z&O1x#K0P3lSqF?vN{5Cl9#!v^(F5}Vnanj{WKtS4a`3<U_es(Jpe=Jr{jz0<%Clb;&!0y3B|F>@99H>_^F+ezwR+%n7ofB&2&Is9hUrR{)LjpZ;1HJYJTyJP9 zfXob#2)8B#Akv;Rr6to;THj(gyi+83%x;_6r#O{)^Av~MX78-GwmREgZr_x-88<|%Ho%MR*OAlEPjEE*hkr>#Cr$idXNb$M9T24}0eZB7nWcelGd zT~Y-uIN`=er%<)K-RWF5D`(k8)hJUd`UtH(qRS*Bz#E3kfLFR z4V-0?upvdm3>!>ek+`8{!wf7+laa__g~JR@;_8 zsWQwo0#42uECO8NFhhsACq+O~WteFMB9J#&1h~Rsh7NO2ih!ibFw+P`Ab+q3aD~GR z9p;`C0ZEl%rV)&QdXRY_pDi3_=y3NW2uP|7GYx!gK*DuiI^1CV#RLgJQOPiaimpG9 zxM79o2b#NQ5fMCCJk0RHD;*?uXxVuIX0MJ!LJkz37ijK|N+j%H=`e#!-i9GrL{y6d zo$T1?uxfFjBQfhqMdy4>z*0E5u|840Q30Loz7grNbNx zQS}lBTH&SV4_sofh``Us7ztD_Ijx454zuUMo1tWjhn5X9Fus9GLJHC_FxdV`9mfX5 zgB`~5_ghIUQ9&47yW3%f8?e|U^!&Hg(rmZ3VfkQ)?Cn_+FCZQ|8b10qx3zg4mi9LG z*!yNUf1YCX+RbgPUO17yd5Y6v@tVC|_~6^qW%hbo{g@EkiN|;3u~enbgxdvEupQ4w zT$q^+U#Dd9wWJ7FDOxVo-CkY@fs8DsD3^ z)X#ykwmL(NqfE;qN{F@DzoZ}0lT={m|d;5>S_QN>vwk48LLzkT0O>Sw;7av zdu@J}-KcS5Y`x3mv#5%*Dw?a<`YndqtU5z=lTBBId4qnZuK~E*yGq;Z3`)+%UD{?< zMdZ_EF;+PB4)E8lX#zi+tja;~+_i1i5@&<0#OddKby;-zO}4_iEVd4p(rMEbx6~P{ zn=oICyVhqZfq0dK#HXv=!TRSi8)|)~;(QTTeOH{hG+W)8ZjQUjV={EM7**;f3$!0@ zkGkwqzqO0y>aR1X!LN!zd{-+iMvZ!q_G~jjT%gUf>To~NfxfOYzf6n!VGtAI={9Cp zG+VMuxpr}pya=86wOzMaSKMXQsh!sB@<@H0(1}}LcU60=*&3(SUE#5)aw73-F&iuV zCSz&0*^u8J@BU`VHaV@XVz;R_udAFJ=j1qmxyRn?`4 z4#HI`u5srqj%!YAac#U9;yMUdskp|SvpBBm*y7r9F~oHcu2ON0JLizNRxFILehc|- zwJ0xw@0L0;SIM}>n{%_()j8Pu(--gcXPv6rX;gJ~46?p!hCNdQtUI|Jq*eBt=9J$7GYdp5%d@t;` z{CMqTFT$BW-yL%{Ab8*Ht7J4(cn_anR1 zV^TF;D2~l0gF4G()JN)*f-fO@acgT=Z9D9D+;En_ujlg}crR+~X4=am_0Q-waT`+* zTi7$%U~kRsi;b#Me9q8NuFZ)!cSf~|Uq5zQU@vFNb~aeyoXP5{ZK!j>xlL`JzpOdp zxlB}>i=ppSaAzarm>i_<@HQ7i-%&?8_Zg(`a5fjd?{tOm+nRgcEqR4 zkFzfO9CE5ICDd2zi!_g04EZf4gFj-s*$iqA<~z);+Gc*wE15Usa@56H z(4RbY?poM;vbpPAod1Q~b(hIdoMp+@@%rD)#F@)`;Kpm>PyD^TS;MtzHB5*Wh;zelKjrxqctM5Kw~e3NU{Bp2q;cG5`l9 zxE=s+@Lq|jX#JRq6;olU75cHM51W=Wf+_q1?^^s`0TnS?mKgRN*9Jd~!DLlh;64L9 zRMmhs!(gJps|!#Img>0vgG*GE_yR<;YWD0rcq=MqE5VNQ_2CE13M>PC9oGkJSk=MT!W`C7MsLaH{MR%7fIsDu{;;9eg?_gySWWF59$(8; zF5>GeDrAEMPQ#CVWS?-uLp(#jYg{;T1O70rGofv;A|H?F{%CoBv}HMhKgP;tzI`}< z*dXGL#Q%!%qb}0*(H&CB)i1k(AP*zw1^%YgRhvYBYmYOoP z5Whu3tgjsNE6NSj=NM{zxIOgVKz*GU3Pg(;gO#IJIQ^|9?+ zm>a0iKGgbf{7Srm`r3wAAKSiVmOy=ZL#+?2&hL<~@LAcfiblulYZ-lEUP%1$LPY#M&|p)I)V&uq zJ0F$g^7eYBy^`ls#m(oiFo7RiANvHW2p}!Kztf^*<_GQPgxSNw2!28PBK)Y6%uTXh zsRQv~8fU|&7qka$4_Ig7^<9%mRok#&5qwHI;M0g}YSdJP*~7vtq&=(gJ6^McJ5;Bzi{7M>HaOwA?K7cnwbInvM~GD+egJND#|Q8zMjde{4{slP`&6;*Q_{iPH)`r;!_Z+vD!ra!$BVdqDj=?l0`^*cn0?Ic zQ{IeQPOIbXwM=;-4Bc~TpRRI7s|wgxMzd$%+OUzf#c=7sN)E{7D z0l(mQWv>>O=(~WXIe=HOh_@H!dr3U@?k!tgIqVsDJRk++VSV0+_dINuVi)_Y9wXGp z%0>L~DA3@0$5>6r+rwY>$IbQ(bDd0);W2w}YPYvtYxQ`pWiN4O@RzhRxa-^S-(`mX TX!Aw#kL=SWGFc`3KcoIX)Pvv% literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json new file mode 100644 index 0000000..d5543af --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json @@ -0,0 +1,57 @@ +{ + "artifact": { + "digest": "sha256:02f3827ee1787a6559f841cd277407d7faa1f2af2dd8031caf39794d8173594e", + "id": 9258329504, + "source_sha": "f092613f5d3b77c54b100c8a07be865ec3173fe3", + "workflow_run_id": 31927426636 + }, + "baseline": "../1028_swd_lr_sweep_20260816/baseline_lr_001", + "candidate_vs_nll": { + "coverage_90_abs_error": { + "baseline_mean": 0.030999999046325687, + "candidate_lower_wins": 2, + "candidate_mean": 0.027999999523162854, + "relative_percent": -9.677418114367365 + }, + "crps": { + "baseline_mean": 0.35507492380808053, + "candidate_lower_wins": 4, + "candidate_mean": 0.353996062777474, + "relative_percent": -0.30384038924407 + }, + "interval_score_90": { + "baseline_mean": 2.559808672169682, + "candidate_lower_wins": 0, + "candidate_mean": 2.6605614850848283, + "relative_percent": 3.9359509173687135 + }, + "pit_ks_stat": { + "baseline_mean": 0.08734850000489737, + "candidate_lower_wins": 4, + "candidate_mean": 0.08087311043727916, + "relative_percent": -7.4132807858808825 + }, + "rmse": { + "baseline_mean": 0.6260427399635591, + "candidate_lower_wins": 3, + "candidate_mean": 0.6255523498852755, + "relative_percent": -0.07833172513302955 + }, + "sharpness": { + "baseline_mean": 0.5801309335762859, + "candidate_lower_wins": 0, + "candidate_mean": 0.5953941579557143, + "relative_percent": 2.630996469251601 + } + }, + "decision": "keep_objective_reject_candidate_configuration", + "file_sha256": { + "benchmark_outcome.json": "cefba05ad79fdb6056f55cba5afda606d79984537856a41868b7e4a88c12e4a1", + "datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "openboost_manifest.json": "2ff8fd620d6ec1cc1f6c142790eebf3249532ca11f6aa0281efd138bd0367c96", + "raw/openboost_cpu/1028_SWD.parquet": "f1f199eb9c48efdcd00e4acdc55132db2e54eedc4817b4a3b90a41ce8614fb3f" + }, + "folds": 5, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 456b153..fac5a93 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -150,6 +150,15 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. carry the `development_tuning` protocol label. The provenance suite passed 10 tests and both NLL/CRPS wrapper contracts passed against the pinned ScoringBench checkout. +- Clean CRPS-objective run `31927426636` completed 5/5 `1028_SWD` rows in + artifact `9258329504` (digest + `sha256:02f3827ee1787a6559f841cd277407d7faa1f2af2dd8031caf39794d8173594e`). + Against the identical NLL configuration it improved mean CRPS by 0.30% + (four of five folds), PIT KS by 7.41%, coverage error by 9.68%, and RMSE by + 0.08%, but worsened 90% interval score by 3.94% on all five folds and widened + sharpness by 2.63%. It still trails every frozen strong baseline on CRPS. + Keep the objective implementation, reject this configuration as the final + wrapper, and next isolate post-fit scale calibration on training folds only. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts From 236c2dfad7c6ea6ea8d406077a59eec3ac32e510 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:17:31 -0700 Subject: [PATCH 32/49] fix: preserve the highest numeric bin --- .../2026-08-15-scoringbench-integration.md | 17 ++++++ src/openboost/_array.py | 51 ++++++++++++++-- src/openboost/_persistence.py | 10 +++- tests/test_binning_correctness.py | 60 ++++++++++++++++++- tests/test_persistence.py | 28 ++++++++- 5 files changed, 157 insertions(+), 9 deletions(-) diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index fac5a93..6cde112 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -159,6 +159,23 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. sharpness by 2.63%. It still trails every frozen strong baseline on CRPS. Keep the objective implementation, reject this configuration as the final wrapper, and next isolate post-fit scale calibration on training folds only. +- The apparent mean-model gap was traced to numeric binning, not histogram + subtraction. `np.searchsorted` over `m` cut edges legitimately returns + indices `0..m`, but both fit and transform clipped the result to `m - 1`. + That silently merged the highest numeric interval into its predecessor; a + binary feature with `n_bins=2` could become constant. The corrected binning + keeps the top index, tests the NaN and out-of-range paths, and records a + binning-semantics version in persistence. Models saved before serialization + version 4 retain the legacy routing so loading them does not silently change + predictions. Histogram subtraction was independently checked against direct + child histograms and selected the same splits; do not replace that optimized + path as part of this fix. +- The binning/persistence slice passed 83 focused tests across array handling, + tree growth, core fitting, and model round trips. Production files and the + changed binning test pass Ruff; `tests/test_persistence.py` still has its + pre-existing import-order/unused-import findings, which are unrelated to this + correctness change. A fresh ScoringBench artifact is still required before + interpreting the quality impact. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts diff --git a/src/openboost/_array.py b/src/openboost/_array.py index dab35a7..191d224 100644 --- a/src/openboost/_array.py +++ b/src/openboost/_array.py @@ -23,6 +23,8 @@ # Reserved bin index for missing values (NaN) MISSING_BIN: int = 255 +_LEGACY_BINNING_VERSION: int = 1 +_CURRENT_BINNING_VERSION: int = 2 @dataclass @@ -51,6 +53,11 @@ class BinnedArray: is_categorical: NDArray[np.bool_] = field(default_factory=lambda: np.array([], dtype=np.bool_)) category_maps: list[dict | None] = field(default_factory=list) n_categories: NDArray[np.int32] = field(default_factory=lambda: np.array([], dtype=np.int32)) + # Version 1 clipped searchsorted outputs to ``len(edges) - 1`` and + # accidentally merged the top numeric bin into its predecessor. Keep the + # version on fitted bin metadata so models saved before the correction + # retain their original prediction routing after loading. + binning_version: int = _CURRENT_BINNING_VERSION def __repr__(self) -> str: n_missing = int(np.sum(self.has_missing)) if len(self.has_missing) > 0 else 0 @@ -91,6 +98,12 @@ def transform(self, X: ArrayLike) -> BinnedArray: >>> X_test_binned = X_train_binned.transform(X_test) >>> predictions = model.predict(X_test_binned) """ + # Direct pickle/joblib payloads from before the version field existed + # bypass PersistenceMixin. Treat their missing attribute as legacy. + binning_version = vars(self).get( + 'binning_version', _LEGACY_BINNING_VERSION + ) + # Convert to numpy X_np = _to_numpy(X) if X_np.ndim == 1: @@ -141,10 +154,13 @@ def transform(self, X: ArrayLike) -> BinnedArray: # No bin edges (constant feature) binned[:, j] = 0 else: - # searchsorted finds the bin index + # ``m`` cut edges define ``m + 1`` bins, indexed 0..m. + # Version 1 incorrectly clipped the upper index to m - 1; + # preserve that behavior only for models trained before + # the top-bin correctness fix. bin_idx = np.searchsorted(edges, col[~nan_mask], side='right') - # Clip to valid range (in case test values exceed training range) - bin_idx = np.clip(bin_idx, 0, len(edges) - 1) + max_bin = _numeric_max_bin(edges, binning_version) + bin_idx = np.clip(bin_idx, 0, max_bin) binned[~nan_mask, j] = bin_idx.astype(np.uint8) # Handle missing values @@ -169,6 +185,7 @@ def transform(self, X: ArrayLike) -> BinnedArray: is_categorical=self.is_categorical, category_maps=self.category_maps, n_categories=self.n_categories, + binning_version=binning_version, ) @@ -288,6 +305,7 @@ def array( is_categorical=is_categorical, category_maps=category_maps, n_categories=n_categories, + binning_version=_CURRENT_BINNING_VERSION, ) @@ -405,21 +423,42 @@ def _bin_numeric_feature( else: edges = np.nanpercentile(valid_col, percentiles) edges = np.unique(edges) - # searchsorted is what digitize calls internally, minus overhead + # ``m`` cut edges create ``m + 1`` bins. The searchsorted output + # already lies in 0..m; allowing m is essential for preserving the + # top interval, especially for low-cardinality integer features. bin_idx = np.searchsorted(edges, valid_col, side='right') - np.clip(bin_idx, 0, len(edges) - 1, out=bin_idx) + np.clip( + bin_idx, + 0, + _numeric_max_bin(edges, _CURRENT_BINNING_VERSION), + out=bin_idx, + ) binned_col[~nan_mask] = bin_idx.astype(np.uint8) binned_col[nan_mask] = MISSING_BIN else: edges = np.percentile(col, percentiles) edges = np.unique(edges) bin_idx = np.searchsorted(edges, col, side='right') - np.clip(bin_idx, 0, len(edges) - 1, out=bin_idx) + np.clip( + bin_idx, + 0, + _numeric_max_bin(edges, _CURRENT_BINNING_VERSION), + out=bin_idx, + ) binned_col = bin_idx.astype(np.uint8) return binned_col, edges, has_nan, False, None, 0 +def _numeric_max_bin(edges: NDArray, binning_version: int) -> int: + """Return the highest valid numeric bin for persisted binning semantics.""" + if binning_version <= _LEGACY_BINNING_VERSION: + return max(len(edges) - 1, 0) + # ``array`` creates at most 253 cut edges, so the current top bin is at + # most 253 and remains safely below MISSING_BIN (255). + return min(len(edges), MISSING_BIN - 1) + + def _bin_categorical_feature( col: NDArray, ) -> tuple[NDArray[np.uint8], NDArray[np.float64], bool, bool, dict, int]: diff --git a/src/openboost/_persistence.py b/src/openboost/_persistence.py index b45a846..1bc33bf 100644 --- a/src/openboost/_persistence.py +++ b/src/openboost/_persistence.py @@ -17,7 +17,7 @@ T = TypeVar("T", bound="PersistenceMixin") -_SERIALIZATION_VERSION = 3 +_SERIALIZATION_VERSION = 4 def _to_numpy(arr: Any) -> np.ndarray | None: @@ -256,6 +256,9 @@ def _to_state_dict(self) -> dict[str, Any]: if value is not None: state["_bin_edges"] = value.bin_edges state["_n_features"] = value.n_features + state["_binning_version"] = vars(value).get( + "binning_version", 1 + ) if hasattr(value, "has_missing"): state["_has_missing"] = _to_numpy(value.has_missing) if hasattr(value, "is_categorical"): @@ -361,6 +364,10 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: is_categorical = state.get("_is_categorical", np.array([], dtype=np.bool_)) category_maps = state.get("_category_maps", []) n_categories = state.get("_n_categories", np.array([], dtype=np.int32)) + # States written before serialization version 4 used the legacy + # top-bin clipping rule. Preserve it so loading an old model does + # not silently change predictions. + binning_version = state.get("_binning_version", 1) # Create placeholder data (empty, just need structure for transform) placeholder_data = np.zeros((n_features, 0), dtype=np.uint8) @@ -375,6 +382,7 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: is_categorical=is_categorical if isinstance(is_categorical, np.ndarray) else np.array(is_categorical, dtype=np.bool_), category_maps=category_maps, n_categories=n_categories if isinstance(n_categories, np.ndarray) else np.array(n_categories, dtype=np.int32), + binning_version=int(binning_version), ) # Reconstruct _loss_fn from stored loss name/config diff --git a/tests/test_binning_correctness.py b/tests/test_binning_correctness.py index b116338..0753340 100644 --- a/tests/test_binning_correctness.py +++ b/tests/test_binning_correctness.py @@ -53,6 +53,27 @@ def test_transform_out_of_range_values(self): assert np.all(test_binned.data < 255), "Out-of-range values should not be missing bin" assert np.all(test_binned.data >= 0), "Bins should be non-negative" + def test_transform_preserves_corrected_top_bin(self): + """The largest and above-range values stay in the final numeric bin.""" + X_train = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + X_test = np.array([[0.0], [1.0], [2.0], [3.0]], dtype=np.float32) + + binned = ob.array(X_train, n_bins=2) + transformed = binned.transform(X_test) + + np.testing.assert_array_equal(transformed.data[0], [0, 0, 1, 1]) + + def test_transform_without_version_uses_legacy_routing(self): + """Directly unpickled pre-version metadata keeps its old top-bin rule.""" + X = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + binned = ob.array(X, n_bins=2) + del binned.binning_version + + transformed = binned.transform(X) + + assert transformed.binning_version == 1 + np.testing.assert_array_equal(transformed.data[0], 0) + class TestBinEdges: """Verify bin edge properties.""" @@ -142,11 +163,48 @@ def test_two_unique_values(self): """Two distinct values should produce two bins.""" X = np.array([[0.0], [0.0], [1.0], [1.0]], dtype=np.float32) - binned = ob.array(X) + binned = ob.array(X, n_bins=2) unique_bins = np.unique(binned.data[0, :]) assert len(unique_bins) == 2, f"Two values should produce 2 bins, got {len(unique_bins)}" + def test_low_cardinality_values_keep_distinct_top_bin(self): + """Quantile binning must not merge the highest ordinal level.""" + X = np.repeat( + np.array([[1.0], [2.0], [3.0], [4.0]], dtype=np.float32), + repeats=4, + axis=0, + ) + + binned = ob.array(X, n_bins=4) + + assert np.unique(binned.data[0]).tolist() == [0, 1, 2, 3] + + def test_top_bin_with_missing_values(self): + """The NaN path preserves both numeric bins plus the missing bin.""" + X = np.array([[1.0], [1.0], [2.0], [2.0], [np.nan]], dtype=np.float32) + + binned = ob.array(X, n_bins=2) + + np.testing.assert_array_equal(binned.data[0], [0, 0, 1, 1, 255]) + + def test_top_bin_enables_binary_tree_split(self): + """A depth-one tree can split the two values retained by binning.""" + X = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + y = np.array([0.0, 0.0, 1.0, 1.0], dtype=np.float32) + binned = ob.array(X, n_bins=2) + + tree = ob.fit_tree( + binned, + -y, + np.ones_like(y), + max_depth=1, + reg_lambda=0.0, + ) + + assert tree.features[0] == 0 + np.testing.assert_array_equal(tree(binned), y) + def test_very_large_values(self): """Large values should not cause overflow.""" X = np.array([[1e10, -1e10], [1e15, -1e15]], dtype=np.float32) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8c4cdd4..8f437c5 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -99,7 +99,8 @@ def test_save_load_preserves_categorical_tree_state(self, tmp_path): ) state = model._to_state_dict() - assert state["_serialization_version"] == 3 + assert state["_serialization_version"] == 4 + assert state["_binning_version"] == 2 assert all("is_categorical_split" in tree for tree in state["trees_"]) assert all("cat_bitsets" in tree for tree in state["trees_"]) @@ -469,6 +470,31 @@ def test_classifier_pickle(self, binary_data, tmp_path): class TestPersistenceEdgeCases: """Test edge cases for persistence.""" + def test_legacy_numeric_binning_routing_survives_load(self): + """Loading a pre-fix model preserves its legacy top-bin routing.""" + import openboost as ob + + X = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + y = np.array([0.0, 0.0, 1.0, 1.0], dtype=np.float32) + metadata = ob.array(X, n_bins=2) + metadata.binning_version = 1 + legacy_binned = metadata.transform(X) + assert np.unique(legacy_binned.data).tolist() == [0] + + model = ob.GradientBoosting(n_trees=1, max_depth=1) + model.fit(legacy_binned, y) + pred_before = model.predict(X) + state = model._to_state_dict() + state["_serialization_version"] = 3 + state.pop("_binning_version") + + loaded = ob.GradientBoosting() + loaded._from_state_dict(state) + + assert loaded.X_binned_.binning_version == 1 + np.testing.assert_array_equal(loaded.X_binned_.transform(X).data, 0) + np.testing.assert_allclose(loaded.predict(X), pred_before, rtol=0, atol=0) + def test_load_wrong_class_raises(self, regression_data, tmp_path): """Test that loading with wrong class raises error.""" import openboost as ob From 1d3faeb5ffc7645ef3e4f2785dbba2eebd71be0f Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:31:21 -0700 Subject: [PATCH 33/49] bench: freeze corrected binning diagnostic --- .../1028_swd_binning_fix_20260816/README.md | 46 ++ .../benchmark_outcome.json | 21 + .../datasets.json | 482 ++++++++++++++++++ .../openboost_manifest.json | 126 +++++ .../raw/openboost_cpu/1028_SWD.parquet | Bin 0 -> 29974 bytes .../summary.json | 74 +++ .../2026-08-15-scoringbench-integration.md | 22 + 7 files changed, 771 insertions(+) create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/raw/openboost_cpu/1028_SWD.parquet create mode 100644 benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md new file mode 100644 index 0000000..fd3d758 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md @@ -0,0 +1,46 @@ +# 1028_SWD corrected numeric-binning diagnostic + +This is development/tuning evidence, not held-out or leaderboard evidence. It +reruns the frozen OpenBoost NLL configuration after commit `236c2df` corrected +numeric binning so that `m` cut edges retain all `m + 1` intervals. The prior +implementation silently merged the highest interval into its predecessor. + +The candidate used 500 rounds, learning rate 0.01, depth 3, seed 42, five +folds, sample cap 3000, 99 quantiles, and pinned ScoringBench commit +`a938a667`. Strong-baseline rows come from the already-frozen, same-dataset, +same-split artifact in `../1028_swd_lr_sweep_20260816/baseline_lr_001/`. + +| Model | CRPS | RMSE | 90% coverage error | 90% interval score | PIT KS | Sharpness | +|---|---:|---:|---:|---:|---:|---:| +| OpenBoost before fix | 0.355075 | 0.626043 | 0.0310 | 2.559809 | **0.087349** | 0.580131 | +| **OpenBoost after fix** | **0.347845** | **0.615779** | **0.0290** | **2.509361** | 0.098027 | 0.560487 | +| XGBoostLSS Gaussian | 0.349289 | 0.617750 | 0.0410 | 2.670105 | 0.100551 | 0.526127 | +| NGBoost Gaussian | 0.348185 | 0.616315 | 0.0320 | 2.572281 | 0.099395 | 0.542749 | +| XGBoost quantile | 0.338359 | 0.636753 | 0.2670 | 2.788130 | 0.302062 | 0.505502 | +| CatBoost quantile | 0.336377 | 0.635947 | 0.2380 | 2.825541 | 0.172461 | **0.491907** | + +Relative to the pre-fix OpenBoost artifact, corrected binning improves CRPS by +2.04% (four of five folds), RMSE by 1.64%, interval score by 1.97%, coverage +error by 6.45%, and sharpness by 3.39%. PIT KS worsens by 12.22%, so the change +is not uniformly better on every diagnostic, although it fixes an unambiguous +representation bug. + +On this consumed development dataset, corrected OpenBoost has 0.41% lower mean +CRPS than XGBoostLSS and wins four of five paired folds. It also improves RMSE, +coverage error, interval score, and PIT KS. Against NGBoost, mean CRPS is only +0.10% lower and OpenBoost wins two of five folds; treat that as parity, not a +win. + +The stated goal is still unmet. Corrected OpenBoost CRPS remains 2.80% worse +than native XGBoost quantile and 3.41% worse than CatBoost quantile, winning +only one of five folds against each. OpenBoost is much better calibrated on +this dataset and has lower interval score and RMSE, but those guardrails do not +erase the pre-registered CRPS loss. The next candidate must target the +remaining distribution-shape/quantile gap and then be evaluated on new data. + +Run: [31928677396](https://github.com/jxucoder/openboost/actions/runs/31928677396), +artifact `9258714239`, digest +`sha256:c75eea2d93bf10d5de1ae48a8ab37eed66c1dc1c326a9810ab8dc808c4656836`. +The run completed 5/5 rows from clean source `236c2df`; the manifest records a +clean pinned ScoringBench checkout and the full Linux environment. Fit times +from separate Actions processes are not used as speed evidence. diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json new file mode 100644 index 0000000..598c1da --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 5, + "observed_rows": 5, + "status": "complete", + "valid_rows": 5 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 5, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 5, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 5 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json new file mode 100644 index 0000000..c66acd1 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json @@ -0,0 +1,126 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "development_run": true, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_cpu" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "nll", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31928677396", + "source_sha": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510", + "tested_sha": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510" + }, + "created_at": "2026-08-16T05:25:28+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 5, + "model_parameters": { + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "model_params": { + "min_child_weight": 1.0, + "reg_lambda": 1.0, + "training_objective": "nll" + }, + "n_quantiles": 99, + "n_trees": 500 + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 5 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 5, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9f07642ee8dea658bf8ae110f0ab6344d1500c65 GIT binary patch literal 29974 zcmd5_3wV>&`A;v@P(eXxNwL$4`TgkuXO;egu3N&rF)ZE&pP0}n$o3`j= zcp1|PZ1cj@|Au%O+jtl#@-P7@f{f=cW6U8f2+}Gzl#7VK6xiSYd%jEZCFe^MYzgFX z_WR!RzUTLQ-*frSIq!K~C0!*?Nld9snO&ZeoFchRBAM{=viB~?#ysDP)@u4*EZKb& z&0F@KbyQg|nmBLSueVA1(2ffe9{y9sVU!|CNlh7>33ek=l2Ve=Wl~;4CgpWfB@;xX zBS%UklA9;Q!@rZ`5)zh_NJ2RzWV+uQx!@Rj+p)84)6UP({pZdsylL%c$oIQt1uv~R zhDNO&?)+T(sR##NLnh^QFb7f^nZwlWrbF?WJ?L=D^uzytrp*Hg!x zn%jd`&cE`))V-gGaNtX1Qoat%fs{t(kiYmo^KDc&TC#L@S%Ufis-o9-bvO^8H|9Ok zx&J5K$d>d>!_cJtA{=-PnUvST97t(o4i?vc>rTCX8l^s*dDmm_pF~T%kALy~Zi$Xm7hk7@~uM>JYR@#;4NiRUI%j^rI9)09{l%$r$6pPzZo`RebbsgbmHi{nZG*S zhhDAzsOtOueduECo(B`mV*P0f=3p5+^P!T;P3UJ2ef6O3z0GLimfKd> z^uCLZPO7i*QCm>;t{0Mu_WnvXRc z`Ucg#GCKW@2U8%WqTW}g zRQhKo#`}584qJP#`#03Fr5lX`_5X_BBp1F_~ z|Hgk%rsgHX?{{59ul0QLpNg+9qT`d+Px~PGTl9}Dv!9#ad|reDuOXB2I+z0~4Z|Tg z3X9&m(2Mxn=9IgeN=^@)WuYG2uV>7)2Ua%sS8$fUdura(%=_#SBZTyV4Z zRo7|kZpK>85NY;G*8^vmw|UFXh&!KwOBm9=S!u9x7;+gFZXbwi1WOLOv>wE3Zzuj`JlDgsP)h2 zj*U8pwmKmiOs^i2CU6E9n680M&|Ix zKd)J`Sbk*NA3w3gNnbp=O}*Ul(N_zPY;$}}Ov>wE z4x}_PhsvIr51qgKHro02Pm`7@HlWHC9lf=E8&P5W_PZ9pvJGu$DBTdhdyB{*tYV1G%|-fmv+vtD&B(ZlWu?hCojE&Qj+ zL^$voGAXZvIgrxG91=(G(xrd18U6C9(eLkCv>Dy^_oWrjoO=f?n(^7?%XOR4@$Hv; z|MnX(4!osI%Ijbbq%<;zJ6<^ckFO@biPC@8w^Y@-9{nt9;zsG8){e$ z`9D1y_c8j#3fnJoRvi@Kz-!2)ybk6-N~7S=^1-U2VF%HRqvmDTJl=zjKDX@N<8Sw% zO|!0?JDGhDJ@wbu7j)j+Ey970f=8#1-Nn1~Jqpv%!>^-3DMvBExl%2MB zqsv>Dyf`Ag2Nhg7J>;H;KN8`}(8$Oucjeg}zAG5jR08-Dn zWb9pi07?2z&zrQY8-4kgz3X%8#rAHzhD^%qU=E}-GKU*s|Cm`b>hOnSx>5JPzPT{w zo&(5p^x#wb$_}90KP!FA25mA~*`qkJ^Byd#81y$Bt8N zw^0Ysp))5JWECDjOK#uuzfG&UQU0#C(k?ZNao{y%QeFphAf=Hx+~a-nm*fA~jV3Jr zqTuh|18Du+>hCT-asZw9cFj5W_#X7)!KdDro5VQq8Zs%bgE^4W$Q;fm-%(Y&pa*Te zTUY*8ayOcH=Nm8m=>HBNTYCxf6?BA&V%UNb@Pw^ZMPT)UPC73bub4~8ks}B@7*Jp^#{?{ z%WjSP?szwPqWf%G+PmH8;`m#>Ty^Uq^t(UnE+k6D#wM>Jlkz&411XKnfj;=S5nZyA`KfESjp*uG{z>2A{E=PcT8_7rNqHR{ z2T~e24y7;8cRzjP{I=Jg8FBE;uPf6j$%zfZQmL|;0;%|2mkM%Si`+ei+) zrA*4}U=E}-G6#MQ8uK-6MbPO+;gH7ULfD#grJP8LlS(BY?1lYJ?{j;SurTB4my<*` z7f`~OvHKgWgM&ay!vx`eKLLqkc$_3|xKt89TzY>3RDgep>>mjke7=OiWwbCZDHX+) zrKFdq#50Fl!!j{hdA6dUZkE0XaBfRU4V++2g*y2%ng5BYb7%&8S{0X!kB`TtP<)Le z3x`9EEiS9m)Y9Vd)|p+cBGnELQji6|k^pH}rsPRvNgi{f)nQDTCrL@ZH(r+DFk0EC zM7P7kJ;u8goXqTYdDzFH_Lhb^kGaLo=?wKW8r?3Z)x(u1TWlVe)$OshI9Y?6%`I(K zx3R%mr^sd{BW+Hv)!k;av-SAp0nOYnn!G%qIW>$XFAr!Igwf>X0ZnBXO(;~ zH8gUPVXVY##W7`TO0hbv?uHI7x+bgFSeKn8@24!MpEAx9(K_?{u~UXElV@@96{%C+ z4^Mekc%JgC@I2*N;d#ol3i{#6`7q4lvea>59N0Jhcr^ONQ=l+RFX_O=x;Dsp6oFIT%lt3+y z2Ou&Tg=bDu3X@QDKrrJ7V3rJ_RwM!pnQy`{q%>+67?&=O0BHG5)JmN8Ba=!Ph?GVR z6qqdqXe)+Nt8p{&%cdxjO3_6xFc%BZR^CjlO9nJDQG|UXrBVAPnCS&@tA|sYh5;Oz z6T)z$G-|lOtgFao@Xo}go{yt`cPrI(3*bSI#d#!uW~7Th91W31jYoc^1YW*2lG-o= zP|4g6_L7uFjT-PWm-Q0H_2co>>QU5|+gL9{GCdd%d=HUEjTh==02OAFbtzQWXh0=% zI~bLeMvWTqa;nhF4P&U?sjQbFnHR|V7Dy75#O1wdqS1p|@NsDb>e zVFB2tiPY{10LJfNqsXm9*JFfhZ2?}_Bx>J%fJbIbuydp|YUhGi@&d%&lc}CeKqT`c z7?G4ljVRbB2;lZjp?b3bj!cDMI8quloNyN+!0VAyC$a&LOnYEFQW`a0&>lws*qcM0 zRWR`k&2g~t43$O=D6%tw9h3n1L>~2RE+CVc4U9}m2Zo#@Kt7vKEy@REGM|BwN$J3l zg|p7LN@|G`kjX3tMkb{LL(UiYyy$*v`Hyk5iQEMfZ9+OQWTgOk$%E922LPGOSYTvQ zIuK;oBMOk07g8$`Ad{O9j7&-ghAiycD^%2K8j#8C1V$#M149<}?UhB;IyE4ZISGtR zN(Y84?AxnLs7=LyOlBf5GASJxvaoNjE2X+LfK28eFfu6}7;=H2hixjOc54Bd%sOCX zQaUhX;i&2=r}j+;WHQ%)kx6OP$bkcw?4ZH4yMpS`0pOtTxP^eilZ$`Kfm{b#@j24*F_Ma6-m+*2VGEr`6QAa{vx+ z^Koi{d1CSi*gsMl(Z59Q0F60+E0M54MS~0* zNMDh-!DWLCEJ%})$RUM;3{B+YNCKiNgG|HkWNtP|Af#}Rp+npgBOt0W$Ta*;PVFxO zY~dh7hqxz3KvZRrY4{^h&|d`D!a;@(aZikZsLCMI@JB$|Uj*2~L52=-PmF-5${^DS zL?Ex9c|ge&4l;D8dm;oxRR);`zBVA@Iwu`$Fz#Z42%w;3kU<64A4uGg!fOM~-m?e` z9w;7U_`sD85<9r;ngBCbN5Ucd3$F<@dq*W4cA#{S!9{PwkSxNg#ehyubaWWC7|;=! z^~9ocEhbGj|I?CwiFlyJF4jjrvLx)p~0UddCbQrZ5(1~hf!Ob$vItFy2 z8d-3eEk<;r8d-3EE=F_;qU$@1S_~R1s(~)gx)uXnc;k=^N@(dI$3jrO$bnX9>9qqF z87uM*+9Ej~;L?!@Ce@>r_K%YxelQh|KZOrkYv4 z2CajW*_AGnqsng8RCj8dS{n57Y>!r5)?`w&%T4+`R+i(jnH(iv!z}GoDD&3p=h({C zw9VkK+fACPda*LIrUbA{I%{&O^30k!fQ$9p+iGiODk{`kjL~Y*%YD{rWwy0OWyjcB zhrwe~lw?;l&1C8~>8rDA^;L})O)=&T`t6>2;BM_GYpvDGSsQ1W*Q^N3r^8fJVb|Kg zU#F@O{A@JK`@wTod(EZxdP}L@$NB0oX_SqYqS|bx4u{-s(UdgT)>JiOz9wh2$5aaO zDh-NHht9_M=P>H4J%$pcfUCA6#$1}rPE{w%-QY6l+nZ|?d5tD$KinR*Ib}X`2gB7@ ztIq?!D*Ev~Pj0GF<@M8^UIWAh+B~}!_Y)21Yuc6NYTOS4m=I5AO-@CVDW{BW7YE6U z(1~5!bs9A#9cE3Q-JGKf*T)H+nDuo>mCKx?vYVY1E|VfR9KU8`O@+@;Q`TwJD_djT z-%L3MyV+6VG*lOK=-6>ijsuu`ERO1qTD`?y)2a2vn4`<&tb!OC8cik2&RT`CqkpdU zR*R;(!{R9M@cs5WaE9^Xk85^xah-W1#I+x;VsVW*XJK3w(Z#j)Mu=-aT*cxVbI!uJ z=0+FSh8rQS{cshFYs@(d;h$_vj` zG_JAc++=pN_qYD^#CrW%tEjTqDB9clS>H9mo~a(zpq;ViX4crVYZPACGgV~6J}KP# z&S$KtsyF6T)SGi)Z^W*#Vzzz{a8xO3_3d)lC*i#uxyEy`}TXo_3XJnh0jVXvN?3paEw`TXn zHHtEP&QPyY=Z2j-Bih8SAKOi^mowzp>& zvk`Jk_S1Jbn;W6;8J0oHrK!Jw3j#0;pUofeaCNe{rgVw514oSHrK!J%=!WI z4sUZq^qs4occAaEHa9}wx%zphxb=pacV^}0UiG}g+uRU+r}+oWJFHFY`c9P(@(O`9 zPS|r+QN7sBFQqETH~7~wZ0Q<}KhW*?oeeyW8TW&G88jUS z;m15OPq^VBp26QWHXQkWf9RG;&^B0+i$`RC)SN%+@?74Zn#v}weOP~(AmWb1{Yvno zCfxZ%;si1Mw@bYzu0S(D{{_k4pKo^a?w3cY>KIXvZXWK2_72 zqI|YJxcP{f-#q}T;?IcL^KM$D_uH36vTqUEGiUWM(`)?px)6I}KFr4UfxJm6-2Eu6 zwu6vvZR7lJD9YyQ{E4rxDlGpqd$C4-2!P_N>D1c+C#>IG5!=zn;N#desch z9^%^spI&DD!hUbX`?oNMoU5Z%ll=DOA@LQp4`@_essf70iu>*QA!tqnnB~jVK z-B=6PNAZFd#y@Yb4Yg+`Xu*0A4>}c-4}}N#K9z$d=BuUMczrJD53sR-Utqj4SBp!v z9YE9Q$E%pl+4J+gC?0e7mZ?q$dj<{i+{HlIVH> literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json new file mode 100644 index 0000000..1fc1f72 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json @@ -0,0 +1,74 @@ +{ + "artifact": { + "digest": "sha256:c75eea2d93bf10d5de1ae48a8ab37eed66c1dc1c326a9810ab8dc808c4656836", + "id": 9258714239, + "source_sha": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510", + "workflow_run_id": 31928677396 + }, + "baseline": "../1028_swd_lr_sweep_20260816/baseline_lr_001", + "candidate_metrics": { + "coverage_90_abs_error": 0.028999996185302756, + "crps": 0.34784456793765095, + "interval_score_90": 2.509361067814866, + "pit_ks_stat": 0.09802668231196801, + "rmse": 0.6157794340555057, + "sharpness": 0.5604867750552687 + }, + "comparisons": { + "catboost_quantile": { + "baseline_crps": 0.3363766926503152, + "candidate_crps_relative_percent": 3.4092359958058482, + "candidate_crps_wins": 1, + "candidate_coverage_error_relative_percent": -87.81512762882153, + "candidate_interval_score_relative_percent": -11.19005295818769, + "candidate_pit_ks_relative_percent": -43.160047784863856, + "candidate_rmse_relative_percent": -3.1713396938279903 + }, + "ngboost": { + "baseline_crps": 0.3481847240643794, + "candidate_crps_relative_percent": -0.09769415577965956, + "candidate_crps_wins": 2, + "candidate_coverage_error_relative_percent": -9.374993015079236, + "candidate_interval_score_relative_percent": -2.4460776539851126, + "candidate_pit_ks_relative_percent": -1.3762692644908037, + "candidate_rmse_relative_percent": -0.08693406957233085 + }, + "openboost_before_fix": { + "baseline_crps": 0.35507492380808053, + "candidate_crps_relative_percent": -2.0362901983857373, + "candidate_crps_wins": 4, + "candidate_coverage_error_relative_percent": -6.4516223308077265, + "candidate_interval_score_relative_percent": -1.9707568344182858, + "candidate_pit_ks_relative_percent": 12.224803295387954, + "candidate_rmse_relative_percent": -1.6393938069868441 + }, + "xgblss": { + "baseline_crps": 0.3492890583364678, + "candidate_crps_relative_percent": -0.41355157407345633, + "candidate_crps_wins": 4, + "candidate_coverage_error_relative_percent": -29.26827566315279, + "candidate_interval_score_relative_percent": -6.020129964847132, + "candidate_pit_ks_relative_percent": -2.510826416548262, + "candidate_rmse_relative_percent": -0.31891960877288295 + }, + "xgboost_quantile": { + "baseline_crps": 0.3383589409871867, + "candidate_crps_relative_percent": 2.8034213970492994, + "candidate_crps_wins": 1, + "candidate_coverage_error_relative_percent": -89.13857803317438, + "candidate_interval_score_relative_percent": -9.99843321488677, + "candidate_pit_ks_relative_percent": -67.54753242799605, + "candidate_rmse_relative_percent": -3.293809336382647 + } + }, + "decision": "correctness_fix_accepted_quality_goal_not_met", + "file_sha256": { + "benchmark_outcome.json": "cefba05ad79fdb6056f55cba5afda606d79984537856a41868b7e4a88c12e4a1", + "datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "openboost_manifest.json": "3f8b2678899ebb79cee25a0b1f9401cc8d5377f09698294907e9bdbf4bb5c9f1", + "raw/openboost_cpu/1028_SWD.parquet": "5c810080aafcc6f561e86ca517ec20988fb2bb9f1a59a1aa6b97c2908efb334d" + }, + "folds": 5, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 6cde112..9e341e9 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -176,6 +176,28 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. pre-existing import-order/unused-import findings, which are unrelated to this correctness change. A fresh ScoringBench artifact is still required before interpreting the quality impact. +- The full non-GPU/non-benchmark CPU suite subsequently passed with 752 tests + and 32 expected skips. Clean Linux development run `31928677396` then + completed all five `1028_SWD` folds from source `236c2df`; artifact + `9258714239` has digest + `sha256:c75eea2d93bf10d5de1ae48a8ab37eed66c1dc1c326a9810ab8dc808c4656836`. + The committed raw artifact is under + `benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/`. +- Correct binning reduces OpenBoost mean CRPS from 0.355075 to 0.347845 + (2.04%, four of five folds) and RMSE from 0.626043 to 0.615779. On this + consumed development dataset it beats XGBoostLSS CRPS by 0.41% in four of + five folds, while NGBoost remains effective parity (0.10% lower mean but only + two OpenBoost fold wins). The primary goal remains unmet: OpenBoost is 2.80% + behind native XGBoost quantile and 3.41% behind CatBoost quantile, winning + only one fold against each. Its coverage error, interval score, PIT KS, and + RMSE are substantially better than those two quantile rows here; report this + as a Pareto trade-off, not a CRPS win. +- Two local, non-artifact follow-ups were rejected before implementation. + Replacing the Normal shape with the training residual empirical shape made + negligible CRPS difference. Independent OpenBoost quantile models improved + with more rounds but did not close the native-XGBoost gap, while a prototype + exact residual-quantile leaf refit was worse. Do not productize either path + without a stronger multi-dataset hypothesis and a clean artifact. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts From f3dbdbfb28795cbbcf1fe33c023f09b1dd0c3a5a Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:42:25 -0700 Subject: [PATCH 34/49] bench: preregister CRPS distribution experiment --- .../protocols/crps_distribution_v1.json | 20 ++++++ .../protocols/crps_distribution_v1.md | 65 +++++++++++++++++++ .../2026-08-15-scoringbench-integration.md | 15 +++++ 3 files changed, 100 insertions(+) create mode 100644 benchmarks/scoringbench/protocols/crps_distribution_v1.json create mode 100644 benchmarks/scoringbench/protocols/crps_distribution_v1.md diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v1.json b/benchmarks/scoringbench/protocols/crps_distribution_v1.json new file mode 100644 index 0000000..54674d0 --- /dev/null +++ b/benchmarks/scoringbench/protocols/crps_distribution_v1.json @@ -0,0 +1,20 @@ +[ + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "target_col": "target", + "abbr": "1CA", + "openboost_role": "development", + "raw_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/537_houses/537_houses.tsv.gz", + "target_col": "target", + "abbr": "5H", + "openboost_role": "untouched_confirmation", + "raw_sha256": "d383fd58bb79760ffaf0b615942fd47d926a19e75a8f980b79fbe9ad479d9cbf" + } +] diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v1.md b/benchmarks/scoringbench/protocols/crps_distribution_v1.md new file mode 100644 index 0000000..b2ddafd --- /dev/null +++ b/benchmarks/scoringbench/protocols/crps_distribution_v1.md @@ -0,0 +1,65 @@ +# CRPS distribution experiment v1 + +This protocol was frozen before loading either dataset or observing any result +on them. It separates architecture development from confirmation after the +`1027_ESL` and `1028_SWD` diagnostics were consumed. + +## Frozen data roles + +- Development: `197_cpu_act`, a continuous regression dataset with 8,192 rows + and 21 machine-performance features. +- Untouched confirmation: `537_houses`, a continuous housing/population + regression dataset with 8 numeric features. Do not load, validate, or run + this entry until one candidate implementation and configuration have been + frozen after the development result. + +Both URLs in `crps_distribution_v1.json` point to PMLB commit +`9cc9017958f2d8284e62d8bc54b77cb6fa1e9592`. The registry also records the +expected SHA-256 of each compressed source file. A run must fail closed if the +downloaded bytes do not match. The mutable PMLB `master` URLs are ineligible +for this experiment. + +## Protocol + +- ScoringBench commit: + `a938a667b7839b41e9272929010573410301c0b4`. +- Five folds, one repeat, seed 42, sample cap 3,000, CPU execution. +- Compare the frozen OpenBoost candidate with ScoringBench's native XGBoost + multi-quantile model and CatBoost MultiQuantile model. +- XGBoost: 100 rounds and 50 quantiles, seed 42, two threads. +- CatBoost: 1,000 rounds and 99 quantiles, seed 42, two threads. +- Use an empty output directory. Exactly 15 finite result rows must exist. +- CRPS is primary. Log score and CRLS are excluded because the pinned evaluator + does not put different finite-support predictions on a common support. + +The candidate family is a non-parametric histogram distribution trained by +direct CRPS gradients with one shared tree and vector leaves per round. Only a +positive-semidefinite Gauss--Newton curvature is eligible; the earlier +absolute-value transform of an indefinite exact diagonal is rejected. The +candidate's exact public API and hyperparameters must be committed before the +development dataset is loaded. + +## Development acceptance + +Let `B` be whichever of XGBoost quantile and CatBoost MultiQuantile has the +lower five-fold mean CRPS. Choose `B` once per dataset, never separately for +each fold. A candidate passes only if every condition holds: + +1. all 15 expected rows are present, finite, and error-free; +2. `mean_CRPS(candidate) / mean_CRPS(B) <= 1.02`; +3. candidate CRPS is no greater than `B` on at least three of five folds; +4. candidate mean absolute 90% coverage error is at most 0.05 and at least + 0.02 lower than `B`; +5. candidate mean 90% interval score is at most `1.05 * B`; +6. candidate mean RMSE is at most `1.05 * B`. + +Passing development allows one frozen run on `537_houses`; it is not evidence +of a general win. + +## Confirmation language + +On `537_houses`, OpenBoost may be described as having lower CRPS on that +dataset only when its mean CRPS ratio to `B` is below 1.00 and it wins at least +four of five folds. A ratio at or below 1.02 that also satisfies the guardrails +is only calibrated non-inferiority. Neither outcome is an overall or SOTA +claim. The full ScoringBench suite remains the product acceptance test. diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 9e341e9..723acb0 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -198,6 +198,21 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. with more rounds but did not close the native-XGBoost gap, while a prototype exact residual-quantile leaf refit was worse. Do not productize either path without a stronger multi-dataset hypothesis and a clean artifact. +- A shared-tree vector-PMF prototype identified a different ceiling: directly + optimizing discretized CRPS can close the shape gap that a two-parameter + Gaussian cannot. The first prototype used an invalid absolute-value + transform of an indefinite exact diagonal Hessian and is rejected. Replacing + it with the positive-semidefinite Gauss--Newton diagonal, 50 bins, 100 rounds, + and learning rate 0.05 produced mean CRPS 0.333588 on the already-consumed + `1028_SWD` folds, versus 0.338359 for the frozen native XGBoost quantile row. + This selects an architecture for implementation; it is not new-dataset or + OpenBoost-core evidence. +- Before loading more data, protocol `crps_distribution_v1` froze + `197_cpu_act` for development and `537_houses` for one untouched confirmation + run. Their PMLB source commit, compressed-file hashes, model budgets, metric + guardrails, fold-win thresholds, and permitted claim language are committed + under `benchmarks/scoringbench/protocols/`. The candidate API and exact + hyperparameters must be committed before loading the development entry. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts From 109646332b30e2f58a4690bc4af59054d80c09bc Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:44:32 -0700 Subject: [PATCH 35/49] bench: verify pinned dataset bytes --- benchmarks/scoringbench/run.py | 69 ++++++++++++++++++- .../2026-08-15-scoringbench-integration.md | 5 ++ tests/test_scoringbench_provenance.py | 54 +++++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 30ad727..ddbf0af 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -53,6 +53,49 @@ def _load_dataset_registry(path: Path) -> list[dict]: return datasets +def _verify_dataset_files(datasets: list[dict], ensure_cached) -> list[dict]: + """Materialize and verify dataset files pinned by a frozen registry. + + ScoringBench's processed cache is intentionally fast but cannot prove which + raw bytes produced an entry. A registry may therefore provide + ``raw_sha256``. Those entries are downloaded through ScoringBench's own raw + cache and checked before validation or fold construction. + """ + verified = [] + for dataset in datasets: + expected = dataset.get("raw_sha256") + if expected is None: + continue + expected = str(expected).lower() + if len(expected) != 64 or any(char not in "0123456789abcdef" for char in expected): + raise ValueError( + f"invalid raw_sha256 for dataset {dataset['name']!r}: {expected!r}" + ) + if dataset.get("source") != "pmlb" or not dataset.get("url"): + raise ValueError( + "raw_sha256 verification currently requires a PMLB URL; " + f"dataset {dataset['name']!r} has source={dataset.get('source')!r}" + ) + + filename = f"{dataset['name']}.tsv.gz" + path = Path(ensure_cached(dataset["name"], dataset["url"], filename)) + actual = _sha256(path) + if actual != expected: + raise ValueError( + f"raw dataset hash mismatch for {dataset['name']!r}: " + f"expected {expected}, got {actual} at {path}" + ) + verified.append( + { + "name": dataset["name"], + "url": dataset["url"], + "sha256": actual, + "size_bytes": path.stat().st_size, + } + ) + return verified + + def _git_state(path: Path) -> dict: def run(*args: str) -> str | None: try: @@ -579,6 +622,7 @@ def _write_provenance( datasets: list[dict], result_rows: int, outcome: dict, + verified_dataset_files: list[dict] | None = None, ) -> Path: import openboost as ob @@ -606,7 +650,7 @@ def _write_provenance( registry_path = output_dir / "datasets.json" manifest = { - "schema_version": 2, + "schema_version": 3, "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "protocol": "ScoringBench", "protocol_mode": protocol_mode, @@ -646,6 +690,7 @@ def _write_provenance( ), "file": "datasets.json" if registry_path.exists() else None, }, + "verified_dataset_files": verified_dataset_files or [], "result_rows": result_rows, "expected_result_rows": outcome["expected_rows"], "outcome": { @@ -716,12 +761,17 @@ def main() -> int: sys.path.insert(0, str(PROJECT_ROOT)) sys.path.insert(0, str(scoringbench_dir)) - from scoringbench.datasets import get_DATASETS_CONFIG, validate_datasets + from scoringbench.datasets import ( + _ensure_cached, + get_DATASETS_CONFIG, + validate_datasets, + ) from scoringbench.runner import run_benchmark from scoringbench.utils import set_seed set_seed(args.seed) output_dir = Path(args.output_dir).expanduser().resolve() + verified_dataset_files = [] if args.smoke: datasets = [ { @@ -751,10 +801,22 @@ def main() -> int: for index, dataset in enumerate(datasets): print(f"{index:3d} {dataset['name']}") return 0 + def validate_with_raw_verification(selected): + nonlocal verified_dataset_files + verified_dataset_files = _verify_dataset_files( + selected, + _ensure_cached, + ) + if verified_dataset_files: + # Force the pinned raw file through preprocessing instead + # of accepting an opaque processed cache entry. + os.environ["SCORINGBENCH_NO_CACHE"] = "1" + return validate_datasets(selected) + datasets = _validate_selected_datasets( all_datasets, args, - validate_datasets, + validate_with_raw_verification, ) if args.lite: @@ -785,6 +847,7 @@ def main() -> int: datasets, result_rows=len(result), outcome=outcome, + verified_dataset_files=verified_dataset_files, ) print(f"OpenBoost outcome: {outcome_path} ({outcome['status']})") print(f"OpenBoost provenance: {manifest}") diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 723acb0..1f34fef 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -213,6 +213,11 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. guardrails, fold-win thresholds, and permitted claim language are committed under `benchmarks/scoringbench/protocols/`. The candidate API and exact hyperparameters must be committed before loading the development entry. +- Frozen registries may now include `raw_sha256`. The launcher materializes + those entries through ScoringBench's raw cache, rejects a mismatch before + validation, bypasses the opaque processed cache for the run, and records the + verified URL, digest, and byte size in manifest schema 3. This makes the + protocol's immutable-data requirement executable rather than documentary. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index 816c636..8cc1e8a 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -11,6 +11,7 @@ _model_parameters, _select_datasets, _validate_selected_datasets, + _verify_dataset_files, _working_directory, ) @@ -144,6 +145,59 @@ def test_load_dataset_registry_accepts_frozen_scoringbench_list(tmp_path): ] +def test_verify_dataset_files_accepts_matching_pinned_bytes(tmp_path): + raw = tmp_path / "pinned.tsv.gz" + raw.write_bytes(b"frozen dataset bytes") + expected = __import__("hashlib").sha256(raw.read_bytes()).hexdigest() + calls = [] + + def ensure_cached(name, url, filename): + calls.append((name, url, filename)) + return raw + + verified = _verify_dataset_files( + [ + { + "name": "example", + "source": "pmlb", + "url": "https://example.test/example.tsv.gz", + "raw_sha256": expected, + } + ], + ensure_cached, + ) + + assert calls == [ + ("example", "https://example.test/example.tsv.gz", "example.tsv.gz") + ] + assert verified == [ + { + "name": "example", + "url": "https://example.test/example.tsv.gz", + "sha256": expected, + "size_bytes": len(b"frozen dataset bytes"), + } + ] + + +def test_verify_dataset_files_fails_closed_on_hash_mismatch(tmp_path): + raw = tmp_path / "pinned.tsv.gz" + raw.write_bytes(b"different bytes") + + with pytest.raises(ValueError, match="raw dataset hash mismatch"): + _verify_dataset_files( + [ + { + "name": "example", + "source": "pmlb", + "url": "https://example.test/example.tsv.gz", + "raw_sha256": "0" * 64, + } + ], + lambda *_args: raw, + ) + + def test_stable_strided_shards_cover_registry_exactly_once(): registry = [{"name": f"dataset_{index}"} for index in range(7)] From d16053c5844386a0246732b137f0970e34517b53 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:45:49 -0700 Subject: [PATCH 36/49] bench: lock confirmation datasets --- .../protocols/crps_distribution_v1.md | 3 ++- benchmarks/scoringbench/run.py | 26 +++++++++++++++++++ .../2026-08-15-scoringbench-integration.md | 3 +++ tests/test_scoringbench_provenance.py | 15 +++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v1.md b/benchmarks/scoringbench/protocols/crps_distribution_v1.md index b2ddafd..b99932b 100644 --- a/benchmarks/scoringbench/protocols/crps_distribution_v1.md +++ b/benchmarks/scoringbench/protocols/crps_distribution_v1.md @@ -11,7 +11,8 @@ on them. It separates architecture development from confirmation after the - Untouched confirmation: `537_houses`, a continuous housing/population regression dataset with 8 numeric features. Do not load, validate, or run this entry until one candidate implementation and configuration have been - frozen after the development result. + frozen after the development result. The launcher enforces this registry + role unless the confirmation run explicitly supplies `--allow-confirmation`. Both URLs in `crps_distribution_v1.json` point to PMLB commit `9cc9017958f2d8284e62d8bc54b77cb6fa1e9592`. The registry also records the diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index ddbf0af..ed40d3d 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -96,6 +96,20 @@ def _verify_dataset_files(datasets: list[dict], ensure_cached) -> list[dict]: return verified +def _enforce_dataset_role_lock(datasets: list[dict], *, allow_confirmation: bool) -> None: + """Prevent accidental observation of preregistered confirmation data.""" + locked = [ + dataset["name"] + for dataset in datasets + if dataset.get("openboost_role") == "untouched_confirmation" + ] + if locked and not allow_confirmation: + raise ValueError( + "confirmation dataset is still locked; freeze the candidate first, " + "then rerun with --allow-confirmation: " + ", ".join(locked) + ) + + def _git_state(path: Path) -> dict: def run(*args: str) -> str | None: try: @@ -463,6 +477,14 @@ def _build_parser() -> argparse.ArgumentParser: "reported as a held-out leaderboard result" ), ) + parser.add_argument( + "--allow-confirmation", + action="store_true", + help=( + "Unlock a registry entry marked untouched_confirmation. Use only " + "after the candidate implementation and configuration are committed." + ), + ) parser.add_argument( "--list-datasets", action="store_true", @@ -803,6 +825,10 @@ def main() -> int: return 0 def validate_with_raw_verification(selected): nonlocal verified_dataset_files + _enforce_dataset_role_lock( + selected, + allow_confirmation=args.allow_confirmation, + ) verified_dataset_files = _verify_dataset_files( selected, _ensure_cached, diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md index 1f34fef..26e5a77 100644 --- a/learnings/2026-08-15-scoringbench-integration.md +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -218,6 +218,9 @@ OpenBoost's exposure-aware API, which still needs a domain benchmark. validation, bypasses the opaque processed cache for the run, and records the verified URL, digest, and byte size in manifest schema 3. This makes the protocol's immutable-data requirement executable rather than documentary. +- A registry entry marked `untouched_confirmation` is rejected before download + unless the launcher receives `--allow-confirmation`. This is a deliberate, + manifest-recorded unlock after the selected candidate is committed. - Integration commit: `a4555bc` (`bench: add ScoringBench integration`). ## Failed Attempts diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index 8cc1e8a..a83fc09 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -7,6 +7,7 @@ _audit_records, _build_parser, _ci_state, + _enforce_dataset_role_lock, _load_dataset_registry, _model_parameters, _select_datasets, @@ -198,6 +199,20 @@ def test_verify_dataset_files_fails_closed_on_hash_mismatch(tmp_path): ) +def test_confirmation_dataset_requires_explicit_unlock(): + datasets = [ + { + "name": "held_out", + "openboost_role": "untouched_confirmation", + } + ] + + with pytest.raises(ValueError, match="confirmation dataset is still locked"): + _enforce_dataset_role_lock(datasets, allow_confirmation=False) + + _enforce_dataset_role_lock(datasets, allow_confirmation=True) + + def test_stable_strided_shards_cover_registry_exactly_once(): registry = [{"name": f"dataset_{index}"} for index in range(7)] From b9db276924b24434074612c7986ab75de8f365b4 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:54:47 -0700 Subject: [PATCH 37/49] feat: add histogram CRPS boosting --- learnings/2026-08-15-histogram-crps-boost.md | 84 ++++ src/openboost/__init__.py | 4 + src/openboost/_core/_vector_tree.py | 277 ++++++++++++ src/openboost/_models/__init__.py | 3 + src/openboost/_models/_histogram_boost.py | 427 +++++++++++++++++++ src/openboost/_persistence.py | 4 +- src/openboost/_validation.py | 4 +- tests/test_histogram_boost.py | 263 ++++++++++++ 8 files changed, 1063 insertions(+), 3 deletions(-) create mode 100644 learnings/2026-08-15-histogram-crps-boost.md create mode 100644 src/openboost/_core/_vector_tree.py create mode 100644 src/openboost/_models/_histogram_boost.py create mode 100644 tests/test_histogram_boost.py diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md new file mode 100644 index 0000000..da87d97 --- /dev/null +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -0,0 +1,84 @@ +# 2026-08-15: Histogram CRPS Boosting + +## Context + +The corrected Gaussian NaturalBoost model reached parity with Gaussian +XGBoostLSS on the consumed `1028_SWD` ScoringBench dataset but remained 2.80% +behind native XGBoost multi-quantile on mean CRPS. CRPS training, early +stopping, scale calibration, Student-t, empirical residual shapes, and +independent quantile trees did not close that gap. The remaining limitation was +distribution shape, not only Gaussian scale optimization. + +## Decision or Result + +Add a separate non-parametric estimator, `HistogramBoost`, rather than forcing +many histogram logits through NaturalBoost's one-tree-per-parameter loop. Each +round learns one shared routing structure with a vector of logit updates in +every leaf. Softmax probabilities imply a valid monotone CDF, so quantile +crossing is impossible. + +The training loss is a discretized CRPS (ranked probability score) on an +ordered target grid. Its curvature is the positive-semidefinite diagonal of the +Gauss--Newton matrix, `2 * diag(J.T W J)`. An earlier prototype that took the +absolute value of an indefinite exact Hessian diagonal is mathematically +invalid and was not carried into OpenBoost. + +The frozen development defaults are 50 distribution bins, 100 shared-vector +trees, learning rate 0.05, depth 6, and curvature scale 1. These defaults were +chosen using only the already-consumed `1028_SWD` diagnostics. They must not be +tuned again before the preregistered `197_cpu_act` run. + +## Changes + +- `src/openboost/_core/_vector_tree.py`: CPU-only level-wise vector histogram + tree with shared splits, vector Newton leaves, missing-value routing, and + output-count-invariant gain/child-weight aggregation. +- `src/openboost/_models/_histogram_boost.py`: sklearn-cloneable + `HistogramBoost`, train-only target support, empirical smoothed base logits, + PSD CRPS gradients/curvature, sample weights, and full histogram distribution + output (`mean`, `variance`, `quantile`, `interval`, and `sample`). +- Persistence reconstructs `VectorLeaves` with its output dimension and can + auto-load `HistogramBoost`. +- Shared sample-weight validation now rejects all non-finite values. +- The model is exported from the public package, while the vector-tree builder + remains internal until its CPU/CUDA contract is complete. + +## Verification + +- Focused model, persistence, growth, and validation suite: 82 passed. +- Ruff lint passed for every changed production/test file; the three new files + pass Ruff formatting. +- Finite differences validate the CRPS logit gradient. +- An explicit Jacobian validates the PSD Gauss--Newton diagonal. +- Brute-force split/leaf checks cover vector gain and missing routing. +- Fit/predict tests cover monotone CDFs, train-only support, moments, quantiles, + deterministic sampling, sample weights, sklearn cloning, and persistence. +- A local, non-artifact `1028_SWD` fold-0 smoke produced CRPS 0.327624 versus + 0.334891 for the frozen native XGBoost quantile row. This is consumed-data + implementation evidence only and cannot support a product claim. + +## Failed Attempts + +- Absolute-value exact Hessian: improved the prototype but has no valid PSD or + majorization interpretation; reject it. +- Gaussian residual-shape calibration and independent scalar quantile models: + remained materially behind the native quantile baseline; do not productize + them as the winning path. +- Reusing NaturalBoost's parameter loop: would fit 50 independent structures + per round and lose the shared-tree scaling and non-crossing design. + +## Risks and Follow-ups + +- The implementation is intentionally CPU/numeric only. CUDA histograms and + prediction kernels are scalar today; a GPU path needs output tiling and exact + CPU/CUDA parity before it is enabled. +- Categorical splits, callbacks, evaluation sets, early stopping, subsampling, + and column sampling are not implemented. +- Fixed train-range support can clip held-out extremes. The preregistered run + must publish failures and all interval/calibration guardrails. +- Benchmark next on `197_cpu_act` using protocol `crps_distribution_v1`. Freeze + one candidate before explicitly unlocking `537_houses`. + +## Commits + +- Pending — `feat: add histogram CRPS boosting` diff --git a/src/openboost/__init__.py b/src/openboost/__init__.py index 950a945..fa57103 100644 --- a/src/openboost/__init__.py +++ b/src/openboost/__init__.py @@ -184,6 +184,8 @@ # Phase 15/16: Distributional GBDT (NaturalBoost) DistributionalGBDT, GradientBoosting, + HistogramBoost, + HistogramDistributionOutput, # Phase 15: Linear Leaf GBDT LinearLeafGBDT, MultiClassGradientBoosting, @@ -333,6 +335,8 @@ def __getattr__(name: str): "MISSING_BIN", # High-level API (recommended) "GradientBoosting", + "HistogramBoost", + "HistogramDistributionOutput", "MultiClassGradientBoosting", "OpenBoostGAM", "DART", diff --git a/src/openboost/_core/_vector_tree.py b/src/openboost/_core/_vector_tree.py new file mode 100644 index 0000000..bc8df30 --- /dev/null +++ b/src/openboost/_core/_vector_tree.py @@ -0,0 +1,277 @@ +"""CPU tree fitting for shared-structure, vector-valued boosting trees. + +This module is intentionally separate from the scalar tree builder. A vector +tree chooses one split structure for all outputs, sums the per-output Newton +gain when comparing splits, and stores one vector in every leaf. The first +consumer is :class:`openboost.HistogramBoost`. + +The implementation is CPU-only. Keeping that boundary explicit avoids +silently routing vector gradients through CUDA kernels whose histogram and +prediction layouts are scalar. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +from numpy.typing import NDArray + +from .._array import MISSING_BIN, BinnedArray +from ._growth import TreeStructure, VectorLeaves + + +@dataclass(frozen=True) +class VectorSplit: + """Best split found for one vector-gradient node.""" + + feature: int = -1 + threshold: int = -1 + gain: float = 0.0 + missing_go_left: bool = True + + +def _soft_threshold(values: NDArray, reg_alpha: float) -> NDArray: + if reg_alpha <= 0.0: + return values + return np.sign(values) * np.maximum(np.abs(values) - reg_alpha, 0.0) + + +def _node_score( + sum_grad: NDArray, + sum_hess: NDArray, + reg_lambda: float, + reg_alpha: float, +) -> NDArray: + """Per-candidate vector Newton score, averaged across outputs.""" + shrunk = _soft_threshold(sum_grad, reg_alpha) + return np.mean(shrunk * shrunk / (sum_hess + reg_lambda), axis=-1) + + +def _leaf_value( + sum_grad: NDArray, + sum_hess: NDArray, + reg_lambda: float, + reg_alpha: float, +) -> NDArray: + shrunk = _soft_threshold(sum_grad, reg_alpha) + return -shrunk / (sum_hess + reg_lambda) + + +def _feature_histogram( + bins: NDArray, + grad: NDArray, + hess: NDArray, +) -> tuple[NDArray, NDArray]: + """Aggregate ``(n_samples, n_outputs)`` statistics into 256 bins.""" + n_outputs = grad.shape[1] + hist_grad = np.zeros((256, n_outputs), dtype=np.float64) + hist_hess = np.zeros((256, n_outputs), dtype=np.float64) + np.add.at(hist_grad, bins, grad) + np.add.at(hist_hess, bins, hess) + return hist_grad, hist_hess + + +def _find_best_vector_split( + binned: NDArray, + grad: NDArray, + hess: NDArray, + *, + min_child_weight: float, + reg_lambda: float, + reg_alpha: float, + min_gain: float, +) -> VectorSplit: + """Find the best numeric split, including both missing-value directions.""" + total_grad = np.sum(grad, axis=0, dtype=np.float64) + total_hess = np.sum(hess, axis=0, dtype=np.float64) + parent_score = float(_node_score(total_grad, total_hess, reg_lambda, reg_alpha)) + best = VectorSplit() + + for feature in range(binned.shape[0]): + hist_grad, hist_hess = _feature_histogram(binned[feature], grad, hess) + missing_grad = hist_grad[MISSING_BIN] + missing_hess = hist_hess[MISSING_BIN] + nonmissing_grad = total_grad - missing_grad + nonmissing_hess = total_hess - missing_hess + + # Threshold 254 cannot leave a non-missing sample on the right, so the + # useful numeric thresholds are 0..253. Empty sides are rejected by + # the child-weight check below. + left_grad = np.cumsum(hist_grad[:MISSING_BIN], axis=0)[:-1] + left_hess = np.cumsum(hist_hess[:MISSING_BIN], axis=0)[:-1] + right_grad = nonmissing_grad - left_grad + right_hess = nonmissing_hess - left_hess + + for missing_go_left in (True, False): + candidate_left_grad = left_grad + (missing_grad if missing_go_left else 0.0) + candidate_left_hess = left_hess + (missing_hess if missing_go_left else 0.0) + candidate_right_grad = right_grad + (0.0 if missing_go_left else missing_grad) + candidate_right_hess = right_hess + (0.0 if missing_go_left else missing_hess) + + # Mean curvature keeps min_child_weight invariant to n_outputs. + valid = (np.mean(candidate_left_hess, axis=1) >= min_child_weight) & ( + np.mean(candidate_right_hess, axis=1) >= min_child_weight + ) + if not np.any(valid): + continue + + gains = ( + _node_score( + candidate_left_grad, + candidate_left_hess, + reg_lambda, + reg_alpha, + ) + + _node_score( + candidate_right_grad, + candidate_right_hess, + reg_lambda, + reg_alpha, + ) + - parent_score + ) + gains = np.where(valid, gains, -np.inf) + threshold = int(np.argmax(gains)) + gain = float(gains[threshold]) + if gain > best.gain and gain > min_gain: + best = VectorSplit(feature, threshold, gain, missing_go_left) + + return best + + +def fit_vector_tree( + X: BinnedArray | NDArray, + grad: NDArray, + hess: NDArray, + *, + max_depth: int = 6, + min_child_weight: float = 1e-3, + reg_lambda: float = 1.0, + reg_alpha: float = 0.0, + min_gain: float = 0.0, +) -> TreeStructure: + """Fit one CPU, level-wise tree with shared structure and vector leaves. + + Parameters + ---------- + X: + CPU ``BinnedArray`` or feature-major uint8 matrix. + grad, hess: + Arrays with shape ``(n_samples, n_outputs)``. + """ + if isinstance(X, BinnedArray): + if X.device != "cpu" or hasattr(X.data, "__cuda_array_interface__"): + raise NotImplementedError("fit_vector_tree currently supports CPU data only") + if X.any_categorical: + raise NotImplementedError( + "fit_vector_tree does not yet support categorical feature splits" + ) + binned = np.asarray(X.data, dtype=np.uint8) + n_features = X.n_features + n_samples = X.n_samples + else: + binned = np.asarray(X, dtype=np.uint8) + if binned.ndim != 2: + raise ValueError("binned X must have shape (n_features, n_samples)") + n_features, n_samples = binned.shape + + grad = np.asarray(grad, dtype=np.float64) + hess = np.asarray(hess, dtype=np.float64) + if grad.ndim != 2 or hess.shape != grad.shape: + raise ValueError("grad and hess must have matching (n_samples, n_outputs) shapes") + if grad.shape[0] != n_samples: + raise ValueError(f"grad has {grad.shape[0]} samples, expected {n_samples}") + if not np.all(np.isfinite(grad)) or not np.all(np.isfinite(hess)): + raise ValueError("grad and hess must contain only finite values") + if np.any(hess < 0.0): + raise ValueError("hess must be non-negative for vector Newton trees") + if max_depth < 0: + raise ValueError("max_depth must be non-negative") + if reg_lambda <= 0.0: + raise ValueError("reg_lambda must be strictly positive") + + n_outputs = grad.shape[1] + max_nodes = 2 ** (max_depth + 1) - 1 + features = np.full(max_nodes, -1, dtype=np.int32) + thresholds = np.zeros(max_nodes, dtype=np.uint8) + left_children = np.full(max_nodes, -1, dtype=np.int32) + right_children = np.full(max_nodes, -1, dtype=np.int32) + missing_go_left = np.ones(max_nodes, dtype=np.bool_) + values = np.zeros((max_nodes, n_outputs), dtype=np.float32) + + node_samples: dict[int, NDArray] = {0: np.arange(n_samples, dtype=np.int32)} + leaves: dict[int, NDArray] = {} + active = [0] + deepest = 0 + + for depth in range(max_depth): + next_active: list[int] = [] + for node_id in active: + sample_idx = node_samples[node_id] + split = _find_best_vector_split( + binned[:, sample_idx], + grad[sample_idx], + hess[sample_idx], + min_child_weight=min_child_weight, + reg_lambda=reg_lambda, + reg_alpha=reg_alpha, + min_gain=min_gain, + ) + if split.feature < 0: + leaves[node_id] = sample_idx + continue + + bins = binned[split.feature, sample_idx] + goes_left = bins <= split.threshold + if split.missing_go_left: + goes_left |= bins == MISSING_BIN + else: + goes_left &= bins != MISSING_BIN + + left_idx = sample_idx[goes_left] + right_idx = sample_idx[~goes_left] + if left_idx.size == 0 or right_idx.size == 0: + leaves[node_id] = sample_idx + continue + + left_id = 2 * node_id + 1 + right_id = left_id + 1 + features[node_id] = split.feature + thresholds[node_id] = split.threshold + left_children[node_id] = left_id + right_children[node_id] = right_id + missing_go_left[node_id] = split.missing_go_left + node_samples[left_id] = left_idx + node_samples[right_id] = right_idx + next_active.extend((left_id, right_id)) + deepest = max(deepest, depth + 1) + active = next_active + if not active: + break + + for node_id in active: + leaves[node_id] = node_samples[node_id] + + for node_id, sample_idx in leaves.items(): + sum_grad = np.sum(grad[sample_idx], axis=0, dtype=np.float64) + sum_hess = np.sum(hess[sample_idx], axis=0, dtype=np.float64) + values[node_id] = _leaf_value( + sum_grad, + sum_hess, + reg_lambda, + reg_alpha, + ).astype(np.float32) + + n_nodes = max([0, *node_samples]) + 1 + return TreeStructure( + features=features[:n_nodes], + thresholds=thresholds[:n_nodes], + left_children=left_children[:n_nodes], + right_children=right_children[:n_nodes], + values=VectorLeaves(values[:n_nodes], n_outputs=n_outputs), + n_nodes=n_nodes, + depth=deepest, + n_features=n_features, + missing_go_left=missing_go_left[:n_nodes], + ) diff --git a/src/openboost/_models/__init__.py b/src/openboost/_models/__init__.py index 810829a..e2fffbb 100644 --- a/src/openboost/_models/__init__.py +++ b/src/openboost/_models/__init__.py @@ -34,6 +34,7 @@ NGBoostTweedie, ) from ._gam import OpenBoostGAM +from ._histogram_boost import HistogramBoost, HistogramDistributionOutput # Phase 15: Linear Leaf GBDT from ._linear_leaf import LinearLeafGBDT, LinearLeafTree @@ -52,6 +53,8 @@ "MultiClassGradientBoosting", "DART", "OpenBoostGAM", + "HistogramBoost", + "HistogramDistributionOutput", # Phase 13: sklearn-compatible wrappers "OpenBoostRegressor", "OpenBoostClassifier", diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py new file mode 100644 index 0000000..9926ec3 --- /dev/null +++ b/src/openboost/_models/_histogram_boost.py @@ -0,0 +1,427 @@ +"""Shared-tree histogram distribution boosting with a CRPS objective.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from .._array import BinnedArray, array +from .._core._growth import TreeStructure +from .._core._vector_tree import fit_vector_tree +from .._persistence import PersistenceMixin +from .._validation import validate_sample_weight, validate_X, validate_y + + +def _softmax(logits: NDArray) -> NDArray: + shifted = np.asarray(logits, dtype=np.float64) + if not np.all(np.isfinite(shifted)): + raise FloatingPointError("histogram logits contain non-finite values") + shifted = shifted - np.max(shifted, axis=1, keepdims=True) + exp = np.exp(shifted) + return exp / np.sum(exp, axis=1, keepdims=True) + + +def _normalized_crps_loss( + logits: NDArray, + labels: NDArray, + spacings: NDArray, +) -> NDArray: + """Per-row ranked probability score on an ordered target grid. + + The positive spacing weights are normalized to mean one. This is CRPS on + the discretized support up to a positive, target-scale-dependent constant; + normalization keeps tree regularization comparable across target units. + """ + probabilities = _softmax(logits) + cdf = np.cumsum(probabilities, axis=1) + target_cdf = np.arange(probabilities.shape[1])[None, :] >= labels[:, None] + weights = np.concatenate([spacings / np.mean(spacings), np.zeros(1, dtype=np.float64)]) + residual = cdf - target_cdf + return np.sum(weights * residual * residual, axis=1) + + +def _crps_grad_gn( + logits: NDArray, + labels: NDArray, + spacings: NDArray, + *, + curvature_scale: float = 1.0, + sample_weight: NDArray | None = None, + curvature_floor: float = 1e-6, +) -> tuple[NDArray, NDArray]: + """Gradient and positive diagonal Gauss-Newton curvature for CRPS. + + For ``p = softmax(z)``, ``C_k = sum_{j<=k} p_j`` and ordered one-hot + target CDF ``T``, the normalized score is ``sum_k w_k (C_k-T_k)^2``. + The returned curvature is the diagonal of ``2 J.T @ W @ J`` multiplied by + ``curvature_scale``; it deliberately excludes the indefinite residual + term in the exact logit Hessian. + """ + logits = np.asarray(logits, dtype=np.float64) + labels = np.asarray(labels, dtype=np.int64).reshape(-1) + spacings = np.asarray(spacings, dtype=np.float64).reshape(-1) + if logits.ndim != 2: + raise ValueError("logits must have shape (n_samples, n_distribution_bins)") + n_samples, n_outputs = logits.shape + if labels.shape != (n_samples,): + raise ValueError("labels must have one entry per logit row") + if np.any((labels < 0) | (labels >= n_outputs)): + raise ValueError("labels must index a distribution bin") + if spacings.shape != (n_outputs - 1,) or np.any(spacings <= 0.0): + raise ValueError("spacings must be positive with length n_distribution_bins - 1") + if curvature_scale <= 0.0: + raise ValueError("curvature_scale must be strictly positive") + + probabilities = _softmax(logits) + cdf = np.cumsum(probabilities, axis=1) + target_cdf = np.arange(n_outputs)[None, :] >= labels[:, None] + weights = np.concatenate([spacings / np.mean(spacings), np.zeros(1, dtype=np.float64)]) + residual = cdf - target_cdf + + # dL/dp_j = 2 * sum_{k>=j} w_k residual_k. + grad_probability = 2.0 * np.flip( + np.cumsum(np.flip(weights * residual, axis=1), axis=1), + axis=1, + ) + centered = grad_probability - np.sum(probabilities * grad_probability, axis=1, keepdims=True) + gradient = probabilities * centered + + # dC_k/dz_j = p_j * (1[j<=k] - C_k). Prefix/suffix sums compute + # diag(2 J.T W J) for every j in O(n_samples * n_outputs). + left_terms = weights * cdf * cdf + left = np.concatenate( + [ + np.zeros((n_samples, 1), dtype=np.float64), + np.cumsum(left_terms, axis=1)[:, :-1], + ], + axis=1, + ) + right = np.flip( + np.cumsum(np.flip(weights * (1.0 - cdf) ** 2, axis=1), axis=1), + axis=1, + ) + curvature = curvature_scale * 2.0 * probabilities * probabilities * (left + right) + curvature = np.maximum(curvature, curvature_floor) + + if sample_weight is not None: + weight = np.asarray(sample_weight, dtype=np.float64).reshape(-1, 1) + if weight.shape[0] != n_samples: + raise ValueError("sample_weight must have one entry per logit row") + gradient *= weight + curvature *= weight + + return gradient.astype(np.float32), curvature.astype(np.float32) + + +@dataclass +class HistogramDistributionOutput: + """Piecewise-uniform predictive distributions on one shared bin grid.""" + + probas: NDArray + bin_edges: NDArray + + def __post_init__(self) -> None: + self.probas = np.asarray(self.probas, dtype=np.float64) + self.bin_edges = np.asarray(self.bin_edges, dtype=np.float64).reshape(-1) + if self.probas.ndim != 2: + raise ValueError("probas must have shape (n_samples, n_bins)") + if self.bin_edges.shape != (self.probas.shape[1] + 1,): + raise ValueError("bin_edges must have n_bins + 1 entries") + if not np.all(np.isfinite(self.bin_edges)): + raise ValueError("bin_edges must contain only finite values") + if np.any(np.diff(self.bin_edges) <= 0.0): + raise ValueError("bin_edges must be strictly increasing") + if not np.all(np.isfinite(self.probas)) or np.any(self.probas < 0.0): + raise ValueError("probas must be finite and non-negative") + totals = np.sum(self.probas, axis=1, keepdims=True) + if np.any(totals <= 0.0): + raise ValueError("each probability row must have positive mass") + self.probas = self.probas / totals + + @property + def bin_midpoints(self) -> NDArray: + return 0.5 * (self.bin_edges[:-1] + self.bin_edges[1:]) + + def mean(self) -> NDArray: + return self.probas @ self.bin_midpoints + + def variance(self) -> NDArray: + mean = self.mean() + widths = np.diff(self.bin_edges) + second_centered = (self.bin_midpoints[None, :] - mean[:, None]) ** 2 + widths[ + None, : + ] ** 2 / 12.0 + return np.sum(self.probas * second_centered, axis=1) + + def std(self) -> NDArray: + return np.sqrt(self.variance()) + + def quantile(self, q: float) -> NDArray: + if not 0.0 <= q <= 1.0: + raise ValueError("q must lie in [0, 1]") + if q == 0.0: + return np.full(self.probas.shape[0], self.bin_edges[0]) + if q == 1.0: + return np.full(self.probas.shape[0], self.bin_edges[-1]) + cdf = np.cumsum(self.probas, axis=1) + indices = np.argmax(cdf >= q, axis=1) + rows = np.arange(self.probas.shape[0]) + previous = np.where(indices == 0, 0.0, cdf[rows, np.maximum(indices - 1, 0)]) + mass = self.probas[rows, indices] + fraction = np.divide( + q - previous, + mass, + out=np.zeros_like(mass), + where=mass > 0.0, + ) + widths = np.diff(self.bin_edges) + return self.bin_edges[indices] + np.clip(fraction, 0.0, 1.0) * widths[indices] + + def interval(self, alpha: float = 0.1) -> tuple[NDArray, NDArray]: + if not 0.0 < alpha < 1.0: + raise ValueError("alpha must lie in (0, 1)") + return self.quantile(alpha / 2.0), self.quantile(1.0 - alpha / 2.0) + + def sample(self, n_samples: int = 1, seed: int | None = None) -> NDArray: + if n_samples < 1: + raise ValueError("n_samples must be at least 1") + rng = np.random.default_rng(seed) + uniforms = rng.random((self.probas.shape[0], n_samples)) + cdf = np.cumsum(self.probas, axis=1) + samples = np.empty_like(uniforms) + widths = np.diff(self.bin_edges) + for row in range(self.probas.shape[0]): + indices = np.searchsorted(cdf[row], uniforms[row], side="left") + previous = np.where(indices == 0, 0.0, cdf[row, np.maximum(indices - 1, 0)]) + mass = self.probas[row, indices] + fraction = np.divide( + uniforms[row] - previous, + mass, + out=np.zeros(n_samples, dtype=np.float64), + where=mass > 0.0, + ) + samples[row] = self.bin_edges[indices] + np.clip(fraction, 0.0, 1.0) * widths[indices] + return samples + + +@dataclass +class HistogramBoost(PersistenceMixin): + """CPU shared-tree boosting for a flexible histogram distribution. + + Each boosting round fits one tree structure with a vector of logit updates + in every leaf. The objective is an ordered, discretized CRPS and therefore + produces a monotone CDF by construction without independent-quantile + crossing. + + This first implementation supports numeric CPU input (including NaNs). + Categorical splits, CUDA, callbacks, and evaluation sets intentionally raise + or remain outside this API until their vector paths are implemented. + """ + + n_distribution_bins: int = 50 + n_trees: int = 100 + learning_rate: float = 0.05 + max_depth: int = 6 + min_child_weight: float = 1e-3 + reg_lambda: float = 1.0 + reg_alpha: float = 0.0 + min_gain: float = 0.0 + n_feature_bins: int = 254 + curvature_scale: float = 1.0 + base_smoothing: float = 1.0 + + trees_: list[TreeStructure] = field(default_factory=list, init=False, repr=False) + X_binned_: BinnedArray | None = field(default=None, init=False, repr=False) + base_logits_: NDArray | None = field(default=None, init=False, repr=False) + target_bin_edges_: NDArray | None = field(default=None, init=False, repr=False) + target_bin_midpoints_: NDArray | None = field(default=None, init=False, repr=False) + n_features_in_: int | None = field(default=None, init=False) + + _PARAM_NAMES = ( + "n_distribution_bins", + "n_trees", + "learning_rate", + "max_depth", + "min_child_weight", + "reg_lambda", + "reg_alpha", + "min_gain", + "n_feature_bins", + "curvature_scale", + "base_smoothing", + ) + + def get_params(self, deep: bool = True) -> dict[str, Any]: # noqa: ARG002 + """Return constructor parameters for sklearn cloning.""" + return {name: getattr(self, name) for name in self._PARAM_NAMES} + + def set_params(self, **params: Any) -> HistogramBoost: + unknown = sorted(set(params) - set(self._PARAM_NAMES)) + if unknown: + raise ValueError(f"Unknown parameter(s): {unknown}") + for name, value in params.items(): + setattr(self, name, value) + return self + + def _validate_params(self) -> None: + if self.n_distribution_bins < 2: + raise ValueError("n_distribution_bins must be at least 2") + if self.n_trees < 0: + raise ValueError("n_trees must be non-negative") + if self.learning_rate <= 0.0: + raise ValueError("learning_rate must be strictly positive") + if self.max_depth < 0: + raise ValueError("max_depth must be non-negative") + if self.min_child_weight < 0.0: + raise ValueError("min_child_weight must be non-negative") + if self.reg_lambda <= 0.0: + raise ValueError("reg_lambda must be strictly positive") + if self.reg_alpha < 0.0 or self.min_gain < 0.0: + raise ValueError("reg_alpha and min_gain must be non-negative") + if not 2 <= self.n_feature_bins <= 254: + raise ValueError("n_feature_bins must lie in [2, 254]") + if self.curvature_scale <= 0.0: + raise ValueError("curvature_scale must be strictly positive") + if self.base_smoothing < 0.0: + raise ValueError("base_smoothing must be non-negative") + + def _make_target_grid(self, y: NDArray) -> tuple[NDArray, NDArray]: + lower = float(np.min(y)) + upper = float(np.max(y)) + if lower == upper: + half_span = max(abs(lower) * 1e-6, 1e-6) + edges = np.linspace( + lower - half_span, + upper + half_span, + self.n_distribution_bins + 1, + ) + else: + spacing = (upper - lower) / (self.n_distribution_bins - 1) + edges = np.linspace( + lower - 0.5 * spacing, + upper + 0.5 * spacing, + self.n_distribution_bins + 1, + ) + return edges, 0.5 * (edges[:-1] + edges[1:]) + + def fit( + self, + X: Any, + y: Any, + sample_weight: Any | None = None, + ) -> HistogramBoost: + """Fit on numeric CPU data; target grid state is learned from train y only.""" + self._validate_params() + if isinstance(X, BinnedArray): + raise TypeError( + "HistogramBoost.fit expects raw numeric X so it can own and persist " + "the training bin transform" + ) + X_valid = validate_X(X, allow_binned=False, allow_nan=True, context="fit") + y_valid = validate_y(y, n_samples=X_valid.shape[0], task="regression") + weights = validate_sample_weight(sample_weight, X_valid.shape[0]) + if weights is not None and float(np.sum(weights)) <= 0.0: + raise ValueError("sample_weight must contain positive total weight") + + self.X_binned_ = array(X_valid, n_bins=self.n_feature_bins, device="cpu") + if self.X_binned_.any_categorical: + raise NotImplementedError("HistogramBoost currently supports numeric features only") + self.n_features_in_ = X_valid.shape[1] + self.target_bin_edges_, self.target_bin_midpoints_ = self._make_target_grid(y_valid) + labels = np.searchsorted(self.target_bin_edges_[1:-1], y_valid, side="right").astype( + np.int64 + ) + labels = np.clip(labels, 0, self.n_distribution_bins - 1) + + count_weights = ( + np.ones_like(y_valid, dtype=np.float64) + if weights is None + else weights.astype(np.float64) + ) + counts = np.bincount( + labels, + weights=count_weights, + minlength=self.n_distribution_bins, + ).astype(np.float64) + counts += self.base_smoothing + probabilities = np.maximum(counts, np.finfo(np.float64).tiny) + probabilities /= np.sum(probabilities) + self.base_logits_ = np.log(probabilities) + self.base_logits_ -= np.mean(self.base_logits_) + + self.trees_ = [] + logits = np.broadcast_to( + self.base_logits_, (X_valid.shape[0], self.n_distribution_bins) + ).copy() + spacings = np.diff(self.target_bin_midpoints_) + for _ in range(self.n_trees): + grad, hess = _crps_grad_gn( + logits, + labels, + spacings, + curvature_scale=self.curvature_scale, + sample_weight=weights, + ) + tree = fit_vector_tree( + self.X_binned_, + grad, + hess, + max_depth=self.max_depth, + min_child_weight=self.min_child_weight, + reg_lambda=self.reg_lambda, + reg_alpha=self.reg_alpha, + min_gain=self.min_gain, + ) + self.trees_.append(tree) + update = self.learning_rate * np.asarray(tree(self.X_binned_)) + if not np.all(np.isfinite(update)): + raise FloatingPointError("HistogramBoost produced a non-finite tree update") + logits += update + if not np.all(np.isfinite(logits)): + raise FloatingPointError("HistogramBoost produced non-finite training logits") + return self + + def _predict_logits(self, X: Any) -> NDArray: + if self.X_binned_ is None or self.base_logits_ is None or self.n_features_in_ is None: + raise ValueError("HistogramBoost is not fitted. Call fit before predict.") + if isinstance(X, BinnedArray): + raise TypeError("HistogramBoost.predict expects raw X") + X_valid = validate_X(X, allow_binned=False, allow_nan=True, context="predict") + if X_valid.shape[1] != self.n_features_in_: + raise ValueError(f"X has {X_valid.shape[1]} features, expected {self.n_features_in_}") + X_binned = self.X_binned_.transform(X_valid) + logits = np.broadcast_to( + self.base_logits_, (X_valid.shape[0], self.n_distribution_bins) + ).copy() + for tree in self.trees_: + update = self.learning_rate * np.asarray(tree(X_binned)) + if not np.all(np.isfinite(update)): + raise FloatingPointError("HistogramBoost produced a non-finite tree update") + logits += update + if not np.all(np.isfinite(logits)): + raise FloatingPointError("HistogramBoost produced non-finite prediction logits") + return logits + + def predict_distribution(self, X: Any) -> HistogramDistributionOutput: + if self.target_bin_edges_ is None: + raise ValueError("HistogramBoost is not fitted. Call fit before predict.") + return HistogramDistributionOutput( + probas=_softmax(self._predict_logits(X)), + bin_edges=self.target_bin_edges_, + ) + + def predict(self, X: Any) -> NDArray: + return self.predict_distribution(X).mean() + + def score(self, X: Any, y: Any) -> float: + y_true = np.asarray(y, dtype=np.float64).reshape(-1) + prediction = self.predict(X) + residual = np.sum((y_true - prediction) ** 2) + total = np.sum((y_true - np.mean(y_true)) ** 2) + return float(1.0 - residual / total) if total > 0.0 else 0.0 + + def __sklearn_is_fitted__(self) -> bool: + return self.base_logits_ is not None and self.X_binned_ is not None diff --git a/src/openboost/_persistence.py b/src/openboost/_persistence.py index 1bc33bf..2993cc0 100644 --- a/src/openboost/_persistence.py +++ b/src/openboost/_persistence.py @@ -155,7 +155,7 @@ def _dict_to_tree(data: dict[str, Any]) -> TreeStructure: if values_type == "scalar": values = ScalarLeaves(values_arr) elif values_type == "vector": - values = VectorLeaves(values_arr) + values = VectorLeaves(values_arr, n_outputs=values_arr.shape[1]) else: values = values_arr @@ -545,6 +545,7 @@ def load(path: str | Path) -> PersistenceMixin: NaturalBoostTweedie, ) from ._models._gam import OpenBoostGAM + from ._models._histogram_boost import HistogramBoost from ._models._linear_leaf import LinearLeafGBDT _CLASS_MAP: dict[str, type[PersistenceMixin]] = { @@ -554,6 +555,7 @@ def load(path: str | Path) -> PersistenceMixin: MultiClassGradientBoosting, DART, OpenBoostGAM, + HistogramBoost, DistributionalGBDT, NaturalBoost, NaturalBoostNormal, diff --git a/src/openboost/_validation.py b/src/openboost/_validation.py index 48c9c84..7983de6 100644 --- a/src/openboost/_validation.py +++ b/src/openboost/_validation.py @@ -272,8 +272,8 @@ def validate_sample_weight( f"Min value: {np.min(sample_weight)}" ) - if np.any(np.isnan(sample_weight)): - raise ValueError("sample_weight contains NaN values.") + if not np.all(np.isfinite(sample_weight)): + raise ValueError("sample_weight must contain only finite values.") return sample_weight.astype(np.float32) diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py new file mode 100644 index 0000000..08eebed --- /dev/null +++ b/tests/test_histogram_boost.py @@ -0,0 +1,263 @@ +"""Correctness tests for shared-vector histogram distribution boosting.""" + +from __future__ import annotations + +import warnings + +import numpy as np +import pytest + +import openboost as ob +from openboost._core._vector_tree import _find_best_vector_split, fit_vector_tree +from openboost._models._histogram_boost import ( + _crps_grad_gn, + _normalized_crps_loss, +) + + +def test_crps_logit_gradient_matches_finite_difference(): + rng = np.random.default_rng(4) + logits = rng.normal(size=(3, 5)) + labels = np.array([0, 2, 4]) + spacings = np.array([0.5, 1.0, 0.75, 1.5]) + grad, _ = _crps_grad_gn(logits, labels, spacings) + + eps = 1e-6 + numerical = np.empty_like(logits) + for row in range(logits.shape[0]): + for output in range(logits.shape[1]): + plus = logits.copy() + minus = logits.copy() + plus[row, output] += eps + minus[row, output] -= eps + numerical[row, output] = ( + _normalized_crps_loss(plus, labels, spacings)[row] + - _normalized_crps_loss(minus, labels, spacings)[row] + ) / (2 * eps) + + np.testing.assert_allclose(grad, numerical, rtol=2e-5, atol=2e-6) + np.testing.assert_allclose(np.sum(grad, axis=1), 0.0, atol=1e-7) + + +def test_crps_gauss_newton_diagonal_matches_explicit_jacobian(): + logits = np.array([[0.4, -0.2, 0.1, 0.7]]) + labels = np.array([2]) + spacings = np.array([0.5, 1.5, 0.75]) + scale = 1.7 + _, hess = _crps_grad_gn( + logits, + labels, + spacings, + curvature_scale=scale, + curvature_floor=0.0, + ) + + p = np.exp(logits[0] - np.max(logits[0])) + p /= p.sum() + cdf = np.cumsum(p) + weights = np.r_[spacings / np.mean(spacings), 0.0] + jacobian = np.empty((len(p), len(p))) + for k in range(len(p)): + for j in range(len(p)): + jacobian[k, j] = p[j] * ((j <= k) - cdf[k]) + expected = scale * 2.0 * np.diag(jacobian.T @ np.diag(weights) @ jacobian) + + np.testing.assert_allclose(hess[0], expected, rtol=2e-6, atol=1e-8) + assert np.all(hess >= 0.0) + + +def test_crps_sample_weight_scales_gradient_and_curvature(): + logits = np.zeros((3, 4)) + labels = np.array([0, 1, 3]) + spacings = np.ones(3) + weights = np.array([0.0, 2.0, 5.0]) + grad, hess = _crps_grad_gn(logits, labels, spacings) + weighted_grad, weighted_hess = _crps_grad_gn( + logits, + labels, + spacings, + sample_weight=weights, + ) + np.testing.assert_allclose(weighted_grad, grad * weights[:, None]) + np.testing.assert_allclose(weighted_hess, hess * weights[:, None]) + + +def _brute_split(binned, grad, hess, reg_lambda): + total_grad = grad.sum(axis=0) + total_hess = hess.sum(axis=0) + + def score(g, h): + return np.mean(g * g / (h + reg_lambda)) + + parent = score(total_grad, total_hess) + best = None + for feature in range(binned.shape[0]): + for threshold in range(254): + for missing_left in (True, False): + left = binned[feature] <= threshold + if missing_left: + left |= binned[feature] == ob.MISSING_BIN + else: + left &= binned[feature] != ob.MISSING_BIN + if not np.any(left) or np.all(left): + continue + gain = ( + score(grad[left].sum(axis=0), hess[left].sum(axis=0)) + + score(grad[~left].sum(axis=0), hess[~left].sum(axis=0)) + - parent + ) + candidate = (gain, feature, threshold, missing_left) + if best is None or candidate[0] > best[0]: + best = candidate + return best + + +def test_vector_split_and_leaf_values_match_brute_force(): + binned = np.array( + [ + [0, 0, 1, 1, ob.MISSING_BIN], + [0, 1, 0, 1, 0], + ], + dtype=np.uint8, + ) + grad = np.array([[-2.0, -1.0], [-1.5, -0.5], [1.0, 2.0], [1.5, 2.5], [-0.5, 0.5]]) + hess = np.ones_like(grad) + reg_lambda = 1.0 + expected = _brute_split(binned, grad, hess, reg_lambda) + split = _find_best_vector_split( + binned, + grad, + hess, + min_child_weight=0.0, + reg_lambda=reg_lambda, + reg_alpha=0.0, + min_gain=0.0, + ) + assert (split.feature, split.threshold, split.missing_go_left) == expected[1:] + assert split.gain == pytest.approx(expected[0]) + + tree = fit_vector_tree( + binned, + grad, + hess, + max_depth=1, + min_child_weight=0.0, + reg_lambda=reg_lambda, + ) + prediction = tree.predict(binned) + assert prediction.shape == grad.shape + left = binned[split.feature] <= split.threshold + if split.missing_go_left: + left |= binned[split.feature] == ob.MISSING_BIN + else: + left &= binned[split.feature] != ob.MISSING_BIN + np.testing.assert_allclose( + prediction[left][0], + -grad[left].sum(axis=0) / (hess[left].sum(axis=0) + reg_lambda), + ) + + +def test_histogram_boost_defaults_are_frozen_and_sklearn_cloneable(): + model = ob.HistogramBoost() + assert model.n_distribution_bins == 50 + assert model.n_trees == 100 + assert model.learning_rate == 0.05 + assert model.max_depth == 6 + assert model.curvature_scale == 1.0 + + sklearn = pytest.importorskip("sklearn.base") + cloned = sklearn.clone(model) + assert cloned.get_params() == model.get_params() + assert not cloned.__sklearn_is_fitted__() + + +def test_histogram_boost_fit_predict_distribution_and_no_crossing(): + rng = np.random.default_rng(12) + X = rng.normal(size=(80, 3)).astype(np.float32) + X[3, 1] = np.nan + y = (X[:, 0] + rng.normal(scale=0.4, size=80)).astype(np.float32) + model = ob.HistogramBoost( + n_distribution_bins=12, + n_trees=4, + learning_rate=0.05, + max_depth=2, + n_feature_bins=16, + ).fit(X, y) + + edges_before = model.target_bin_edges_.copy() + dist = model.predict_distribution(np.array([[100.0, 0.0, 0.0], X[0]])) + assert dist.probas.shape == (2, 12) + np.testing.assert_allclose(dist.probas.sum(axis=1), 1.0) + assert np.all(np.diff(np.cumsum(dist.probas, axis=1), axis=1) >= -1e-12) + assert np.all(np.diff(np.column_stack([dist.quantile(q) for q in [0.1, 0.5, 0.9]])) >= 0) + np.testing.assert_array_equal(model.target_bin_edges_, edges_before) + assert model.predict(X[:5]).shape == (5,) + + +def test_histogram_distribution_output_moments_quantiles_and_sampling(): + dist = ob.HistogramDistributionOutput( + probas=np.array([[0.25, 0.75], [1.0, 0.0]]), + bin_edges=np.array([0.0, 1.0, 2.0]), + ) + np.testing.assert_allclose(dist.mean(), [1.25, 0.5]) + np.testing.assert_allclose(dist.quantile(0.5), [4.0 / 3.0, 0.5]) + lower, upper = dist.interval(0.2) + assert np.all(lower <= upper) + samples1 = dist.sample(20, seed=7) + samples2 = dist.sample(20, seed=7) + np.testing.assert_array_equal(samples1, samples2) + assert samples1.shape == (2, 20) + assert np.all((samples1 >= 0.0) & (samples1 <= 2.0)) + + with pytest.raises(ValueError, match="bin_edges must contain only finite"): + ob.HistogramDistributionOutput( + probas=np.array([[0.5, 0.5]]), + bin_edges=np.array([0.0, np.nan, 2.0]), + ) + + +def test_histogram_boost_sample_weight_controls_base_distribution(): + X = np.arange(8, dtype=np.float32).reshape(-1, 1) + y = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]) + unweighted = ob.HistogramBoost( + n_distribution_bins=3, + n_trees=0, + base_smoothing=0.0, + ).fit(X, y) + weighted = ob.HistogramBoost( + n_distribution_bins=3, + n_trees=0, + base_smoothing=0.0, + ).fit(X, y, sample_weight=np.array([10, 10, 10, 10, 1, 1, 1, 1])) + + p_unweighted = unweighted.predict_distribution(X[:1]).probas[0] + p_weighted = weighted.predict_distribution(X[:1]).probas[0] + assert p_unweighted[0] == pytest.approx(p_unweighted[-1]) + assert p_weighted[0] / p_weighted[-1] == pytest.approx(10.0) + with pytest.raises(ValueError, match="positive total weight"): + weighted.fit(X, y, sample_weight=np.zeros(len(y))) + with pytest.raises(ValueError, match="only finite"): + weighted.fit(X, y, sample_weight=np.full(len(y), np.inf)) + + +def test_histogram_boost_persistence_preserves_vector_predictions(tmp_path): + rng = np.random.default_rng(2) + X = rng.normal(size=(50, 2)).astype(np.float32) + y = (X[:, 0] - 0.5 * X[:, 1]).astype(np.float32) + model = ob.HistogramBoost( + n_distribution_bins=8, + n_trees=3, + max_depth=2, + n_feature_bins=10, + ).fit(X, y) + expected = model.predict_distribution(X[:7]).probas + path = tmp_path / "histogram.joblib" + model.save(path) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + restored = ob.HistogramBoost.load(path) + auto_restored = ob.load(path) + np.testing.assert_allclose(restored.predict_distribution(X[:7]).probas, expected) + np.testing.assert_allclose(auto_restored.predict_distribution(X[:7]).probas, expected) + assert restored.trees_[0].leaf_values_array.ndim == 2 From a4597ca395fbddd9ae6609bd675f2d77368fbd76 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 22:57:18 -0700 Subject: [PATCH 38/49] bench: add histogram CRPS candidate --- .github/workflows/scoringbench.yml | 39 ++++++++- benchmarks/scoringbench/README.md | 7 ++ benchmarks/scoringbench/openboost_wrapper.py | 84 ++++++++++++++++++- .../protocols/crps_distribution_v1.md | 17 ++++ benchmarks/scoringbench/run.py | 28 ++++++- .../scoringbench/test_openboost_wrapper.py | 26 +++++- learnings/2026-08-15-histogram-crps-boost.md | 4 + tests/test_scoringbench_provenance.py | 21 +++-- 8 files changed, 212 insertions(+), 14 deletions(-) diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index 4a0f267..bc627e8 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -20,6 +20,7 @@ on: - quality_shard - strong_shard - development_shard + - crps_distribution_shard dataset_name: description: Exact ScoringBench dataset name for a quality/strong shard required: true @@ -58,6 +59,11 @@ on: required: true default: "1.0" type: string + allow_confirmation: + description: Unlock the preregistered confirmation dataset + required: true + default: false + type: boolean concurrency: group: scoringbench-${{ github.workflow }}-${{ github.ref }} @@ -98,7 +104,7 @@ jobs: python -m pip install -e . - name: Install strong comparison models - if: github.event_name == 'workflow_dispatch' && inputs.mode == 'strong_shard' + if: github.event_name == 'workflow_dispatch' && (inputs.mode == 'strong_shard' || inputs.mode == 'crps_distribution_shard') run: | python -m pip install -r benchmarks/scoringbench/requirements-strong-baselines.txt @@ -207,6 +213,33 @@ jobs: --development-run \ --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Run preregistered CRPS distribution shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'crps_distribution_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + ALLOW_CONFIRMATION: ${{ inputs.allow_confirmation }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + CONFIRMATION_FLAG=() + if [[ "${ALLOW_CONFIRMATION}" == "true" ]]; then + CONFIRMATION_FLAG=(--allow-confirmation) + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_histogram_cpu,xgboost_quantile,catboost_quantile \ + --dataset-registry \ + benchmarks/scoringbench/protocols/crps_distribution_v1.json \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --development-run \ + "${CONFIRMATION_FLAG[@]}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Verify smoke artifact if: github.event_name == 'pull_request' || inputs.mode == 'smoke' run: | @@ -217,14 +250,14 @@ jobs: python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4 and m["outcome"]["status"] == "complete"' "${MANIFEST}" - name: Verify quality-shard artifact - if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' || inputs.mode == 'development_shard' + if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' || inputs.mode == 'development_shard' || inputs.mode == 'crps_distribution_shard' run: | MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" test -s "${MANIFEST}" test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); mode=sys.argv[2]; expected=25 if mode == "strong_shard" else (5 if mode == "development_shard" else 10); protocol="development_tuning" if mode == "development_shard" else "official_quality_shard"; compatible=mode != "development_shard"; assert m["protocol_mode"] == protocol and m["official_protocol_compatible"] is compatible and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False' "${MANIFEST}" "${{ inputs.mode }}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); mode=sys.argv[2]; dev=mode in {"development_shard", "crps_distribution_shard"}; expected=25 if mode == "strong_shard" else (15 if mode == "crps_distribution_shard" else (5 if mode == "development_shard" else 10)); protocol="development_tuning" if dev else "official_quality_shard"; assert m["protocol_mode"] == protocol and m["official_protocol_compatible"] is (not dev) and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False and (mode != "crps_distribution_shard" or len(m["verified_dataset_files"]) == 1)' "${MANIFEST}" "${{ inputs.mode }}" - name: Upload benchmark artifact if: always() diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md index e2a9d96..55c2b52 100644 --- a/benchmarks/scoringbench/README.md +++ b/benchmarks/scoringbench/README.md @@ -51,6 +51,13 @@ OpenBoost/NGBoost. These choices and resolved package versions are recorded in the manifest. Its timing is descriptive; a later speed claim requires a quality-matched compute sweep. +The preregistered `crps_distribution_v1` development experiment adds +`openboost_histogram_cpu`: 50 target bins, 100 shared-vector trees, learning +rate 0.05, depth 6, and positive-semidefinite Gauss--Newton curvature scale 1. +It is a development candidate, not an official leaderboard row. Its immutable +dataset roles, raw-file hashes, acceptance thresholds, and confirmation lock +are recorded under `benchmarks/scoringbench/protocols/`. + ## Environment Use a separate Linux environment because ScoringBench currently constrains diff --git a/benchmarks/scoringbench/openboost_wrapper.py b/benchmarks/scoringbench/openboost_wrapper.py index afe8822..29dff28 100644 --- a/benchmarks/scoringbench/openboost_wrapper.py +++ b/benchmarks/scoringbench/openboost_wrapper.py @@ -1,4 +1,4 @@ -"""ScoringBench wrapper for OpenBoost NaturalBoost. +"""ScoringBench wrappers for OpenBoost distributional models. This module intentionally lives in OpenBoost's repository while the integration is being validated. It is also shaped as an upstream-ready ScoringBench wrapper: @@ -164,3 +164,85 @@ def predict_distribution(self, X) -> DistributionPrediction: mean=mean, y_range=self._y_range, ) + + +class OpenBoostHistogramWrapper(ProbabilisticWrapper): + """OpenBoost shared-tree histogram distribution trained by CRPS. + + The defaults are the candidate frozen before the + ``crps_distribution_v1`` development run: 50 target bins, 100 trees, + learning rate 0.05, depth 6, and Gauss--Newton curvature scale 1. + """ + + def __init__( + self, + *, + n_distribution_bins: int = 50, + n_trees: int = 100, + learning_rate: float = 0.05, + max_depth: int = 6, + n_feature_bins: int = 254, + curvature_scale: float = 1.0, + model_params: dict | None = None, + ) -> None: + self.n_distribution_bins = n_distribution_bins + self.n_trees = n_trees + self.learning_rate = learning_rate + self.max_depth = max_depth + self.n_feature_bins = n_feature_bins + self.curvature_scale = curvature_scale + self.model_params = dict(model_params or {}) + self._model = None + + @staticmethod + def _sanitize_X(X) -> np.ndarray: + X = np.asarray(X, dtype=np.float32) + if X.ndim != 2: + raise ValueError(f"X must be 2-dimensional, got shape {X.shape}") + # HistogramBoost handles NaN explicitly; only infinities need a finite + # sentinel to match the other ScoringBench wrappers. + return np.nan_to_num(X, nan=np.nan, posinf=1e7, neginf=-1e7) + + def _require_fitted(self) -> None: + if self._model is None: + raise RuntimeError("Model not fitted. Call fit() first.") + + def fit(self, X, y) -> OpenBoostHistogramWrapper: + import openboost as ob + + X = self._sanitize_X(X) + y = np.asarray(y, dtype=np.float32).reshape(-1) + valid = np.isfinite(y) + X, y = X[valid], y[valid] + if len(y) == 0: + raise ValueError("No valid finite training samples") + + params = { + "n_distribution_bins": self.n_distribution_bins, + "n_trees": self.n_trees, + "learning_rate": self.learning_rate, + "max_depth": self.max_depth, + "n_feature_bins": self.n_feature_bins, + "curvature_scale": self.curvature_scale, + **self.model_params, + } + self._model = ob.HistogramBoost(**params).fit(X, y) + return self + + def predict(self, X) -> np.ndarray: + self._require_fitted() + return np.asarray( + self._model.predict(self._sanitize_X(X)), + dtype=np.float64, + ).reshape(-1) + + def predict_distribution(self, X) -> DistributionPrediction: + self._require_fitted() + output = self._model.predict_distribution(self._sanitize_X(X)) + return DistributionPrediction( + probas=np.asarray(output.probas, dtype=np.float64), + bin_edges=np.asarray(output.bin_edges, dtype=np.float64), + bin_midpoints=np.asarray(output.bin_midpoints, dtype=np.float64), + mean=np.asarray(output.mean(), dtype=np.float64), + is_natively_gridded_model=True, + ) diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v1.md b/benchmarks/scoringbench/protocols/crps_distribution_v1.md index b99932b..535f819 100644 --- a/benchmarks/scoringbench/protocols/crps_distribution_v1.md +++ b/benchmarks/scoringbench/protocols/crps_distribution_v1.md @@ -40,6 +40,23 @@ absolute-value transform of an indefinite exact diagonal is rejected. The candidate's exact public API and hyperparameters must be committed before the development dataset is loaded. +The frozen candidate is `openboost_histogram_cpu`, backed by: + +```python +openboost.HistogramBoost( + n_distribution_bins=50, + n_trees=100, + learning_rate=0.05, + max_depth=6, + n_feature_bins=254, + curvature_scale=1.0, +) +``` + +All other constructor values remain at the committed defaults. The wrapper +passes its regular shared grid and PMF directly to ScoringBench as a natively +gridded prediction; it does not derive or regrid quantiles. + ## Development acceptance Let `B` be whichever of XGBoost quantile and CatBoost MultiQuantile has the diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index ed40d3d..458e300 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -378,8 +378,9 @@ def _build_parser() -> argparse.ArgumentParser: type=_csv, default=["openboost_cpu", "ngboost"], help=( - "Comma-separated models: openboost_cpu, openboost_cuda, ngboost, " - "xgboost_quantile, xgblss, catboost_quantile" + "Comma-separated models: openboost_cpu, openboost_cuda, " + "openboost_histogram_cpu, ngboost, xgboost_quantile, xgblss, " + "catboost_quantile" ), ) parser.add_argument("--n-trees", type=int, default=500) @@ -397,6 +398,11 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--reg-lambda", type=float, default=1.0) parser.add_argument("--min-child-weight", type=float, default=1.0) parser.add_argument("--n-quantiles", type=int, default=99) + parser.add_argument("--histogram-rounds", type=int, default=100) + parser.add_argument("--histogram-bins", type=int, default=50) + parser.add_argument("--histogram-learning-rate", type=float, default=0.05) + parser.add_argument("--histogram-max-depth", type=int, default=6) + parser.add_argument("--histogram-curvature-scale", type=float, default=1.0) parser.add_argument( "--xgboost-rounds", type=int, @@ -557,6 +563,14 @@ def _model_parameters(args) -> dict[str, dict]: return { "openboost_cpu": {"backend": "cpu", **openboost_common}, "openboost_cuda": {"backend": "cuda", **openboost_common}, + "openboost_histogram_cpu": { + "n_distribution_bins": args.histogram_bins, + "n_trees": args.histogram_rounds, + "learning_rate": args.histogram_learning_rate, + "max_depth": args.histogram_max_depth, + "n_feature_bins": 254, + "curvature_scale": args.histogram_curvature_scale, + }, "ngboost": { "dist": "normal", "n_estimators": args.n_trees, @@ -588,13 +602,19 @@ def _model_parameters(args) -> dict[str, dict]: def _model_factories(args): - from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper + from benchmarks.scoringbench.openboost_wrapper import ( + OpenBoostHistogramWrapper, + OpenBoostWrapper, + ) parameters = _model_parameters(args) factories = { "openboost_cpu": lambda: OpenBoostWrapper(**parameters["openboost_cpu"]), "openboost_cuda": lambda: OpenBoostWrapper(**parameters["openboost_cuda"]), + "openboost_histogram_cpu": lambda: OpenBoostHistogramWrapper( + **parameters["openboost_histogram_cpu"] + ), } if "ngboost" in args.models: @@ -627,6 +647,7 @@ def _model_factories(args): allowed = [ "openboost_cpu", "openboost_cuda", + "openboost_histogram_cpu", "ngboost", "xgboost_quantile", "xgblss", @@ -823,6 +844,7 @@ def main() -> int: for index, dataset in enumerate(datasets): print(f"{index:3d} {dataset['name']}") return 0 + def validate_with_raw_verification(selected): nonlocal verified_dataset_files _enforce_dataset_role_lock( diff --git a/benchmarks/scoringbench/test_openboost_wrapper.py b/benchmarks/scoringbench/test_openboost_wrapper.py index a01b22e..694975b 100644 --- a/benchmarks/scoringbench/test_openboost_wrapper.py +++ b/benchmarks/scoringbench/test_openboost_wrapper.py @@ -3,7 +3,10 @@ import numpy as np from scoringbench.wrappers.base import DistributionPrediction -from benchmarks.scoringbench.openboost_wrapper import OpenBoostWrapper +from benchmarks.scoringbench.openboost_wrapper import ( + OpenBoostHistogramWrapper, + OpenBoostWrapper, +) def test_openboost_wrapper_distribution_contract(): @@ -50,3 +53,24 @@ def test_openboost_wrapper_forwards_crps_training_objective(): assert model._model.training_objective == "crps" assert np.all(np.isfinite(model.predict(X[80:]))) + + +def test_openboost_histogram_wrapper_preserves_native_distribution_grid(): + rng = np.random.default_rng(11) + X = rng.normal(size=(100, 3)).astype(np.float32) + y = (X[:, 0] + rng.normal(scale=0.5, size=100)).astype(np.float32) + model = OpenBoostHistogramWrapper( + n_distribution_bins=8, + n_trees=3, + max_depth=2, + n_feature_bins=12, + ).fit(X[:80], y[:80]) + + distribution = model.predict_distribution(X[80:]) + + assert isinstance(distribution, DistributionPrediction) + assert distribution.is_natively_gridded_model is True + assert distribution.probas.shape == (20, 8) + assert distribution.bin_edges.shape == (9,) + np.testing.assert_allclose(distribution.probas.sum(axis=1), 1.0) + np.testing.assert_allclose(distribution.mean, model.predict(X[80:])) diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index da87d97..ddf4b47 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -56,6 +56,10 @@ tuned again before the preregistered `197_cpu_act` run. - A local, non-artifact `1028_SWD` fold-0 smoke produced CRPS 0.327624 versus 0.334891 for the frozen native XGBoost quantile row. This is consumed-data implementation evidence only and cannot support a product claim. +- The ScoringBench adapter exposes the frozen model as + `openboost_histogram_cpu` and passes the model's regular PMF grid through the + benchmark's native-grid path. The three wrapper contracts and 13 provenance + tests pass against the pinned local ScoringBench checkout. ## Failed Attempts diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index a83fc09..5e9a876 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -130,9 +130,7 @@ def test_outcome_audit_publishes_missing_error_and_invalid_metric_rows(): assert outcome["status"] == "incomplete" assert outcome["expected_rows"] == 4 assert outcome["valid_rows"] == 1 - assert outcome["missing_rows"] == [ - {"dataset": "example", "model": "ngboost", "fold": 1} - ] + assert outcome["missing_rows"] == [{"dataset": "example", "model": "ngboost", "fold": 1}] assert outcome["error_rows"][0]["error"] == "model exploded" assert outcome["invalid_metric_rows"][0]["metrics"] == ["log_score"] @@ -168,9 +166,7 @@ def ensure_cached(name, url, filename): ensure_cached, ) - assert calls == [ - ("example", "https://example.test/example.tsv.gz", "example.tsv.gz") - ] + assert calls == [("example", "https://example.test/example.tsv.gz", "example.tsv.gz")] assert verified == [ { "name": "example", @@ -246,6 +242,11 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): "catboost_quantile", ] assert args.n_trees == 500 + assert args.histogram_rounds == 100 + assert args.histogram_bins == 50 + assert args.histogram_learning_rate == 0.05 + assert args.histogram_max_depth == 6 + assert args.histogram_curvature_scale == 1.0 assert args.xgboost_rounds == 100 assert args.xgboost_quantiles == 50 assert args.xgblss_rounds == 100 @@ -261,6 +262,14 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): "min_child_weight": 1.0, "training_objective": "nll", } + assert parameters["openboost_histogram_cpu"] == { + "n_distribution_bins": 50, + "n_trees": 100, + "learning_rate": 0.05, + "max_depth": 6, + "n_feature_bins": 254, + "curvature_scale": 1.0, + } assert parameters["xgboost_quantile"]["num_boost_round"] == 100 assert parameters["xgblss"]["num_boost_round"] == 100 assert parameters["catboost_quantile"]["iterations"] == 1000 From 56b875787036c08ddbd305d58c4b6a35b6a4e878 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:02:49 -0700 Subject: [PATCH 39/49] docs: document histogram CRPS boosting --- docs/api/models.md | 14 ++++ docs/user-guide/model-persistence.md | 1 + docs/user-guide/models/histogram-boost.md | 67 ++++++++++++++++++++ learnings/2026-08-15-histogram-crps-boost.md | 6 ++ mkdocs.yml | 1 + 5 files changed, 89 insertions(+) create mode 100644 docs/user-guide/models/histogram-boost.md diff --git a/docs/api/models.md b/docs/api/models.md index 6a1958b..db707c1 100644 --- a/docs/api/models.md +++ b/docs/api/models.md @@ -63,6 +63,20 @@ All OpenBoost model classes. ## Probabilistic Models (NaturalBoost) +### HistogramBoost + +::: openboost.HistogramBoost + options: + show_root_heading: true + show_source: true + +### HistogramDistributionOutput + +::: openboost.HistogramDistributionOutput + options: + show_root_heading: true + show_source: true + ### NaturalBoost ::: openboost.NaturalBoost diff --git a/docs/user-guide/model-persistence.md b/docs/user-guide/model-persistence.md index 30579ec..a278f76 100644 --- a/docs/user-guide/model-persistence.md +++ b/docs/user-guide/model-persistence.md @@ -28,6 +28,7 @@ All models support save/load: - `DART` - `OpenBoostGAM` - `NaturalBoostNormal`, `NaturalBoostGamma`, etc. +- `HistogramBoost` - `LinearLeafGBDT` ## Using joblib/pickle Directly diff --git a/docs/user-guide/models/histogram-boost.md b/docs/user-guide/models/histogram-boost.md new file mode 100644 index 0000000..b573f47 --- /dev/null +++ b/docs/user-guide/models/histogram-boost.md @@ -0,0 +1,67 @@ +# HistogramBoost + +`HistogramBoost` predicts a flexible probability histogram instead of assuming +a Normal, Gamma, or other parametric family. It trains the complete CDF with a +discretized continuous ranked probability score (CRPS). + +Each boosting round learns one tree structure. Every leaf stores a vector of +updates for the ordered histogram logits. A softmax converts those logits into +non-negative probabilities that sum to one, so predicted CDFs are monotone and +quantiles cannot cross. + +```python +import openboost as ob + +model = ob.HistogramBoost( + n_distribution_bins=50, + n_trees=100, + learning_rate=0.05, + max_depth=6, +) +model.fit(X_train, y_train) + +distribution = model.predict_distribution(X_test) +mean = distribution.mean() +lower, upper = distribution.interval(alpha=0.1) +samples = distribution.sample(n_samples=100, seed=42) +``` + +## When to use it + +Use `HistogramBoost` when a two-parameter distribution is too restrictive—for +example when conditional outcomes may be skewed or have more than one mode—and +CRPS is the primary quality target. Use NaturalBoost when a named distribution, +analytic likelihood, exposure offset, or distribution-specific interpretation +is important. + +## Current boundary + +This is a CPU-only development model. It currently supports numeric features, +including missing values, and sample weights. CUDA, categorical splits, +callbacks, evaluation sets, early stopping, and row/column sampling are not yet +implemented. + +The target grid is learned from the training target range. Predictions cannot +put mass outside that finite range plus its half-bin padding, so held-out +extremes may be clipped. Always evaluate CRPS together with coverage, interval +score, RMSE, and failure rate. + +The default configuration is frozen for the repository's preregistered +ScoringBench development experiment. It is not yet an overall leaderboard or +state-of-the-art claim. + +## Parameters + +| Parameter | Default | Meaning | +|---|---:|---| +| `n_distribution_bins` | 50 | Number of ordered target bins | +| `n_trees` | 100 | Shared-vector boosting rounds | +| `learning_rate` | 0.05 | Shrinkage applied to each tree | +| `max_depth` | 6 | Maximum routing depth | +| `n_feature_bins` | 254 | Numeric feature histogram bins | +| `curvature_scale` | 1.0 | Scale of the PSD Gauss–Newton diagonal | +| `reg_lambda` | 1.0 | L2 regularization for vector leaf values | +| `reg_alpha` | 0.0 | L1 regularization for vector leaf values | + +`predict_distribution()` returns `HistogramDistributionOutput`, which provides +`mean()`, `variance()`, `std()`, `quantile()`, `interval()`, and `sample()`. diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index ddf4b47..29d6b22 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -60,6 +60,12 @@ tuned again before the preregistered `197_cpu_act` run. `openboost_histogram_cpu` and passes the model's regular PMF grid through the benchmark's native-grid path. The three wrapper contracts and 13 provenance tests pass against the pinned local ScoringBench checkout. +- The complete non-GPU/non-benchmark CPU suite passed with 764 tests and 32 + expected skips. Public documentation marks the estimator CPU/numeric only, + explains its finite-support risk, and makes no benchmark-win claim. +- The normal MkDocs build passes. Strict mode still aborts on 29 pre-existing + Griffe warnings in older callbacks, distributions, losses, models, array, + tree, and importance docstrings; none originate from `HistogramBoost`. ## Failed Attempts diff --git a/mkdocs.yml b/mkdocs.yml index 11e3b6f..a741c89 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - DART: user-guide/models/dart.md - OpenBoostGAM: user-guide/models/gam.md - Linear Leaf GBDT: user-guide/models/linear-leaf.md + - HistogramBoost: user-guide/models/histogram-boost.md - NaturalBoost: - Overview: user-guide/naturalboost/overview.md - Distributions: user-guide/naturalboost/distributions.md From 835335aa9b5fe89708eb757aef16c4fc60472574 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:03:45 -0700 Subject: [PATCH 40/49] fix: resolve frozen registry before benchmark chdir --- benchmarks/scoringbench/run.py | 11 +++++++++-- learnings/2026-08-15-histogram-crps-boost.md | 5 +++++ tests/test_scoringbench_provenance.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 458e300..26ddc88 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -53,6 +53,13 @@ def _load_dataset_registry(path: Path) -> list[dict]: return datasets +def _resolve_dataset_registry_path(value: str | None) -> Path | None: + """Resolve a CLI registry path before changing into the artifact directory.""" + if value is None: + return None + return Path(value).expanduser().resolve() + + def _verify_dataset_files(datasets: list[dict], ensure_cached) -> list[dict]: """Materialize and verify dataset files pinned by a frozen registry. @@ -785,6 +792,7 @@ def _write_provenance( def main() -> int: args = _build_parser().parse_args() + dataset_registry_path = _resolve_dataset_registry_path(args.dataset_registry) if sys.platform == "darwin" and platform.machine() == "x86_64": raise SystemExit( "The complete ScoringBench runner is unsupported on Intel macOS: " @@ -832,8 +840,7 @@ def main() -> int: # the OpenBoost checkout. with _working_directory(output_dir): if args.dataset_registry: - registry_path = Path(args.dataset_registry).expanduser().resolve() - all_datasets = _load_dataset_registry(registry_path) + all_datasets = _load_dataset_registry(dataset_registry_path) Path("datasets.json").write_text( json.dumps(all_datasets, indent=2, ensure_ascii=False) + "\n" ) diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index 29d6b22..ba65a62 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -69,6 +69,11 @@ tuned again before the preregistered `197_cpu_act` run. ## Failed Attempts +- The first preregistered Actions run (`31930232251`) failed before dataset + download because the launcher resolved a relative frozen-registry path after + changing into its artifact directory. Registry inputs are now resolved + before that directory change and covered by a regression test. The failed + run observed no benchmark outcome and remains part of the audit trail. - Absolute-value exact Hessian: improved the prototype but has no valid PSD or majorization interpretation; reject it. - Gaussian residual-shape calibration and independent scalar quantile models: diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index 5e9a876..ec4e6d0 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -10,6 +10,7 @@ _enforce_dataset_role_lock, _load_dataset_registry, _model_parameters, + _resolve_dataset_registry_path, _select_datasets, _validate_selected_datasets, _verify_dataset_files, @@ -144,6 +145,18 @@ def test_load_dataset_registry_accepts_frozen_scoringbench_list(tmp_path): ] +def test_registry_path_is_resolved_before_artifact_working_directory( + tmp_path, monkeypatch +): + registry = tmp_path / "datasets.json" + registry.write_text('[{"name": "alpha"}]') + monkeypatch.chdir(tmp_path) + + resolved = _resolve_dataset_registry_path("datasets.json") + with _working_directory(tmp_path / "artifact"): + assert _load_dataset_registry(resolved) == [{"name": "alpha"}] + + def test_verify_dataset_files_accepts_matching_pinned_bytes(tmp_path): raw = tmp_path / "pinned.tsv.gz" raw.write_bytes(b"frozen dataset bytes") From 3cbd76393043654429b8edb792d688b37a9de46f Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:11:46 -0700 Subject: [PATCH 41/49] bench: automate CRPS candidate acceptance --- .../scoringbench/evaluate_crps_candidate.py | 177 ++++++++++++++++++ learnings/2026-08-15-histogram-crps-boost.md | 8 +- .../test_scoringbench_candidate_evaluation.py | 80 ++++++++ 3 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 benchmarks/scoringbench/evaluate_crps_candidate.py create mode 100644 tests/test_scoringbench_candidate_evaluation.py diff --git a/benchmarks/scoringbench/evaluate_crps_candidate.py b/benchmarks/scoringbench/evaluate_crps_candidate.py new file mode 100644 index 0000000..739323c --- /dev/null +++ b/benchmarks/scoringbench/evaluate_crps_candidate.py @@ -0,0 +1,177 @@ +"""Evaluate a preregistered ScoringBench CRPS candidate from raw Parquet rows.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path + +METRICS = ("crps", "coverage_90", "interval_score_90", "rmse") + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) + + +def _finite_number(value) -> bool: + if value is None or isinstance(value, bool): + return False + try: + return math.isfinite(float(value)) + except (TypeError, ValueError): + return False + + +def evaluate_records( + records: list[dict], + *, + candidate: str, + baselines: tuple[str, str], + n_folds: int = 5, + phase: str = "development", +) -> dict: + """Apply the frozen CRPS/coverage/interval/RMSE acceptance rules.""" + if phase not in {"development", "confirmation"}: + raise ValueError("phase must be 'development' or 'confirmation'") + models = (candidate, *baselines) + datasets = {str(record.get("dataset")) for record in records} + if len(datasets) != 1: + raise ValueError(f"expected exactly one dataset, got {sorted(datasets)}") + dataset = datasets.pop() + + indexed: dict[tuple[str, int], dict] = {} + for record in records: + model = str(record.get("model")) + if model not in models: + continue + try: + fold = int(record["fold"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"invalid fold identity: {record.get('fold')!r}") from exc + key = (model, fold) + if key in indexed: + raise ValueError(f"duplicate result row: model={model}, fold={fold}") + error = record.get("error") + if error is not None and str(error).strip() and str(error).lower() != "nan": + raise ValueError(f"captured model error: model={model}, fold={fold}: {error}") + invalid = [metric for metric in METRICS if not _finite_number(record.get(metric))] + if invalid: + raise ValueError(f"non-finite metrics for model={model}, fold={fold}: {invalid}") + indexed[key] = record + + expected = {(model, fold) for model in models for fold in range(n_folds)} + missing = sorted(expected - set(indexed)) + unexpected = sorted(set(indexed) - expected) + if missing or unexpected or len(records) != len(expected): + raise ValueError( + f"incomplete rows: expected={len(expected)}, observed={len(records)}, " + f"missing={missing}, unexpected={unexpected}" + ) + + summaries = {} + for model in models: + rows = [indexed[(model, fold)] for fold in range(n_folds)] + summaries[model] = { + "mean_crps": _mean([float(row["crps"]) for row in rows]), + "mean_abs_coverage_90_error": _mean( + [abs(float(row["coverage_90"]) - 0.9) for row in rows] + ), + "mean_interval_score_90": _mean([float(row["interval_score_90"]) for row in rows]), + "mean_rmse": _mean([float(row["rmse"]) for row in rows]), + } + + baseline = min(baselines, key=lambda name: summaries[name]["mean_crps"]) + candidate_summary = summaries[candidate] + baseline_summary = summaries[baseline] + crps_ratio = candidate_summary["mean_crps"] / baseline_summary["mean_crps"] + fold_wins = sum( + float(indexed[(candidate, fold)]["crps"]) <= float(indexed[(baseline, fold)]["crps"]) + for fold in range(n_folds) + ) + coverage_improvement = ( + baseline_summary["mean_abs_coverage_90_error"] + - candidate_summary["mean_abs_coverage_90_error"] + ) + interval_ratio = ( + candidate_summary["mean_interval_score_90"] / baseline_summary["mean_interval_score_90"] + ) + rmse_ratio = candidate_summary["mean_rmse"] / baseline_summary["mean_rmse"] + + guardrails = { + "complete_15_rows": len(indexed) == 3 * n_folds, + "crps_ratio_at_most_1_02": crps_ratio <= 1.02, + "at_least_3_of_5_fold_wins": fold_wins >= 3, + "coverage_error_at_most_0_05": (candidate_summary["mean_abs_coverage_90_error"] <= 0.05), + "coverage_error_improves_by_0_02": coverage_improvement >= 0.02, + "interval_score_ratio_at_most_1_05": interval_ratio <= 1.05, + "rmse_ratio_at_most_1_05": rmse_ratio <= 1.05, + } + development_pass = all(guardrails.values()) + confirmation_win = development_pass and crps_ratio < 1.0 and fold_wins >= 4 + + return { + "schema_version": 1, + "dataset": dataset, + "phase": phase, + "candidate": candidate, + "baselines": list(baselines), + "selected_strong_baseline": baseline, + "summaries": summaries, + "comparisons": { + "crps_ratio": crps_ratio, + "candidate_fold_wins": fold_wins, + "coverage_error_improvement": coverage_improvement, + "interval_score_ratio": interval_ratio, + "rmse_ratio": rmse_ratio, + }, + "guardrails": guardrails, + "development_pass": development_pass, + "confirmation_dataset_win": confirmation_win, + "accepted": development_pass if phase == "development" else confirmation_win, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("result_dir", type=Path) + parser.add_argument("--candidate", default="openboost_histogram_cpu") + parser.add_argument( + "--baselines", + default="xgboost_quantile,catboost_quantile", + help="Exactly two comma-separated baseline model names", + ) + parser.add_argument("--phase", choices=("development", "confirmation"), default="development") + parser.add_argument("--output", type=Path) + return parser + + +def main() -> int: + args = _parser().parse_args() + baselines = tuple(part.strip() for part in args.baselines.split(",") if part.strip()) + if len(baselines) != 2: + raise SystemExit("--baselines must contain exactly two model names") + + import pandas as pd + + parquet_files = sorted((args.result_dir / "raw").glob("*/*.parquet")) + if not parquet_files: + raise SystemExit(f"no raw Parquet files found under {args.result_dir / 'raw'}") + records = [] + for path in parquet_files: + records.extend(pd.read_parquet(path).to_dict(orient="records")) + result = evaluate_records( + records, + candidate=args.candidate, + baselines=baselines, + phase=args.phase, + ) + payload = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(payload) + print(payload, end="") + return 0 if result["accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index ba65a62..be97772 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -66,6 +66,11 @@ tuned again before the preregistered `197_cpu_act` run. - The normal MkDocs build passes. Strict mode still aborts on 29 pre-existing Griffe warnings in older callbacks, distributions, losses, models, array, tree, and importance docstrings; none originate from `HistogramBoost`. +- `evaluate_crps_candidate.py` turns the preregistered thresholds into a + fail-closed machine-readable decision: exact rows, one globally selected + strong baseline, paired fold wins, CRPS ratio, coverage-error improvement, + interval-score ratio, and RMSE ratio. This evaluator was committed before + observing the new development outcome. ## Failed Attempts @@ -96,4 +101,5 @@ tuned again before the preregistered `197_cpu_act` run. ## Commits -- Pending — `feat: add histogram CRPS boosting` +- `b9db276` — `feat: add histogram CRPS boosting` +- This change — `bench: automate CRPS candidate acceptance` diff --git a/tests/test_scoringbench_candidate_evaluation.py b/tests/test_scoringbench_candidate_evaluation.py new file mode 100644 index 0000000..b90eb1f --- /dev/null +++ b/tests/test_scoringbench_candidate_evaluation.py @@ -0,0 +1,80 @@ +"""Tests for the preregistered HistogramBoost acceptance evaluator.""" + +import pytest +from benchmarks.scoringbench.evaluate_crps_candidate import evaluate_records + + +def _records( + candidate_crps=0.99, + xgb_crps=1.0, + cat_crps=1.02, + candidate_coverage=0.9, + baseline_coverage=0.8, +): + result = [] + for model, crps, coverage, interval, rmse in ( + ("openboost_histogram_cpu", candidate_crps, candidate_coverage, 2.0, 1.0), + ("xgboost_quantile", xgb_crps, baseline_coverage, 2.0, 1.0), + ("catboost_quantile", cat_crps, baseline_coverage, 2.1, 1.02), + ): + for fold in range(5): + result.append( + { + "dataset": "example", + "model": model, + "fold": fold, + "crps": crps, + "coverage_90": coverage, + "interval_score_90": interval, + "rmse": rmse, + "error": None, + } + ) + return result + + +def test_development_candidate_passes_every_frozen_guardrail(): + result = evaluate_records( + _records(), + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + ) + + assert result["selected_strong_baseline"] == "xgboost_quantile" + assert result["comparisons"]["candidate_fold_wins"] == 5 + assert result["development_pass"] is True + assert result["accepted"] is True + + +def test_confirmation_requires_strict_crps_win_and_four_folds(): + result = evaluate_records( + _records(candidate_crps=1.0), + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + phase="confirmation", + ) + + assert result["development_pass"] is True + assert result["confirmation_dataset_win"] is False + assert result["accepted"] is False + + +def test_candidate_with_good_crps_but_bad_coverage_fails(): + result = evaluate_records( + _records(candidate_coverage=0.7), + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + ) + + assert result["guardrails"]["coverage_error_at_most_0_05"] is False + assert result["guardrails"]["coverage_error_improves_by_0_02"] is False + assert result["accepted"] is False + + +def test_incomplete_rows_fail_closed(): + with pytest.raises(ValueError, match="incomplete rows"): + evaluate_records( + _records()[:-1], + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + ) From 5ec4743dbbc911498ee25e13149360edbe8cdb6f Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:31:21 -0700 Subject: [PATCH 42/49] bench: record HistogramBoost development result --- .../README.md | 51 ++++++ .../benchmark_outcome.json | 21 +++ .../candidate_evaluation.json | 50 ++++++ .../datasets.json | 20 +++ .../openboost_manifest.json | 156 ++++++++++++++++++ .../raw/catboost_quantile/197_cpu_act.parquet | Bin 0 -> 30023 bytes .../197_cpu_act.parquet | Bin 0 -> 30027 bytes .../raw/xgboost_quantile/197_cpu_act.parquet | Bin 0 -> 30030 bytes .../summary.json | 79 +++++++++ learnings/2026-08-15-histogram-crps-boost.md | 16 ++ 10 files changed, 393 insertions(+) create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/catboost_quantile/197_cpu_act.parquet create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/openboost_histogram_cpu/197_cpu_act.parquet create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/xgboost_quantile/197_cpu_act.parquet create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md new file mode 100644 index 0000000..d78397a --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md @@ -0,0 +1,51 @@ +# 197_cpu_act HistogramBoost preregistered development result + +This is development/tuning evidence, not held-out, confirmation, or +leaderboard evidence. It is the first run of the frozen HistogramBoost +candidate on the preregistered `197_cpu_act` development dataset. The candidate +used 50 distribution bins, 100 shared-vector trees, learning rate 0.05, depth +6, and PSD Gauss--Newton curvature scale 1. The benchmark used five folds, +seed 42, a 3000-row cap, and pinned ScoringBench commit `a938a667`. + +The strong baseline was selected once per dataset as the lower-mean-CRPS model +between native XGBoost quantile and CatBoost MultiQuantile. CatBoost was the +selected baseline. + +| Model | CRPS | RMSE | 90% coverage | 90% interval score | Sharpness | Mean fit time (s) | +|---|---:|---:|---:|---:|---:|---:| +| CatBoost quantile | **1.326236** | **2.582497** | 71.73% | **13.329529** | **1.556934** | 144.43 | +| HistogramBoost | 1.354233 | 2.591390 | **98.23%** | 13.496925 | 6.476879 | 84.34 | +| XGBoost quantile | 1.620136 | 3.698760 | 67.47% | 17.979673 | 1.619687 | **13.59** | + +HistogramBoost did **not** pass the frozen development gate. Its CRPS was +2.111% worse than CatBoost, just beyond the 2% non-inferiority threshold, and +it won only two of five folds rather than the required three. Its absolute +90% coverage error was 8.23 percentage points rather than at most five. The +interval-score and RMSE guardrails passed. Therefore the untouched +`537_houses` confirmation dataset remains locked. + +There is a useful but limited positive signal: HistogramBoost beat the native +XGBoost quantile baseline on all five folds, lowering mean CRPS by 16.41%, +90% interval score by 24.93%, and RMSE by 29.94%. This is one development +dataset and cannot support an overall win claim. + +The failure shape is informative. HistogramBoost's mean prediction RMSE was +within 0.34% of CatBoost, but its distribution was much wider: sharpness +6.48 versus 1.56 and 98.23% empirical coverage at the nominal 90% interval. +The next development work should diagnose why the finite-bin PMF retains too +much tail mass before changing model capacity. It should not use the untouched +confirmation dataset. + +Within this single four-core Actions run, HistogramBoost fit in 84.34 seconds +per fold on average: 1.71x faster than CatBoost but 6.21x slower than XGBoost. +These timings are implementation diagnostics, not a general speed claim. + +Run: [31930502694](https://github.com/jxucoder/openboost/actions/runs/31930502694), +artifact `9259424361`, digest +`sha256:9b8d5676727cbece7a61ac1067c6705089cb69b9e8cc3718bd0e79e34b056f6a`. +The artifact contains 15/15 valid rows with no errors, missing rows, duplicates, +or non-finite metrics. OpenBoost source `835335a` and ScoringBench were clean; +the compressed dataset matched the preregistered SHA-256 +`d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc`. +The fail-closed evaluator was committed as `3cbd763` before this result was +observed. diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json new file mode 100644 index 0000000..9c8629e --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "197_cpu_act", + "expected_rows": 15, + "observed_rows": 15, + "status": "complete", + "valid_rows": 15 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 15, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 15, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 15 +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json new file mode 100644 index 0000000..ae5b6de --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json @@ -0,0 +1,50 @@ +{ + "accepted": false, + "baselines": [ + "xgboost_quantile", + "catboost_quantile" + ], + "candidate": "openboost_histogram_cpu", + "comparisons": { + "candidate_fold_wins": 2, + "coverage_error_improvement": 0.10033333301544194, + "crps_ratio": 1.02111053670206, + "interval_score_ratio": 1.0125582707407659, + "rmse_ratio": 1.0034435540465978 + }, + "confirmation_dataset_win": false, + "dataset": "197_cpu_act", + "development_pass": false, + "guardrails": { + "at_least_3_of_5_fold_wins": false, + "complete_15_rows": true, + "coverage_error_at_most_0_05": false, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": false, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "phase": "development", + "schema_version": 1, + "selected_strong_baseline": "catboost_quantile", + "summaries": { + "catboost_quantile": { + "mean_abs_coverage_90_error": 0.1826666831970215, + "mean_crps": 1.3262356982847625, + "mean_interval_score_90": 13.329529331871402, + "mean_rmse": 2.5824970354629198 + }, + "openboost_histogram_cpu": { + "mean_abs_coverage_90_error": 0.08233335018157957, + "mean_crps": 1.3542332456689852, + "mean_interval_score_90": 13.496925170068025, + "mean_rmse": 2.591390003579715 + }, + "xgboost_quantile": { + "mean_abs_coverage_90_error": 0.22533334493637086, + "mean_crps": 1.6201360866516095, + "mean_interval_score_90": 17.979672910724513, + "mean_rmse": 3.698760308978361 + } + } +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json new file mode 100644 index 0000000..54674d0 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json @@ -0,0 +1,20 @@ +[ + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "target_col": "target", + "abbr": "1CA", + "openboost_role": "development", + "raw_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/537_houses/537_houses.tsv.gz", + "target_col": "target", + "abbr": "5H", + "openboost_role": "untouched_confirmation", + "raw_sha256": "d383fd58bb79760ffaf0b615942fd47d926a19e75a8f980b79fbe9ad479d9cbf" + } +] diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json new file mode 100644 index 0000000..3d9527b --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json @@ -0,0 +1,156 @@ +{ + "arguments": { + "allow_confirmation": false, + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "197_cpu_act" + ], + "dataset_registry": "benchmarks/scoringbench/protocols/crps_distribution_v1.json", + "development_run": true, + "histogram_bins": 50, + "histogram_curvature_scale": 1.0, + "histogram_learning_rate": 0.05, + "histogram_max_depth": 6, + "histogram_rounds": 100, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_histogram_cpu", + "xgboost_quantile", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "nll", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31930502694", + "source_sha": "835335aa9b5fe89708eb757aef16c4fc60472574", + "tested_sha": "835335aa9b5fe89708eb757aef16c4fc60472574" + }, + "created_at": "2026-08-16T06:28:35+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "frozen_file", + "resolved_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "source_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1" + }, + "datasets": [ + { + "id": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "name": "197_cpu_act", + "source": "pmlb" + } + ], + "expected_result_rows": 15, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "openboost_histogram_cpu": { + "curvature_scale": 1.0, + "learning_rate": 0.05, + "max_depth": 6, + "n_distribution_bins": 50, + "n_feature_bins": 254, + "n_trees": 100 + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "835335aa9b5fe89708eb757aef16c4fc60472574", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 15 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 15, + "schema_version": 3, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "verified_dataset_files": [ + { + "name": "197_cpu_act", + "sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc", + "size_bytes": 381809, + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz" + } + ], + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/catboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/catboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6fff9b28d284ed8d60a3a4ea03c62fda715658ec GIT binary patch literal 30023 zcmd5_3w#q*)=wYQSU{kV5)hHCGV-!CX-iuKW+rKyq_icJHff`5&8uzNB#lYhw6LP0 zBJ8^2;^WgrK3PRq`IJSMU04OFimtf00^*B@@=`z%6j4~V>UZuul1c8Q3A6=Y z#_#)jyZ@v}9C!`6l&=GEz@@P{R5cyxc%=JBQvJ;GIjahOARoK=%oDSdPLsDSxc!$G zou|p;uPgFpfgeOT@Ro8ZZ-Y4C(%2lFJJ!xHFHNGpno&G6V^R`jpRjKCTklG!g|h~9 zeEwSrHFw<|PtVt$nIvUB85!@kQ<)SV^& znbQ8iTeh?0=IrEMl5J;2;=o(TrMwN|fJ;eEf7534d-~`o>#H z(gD<;2X0A{=G>aH;%y`yGjqY~A`Fl|(vVBxiKziGz@Ttc|S}h7tTva9~4>d8B5Ol2q}Y{emy%cC8s=` z5;-k(47=aGrz4Y^u=#I?JN`M08qsge>93a#r3PghoBflAQ7_Nk{@-^~Wg^b=mU1a? zgDBuqQRnrt@Gh|O@y9cBb~Qvq(~Ha zOSzP{K^$;t6b_YP1L4-KlMgacaIHK9J)!H>SNlN_9_?m?lJU135)T6C9b$vPRKWLy zaA_0*f*{=f$NMInIeIh8ALjfZypiu%jvx4roMfGkQo^;qY*hU-Lv zz+1|tyba=jOQUco3kCs=gOvmQU&1&@)g0(2m4fpSg8e5>e2L=HHTvEoQ9^)9n5lp< z=R*)KjRG$SfpzOO-|k#;BlYNjf4t+}HG!($wdJeq|BRzj7qt)39veqhHE(!oPx?fW z5b&0ADX)t-;8IZz!l@t+{Dkg*jOq8?7Vuu;-+h#Mo3j0w_*AeGm9P^b(&fC5kTS@j zso+6uDmc5^^!nWyc~s4(+hqGvbE&rBb;F*LD=F2CZz|XRUO_EX{+RmO$XpTUc}uyJ z*F_X?si^Z|D>c*j=fAF#45C~g-*Mk-_YR-}vOhlmW@>-x(I?J+*jYb-y1(l^X}`3A zA{=;2xsN$lpYcdurqGKWR3Rdl#2IlU4IN z*>#ot*V#MY6yd;^$fdk3;($wIbC|qI|39DKb87m$<9kkepM7Y?W81ghbkpQzGuC}l zw&2Ghk{Jy%o;h0g8}oF0#mZaArMwQ}fJ?efB#oA-&tfiICuc^kw5m&WEWV{HBZmap4G{_W`0A9rMZN^ae> z>aL9Lk4SoF`&Y6LJ|1PicLzWFo^1!Y_Ub&xPDgDhO_f#{%B8#w;($wIb9m>zCwG7K z%?a|!>2LPmQU49uKEANCVCHeM(3CkV@bYnT{qX%;lK%3INL%ohaw%_vIN;LQ9Ck?x zzq;?I?@8J_U#`mdwczl^QW4wtvsVvy!PZKh%>o6D5@S+?_k#c>5=^_00If50?B)Za=d}u5Ffx zd^dSZxsq+aM0OG&YCJ;glus>D)mpHecD5I`)A(?zlR=Yxi%BFYdi5 zy=%qNBU`sGPVYMYcWs{GsI&`T!|)|AM{&9+4!AUS9KLmDJ!d{}X!D)^A-}sT?~Bc? zBecWi`wnm3m3_lAXOv%Ve(4VreqOfvz-F8SUm}09NsPMNvV$)zemQw)z1o zLzWpj&6y$@*I!U97rro+Upa<7ze!3*r^*v#*?iS9Jvw%_n*Em6R-eEAwl=fd?{L~h z0ES>na$JOijp&}7FOjGDtW9>8S$2y=mOeIFp5ijw*{4*m%f~$?dzGBb>h<{8$19z! zjrBfjtC!Qc(${46dfav&SDtQj_&j#6&(Z2;4fa-RxxR z3Ccs7c~LZZc}R0g6ir?p(kzUk$;(5UMNu?)xj-{t+@m5+o@(_vxwuHJHhaCZl@C*Y zkHcTz;;Z-h&3?9yft3*--;jZ^PT+qa+Fz`|vTAenDYydo7VKe*9 zKD&Rcq#T`@F|vNHR-4_)Nix$%j2h9MO-M$`(nsV=(gu%U#;q(lGd=Up5fhzFAa2+voc=@gg3BorSI%s2v= z#r=rKQUL}}Ng)_q8aE7#OOH41jkk`Nj$jT=oc(+l998%S&z0C0HP2*KgfxZy&xt}=(myC<1= zVGz-E72rXSjW=}^U(?}VDIOKZ*}5Ubmcf9Er-hK0xbyzA}P%R}QFneg~oA(zsDWUKR$uT$oI3yqefKl55NGbPnH^;nKMA zBE1Zuf|pyaA$E-dR6K!$P;qJ8s39+lf?ocdM7(h=v3E4*Wq8Jh_cB}>H(sQdAyn}4 z-Cq&A#{epxqd};+G;UOZm-);*z>dq;t|Ja+v0g@GW;ib+q;cb6Ub3j{xZE<9*mXUi z;>j3QvM#9jh`@Qe!r!liKo1y{oYuwCPc!{Y!9Ppu#@TpBl+aIG!C z+dGjsJ^}FXGz!AQrE%khujB=Yhi@cKPXa_dX@U@OY21i{eS!e)_++9x8{qK#2!g|< zal;9BAp*S93gR9G;NiItgojJx#tYly2mrftiTiT_5Knv{KwKI(pvcYyc2EN3d!`T% z=L0gH?m)=6w0Fq40_6J(i6;sG8BcN`WL(-iWZ|sy@Kj<|5g_BK4TOwKdxu;g@cD^p z#0xh8GM>;t$hfq3$VCF=RWpcn(*YSzV<2Q)+8bopBMOjTpoonmAoux}M5J%Su3Lb- zPDN~?0U1wOAfIt*?|c^a?Ty96HZ>sQi3)^_OM8bb?Au#PiCratjHf3MGA`{MvaoM& z(-M0%fQ%<45Hc?99dewCF6|w%a8&IrCyvhqWIO?Zka20;$e{z5 zoUp-kxRN+s0f6}48Ufq^k z02n?$bypMjR|6ovE<=F0G;Tn_2^rs6Vbk=pfq2*eaIn>dJVRWGgPynfsoVh?bNY%)ciQTTH|v)2JyurwTAPGTkGDXLNl1 z>`5V0`_81y-ATlvTZoM}quTMLfOwt&M_Qc}gv#7&%1kGunGei`qda+<$LzM5eexf& z=Vv+GHhV|Cwbj|?a{IETj=6d6q%m3SYduqDkB~C2^RwJ$mmSozAoGv~77Y%&(^elL zWMS(4ogPND!P#o|Psu~-?lzaFQ>?%R=iTTa6{>c(Ii0sp8WVy-sSO;9m-aQ7QUr>T z^)V1S%*3I1X&-|H(?2+3SWzFt2D3~!Y*|^K%_rwT@s`N39pp#R2iU3>K$Iub(i4hQ0>0=tf2o&}d0k*J@ zp(ETABOt2M$25WwDC#K!Y+)ZmN4O_OKvboVX@nw>-@`mm#1!^1bfkMC1VmN(m@5bOPqN*i;PHud37_|h@5t;SGqH{4OU@@G6_@cw8B_KLQ@zG(_ zE;=1JqKStNr-(msxbpn8!5t;o`g2QD&L1mG8Aj0CF}omL}D``Gi)JyE>H z!^`>@7~Mz3A$e&87;JyUj$?!3o(^NV+paj4pdbRS&F!$leOP1?e*W8PX|`MaNZu17 zbC(v!3yOP>hK|0?e!tgYY4bD3-t*!7Im_y`oBgd`IFX*8<#brQW^X4t`1W*~z24RU zA_O<&(T#Z|Rpw{IU4txS$MF#sW-H!ql2dD+3-K~ zn@Q7ow5pLVrRhdm-N?vPs<83^5|#wZX9jB7clb9`3sbJ6=>QFO@E290E-3{uywkL{ z2J}@0I+wG}q%q{xDl6M9T7%!>u5iq9s4TT}0*yKsCvz5gEUqf2U1ON1Yi?~cDsp@} zby>4T*`cr)^I2K0&tY+u`b~3mQ=rUWXRL9Qt7(VH<#bv!RSjZgR!u2jm(Hurt;)A* zY5*7MceK~l&Q?~cbqJ%)W>f_1hN2vMt;&h8buN?7qAbm+Y@W^3Z!sEj>Wo!QHcbiQ z4f-9v2HQ>ETxtlyDV@FG^GQY_J?T6ZWul|S7i^r=PN9=s{9_>({F;fK%3{( zp?;zPeN9JExf=Du5GKUaU7K6kY{@NS+r@?R!gLbXcJs`d(oU-;-)YUQh}Oq3orLvu zSCz+_t8!Z1l^%;SFB-oVb8Th7R9iOBY%FR^bbqttnw(ZwsoP{I?5tqNIX(_x?yjPQXk)MF9By5FTuFx#244u zmqT28;3^i^gmV_gRT*Dg>n?}5_P|vvt_kNXjB8$eac#UD;@SgOvA8CjvoNmt@x`^} za)@gWT*cy=aL!?It(+5O{TBA!YEfJU->v-UTt(xWXwJ=6S4U6lPhX<%HVGS35L?(Y*!G;wOce%_k%9Td$vSB*MT#YU@j2Phk@Vcm%D^~B&_r2nWFFYnOvh)X^6h> za~s;s#>#dRrRugG{=&wLwRAyFLr=V zS(DY>!sIm}hYQbT?KXD(t_#7-r+~F3tk23IH!19QkYC~QYA82a;mGSOZ?14O zF!}a6NUE#2F;oGm_B?nm&Qn8K8{GmK_&^zyThaSW^sb)`Ir(B*4O-18H(!c!^t2Vd z=h0SB3TRP%D#+LC(K}@9LB9g-1b{VQl?F5*YeesbbPY|H2jH%NTBm~gYtXw2y%)El zT)z)p2+*Rt0(1bq7t+A59Kb;dt_Q#yyz3DabpTPZA}S2E;s7%BAyaK5n8H8su14>b zP!X+Wh#}8WZSX@FbdKHv_Zi?}b`5CLG$IWS|s6JrBsFvv)I6NJLhXNUfhh8?QE<2sdC#gMA%qO-w-x~t z^gwr@cQ)`SX50_%Wzclk`~`?L?Hqoix+20JqKPUC{s5Pz!@gy6@)$>Hoh6s`Uq=Um z{#1zi!-QHB{@t!*G}Ti$d^N4F=ISdcW`YDx!;g7no>0R>Jj1_hY&Z&n{?M%xp>2>N z7mwKfs5yVs<$1h6wfbhReOP~(Afk@M{YufJCffN-N!u(`URn^W&(+)d5N??_Sf8So z^;M!!R$7AfmFV5Q@|S5#Re3%izv^DrSAqDI6a?#Y^tL|K9y)KZzK-732i4IPmSBCf z-rI}#(QJRA%ct=1)AzDIrhSVGg7w*ZTOW#_)*GzP-^==#_AR#r>nrSSeTbhv6u)wP zdsur!&acdTq~*`jE1k^U31+6CT@%^)R84D&3)uGH<|Ax=_W`JiKO<((yJ?j%XkQk~ zzExS63*}C=slGG zU~)9{l%PG`GC6!cjkIS%E66{g>S#^-JU0G~s+tIU=F=!72hs3-9l=DCn6X z`zaCjurPvO$i4(UYD9CBj92-=c+icL;nNG+J+=p|Gtv64$)q$i%&LY@NjrQRQB94i z*%9`zFbiwXD)f%l>|l>_S;6z`Sxl!_&Eo7KzRmFIW!5k3_g1`r3vmuyqZl8)~)N*PKXRoF!iec!USNn8U&}wDSULVVznSa9W)xo0J z6|`3u!%Wx{K5|VZyf=s(FoF?^pPDX>%O38=+PFT77PK(_d3#-?Ju^WI)`Mu!shE5y zJizy<94rxE9qmQyb3uQAjRpKd+>hvYNB!=X+5+=I$+1T?OnJI6NQ) zyrZ?t+m kH!_#F$8eXl$FSG8;lI)h{|V>IZ(pb-IyQ9yQK!8tck~rfE|Dk8C(Z`DG4f&ZVUray-cljsb(1ACMWkcL zN+gmIGvnajNz(ZEUX3J}!?zFiPT%y$VN}AXg~_AW#Z#(jJ6~JAJ(2qTx3)u{KN?Rx z^K{1Zd*c!)oC9yEkny^R11^osVT`7^Y5mwUc5|!|NWU4$j9;%c57T;lY8Ux z=AHTSUt~?$_{-~V`&xtpuc?snx`+cVjm+U!Wd8n9?@6iuRdj7mmc~)$iV6QZsg0vH z_O_HQ9T`V$`|84#_;GO}ZNY0QWPBZn11^os;ry`$yY9YYB9(c*BK=-!B4s`BVD?)X zDb!P02PdC?G=+-4^NEhO?@turz?Udwyba=jOCxireMM^8YK*5w&uO{qt+p#<>fR4W zoaRN2A`=R;y zB@-y?XQpv2Q{>c~-5b7q<-mCA)w@)yk9;ygq#bxog^brl9B^p}4rKu`SUF+i0E)q3 zijBb+iDPE>_I`mvK<%{-h=*V$(qKZc@-Q_(av=zkhJY7@;FbF)e?Fs6LG3(~^dHI% z3hMUn9lCVoL<%)Ld(XV4`xVrEU!VMk#+4!x0$x)g<8=`STpEIdKLk0fpI;(>w?9}p zT$b?08;P>4mB0Ip5mENOFGGX?X)qxGvoC!i2$Y6^7ldH<7oE2cTQHToHLyoKJG^ck1U!&t=)He%E;sy)Py@9`0G;>XNWk@Yr>Sw*&+(K zRMhz_c}_WeX3-~Fu?(D9>GhuB`aAPia`!UkQ-r(CXfln?=x}ZaNrT*8jeU`K9g;2T zrPHbYlg4$=SIweINA!FX|42IZ+kN26+F8_8)3owCHR&R5^O_17Z;L44(x}{~k`6NN zZhzwq)@{@75NUO|_YY;cr)sr#?tkQHjKOYK~5w zH|^MX$~ER+>(}Rvr-pxfeB6)U8ZW|u*Hp-OTf_mEM&^+8UgH!0zISJT+dXULk4ks; z-?#9rllW*y|3BkSZ+$y|XaD2>{)}FmzOx@+knx%d8D9tDfJ-BDP&~G@Z`CXPWbdCO zclPf2BdIu4vGCxPx5@2S-?v|B-9fHgwO931a+k=|$CoH%d>x1bE{)7#)=PiP{iJ`< zJ!dyxeQfcunS<_u z7k={G8y}Hp-`MqN{PQ1@7oO~Sd)EUWlhSGXj&EP{A^Fz$f^~PS`bZ=Wyrx3N>mm-g zG%|;czupjE`i~FDUp$aGZCCmS}k6m%YW}E`J3XW&(0V)O0KNgcyjVzj*`bro2Q@o zXcwNK+mqzB0{nyLu9yxc8)ZcUQ(+LlsBX_6%-?$k)=Sj`w zyW&nRJV)+Z{OaekGtP@};58L8UKeq|rI9(@3Y*5hwK;G9x!@u>^We#6=T&_}I=38H zGjaGO(qdYZ_w%H0$Q8Yo!+*Q)qDUNgO@)ltMRCBTk>l{n$p>ufiltQQ-!3eFtqit6 ze||jvZ~LXx=&7#x|MMLQ_0vS@mOtf5MZTN7rb5Q+A`ZATGKW*r%++nRGV0Bxhkx0k z97feWk@&N;FJ#p6l)JKbeUnI?R_vT~b^UE39C%HIjMqgRaA{-?6PNT1JNoKa>d0@G zFYbG8G}Z20w*Kkgjiv5Bo4@mD&1kChPiB2)#TXF|yrn|M>mm-gG%^Rv`xpDR{Yp-K zxoKP9Bk|*?wcBPjU&@kG|IGQ>>`5Pwr7X`=Ti?7>Yz@O(DrCGa;($vdb9j1r_WLLQ zFp;`2{rFRcxpHb>=l3t|Zcm{u-#_n&Zj792UvT=eetEJ;9C%BGjMqgRaA{-?{6Wag zJWNR^yleU=CvD$y--L}_iAmef-fPM{|LO4UTQ>Flu6kI~_6O4sZu#Kq@a^~-hSyZc z_&QJ=aB1W?oKViP_p1l`pIuS-i}bMr{W~6CH96zs6a6PvTlT*&ZJ_^?U#6}7u6&>$ z=fGoipkD&}M}o(mFJX6+NDblQ zQc>I~^0ab!9J7PHyhM@xodz3Qk5BH`%n6~%%l(>pAvAfpU$Y>D zCNKAE7KYH|_P0++S9g_DeAB_<1sDO;1=;;^`yySeBZEgnN-R)#W6S$3E*&J)o( z^TXH`hAdNNaPbwXQyGS*G9xrkWkzV8%8bxFl^F$Lcyd0BG&{|WTo^g(LJ>Dcgd&cN zU*z*S$>TCu9gQBV9hx`+23|r_yUi?-kA%nD%m$CaZSl;Il%tb#MwV!AH(P9+BsnQ% za?0flLNZyNl#(kMHYSA`xAM5;q~w)x$-Qwz7a_a0#8%2|n2#j#I5c~Xfj{%^9+pB# zk^o+a!psQ*n3eIw>No(x^HgYp#icL_MF#{kjsT`Nk$5fvVDMBFg2AOx!@#(7x&=V1 zZzK9pMi9?8As}2DHIRR{6repfoY;Vx3C}DcG+Y`znr|)^p!JO)HYEWXoJGck@YetnZtP*B#jy`*h>zT{W84%9patIfQsjD5GpQ>8rAP*A?xK4 zY4Qmv@r$X%yHmKfM7bY!4UV!td|L)eqs9yNl0#*^eDO}=wNyaG(=`Ycmqv{$@G_T~ z2UstEJdN0v#(Ei)s^Pp0l17b(dC8)(UOqE}cwsuA;@KI5ic6zLW!9^LX>RRZ#LEN# z;VBpdgiE6a@~wsiU@zQFY@P{Vc&-J3;nJwVglla9-plF4)>(ju=T#6ME{z&5a3wE5 z+&qWaF&hx^EDA!zrBNda_6Y*Gt(nB041mKECI}9fMhz$2g$VF=D2e@9fQKhX5FRd# z8ZTgvBLLi!O&nGMAf5?9fVebjK#`pZ?4Sh5`*Vq7Ie?7kKM*o59UOAD0Qqn}aWW5( z@vH|z#-)Qp7S1}y3W;+CfQ;ul5Hc=GAAY@!RIAmepe!Q4i zTLj2><^mz((!n7M`}Q*$;)N1G#`6^j8J7+YS=hJNmJ%;(0U6IyAY@!RIOGCB4|}1E z*sKF&JU4-map~ZYg`?`_a$@TOK*lo?2pN|~jqE>g$qE=un=6SO6#$6utq~wDy)HoE z$;H+xV$VVV#B&V@5SLyTps)k)s3G=O10bGQK!CXPIsgObr#*{_!;1hAUzZ_3TpBf? z;Dn6ttgy}BQ%f9M0&uX^jW(4Kew&}b9iTDiZ#>XHrNq{H;#eI3;aLLY7cPzH*GLYC zKiy-nHcFFUmlCV>#QuiKdEt`>m>bEC6yX^iUq5?N$khI|G#Snejw~g5m!jH3r3S*~ z3~M&cn8cprC4!gshHaB(Y_hzT2 zv9FCxnI%Zdysk)d80;2MPlGH(8dx-0EjDvwkdTFG^mIEJ)h1iJ!IPJR)EymmXSY~^ z9nQYdp(<4E=&;%DpPlN5LTL^hi`BFhRVnERl!}?A+ zaoB*OA%^uQws73QvLOZ*B+zi=pu!=B#xiv{0a2A9rr~okCkrPKR5--YLGFnW5LFps z8a^lU!bN~B9AfAo_rwT@sthp=Ujz!mMSv|FV(1|E#0ZG03^5H~1Pa4NfGr$i=pgsR z2#Bf-F%5qNa>L96g-qcPLkGJjLO@hyh-sh;1tP9<(xC?9t}KWE3QC3;RB$l@#|_E*;`n2&xx3 z&^RmZ4tE&K z-F(Hd1O-8G9S*Ar?#3dM!1LdBW2?pFLGo~j%)MG1&nF%{8anzmcswqvvBSe0d#`}= z=QNYcV(_%P;6!>wn$2o-8C>1y;M>`4aJkyOh!EVCN4MvZRF#_n_YTsK9mhvln4yBN z)BO^~NV=wGVclHzLcijaJ`?@C%~qPGtKom>w}7T|XmvBKq3LG2sF{(e)dA&RBrNfj zFYwm0@9=M?7N%TB(_R|v;4iAAs6+!|c&F*oC7`d)*V%0ydTnh^y{fX)SX%2bIx4IS zt!iWaB5$+K&dF?rPNTiXX3^I6=vv#G>y%k;T~S%9QPrh1*5$IYY`4{D*Ld`cba_zb zX{cLbEia<2db`bL)YdeKm6@~}z}EECXV>JKv`YXN>34ND)K{x2i*yL1!(6BIT51ck zEcI#|!q(aKZlg+*RoPn2)Nib-&1$HtX)$X{5O2`$ayJ2YOLtjEL!FYfag=#Xs*rrT zjrEl_ofZ6bs9V6#7Lzgzo}<=dDz!D4OKo1xSG!SL*kUej$YScSD{W@2rmdmArUmge zI%?go%*`2wt7`=ixJunwMRpC znb*|KaP>CS<$_<8VSLY38tc`$VcOH9hqyqSXEmUHq6K|zS7CV(>W6+zh^M1IyRy}o zUBoL?7cEq~B8ME~^lU?J`*A{eFu;Uyb2Qc@T?X}$vb!J(2&NjjdkQ)fr}e*9v>4CRl^^#G0E)Ys;!vd0@{} znFaf#Q0qIdp}wZckX_ki%7(oWyT*#y`rXe_t!k+2Qo=q7?d9+_ou z%gy50s;|q<(%0)k^-12B0KJ&CwY|0j_B#$ZOW@Y?g;ul|t?!{bDnj+o$Tl$>QxIF& zGnrv;&F+iqRb}X$p{b%MC*<52(I$5N*k**ioIcyuWP)=hlfAa7!4Bs(wFTbt){y5i z5p8aTzEjDbjo@Q4OyA*bZiT*+8|vIAOy6N`ZhqhCDsQDj%{8I=j?d=i_nngKn0I_O zH^1*Jx{i5=x49+y&Na_F(05pyTcPh<`@B=qam&m*i*jug~NfO{&_^`#wi)hoP>rQx7>-6i3KO2Inw48|+S7 zL-sY~RP9Qruht!E9yit%w(0A+3XG#%vwR3qyXf3v|j_OpZ1tuL(I^crNQSv+H-2FBgV# z%&j^ltSw=ERtC9AVZVd?3YS+yxzP%1PIq~0g|&&vw>LmiUCoW53P`o*zZ2~24DXLEm`T8pK4jFsUuYfxN zU=3IruNGv@=)HhmLeu45xGPYkQ$zhr(7PJF7q_EazZ+c$C`ES#XfJv%pn+dGfP)fT z4}dp#uR>IcyoibkQDLYRdy%OdnU*$#Df|QPMd-Z}Dx!-RV#sq;8~hLkomFLo`wZ|< zy#%yr8WHthU4UAURLk}sRHCXv7a&?ynVALfR-$1lL5_3vp$EhYEPZ_))dy@C)v~H4 z4o}D6p+E-Up;s+4rqL5}6dZ0w-m|Jm2qA;|O@+V&J zP7c4hs3OQ7qKPW={Q#Gy1HNU8au`QxoiUsBUq^d={#1zi!-QHJ_}#8#G>h^${35z) zF;`znF%u+k8h*?p^Mo26;u-i|W5bc}^M`IvhqghATs$KCQ^fgGRG!28Q(x7}wGZnL z6GYUJxSs|+YD1mRlyoda<(hn7efGiDhj7bWzWS7dtgjMxH`k*?x!sx4yu5t|`el*)(=<+;1epQ34k7?iH zd|!Q*!PbZ3SL*WB=NV*uO#7A_ef1R#wm!tK${)Y-s?LD+2%cY=`KXjXORuyscPE&c zg0`o#^Jx*SEzW1#gPV`A`P~hmYW|FvJ@2N~bw2yDNcQbQd*-YjW_qp9z9PsTn-8;) zy+3bK3U@zBYi%H;J32Z4n~Sr!`uO<}eUyardpUad=RcSn4V~w+r`s|E=hI+&CbWY5 z6RM8ZcJ{FGZ&oh}vS&WMg6|76zeB#lZDPJEYAvTsyIH??0gOj6ZkRpF;B3H0Mf$yJ8epOf!{ta$Q~9(@blZ3phvA}Zj$jT z*B1}EITJp;pdGe7V4aE9cP)BVZPUWV@G0qpPa~?SSzR4u4-2z^_N+ngXw44xD3|3s zzh206di6ri9^%^ypI&DD!hUbX`?oNMoKr!o(|z{kLH5zNkBV7Tlp8sG9bHhx`4@Hj zGlN`>5Ne zq7}7Vk(RSBqAQAF=w4I%bXU+tDxZB-BztE53Ak4Wi(x!f^P=HMQ{GAabA% zj8OcFXiZf1a5vV>^-;8-h4IhZ>w@i>30kloM1xMvBzE9;~iTLVh7h0bS`U7k% z;O8H&%+=yjT{qA)`0y$hbN2jvFN(+9y=AJafIS0;2c&>Jtk0X#o`>mD%wnI>qxt$6 zxqv?!1!{Ef7^!JFd-%)z*x8<@=Svi+PJ`>d4vS}w$?2TKT;fjUE@`K-*SF!n*bM(U S=Ue0-iR501qze9TRR13Xo|mKm literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/xgboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/xgboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d0d04f5529e5a85836b28d8e5c09589638317050 GIT binary patch literal 30030 zcmd5_34Bvk)=x_dp@6cb2B=sHyg|Xzq)po};CnA=o2Im-6xuWy{YtjBX_GW&Y1#sQ z3^EMj3^F)6gChnxA9e zJNKOX|DSW0ckey-=Ty)nUROg#(CO9M#YUbaE zJaFd8AIZ-bJ@bF_zDT0ZubfF1W&B9WB(ikb=qX?~RMubCKSM6%HRMuWCtWg0L^^Dk zL?Rh9DH;BqmejA`vJy!ohqNEZ_Mhasfx7w9ri^31lTzMewc3GS4W$0~-!mU7ST>Tn zx9Z7tTQ3fza1OkNT*})b4!ATn2WRo_4?n9pNxpc$$6P_3BFi`4ZGL&pN%F}_ZE6FOYkFJ@l=kCohr1(~Za9 z-0>aR_Uj)u+@E}jTx9hup0V$eNE~yW26sX-Xg*P^pPSdmvWXQVt`AdFc8iTAInaiSUi$4ZhL)4 z)A8ZdJxgwQ=7T{asC9G3UG)7$M$NDtD}3#?5hA^T*N{tjTf_mEM&VG|B?^lw$9%x} zyFvG1s#JRC13MuIYwM?ch=h_~O?+QG2$`VB27%Gxf)F8%LO>9N|2_Z2u^qcdQ5W*> z{^RZkZ=!}ccKqVf!#7diot^N~Z@(W+`BOLU%Q$e8NDz1pxs9B`>92TG+ZmOCz{QfcHsIAN8uW8SWr8Yd1w)(I4j-}2zPo^EaIF4Fqs(2`G#w{Y=^BQs~ zZ-Xe{Qc>@#WY|F1kKCPCQ*!_fgy#+)f*|bg9C;8)p)VW|?*mAY?E^5B_J;;SS7{Um zf<7?t)#vt1&c2PZr_65)Jopo8{l-i8uYQ4`-WjJo^p}?=QvvhvZwp8NL?jBlhFr?q zpeW$dC>+8AArJhNOmiJZ{!(L(rb-8|efsGmNkeloH`N_sg0On^SK>iHirhdz6>vQv zLK=mDAP7rtH+_0?K`xbl$3ycT%~DWDboqvYMg?_fS@WIuuE?VvIj&sY{CKuV5O@u_ zl-EHVaA_0{;UJ)aFmLTgjK5Ucr(Z)?*uQu0u~g|4N55kz2$D~qJSrXpq{jw1P5ENefWnE4L**>3mLSIV}L>;9FrOuFn1a&F5X0yiArO0Knj zeZly{TO$2|x0FkH8^i&Z#^x~omCdI|*KYsm&~Jb9{@9%FrcW6;T$@UMJLAhYK56-2 z#mE^)Ui+K=o*lvI_~Mncun?uz%10IyG*iWu&pT8z`@0a9w&%C{Px9umF zoIGOx=1&L6pD%bQ^Pie03(-^KaOPwf=pz-!2*yba=jOJj35t2zI`O7{WsH2J+CLl!nn`)E}DX%P;*hFr?qAP%@RHirjJZrpeG zEf>h;Z?Ey@o6eDcGRl|x6z53ljV*tfxBCKl=mwk0S#eH;1Fs>M@-~PAE{)BhFKjFi zANs`N8ziC5|8T~+?#0XGt_^{fTb{X0rq_NmFVB=jogI>N-y{1ji^PG~kV`{05C>cu zI}XcA?)V}zYXG(D&mVrb@1uUy(R%WwH~Xbf#&>?{ednt*s$txu9ZMBrn_FH3IB>S8 zEpTaU4)0A_|6Wb~FzV<_%f6aFelYcir@oxD=Ci@n=y&gX{p~sF)P3ila@~_XMC7~4 zYsjU%E#iPnV{8O7xbWF{s-|(}oYZ&3 zzTLcrT*})b4!ATnhvkD-%zd9B^rD4%;rB-SBt`LEU;+>I;vp8%w=(Q9W|~eZ?_Pmm}TmTt0 za59Y(r@&rUx@-On6a`!wB?{9+2QCuHkR(ac5UC`2h;({C zr~v&^*gq0H`g{p{(16qsE-n?t9V^Q!mnAc295)sY0$YW;px){U)SIn-fEq4K51(n| zOC}8x6w8G#OyvVJ*;9<9G<3o|Nj8PAdSv-fsJgw;(%S0t*WVp5yZsKQU8Me@kveir zf+LN(JXIx;_xD+w>@KtHPKhjSVzRuS%WP+#QoJr7_n7Qeax$ye<6|EOI$Im-eb!bl zr!&ykWcGU8b{|)sW^?#FcCXLT>ShfFSzFueUUQ?pUYWy6hB@4RySL5kWa|mZ!uak?5)M~TWJ6riM4e~hr^)0@7 zpWp0f>lngGJP?=w7u>WU3VglO-oPmhVg+7DV-qJC%u1|w6jQb)ncZ#oHg<5)wb=dU z`kZV+VvojPinBAbh^NS%sqcq+1^^HgL<=c&k!&QpT0#wotz{+ZB)jn%h`k^LzXs5CFwtO6f8;L^BZU|f290-%)xh_xt1h$o;B5H5`yC_GyV(0)IVScjSkPcn+9ljQVF5q(zwwCGra(A-4Nn$g8>fDA0aqg8aG^c)>Y>4cuSIr^*0imZvZ^# zvGL}R;_Hg|SBgi4arWXcV(U;q#q&eROI&&tUgm|oT%1h2Jc8IUob@sy9fa_}_Xug+ zc#&R)QNhc9$cXnw0xF&WLa4YjZq#sF=CfX&OiKS(67k|q#3$*jml0VX&dUgC+<1{* za;WT_`RXX*T{)oQi5`TCOXEfjds!H2OX#ux7(?tF&3YMcvghi0Y#0^V@~xYRPci@% zPvjskap_ffsSbGwJ@)Neh<#&OFC+3cTw6v+`1%B)v;_Wz;J2YV8XSw0PpN%V)10a!;>fo50}P` z*L5W?K)iH2vHW&G#8V~+5tqh|DA*?m;1=f)tEK`Ro*hAOxHN7!;Vwjgw_Hg)q5wQR z6N2z?Y20{S_BaB-Re8h{xd4c#JrE!+jT=y8X97DY0rDgH#M3H3#*-Zg8JG49Iah%E zL?Q8f0U+Zk4up(Ldxk8Wb)KF^ys8FdJh6e0acR$x3j{ttKb_d}Q$WVk83-Ad_6%7q zKz{WO;@ug5j3+S=GA`{2GVBor$Xh64Cke>-<^v()(w-p;`}VsU;&U31@q`6J#-%+& z7WVC(MZ`fZAmeEYgp5mjhAiycpO+BFivbx=P9S7l+B0Nf-#%DMoYes`o{~VwxU^@; zg@PV-yo|V{2V^`Efsk=&&yaTEf&xEzr2^aDc1rEw#N4_tD(45mw!#PUi2#P`++ z5SLyRpz!2k@oZvM6#(Li1q6souL@AuftSxA9;pUEJd1z;ap_e6cAcMA)euk21wedV zh5&JC+<<};GQP9Irs-h=@pLV~!B#iktU>pirWEb~jX8hgflen8OO3>aIsn2G0?031 z8q=@A91wrH$6#$pN`E=9Dtb$*uH?6QM;7UUhWz@ounciQSBge*+Gzr(|*HaJ_&{`@?o z?hd#-9byG8IPFHqs8F>#;B+pWk{O0Vi47c!m-aT8QUr>T^)e7T%EX~~X)l9?l0P_N zm!e*V4V`P^uw9CJ88)2C!g0Hn^)j#^Wrib16!tPSmYc%~h^q85jgXUhIXHoc!d`}s za8HbYs7f!>2sxSGT?E*|UWSfvPmF-5N-xt0MWC>|2(X2{3?1Q~7y(h0UZxR>fV#T~ zu!X%09pRoB0a2A+rV);Ss+)O0%@p=BbfkMC1VmMOnFhK-AmTbF?QJmbl7a}Jprn^U z1y><(+=#+!1I^yKhzcGq?q&G!^$r}nYuPmcX0DM$Lk<;Q6KM9nN;K?nX)l9|-i*On zL{&=wo!t27Flq^)BQooWMdw;fz+yNB@kNJGOF(qg@zG(_t~nhzpoxbLr- z(q4{*pn8!5t;o`A2QD&L1mM?VjD)HeomL}Dd)f2wEm6G1yO#AbFuIM3L-NuHFxdWx z9mj^m-5thqcU^HTK|usu!0oWYZCGT|_58Qh(rmZ-k-R%Z<|ZwU7ZUdz4IO=({eG{* z67Vy}-t*!7Im_y`oBgd`IFX*8<#brQW^V^N`1W*|z24R!A_Vv2(S3O&RjRV#ra>06 z*vz735S}2hZrmL%G8K$vU_~j=|Q_(Nj=%8tOHvA9$X3}&Xt!bo7Xu6TsHZn4e zrb~Gc35!GJGlO;PJN%oeg(=t5bdUx+_=_sh7MFk+-f6nD2J|%rdY3a`($(hGDJ$D7 zrL}&GyTUQcp|RA>2{!6ooXn~ASX|XkyRNoV-`v`0P~`aZ+OlSgvRz>@s90I9&tY+u z_)T;4`B3IJ8fqNnTH0Z9Ih__=b%R)$RaXMoC7pG-)hes525^yndz-Otwz5*IM;HN{ zK@qgqs&njh8YjZmyG%ZdvLvUnc{WqO#Za4LG*ma)bj64_=(qbCfV;h;EMPP!SQ~el z->Qttr^8ZL>C`*GU$>?S{A{u+y1{eT`mLqT23x5!$ocBB=+sTNB4Z9yhfCqK=}KCR zb=6IXuf<*Kvy?)-O1s3Tqr$=X=Q0~=eWnt%fUCYE!CacHZcQi4-Q+PD+FR+7y+k2P20w7M%j7G+*Eel6y@ z%Al#PtkY~z2NK=iEV(A9)m7p))fRSCu;Uyb2Qc^8T(uoWgUwmjsShTYqsQW|h8UWf zEG6nrqf*_`Jy&PIrmO9+xk`L|zr7BeVZ4OmniF4KXZMA;cEeRHt_kNXjH@!fxElLH zT)W{a7T1Jx7REI%zPL8_g}8RZRV=Ov=PZn?D!#b3^o6)~!&NM<3Fq7;u9b75tlzqP zw^|f^;Jc-Y&Q&z7iRRpFb+vc5{`4h!{b^KIJL{C~ZQZQznqkk>0Bg|BM02z1oH=z$ zKkS(*b6}qoZG9Is*Ht%|b1NIHxv)24*H{T#zlS+$ltx3l0`^I0FNd%3+!~LqMD2zB zRuHY7Y}}qPVQXc#joJG^TRTnesIkmt_CN}?&tx#Md%VQ$HF)d}v%%KNwWCNJIX`mC zJSJt+_2SrUGN^J)b^2(1lJ}*HUc%biRU3f)jvLMrxb?i+f%c+xophigTK|k~ldv%b zv4uU84ffXTzPL_VhRzupDzte~=gycmiR;Hs3+&}gxy}YFoHJQnwGBoWoZHkE2FshH zp3B6v=?i_Ql06&2$7DBshqLJmeMc4T+^3tq!`fW`zSCabOh=n*qV=7S&Gqj)#aA)! zglw*V-u-WAU~yiK3zJ1ti+@31zB z>pRs!$SZ``I8o18MfDOlzm#eq-w;~Mu(=PB^L~?))i!SPZ&;@zVmgwg? zaHbN<1>*TI@cYVgmynNyb$+KQ`d**OHCmOm(f57s+JMJ zY0SNXoT^I!_0{^K&Epn>y2WG&Mr}8nLFIwGN(uj2 ztO2Ver~_FedM~7FXu3QIcLlV14b)$Q-ZkjGs1@bW4gAUh9F*XC z0KCC_6{4aIA}Uryg`rjyM5aDuTG|Mv@DIFe(R(FSL~9vh$a7R1{1676Q)Pks4Dc|! z2DE7!5e;8mfLf4L$MzpoqO3v}Aexm^rxwCnaS2lia-6FVJs?(K8S3k(K48PBmQ^)y zczOm81+ohsdet#w8a*LL!Qn>aJ*$d@5HhIWss<+Lfo@0dY~WGMxF6ihpy@927a-QO zarlkeiU@m%CaNs-16-Q!@-3T_$2dysExD}!dO8^Lr$W>pCe*sF-|b39Q=8A>Yw4=F zTz$nwOpw57_%V;n6KZ&fXV>o<8;*jIKXmJ4Xd9%+#Ur*qTFxJBc^>aiT~#yJKCC}X z5K%|sekJHp7wvqeI4}#9mlTBRbM>@7gj?nf)u-rTeU&Jbm6lL_#Z~Se`OCDWrd-9x zZ*C9kt3dpU3qtiddRiZ954|^3UwcpMgX-uCOQ=4&%G-nZ(QJRA%k%m8RrRnwrhSVF zLiO2uS|5sEsW()gzlZfP?OSdM)mPZl`VhaWaQw=v+Pbty+$4@MpyAc{i;wgzU>=*|!SqnX`JB>2)FdiU@mbKFmS( z;k-#H-2Eu6b%Kx%v~m777UgjD@$(`2C<*8Ha`Ya~e=s>3IzMDjw@mFipGMjsd^v*UaMVA->J<>1Eb0?Dtl@e+zTSc@?x~a>%|s!an}?Q8J5)atnvArwgk%|Ke^R zW00N|w%1p|P(|_K?Bj2r*-ZPCwsH228sk(LI!s8#*Hg@R5w=ey#C2}SUaOC=kGp*; zno-MXb)3DHt|)?`dqwTjQ9)~!A^WOW_RRd#$N>Ww zq4;U(lDO>QZmf;#qi8`3_KH0a(jQqytv@R#|qvpr4EkjOJVX7AkryZ?5p$8$S#i93_Kq@BrL--iE2GyDgf R`^Z0%vv)}(Rq+3$`X9JOsqp{+ literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json new file mode 100644 index 0000000..79ae949 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json @@ -0,0 +1,79 @@ +{ + "artifact": { + "digest": "sha256:9b8d5676727cbece7a61ac1067c6705089cb69b9e8cc3718bd0e79e34b056f6a", + "id": 9259424361, + "source_sha": "835335aa9b5fe89708eb757aef16c4fc60472574", + "workflow_run_id": 31930502694 + }, + "comparisons": { + "catboost_quantile": { + "crps_fold_wins": 2, + "crps_relative_percent": 2.111053670206009, + "interval_score_90_relative_percent": 1.2558270740765876, + "rmse_relative_percent": 0.34435540465977965, + "train_time_ratio": 0.583997739143586 + }, + "xgboost_quantile": { + "crps_fold_wins": 5, + "crps_relative_percent": -16.41237690916291, + "interval_score_90_relative_percent": -24.93230974175631, + "rmse_relative_percent": -29.938958269629378, + "train_time_ratio": 6.205556146272291 + } + }, + "dataset": "197_cpu_act", + "decision": "reject_candidate_keep_confirmation_locked", + "evaluator": { + "commit": "3cbd76393043654429b8edb792d688b37a9de46f", + "development_pass": false, + "selected_strong_baseline": "catboost_quantile" + }, + "file_sha256": { + "benchmark_outcome.json": "36da3bc6bac10df15a1860f2b2c0dfecca6031db1e3dbd6c0a8e69a628cbd06e", + "candidate_evaluation.json": "8f2b2ab2b548ec580a49af2854d105b261fb8f733a0851cb8984544e9bb7727c", + "datasets.json": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "openboost_manifest.json": "dad19f279b8ad37c64313d2dbc4f30c4dcf46cb5135a8e53ce5063677f046dbe", + "raw/catboost_quantile/197_cpu_act.parquet": "e9ced85f09ce1125a3ff7a18dd1053ee091b3496c4433330df6c5aecbf6c8e51", + "raw/openboost_histogram_cpu/197_cpu_act.parquet": "a9e7b244b4ed7b1f10a3a28123ea05939f06d8c375f01b747dd0e33c74a300cc", + "raw/xgboost_quantile/197_cpu_act.parquet": "43af95887d4706021fc626f80af9b1030c45bb0962c1e5f0b6fab7c1f1185158" + }, + "folds": 5, + "guardrails": { + "at_least_3_of_5_fold_wins": false, + "complete_15_rows": true, + "coverage_error_at_most_0_05": false, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": false, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4", + "summaries": { + "catboost_quantile": { + "coverage_90": 0.7173333168029785, + "crps": 1.3262356982847625, + "interval_score_90": 13.329529331871402, + "rmse": 2.5824970354629198, + "sharpness": 1.5569338635111154, + "train_time": 144.42591681480408 + }, + "openboost_histogram_cpu": { + "coverage_90": 0.9823333501815796, + "crps": 1.3542332456689852, + "interval_score_90": 13.496925170068025, + "rmse": 2.591390003579715, + "sharpness": 6.476878909743213, + "train_time": 84.34440889358521 + }, + "xgboost_quantile": { + "coverage_90": 0.6746666550636291, + "crps": 1.6201360866516095, + "interval_score_90": 17.979672910724513, + "rmse": 3.698760308978361, + "sharpness": 1.6196870388235536, + "train_time": 13.591756629943848 + } + }, + "verified_dataset_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" +} diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index be97772..b9a6a32 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -99,6 +99,22 @@ tuned again before the preregistered `197_cpu_act` run. - Benchmark next on `197_cpu_act` using protocol `crps_distribution_v1`. Freeze one candidate before explicitly unlocking `537_houses`. +## First preregistered development result + +The frozen `197_cpu_act` run completed 15/15 rows from clean source with the +preregistered dataset hash, but the candidate failed the development gate. +CatBoost was the strong baseline. HistogramBoost was 2.111% worse on mean CRPS +and won only two of five folds; its 98.23% coverage produced 8.23 percentage +points of absolute 90% coverage error. The interval-score and RMSE guardrails +passed. The confirmation dataset therefore remains locked. + +HistogramBoost did beat native XGBoost quantile on all five folds, with 16.41% +lower mean CRPS, 24.93% lower 90% interval score, and 29.94% lower RMSE. This is +a one-dataset development signal, not an overall win. The candidate was badly +over-dispersed: mean sharpness 6.48 versus CatBoost's 1.56, while RMSE was only +0.34% worse. Diagnose retained tail mass on this consumed dataset before +changing capacity or touching confirmation data. + ## Commits - `b9db276` — `feat: add histogram CRPS boosting` From 8ddd276325f9d3d6297d93a2d233793c446b351c Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:46:55 -0700 Subject: [PATCH 43/49] fix: align histogram training with continuous CRPS --- docs/user-guide/models/histogram-boost.md | 6 +- learnings/2026-08-15-histogram-crps-boost.md | 9 ++ src/openboost/_models/_histogram_boost.py | 151 +++++++++++-------- tests/test_histogram_boost.py | 72 ++++++--- 4 files changed, 152 insertions(+), 86 deletions(-) diff --git a/docs/user-guide/models/histogram-boost.md b/docs/user-guide/models/histogram-boost.md index b573f47..b7c3082 100644 --- a/docs/user-guide/models/histogram-boost.md +++ b/docs/user-guide/models/histogram-boost.md @@ -1,8 +1,9 @@ # HistogramBoost `HistogramBoost` predicts a flexible probability histogram instead of assuming -a Normal, Gamma, or other parametric family. It trains the complete CDF with a -discretized continuous ranked probability score (CRPS). +a Normal, Gamma, or other parametric family. Each bin represents uniform +density, and the model trains the complete CDF with that histogram's exact +continuous ranked probability score (CRPS). Each boosting round learns one tree structure. Every leaf stores a vector of updates for the ordered histogram logits. A softmax converts those logits into @@ -61,6 +62,7 @@ state-of-the-art claim. | `n_feature_bins` | 254 | Numeric feature histogram bins | | `curvature_scale` | 1.0 | Scale of the PSD Gauss–Newton diagonal | | `reg_lambda` | 1.0 | L2 regularization for vector leaf values | +| `base_smoothing` | 1.0 | Total Dirichlet prior weight, spread evenly across target bins | | `reg_alpha` | 0.0 | L1 regularization for vector leaf values | `predict_distribution()` returns `HistogramDistributionOutput`, which provides diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index b9a6a32..50d3a1d 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -115,6 +115,15 @@ over-dispersed: mean sharpness 6.48 versus CatBoost's 1.56, while RMSE was only 0.34% worse. Diagnose retained tail mass on this consumed dataset before changing capacity or touching confirmation data. +The follow-up audit found two semantic problems for V2. First, V1 trained a +midpoint ranked-probability approximation while ScoringBench evaluated the +exact piecewise-uniform histogram CRPS; the within-bin term is not a constant. +Second, `base_smoothing=1` added one pseudocount per bin, so prior strength grew +with output resolution. V2 uses the exact energy-form CRPS gradient and PSD +simplex-tangent curvature, and treats `base_smoothing` as a total Dirichlet +concentration spread evenly across bins. Both changes have analytic and +finite-difference tests; neither is yet benchmark evidence. + ## Commits - `b9db276` — `feat: add histogram CRPS boosting` diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py index 9926ec3..5080a17 100644 --- a/src/openboost/_models/_histogram_boost.py +++ b/src/openboost/_models/_histogram_boost.py @@ -24,86 +24,116 @@ def _softmax(logits: NDArray) -> NDArray: return exp / np.sum(exp, axis=1, keepdims=True) -def _normalized_crps_loss( +def _continuous_crps_terms( + y: NDArray, + bin_edges: NDArray, +) -> tuple[NDArray, NDArray]: + """Return normalized terms for exact piecewise-uniform histogram CRPS. + + CRPS has the energy representation + ``E|X-y| - 0.5 E|X-X'|``. Each histogram bin represents a uniform + conditional density, not a point mass at its midpoint. The returned first + term has shape ``(n_samples, n_bins)`` and the pairwise-distance matrix has + shape ``(n_bins, n_bins)``. Both are divided by mean bin width so tree + regularization remains invariant to target units. + """ + y = np.asarray(y, dtype=np.float64).reshape(-1) + bin_edges = np.asarray(bin_edges, dtype=np.float64).reshape(-1) + if bin_edges.size < 3: + raise ValueError("bin_edges must describe at least two bins") + if not np.all(np.isfinite(y)) or not np.all(np.isfinite(bin_edges)): + raise ValueError("y and bin_edges must contain only finite values") + widths = np.diff(bin_edges) + if np.any(widths <= 0.0): + raise ValueError("bin_edges must be strictly increasing") + midpoints = 0.5 * (bin_edges[:-1] + bin_edges[1:]) + + y_column = y[:, None] + lower = bin_edges[:-1][None, :] + upper = bin_edges[1:][None, :] + distance_to_target = np.where( + y_column < lower, + midpoints[None, :] - y_column, + np.where( + y_column > upper, + y_column - midpoints[None, :], + ((y_column - lower) ** 2 + (upper - y_column) ** 2) / (2.0 * widths[None, :]), + ), + ) + + pairwise_distance = np.abs(midpoints[:, None] - midpoints[None, :]) + np.fill_diagonal(pairwise_distance, widths / 3.0) + normalization = float(np.mean(widths)) + return distance_to_target / normalization, pairwise_distance / normalization + + +def _continuous_crps_loss( logits: NDArray, - labels: NDArray, - spacings: NDArray, + y: NDArray, + bin_edges: NDArray, ) -> NDArray: - """Per-row ranked probability score on an ordered target grid. - - The positive spacing weights are normalized to mean one. This is CRPS on - the discretized support up to a positive, target-scale-dependent constant; - normalization keeps tree regularization comparable across target units. - """ + """Exact per-row CRPS for the represented piecewise-uniform histogram.""" probabilities = _softmax(logits) - cdf = np.cumsum(probabilities, axis=1) - target_cdf = np.arange(probabilities.shape[1])[None, :] >= labels[:, None] - weights = np.concatenate([spacings / np.mean(spacings), np.zeros(1, dtype=np.float64)]) - residual = cdf - target_cdf - return np.sum(weights * residual * residual, axis=1) + distance_to_target, pairwise_distance = _continuous_crps_terms(y, bin_edges) + if distance_to_target.shape != probabilities.shape: + raise ValueError("y and bin_edges must match the logit rows and outputs") + first = np.sum(probabilities * distance_to_target, axis=1) + second = 0.5 * np.einsum( + "ni,ij,nj->n", + probabilities, + pairwise_distance, + probabilities, + ) + return first - second def _crps_grad_gn( logits: NDArray, - labels: NDArray, - spacings: NDArray, + y: NDArray, + bin_edges: NDArray, *, curvature_scale: float = 1.0, sample_weight: NDArray | None = None, curvature_floor: float = 1e-6, ) -> tuple[NDArray, NDArray]: - """Gradient and positive diagonal Gauss-Newton curvature for CRPS. - - For ``p = softmax(z)``, ``C_k = sum_{j<=k} p_j`` and ordered one-hot - target CDF ``T``, the normalized score is ``sum_k w_k (C_k-T_k)^2``. - The returned curvature is the diagonal of ``2 J.T @ W @ J`` multiplied by - ``curvature_scale``; it deliberately excludes the indefinite residual - term in the exact logit Hessian. + """Gradient and PSD diagonal curvature for exact continuous CRPS. + + If ``a_j = E|U_j-y|`` and ``D_jk = E|U_j-U_k|`` for uniform histogram + bins, CRPS is ``a.T @ p - 0.5 * p.T @ D @ p``. The gradient is exact. + Curvature is the diagonal of ``J.T @ (-D) @ J`` for the softmax Jacobian + ``J``; ``-D`` is positive semidefinite on the probability-simplex tangent + space. The indefinite residual term from differentiating ``J`` is + deliberately excluded. """ logits = np.asarray(logits, dtype=np.float64) - labels = np.asarray(labels, dtype=np.int64).reshape(-1) - spacings = np.asarray(spacings, dtype=np.float64).reshape(-1) if logits.ndim != 2: raise ValueError("logits must have shape (n_samples, n_distribution_bins)") n_samples, n_outputs = logits.shape - if labels.shape != (n_samples,): - raise ValueError("labels must have one entry per logit row") - if np.any((labels < 0) | (labels >= n_outputs)): - raise ValueError("labels must index a distribution bin") - if spacings.shape != (n_outputs - 1,) or np.any(spacings <= 0.0): - raise ValueError("spacings must be positive with length n_distribution_bins - 1") if curvature_scale <= 0.0: raise ValueError("curvature_scale must be strictly positive") probabilities = _softmax(logits) - cdf = np.cumsum(probabilities, axis=1) - target_cdf = np.arange(n_outputs)[None, :] >= labels[:, None] - weights = np.concatenate([spacings / np.mean(spacings), np.zeros(1, dtype=np.float64)]) - residual = cdf - target_cdf - - # dL/dp_j = 2 * sum_{k>=j} w_k residual_k. - grad_probability = 2.0 * np.flip( - np.cumsum(np.flip(weights * residual, axis=1), axis=1), - axis=1, - ) + distance_to_target, pairwise_distance = _continuous_crps_terms(y, bin_edges) + if distance_to_target.shape != (n_samples, n_outputs): + raise ValueError("y and bin_edges must match the logit rows and outputs") + + probability_distance = probabilities @ pairwise_distance + grad_probability = distance_to_target - probability_distance centered = grad_probability - np.sum(probabilities * grad_probability, axis=1, keepdims=True) gradient = probabilities * centered - # dC_k/dz_j = p_j * (1[j<=k] - C_k). Prefix/suffix sums compute - # diag(2 J.T W J) for every j in O(n_samples * n_outputs). - left_terms = weights * cdf * cdf - left = np.concatenate( - [ - np.zeros((n_samples, 1), dtype=np.float64), - np.cumsum(left_terms, axis=1)[:, :-1], - ], + probability_quadratic = np.sum( + probabilities * probability_distance, axis=1, + keepdims=True, ) - right = np.flip( - np.cumsum(np.flip(weights * (1.0 - cdf) ** 2, axis=1), axis=1), - axis=1, + distance_diagonal = np.diag(pairwise_distance)[None, :] + curvature = ( + curvature_scale + * probabilities + * probabilities + * (2.0 * probability_distance - distance_diagonal - probability_quadratic) ) - curvature = curvature_scale * 2.0 * probabilities * probabilities * (left + right) curvature = np.maximum(curvature, curvature_floor) if sample_weight is not None: @@ -212,9 +242,9 @@ class HistogramBoost(PersistenceMixin): """CPU shared-tree boosting for a flexible histogram distribution. Each boosting round fits one tree structure with a vector of logit updates - in every leaf. The objective is an ordered, discretized CRPS and therefore - produces a monotone CDF by construction without independent-quantile - crossing. + in every leaf. The objective is the exact continuous CRPS of the + represented piecewise-uniform histogram and therefore produces a monotone + CDF by construction without independent-quantile crossing. This first implementation supports numeric CPU input (including NaNs). Categorical splits, CUDA, callbacks, and evaluation sets intentionally raise @@ -346,7 +376,9 @@ def fit( weights=count_weights, minlength=self.n_distribution_bins, ).astype(np.float64) - counts += self.base_smoothing + # ``base_smoothing`` is a total Dirichlet concentration, distributed + # evenly so changing the number of bins does not change prior strength. + counts += self.base_smoothing / self.n_distribution_bins probabilities = np.maximum(counts, np.finfo(np.float64).tiny) probabilities /= np.sum(probabilities) self.base_logits_ = np.log(probabilities) @@ -356,12 +388,11 @@ def fit( logits = np.broadcast_to( self.base_logits_, (X_valid.shape[0], self.n_distribution_bins) ).copy() - spacings = np.diff(self.target_bin_midpoints_) for _ in range(self.n_trees): grad, hess = _crps_grad_gn( logits, - labels, - spacings, + y_valid, + self.target_bin_edges_, curvature_scale=self.curvature_scale, sample_weight=weights, ) diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py index 08eebed..fac727a 100644 --- a/tests/test_histogram_boost.py +++ b/tests/test_histogram_boost.py @@ -10,17 +10,18 @@ import openboost as ob from openboost._core._vector_tree import _find_best_vector_split, fit_vector_tree from openboost._models._histogram_boost import ( + _continuous_crps_loss, + _continuous_crps_terms, _crps_grad_gn, - _normalized_crps_loss, ) -def test_crps_logit_gradient_matches_finite_difference(): +def test_continuous_crps_logit_gradient_matches_finite_difference(): rng = np.random.default_rng(4) logits = rng.normal(size=(3, 5)) - labels = np.array([0, 2, 4]) - spacings = np.array([0.5, 1.0, 0.75, 1.5]) - grad, _ = _crps_grad_gn(logits, labels, spacings) + y = np.array([-0.2, 1.3, 4.8]) + bin_edges = np.array([-1.0, 0.0, 0.7, 2.0, 3.5, 5.0]) + grad, _ = _crps_grad_gn(logits, y, bin_edges) eps = 1e-6 numerical = np.empty_like(logits) @@ -31,36 +32,43 @@ def test_crps_logit_gradient_matches_finite_difference(): plus[row, output] += eps minus[row, output] -= eps numerical[row, output] = ( - _normalized_crps_loss(plus, labels, spacings)[row] - - _normalized_crps_loss(minus, labels, spacings)[row] + _continuous_crps_loss(plus, y, bin_edges)[row] + - _continuous_crps_loss(minus, y, bin_edges)[row] ) / (2 * eps) np.testing.assert_allclose(grad, numerical, rtol=2e-5, atol=2e-6) np.testing.assert_allclose(np.sum(grad, axis=1), 0.0, atol=1e-7) -def test_crps_gauss_newton_diagonal_matches_explicit_jacobian(): +def test_continuous_crps_includes_uniform_within_bin_distance(): + # A Uniform(0, 1) forecast observed at 0.5 has CRPS 1/12. The second + # far-away bin has negligible softmax mass at these logits. + loss = _continuous_crps_loss( + np.array([[50.0, -50.0]]), + np.array([0.5]), + np.array([0.0, 1.0, 2.0]), + ) + assert loss[0] == pytest.approx(1.0 / 12.0) + + +def test_continuous_crps_psd_diagonal_matches_explicit_jacobian(): logits = np.array([[0.4, -0.2, 0.1, 0.7]]) - labels = np.array([2]) - spacings = np.array([0.5, 1.5, 0.75]) + y = np.array([1.2]) + bin_edges = np.array([-0.5, 0.0, 1.5, 2.25, 4.0]) scale = 1.7 _, hess = _crps_grad_gn( logits, - labels, - spacings, + y, + bin_edges, curvature_scale=scale, curvature_floor=0.0, ) p = np.exp(logits[0] - np.max(logits[0])) p /= p.sum() - cdf = np.cumsum(p) - weights = np.r_[spacings / np.mean(spacings), 0.0] - jacobian = np.empty((len(p), len(p))) - for k in range(len(p)): - for j in range(len(p)): - jacobian[k, j] = p[j] * ((j <= k) - cdf[k]) - expected = scale * 2.0 * np.diag(jacobian.T @ np.diag(weights) @ jacobian) + _, pairwise_distance = _continuous_crps_terms(y, bin_edges) + jacobian = np.diag(p) - np.outer(p, p) + expected = scale * np.diag(jacobian.T @ (-pairwise_distance) @ jacobian) np.testing.assert_allclose(hess[0], expected, rtol=2e-6, atol=1e-8) assert np.all(hess >= 0.0) @@ -68,14 +76,14 @@ def test_crps_gauss_newton_diagonal_matches_explicit_jacobian(): def test_crps_sample_weight_scales_gradient_and_curvature(): logits = np.zeros((3, 4)) - labels = np.array([0, 1, 3]) - spacings = np.ones(3) + y = np.array([-0.5, 1.2, 4.0]) + bin_edges = np.array([-1.0, 0.0, 1.0, 2.0, 3.0]) weights = np.array([0.0, 2.0, 5.0]) - grad, hess = _crps_grad_gn(logits, labels, spacings) + grad, hess = _crps_grad_gn(logits, y, bin_edges) weighted_grad, weighted_hess = _crps_grad_gn( logits, - labels, - spacings, + y, + bin_edges, sample_weight=weights, ) np.testing.assert_allclose(weighted_grad, grad * weights[:, None]) @@ -240,6 +248,22 @@ def test_histogram_boost_sample_weight_controls_base_distribution(): weighted.fit(X, y, sample_weight=np.full(len(y), np.inf)) +def test_histogram_boost_base_smoothing_is_total_prior_weight(): + X = np.arange(4, dtype=np.float32).reshape(-1, 1) + y = np.zeros(4, dtype=np.float32) + model = ob.HistogramBoost( + n_distribution_bins=5, + n_trees=0, + base_smoothing=1.0, + ).fit(X, y) + + probabilities = model.predict_distribution(X[:1]).probas[0] + labels = np.searchsorted(model.target_bin_edges_[1:-1], y, side="right") + counts = np.bincount(labels, minlength=5) + expected = (counts + 1.0 / 5.0) / (len(y) + 1.0) + np.testing.assert_allclose(probabilities, expected) + + def test_histogram_boost_persistence_preserves_vector_predictions(tmp_path): rng = np.random.default_rng(2) X = rng.normal(size=(50, 2)).astype(np.float32) From 35ae55dcaaba926f3612e4e53c053b1e2c35d2e4 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:47:40 -0700 Subject: [PATCH 44/49] feat: separate vector split and leaf regularization --- docs/user-guide/models/histogram-boost.md | 3 ++- learnings/2026-08-15-histogram-crps-boost.md | 7 +++++++ src/openboost/_core/_vector_tree.py | 11 ++++++++++- src/openboost/_models/_histogram_boost.py | 5 +++++ tests/test_histogram_boost.py | 19 +++++++++++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/models/histogram-boost.md b/docs/user-guide/models/histogram-boost.md index b7c3082..7df421d 100644 --- a/docs/user-guide/models/histogram-boost.md +++ b/docs/user-guide/models/histogram-boost.md @@ -61,7 +61,8 @@ state-of-the-art claim. | `max_depth` | 6 | Maximum routing depth | | `n_feature_bins` | 254 | Numeric feature histogram bins | | `curvature_scale` | 1.0 | Scale of the PSD Gauss–Newton diagonal | -| `reg_lambda` | 1.0 | L2 regularization for vector leaf values | +| `reg_lambda` | 1.0 | L2 regularization used to select split structures | +| `leaf_reg_lambda` | `None` | L2 regularization for vector leaves; `None` reuses `reg_lambda` | | `base_smoothing` | 1.0 | Total Dirichlet prior weight, spread evenly across target bins | | `reg_alpha` | 0.0 | L1 regularization for vector leaf values | diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index 50d3a1d..e7b3652 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -124,6 +124,13 @@ simplex-tangent curvature, and treats `base_smoothing` as a total Dirichlet concentration spread evenly across bins. Both changes have analytic and finite-difference tests; neither is yet benchmark evidence. +The training audit also showed that one absolute `reg_lambda` controlled both +split structure and leaf updates. Lowering it from 1 to 0.1 on a consumed fold +reduced sharpness strongly but changed the learned structure enough to worsen +RMSE. V2 therefore adds an optional `leaf_reg_lambda`: split regularization +stays at 1, while leaf shrinkage can be studied independently. `None` preserves +the original coupled behavior. + ## Commits - `b9db276` — `feat: add histogram CRPS boosting` diff --git a/src/openboost/_core/_vector_tree.py b/src/openboost/_core/_vector_tree.py index bc8df30..a7497d2 100644 --- a/src/openboost/_core/_vector_tree.py +++ b/src/openboost/_core/_vector_tree.py @@ -148,6 +148,7 @@ def fit_vector_tree( max_depth: int = 6, min_child_weight: float = 1e-3, reg_lambda: float = 1.0, + leaf_reg_lambda: float | None = None, reg_alpha: float = 0.0, min_gain: float = 0.0, ) -> TreeStructure: @@ -159,6 +160,11 @@ def fit_vector_tree( CPU ``BinnedArray`` or feature-major uint8 matrix. grad, hess: Arrays with shape ``(n_samples, n_outputs)``. + reg_lambda: + L2 regularization used to compare split structures. + leaf_reg_lambda: + L2 regularization for the final vector leaf update. ``None`` reuses + ``reg_lambda`` for backward-compatible behavior. """ if isinstance(X, BinnedArray): if X.device != "cpu" or hasattr(X.data, "__cuda_array_interface__"): @@ -190,6 +196,9 @@ def fit_vector_tree( raise ValueError("max_depth must be non-negative") if reg_lambda <= 0.0: raise ValueError("reg_lambda must be strictly positive") + if leaf_reg_lambda is not None and leaf_reg_lambda <= 0.0: + raise ValueError("leaf_reg_lambda must be strictly positive or None") + resolved_leaf_reg_lambda = reg_lambda if leaf_reg_lambda is None else leaf_reg_lambda n_outputs = grad.shape[1] max_nodes = 2 ** (max_depth + 1) - 1 @@ -259,7 +268,7 @@ def fit_vector_tree( values[node_id] = _leaf_value( sum_grad, sum_hess, - reg_lambda, + resolved_leaf_reg_lambda, reg_alpha, ).astype(np.float32) diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py index 5080a17..6242c4b 100644 --- a/src/openboost/_models/_histogram_boost.py +++ b/src/openboost/_models/_histogram_boost.py @@ -257,6 +257,7 @@ class HistogramBoost(PersistenceMixin): max_depth: int = 6 min_child_weight: float = 1e-3 reg_lambda: float = 1.0 + leaf_reg_lambda: float | None = None reg_alpha: float = 0.0 min_gain: float = 0.0 n_feature_bins: int = 254 @@ -277,6 +278,7 @@ class HistogramBoost(PersistenceMixin): "max_depth", "min_child_weight", "reg_lambda", + "leaf_reg_lambda", "reg_alpha", "min_gain", "n_feature_bins", @@ -309,6 +311,8 @@ def _validate_params(self) -> None: raise ValueError("min_child_weight must be non-negative") if self.reg_lambda <= 0.0: raise ValueError("reg_lambda must be strictly positive") + if self.leaf_reg_lambda is not None and self.leaf_reg_lambda <= 0.0: + raise ValueError("leaf_reg_lambda must be strictly positive or None") if self.reg_alpha < 0.0 or self.min_gain < 0.0: raise ValueError("reg_alpha and min_gain must be non-negative") if not 2 <= self.n_feature_bins <= 254: @@ -403,6 +407,7 @@ def fit( max_depth=self.max_depth, min_child_weight=self.min_child_weight, reg_lambda=self.reg_lambda, + leaf_reg_lambda=self.leaf_reg_lambda, reg_alpha=self.reg_alpha, min_gain=self.min_gain, ) diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py index fac727a..1d23c53 100644 --- a/tests/test_histogram_boost.py +++ b/tests/test_histogram_boost.py @@ -164,6 +164,23 @@ def test_vector_split_and_leaf_values_match_brute_force(): -grad[left].sum(axis=0) / (hess[left].sum(axis=0) + reg_lambda), ) + leaf_reg_lambda = 0.1 + decoupled = fit_vector_tree( + binned, + grad, + hess, + max_depth=1, + min_child_weight=0.0, + reg_lambda=reg_lambda, + leaf_reg_lambda=leaf_reg_lambda, + ) + np.testing.assert_array_equal(decoupled.features, tree.features) + decoupled_prediction = decoupled.predict(binned) + np.testing.assert_allclose( + decoupled_prediction[left][0], + -grad[left].sum(axis=0) / (hess[left].sum(axis=0) + leaf_reg_lambda), + ) + def test_histogram_boost_defaults_are_frozen_and_sklearn_cloneable(): model = ob.HistogramBoost() @@ -172,6 +189,8 @@ def test_histogram_boost_defaults_are_frozen_and_sklearn_cloneable(): assert model.learning_rate == 0.05 assert model.max_depth == 6 assert model.curvature_scale == 1.0 + assert model.reg_lambda == 1.0 + assert model.leaf_reg_lambda is None sklearn = pytest.importorskip("sklearn.base") cloned = sklearn.clone(model) From ec810fd0e41de97cb902020a768a6c7c424c0d01 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sat, 15 Aug 2026 23:58:07 -0700 Subject: [PATCH 45/49] feat: select histogram temperature on inner validation --- benchmarks/scoringbench/openboost_wrapper.py | 53 ++++++++++++++++--- .../scoringbench/test_openboost_wrapper.py | 21 ++++++++ docs/user-guide/models/histogram-boost.md | 3 +- learnings/2026-08-15-histogram-crps-boost.md | 10 ++++ src/openboost/_models/_histogram_boost.py | 39 +++++++++++++- tests/test_histogram_boost.py | 18 +++++++ 6 files changed, 135 insertions(+), 9 deletions(-) diff --git a/benchmarks/scoringbench/openboost_wrapper.py b/benchmarks/scoringbench/openboost_wrapper.py index 29dff28..0f57924 100644 --- a/benchmarks/scoringbench/openboost_wrapper.py +++ b/benchmarks/scoringbench/openboost_wrapper.py @@ -183,6 +183,9 @@ def __init__( max_depth: int = 6, n_feature_bins: int = 254, curvature_scale: float = 1.0, + temperature_grid: tuple[float, ...] = (1.0,), + calibration_fraction: float = 0.2, + calibration_seed: int = 42, model_params: dict | None = None, ) -> None: self.n_distribution_bins = n_distribution_bins @@ -191,8 +194,13 @@ def __init__( self.max_depth = max_depth self.n_feature_bins = n_feature_bins self.curvature_scale = curvature_scale + self.temperature_grid = tuple(float(value) for value in temperature_grid) + self.calibration_fraction = calibration_fraction + self.calibration_seed = calibration_seed self.model_params = dict(model_params or {}) self._model = None + self._selected_temperature = 1.0 + self._temperature_scores: dict[float, float] = {} @staticmethod def _sanitize_X(X) -> np.ndarray: @@ -216,6 +224,12 @@ def fit(self, X, y) -> OpenBoostHistogramWrapper: X, y = X[valid], y[valid] if len(y) == 0: raise ValueError("No valid finite training samples") + if not self.temperature_grid or any( + not np.isfinite(value) or value <= 0.0 for value in self.temperature_grid + ): + raise ValueError("temperature_grid must contain positive finite values") + if not 0.0 < self.calibration_fraction < 1.0: + raise ValueError("calibration_fraction must lie in (0, 1)") params = { "n_distribution_bins": self.n_distribution_bins, @@ -226,19 +240,46 @@ def fit(self, X, y) -> OpenBoostHistogramWrapper: "curvature_scale": self.curvature_scale, **self.model_params, } + + self._selected_temperature = 1.0 + self._temperature_scores = {} + if len(self.temperature_grid) > 1 and len(y) >= 4: + rng = np.random.default_rng(self.calibration_seed) + indices = rng.permutation(len(y)) + n_calibration = min( + max(1, int(round(self.calibration_fraction * len(y)))), + len(y) - 2, + ) + calibration_idx = indices[:n_calibration] + inner_train_idx = indices[n_calibration:] + calibration_model = ob.HistogramBoost(**params).fit( + X[inner_train_idx], + y[inner_train_idx], + ) + calibration_output = calibration_model.predict_distribution(X[calibration_idx]) + for temperature in self.temperature_grid: + score = np.mean(calibration_output.tempered(temperature).crps(y[calibration_idx])) + self._temperature_scores[temperature] = float(score) + self._selected_temperature = min( + self.temperature_grid, + key=lambda value: ( + self._temperature_scores[value], + abs(value - 1.0), + value, + ), + ) + self._model = ob.HistogramBoost(**params).fit(X, y) return self def predict(self, X) -> np.ndarray: - self._require_fitted() - return np.asarray( - self._model.predict(self._sanitize_X(X)), - dtype=np.float64, - ).reshape(-1) + return np.asarray(self.predict_distribution(X).mean, dtype=np.float64).reshape(-1) def predict_distribution(self, X) -> DistributionPrediction: self._require_fitted() - output = self._model.predict_distribution(self._sanitize_X(X)) + output = self._model.predict_distribution(self._sanitize_X(X)).tempered( + self._selected_temperature + ) return DistributionPrediction( probas=np.asarray(output.probas, dtype=np.float64), bin_edges=np.asarray(output.bin_edges, dtype=np.float64), diff --git a/benchmarks/scoringbench/test_openboost_wrapper.py b/benchmarks/scoringbench/test_openboost_wrapper.py index 694975b..39205bd 100644 --- a/benchmarks/scoringbench/test_openboost_wrapper.py +++ b/benchmarks/scoringbench/test_openboost_wrapper.py @@ -74,3 +74,24 @@ def test_openboost_histogram_wrapper_preserves_native_distribution_grid(): assert distribution.bin_edges.shape == (9,) np.testing.assert_allclose(distribution.probas.sum(axis=1), 1.0) np.testing.assert_allclose(distribution.mean, model.predict(X[80:])) + + +def test_openboost_histogram_wrapper_selects_temperature_on_inner_validation(): + rng = np.random.default_rng(19) + X = rng.normal(size=(80, 3)).astype(np.float32) + y = (X[:, 0] + rng.normal(scale=0.3, size=80)).astype(np.float32) + model = OpenBoostHistogramWrapper( + n_distribution_bins=6, + n_trees=2, + max_depth=1, + n_feature_bins=10, + temperature_grid=(0.7, 1.0, 1.2), + calibration_fraction=0.2, + calibration_seed=3, + ).fit(X[:60], y[:60]) + + assert model._selected_temperature in model.temperature_grid + assert set(model._temperature_scores) == set(model.temperature_grid) + assert all(np.isfinite(list(model._temperature_scores.values()))) + distribution = model.predict_distribution(X[60:]) + np.testing.assert_allclose(distribution.mean, model.predict(X[60:])) diff --git a/docs/user-guide/models/histogram-boost.md b/docs/user-guide/models/histogram-boost.md index 7df421d..f7d3238 100644 --- a/docs/user-guide/models/histogram-boost.md +++ b/docs/user-guide/models/histogram-boost.md @@ -67,4 +67,5 @@ state-of-the-art claim. | `reg_alpha` | 0.0 | L1 regularization for vector leaf values | `predict_distribution()` returns `HistogramDistributionOutput`, which provides -`mean()`, `variance()`, `std()`, `quantile()`, `interval()`, and `sample()`. +`mean()`, `variance()`, `std()`, exact `crps()`, `tempered()`, `quantile()`, +`interval()`, and `sample()`. diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index e7b3652..9449933 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -131,6 +131,16 @@ RMSE. V2 therefore adds an optional `leaf_reg_lambda`: split regularization stays at 1, while leaf shrinkage can be studied independently. `None` preserves the original coupled behavior. +On consumed `197_cpu_act` fold 0, the exact-objective model at 100 rounds +improved CRPS from 1.3614 to 1.3187 and RMSE from 2.8446 to 2.7647, but official +90% coverage remained 97.67%. A diagnostic temperature sweep found that 0.7 +improved CRPS again to 1.2957, RMSE to 2.7271, coverage to 94.0%, and interval +score to 10.9102. Because that temperature used an already-consumed outer fold, +it cannot be frozen directly. The ScoringBench wrapper now supports selecting +temperature by exact CRPS on an inner training-only split, then refitting the +base model on the complete outer training fold. Its default grid remains +`(1.0,)`, so V1 behavior does not silently change. + ## Commits - `b9db276` — `feat: add histogram CRPS boosting` diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py index 6242c4b..5b09e5f 100644 --- a/src/openboost/_models/_histogram_boost.py +++ b/src/openboost/_models/_histogram_boost.py @@ -73,10 +73,27 @@ def _continuous_crps_loss( bin_edges: NDArray, ) -> NDArray: """Exact per-row CRPS for the represented piecewise-uniform histogram.""" - probabilities = _softmax(logits) + return _continuous_crps_from_probabilities(_softmax(logits), y, bin_edges) + + +def _continuous_crps_from_probabilities( + probabilities: NDArray, + y: NDArray, + bin_edges: NDArray, +) -> NDArray: + """Exact per-row CRPS for already-normalized histogram probabilities.""" + probabilities = np.asarray(probabilities, dtype=np.float64) + if probabilities.ndim != 2: + raise ValueError("probabilities must have shape (n_samples, n_bins)") + if not np.all(np.isfinite(probabilities)) or np.any(probabilities < 0.0): + raise ValueError("probabilities must be finite and non-negative") + totals = np.sum(probabilities, axis=1, keepdims=True) + if np.any(totals <= 0.0): + raise ValueError("each probability row must have positive mass") + probabilities = probabilities / totals distance_to_target, pairwise_distance = _continuous_crps_terms(y, bin_edges) if distance_to_target.shape != probabilities.shape: - raise ValueError("y and bin_edges must match the logit rows and outputs") + raise ValueError("y and bin_edges must match the probability rows and bins") first = np.sum(probabilities * distance_to_target, axis=1) second = 0.5 * np.einsum( "ni,ij,nj->n", @@ -189,6 +206,24 @@ def variance(self) -> NDArray: def std(self) -> NDArray: return np.sqrt(self.variance()) + def crps(self, y: NDArray) -> NDArray: + """Return exact per-row continuous ranked probability scores.""" + return _continuous_crps_from_probabilities(self.probas, y, self.bin_edges) + + def tempered(self, temperature: float) -> HistogramDistributionOutput: + """Return the same histogram grid with temperature-scaled probabilities.""" + if not np.isfinite(temperature) or temperature <= 0.0: + raise ValueError("temperature must be finite and strictly positive") + if temperature == 1.0: + probabilities = self.probas.copy() + else: + log_probabilities = np.full_like(self.probas, -np.inf) + positive = self.probas > 0.0 + log_probabilities[positive] = np.log(self.probas[positive]) / temperature + log_probabilities -= np.max(log_probabilities, axis=1, keepdims=True) + probabilities = np.exp(log_probabilities) + return HistogramDistributionOutput(probabilities, self.bin_edges.copy()) + def quantile(self, q: float) -> NDArray: if not 0.0 <= q <= 1.0: raise ValueError("q must lie in [0, 1]") diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py index 1d23c53..9f402f1 100644 --- a/tests/test_histogram_boost.py +++ b/tests/test_histogram_boost.py @@ -10,6 +10,7 @@ import openboost as ob from openboost._core._vector_tree import _find_best_vector_split, fit_vector_tree from openboost._models._histogram_boost import ( + _continuous_crps_from_probabilities, _continuous_crps_loss, _continuous_crps_terms, _crps_grad_gn, @@ -49,6 +50,12 @@ def test_continuous_crps_includes_uniform_within_bin_distance(): np.array([0.0, 1.0, 2.0]), ) assert loss[0] == pytest.approx(1.0 / 12.0) + probability_loss = _continuous_crps_from_probabilities( + np.array([[1.0, 0.0]]), + np.array([0.5]), + np.array([0.0, 1.0, 2.0]), + ) + assert probability_loss[0] == pytest.approx(1.0 / 12.0) def test_continuous_crps_psd_diagonal_matches_explicit_jacobian(): @@ -236,6 +243,17 @@ def test_histogram_distribution_output_moments_quantiles_and_sampling(): assert samples1.shape == (2, 20) assert np.all((samples1 >= 0.0) & (samples1 <= 2.0)) + sharper = dist.tempered(0.5) + np.testing.assert_allclose(sharper.probas.sum(axis=1), 1.0) + assert sharper.probas[0, 1] > dist.probas[0, 1] + np.testing.assert_array_equal(dist.tempered(1.0).probas, dist.probas) + np.testing.assert_allclose( + dist.crps(np.array([0.5, 0.5])), + [25.0 / 48.0, 1.0 / 12.0], + ) + with pytest.raises(ValueError, match="strictly positive"): + dist.tempered(0.0) + with pytest.raises(ValueError, match="bin_edges must contain only finite"): ob.HistogramDistributionOutput( probas=np.array([[0.5, 0.5]]), From 5f376e45acdcc4bbce9d46ead699d1678a93f209 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sun, 16 Aug 2026 00:02:06 -0700 Subject: [PATCH 46/49] feat: refine histogram evaluation grids losslessly --- benchmarks/scoringbench/openboost_wrapper.py | 9 ++++ .../scoringbench/test_openboost_wrapper.py | 2 + docs/user-guide/models/histogram-boost.md | 4 +- learnings/2026-08-15-histogram-crps-boost.md | 8 ++++ src/openboost/_models/_histogram_boost.py | 43 ++++++++++++++++--- tests/test_histogram_boost.py | 10 +++++ 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/benchmarks/scoringbench/openboost_wrapper.py b/benchmarks/scoringbench/openboost_wrapper.py index 0f57924..e38691e 100644 --- a/benchmarks/scoringbench/openboost_wrapper.py +++ b/benchmarks/scoringbench/openboost_wrapper.py @@ -186,6 +186,7 @@ def __init__( temperature_grid: tuple[float, ...] = (1.0,), calibration_fraction: float = 0.2, calibration_seed: int = 42, + evaluation_subdivisions: int = 1, model_params: dict | None = None, ) -> None: self.n_distribution_bins = n_distribution_bins @@ -197,6 +198,7 @@ def __init__( self.temperature_grid = tuple(float(value) for value in temperature_grid) self.calibration_fraction = calibration_fraction self.calibration_seed = calibration_seed + self.evaluation_subdivisions = evaluation_subdivisions self.model_params = dict(model_params or {}) self._model = None self._selected_temperature = 1.0 @@ -230,6 +232,12 @@ def fit(self, X, y) -> OpenBoostHistogramWrapper: raise ValueError("temperature_grid must contain positive finite values") if not 0.0 < self.calibration_fraction < 1.0: raise ValueError("calibration_fraction must lie in (0, 1)") + if ( + isinstance(self.evaluation_subdivisions, bool) + or not isinstance(self.evaluation_subdivisions, (int, np.integer)) + or self.evaluation_subdivisions < 1 + ): + raise ValueError("evaluation_subdivisions must be a positive integer") params = { "n_distribution_bins": self.n_distribution_bins, @@ -280,6 +288,7 @@ def predict_distribution(self, X) -> DistributionPrediction: output = self._model.predict_distribution(self._sanitize_X(X)).tempered( self._selected_temperature ) + output = output.subdivide(self.evaluation_subdivisions) return DistributionPrediction( probas=np.asarray(output.probas, dtype=np.float64), bin_edges=np.asarray(output.bin_edges, dtype=np.float64), diff --git a/benchmarks/scoringbench/test_openboost_wrapper.py b/benchmarks/scoringbench/test_openboost_wrapper.py index 39205bd..c9a186b 100644 --- a/benchmarks/scoringbench/test_openboost_wrapper.py +++ b/benchmarks/scoringbench/test_openboost_wrapper.py @@ -88,10 +88,12 @@ def test_openboost_histogram_wrapper_selects_temperature_on_inner_validation(): temperature_grid=(0.7, 1.0, 1.2), calibration_fraction=0.2, calibration_seed=3, + evaluation_subdivisions=2, ).fit(X[:60], y[:60]) assert model._selected_temperature in model.temperature_grid assert set(model._temperature_scores) == set(model.temperature_grid) assert all(np.isfinite(list(model._temperature_scores.values()))) distribution = model.predict_distribution(X[60:]) + assert distribution.probas.shape == (20, 12) np.testing.assert_allclose(distribution.mean, model.predict(X[60:])) diff --git a/docs/user-guide/models/histogram-boost.md b/docs/user-guide/models/histogram-boost.md index f7d3238..a914b88 100644 --- a/docs/user-guide/models/histogram-boost.md +++ b/docs/user-guide/models/histogram-boost.md @@ -67,5 +67,5 @@ state-of-the-art claim. | `reg_alpha` | 0.0 | L1 regularization for vector leaf values | `predict_distribution()` returns `HistogramDistributionOutput`, which provides -`mean()`, `variance()`, `std()`, exact `crps()`, `tempered()`, `quantile()`, -`interval()`, and `sample()`. +`mean()`, `variance()`, `std()`, exact `crps()`, `tempered()`, density-preserving +`subdivide()`, `quantile()`, `interval()`, and `sample()`. diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md index 9449933..db607d9 100644 --- a/learnings/2026-08-15-histogram-crps-boost.md +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -141,6 +141,14 @@ temperature by exact CRPS on an inner training-only split, then refitting the base model on the complete outer training fold. Its default grid remains `(1.0,)`, so V1 behavior does not silently change. +The inner split selected temperature 0.85 on consumed fold 0. Outer metrics +were CRPS 1.2986, RMSE 2.7148, coverage 95.67%, and interval score 11.7174. +Exact within-bin coverage was 93.17%, confirming that the remaining official +coverage excess came partly from whole-bin interval envelopes. The distribution +output now supports density-preserving subdivision, and the wrapper can refine +50 training bins to 100 evaluation bins. Subdivision leaves exact CRPS, mean, +and variance unchanged; it only reduces evaluator quantile-grid error. + ## Commits - `b9db276` — `feat: add histogram CRPS boosting` diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py index 5b09e5f..d893f18 100644 --- a/src/openboost/_models/_histogram_boost.py +++ b/src/openboost/_models/_histogram_boost.py @@ -27,6 +27,8 @@ def _softmax(logits: NDArray) -> NDArray: def _continuous_crps_terms( y: NDArray, bin_edges: NDArray, + *, + normalize: bool = True, ) -> tuple[NDArray, NDArray]: """Return normalized terms for exact piecewise-uniform histogram CRPS. @@ -34,8 +36,8 @@ def _continuous_crps_terms( ``E|X-y| - 0.5 E|X-X'|``. Each histogram bin represents a uniform conditional density, not a point mass at its midpoint. The returned first term has shape ``(n_samples, n_bins)`` and the pairwise-distance matrix has - shape ``(n_bins, n_bins)``. Both are divided by mean bin width so tree - regularization remains invariant to target units. + shape ``(n_bins, n_bins)``. With ``normalize=True``, both are divided by + mean bin width so tree regularization remains invariant to target units. """ y = np.asarray(y, dtype=np.float64).reshape(-1) bin_edges = np.asarray(bin_edges, dtype=np.float64).reshape(-1) @@ -63,7 +65,7 @@ def _continuous_crps_terms( pairwise_distance = np.abs(midpoints[:, None] - midpoints[None, :]) np.fill_diagonal(pairwise_distance, widths / 3.0) - normalization = float(np.mean(widths)) + normalization = float(np.mean(widths)) if normalize else 1.0 return distance_to_target / normalization, pairwise_distance / normalization @@ -73,13 +75,20 @@ def _continuous_crps_loss( bin_edges: NDArray, ) -> NDArray: """Exact per-row CRPS for the represented piecewise-uniform histogram.""" - return _continuous_crps_from_probabilities(_softmax(logits), y, bin_edges) + return _continuous_crps_from_probabilities( + _softmax(logits), + y, + bin_edges, + normalize=True, + ) def _continuous_crps_from_probabilities( probabilities: NDArray, y: NDArray, bin_edges: NDArray, + *, + normalize: bool = False, ) -> NDArray: """Exact per-row CRPS for already-normalized histogram probabilities.""" probabilities = np.asarray(probabilities, dtype=np.float64) @@ -91,7 +100,11 @@ def _continuous_crps_from_probabilities( if np.any(totals <= 0.0): raise ValueError("each probability row must have positive mass") probabilities = probabilities / totals - distance_to_target, pairwise_distance = _continuous_crps_terms(y, bin_edges) + distance_to_target, pairwise_distance = _continuous_crps_terms( + y, + bin_edges, + normalize=normalize, + ) if distance_to_target.shape != probabilities.shape: raise ValueError("y and bin_edges must match the probability rows and bins") first = np.sum(probabilities * distance_to_target, axis=1) @@ -224,6 +237,26 @@ def tempered(self, temperature: float) -> HistogramDistributionOutput: probabilities = np.exp(log_probabilities) return HistogramDistributionOutput(probabilities, self.bin_edges.copy()) + def subdivide(self, factor: int) -> HistogramDistributionOutput: + """Refine each uniform bin without changing the represented density.""" + if isinstance(factor, bool) or not isinstance(factor, (int, np.integer)) or factor < 1: + raise ValueError("factor must be a positive integer") + factor = int(factor) + if factor == 1: + return HistogramDistributionOutput(self.probas.copy(), self.bin_edges.copy()) + n_bins = self.probas.shape[1] + refined_edges = np.empty(n_bins * factor + 1, dtype=np.float64) + fractions = np.arange(factor, dtype=np.float64) / factor + for index, (lower, upper) in enumerate( + zip(self.bin_edges[:-1], self.bin_edges[1:], strict=True) + ): + refined_edges[index * factor : (index + 1) * factor] = lower + fractions * ( + upper - lower + ) + refined_edges[-1] = self.bin_edges[-1] + refined_probabilities = np.repeat(self.probas / factor, factor, axis=1) + return HistogramDistributionOutput(refined_probabilities, refined_edges) + def quantile(self, q: float) -> NDArray: if not 0.0 <= q <= 1.0: raise ValueError("q must lie in [0, 1]") diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py index 9f402f1..4644771 100644 --- a/tests/test_histogram_boost.py +++ b/tests/test_histogram_boost.py @@ -251,8 +251,18 @@ def test_histogram_distribution_output_moments_quantiles_and_sampling(): dist.crps(np.array([0.5, 0.5])), [25.0 / 48.0, 1.0 / 12.0], ) + refined = dist.subdivide(4) + assert refined.probas.shape == (2, 8) + np.testing.assert_allclose(refined.mean(), dist.mean()) + np.testing.assert_allclose(refined.variance(), dist.variance()) + np.testing.assert_allclose( + refined.crps(np.array([0.5, 0.5])), + dist.crps(np.array([0.5, 0.5])), + ) with pytest.raises(ValueError, match="strictly positive"): dist.tempered(0.0) + with pytest.raises(ValueError, match="positive integer"): + dist.subdivide(0) with pytest.raises(ValueError, match="bin_edges must contain only finite"): ob.HistogramDistributionOutput( From 39bdb632c3cc374161e12e2e6230d649a933b576 Mon Sep 17 00:00:00 2001 From: J Xu Date: Sun, 16 Aug 2026 00:03:55 -0700 Subject: [PATCH 47/49] bench: freeze HistogramBoost V2 protocol --- .github/workflows/scoringbench.yml | 34 ++++++- .../protocols/crps_distribution_v2.md | 92 +++++++++++++++++++ benchmarks/scoringbench/run.py | 55 ++++++++--- tests/test_scoringbench_provenance.py | 20 +++- 4 files changed, 180 insertions(+), 21 deletions(-) create mode 100644 benchmarks/scoringbench/protocols/crps_distribution_v2.md diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml index bc627e8..ebdfd2a 100644 --- a/.github/workflows/scoringbench.yml +++ b/.github/workflows/scoringbench.yml @@ -21,6 +21,7 @@ on: - strong_shard - development_shard - crps_distribution_shard + - crps_distribution_v2_shard dataset_name: description: Exact ScoringBench dataset name for a quality/strong shard required: true @@ -104,7 +105,7 @@ jobs: python -m pip install -e . - name: Install strong comparison models - if: github.event_name == 'workflow_dispatch' && (inputs.mode == 'strong_shard' || inputs.mode == 'crps_distribution_shard') + if: github.event_name == 'workflow_dispatch' && (inputs.mode == 'strong_shard' || inputs.mode == 'crps_distribution_shard' || inputs.mode == 'crps_distribution_v2_shard') run: | python -m pip install -r benchmarks/scoringbench/requirements-strong-baselines.txt @@ -240,6 +241,33 @@ jobs: "${CONFIRMATION_FLAG[@]}" \ --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Run preregistered CRPS distribution V2 shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'crps_distribution_v2_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + ALLOW_CONFIRMATION: ${{ inputs.allow_confirmation }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + CONFIRMATION_FLAG=() + if [[ "${ALLOW_CONFIRMATION}" == "true" ]]; then + CONFIRMATION_FLAG=(--allow-confirmation) + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_histogram_cpu_v2,xgboost_quantile,catboost_quantile \ + --dataset-registry \ + benchmarks/scoringbench/protocols/crps_distribution_v1.json \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --development-run \ + "${CONFIRMATION_FLAG[@]}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + - name: Verify smoke artifact if: github.event_name == 'pull_request' || inputs.mode == 'smoke' run: | @@ -250,14 +278,14 @@ jobs: python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4 and m["outcome"]["status"] == "complete"' "${MANIFEST}" - name: Verify quality-shard artifact - if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' || inputs.mode == 'development_shard' || inputs.mode == 'crps_distribution_shard' + if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' || inputs.mode == 'development_shard' || inputs.mode == 'crps_distribution_shard' || inputs.mode == 'crps_distribution_v2_shard' run: | MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" test -s "${MANIFEST}" test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" - python -c 'import json, sys; m=json.load(open(sys.argv[1])); mode=sys.argv[2]; dev=mode in {"development_shard", "crps_distribution_shard"}; expected=25 if mode == "strong_shard" else (15 if mode == "crps_distribution_shard" else (5 if mode == "development_shard" else 10)); protocol="development_tuning" if dev else "official_quality_shard"; assert m["protocol_mode"] == protocol and m["official_protocol_compatible"] is (not dev) and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False and (mode != "crps_distribution_shard" or len(m["verified_dataset_files"]) == 1)' "${MANIFEST}" "${{ inputs.mode }}" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); mode=sys.argv[2]; distribution=mode in {"crps_distribution_shard", "crps_distribution_v2_shard"}; dev=mode == "development_shard" or distribution; expected=25 if mode == "strong_shard" else (15 if distribution else (5 if mode == "development_shard" else 10)); protocol="development_tuning" if dev else "official_quality_shard"; assert m["protocol_mode"] == protocol and m["official_protocol_compatible"] is (not dev) and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False and (not distribution or len(m["verified_dataset_files"]) == 1)' "${MANIFEST}" "${{ inputs.mode }}" - name: Upload benchmark artifact if: always() diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v2.md b/benchmarks/scoringbench/protocols/crps_distribution_v2.md new file mode 100644 index 0000000..b1d972f --- /dev/null +++ b/benchmarks/scoringbench/protocols/crps_distribution_v2.md @@ -0,0 +1,92 @@ +# CRPS distribution experiment V2 + +This protocol freezes the second HistogramBoost candidate before its full +five-fold `197_cpu_act` run. The V1 five-fold artifact and V2 fold-0 diagnostics +are consumed development evidence. `537_houses` remains untouched and locked. + +## Why V2 exists + +V1 beat the frozen native XGBoost quantile baseline on all five folds, but was +2.111% worse than CatBoost on mean CRPS, won only two of five CatBoost folds, +and retained an over-wide PMF. Audits identified three correctable causes: + +1. training used a midpoint ranked-probability approximation while evaluation + used exact piecewise-uniform histogram CRPS; +2. additive smoothing was applied once per bin, so total prior strength grew + with output resolution; +3. whole-bin interval extraction made the 50-bin representation coarser than + the CatBoost representation. + +V2 aligns the training score with evaluation, treats smoothing as total +Dirichlet concentration, selects probability temperature using only an inner +split of each outer training fold, and losslessly subdivides bins for evaluation. + +## Frozen data and benchmark + +- Development: `197_cpu_act` from the commit-pinned registry + `crps_distribution_v1.json`. +- Untouched confirmation: `537_houses` from the same registry. The launcher + must reject it unless `--allow-confirmation` is explicit. +- Dataset bytes and SHA-256 values are unchanged from V1. +- ScoringBench commit: + `a938a667b7839b41e9272929010573410301c0b4`. +- Five folds, one repeat, seed 42, sample cap 3000, CPU execution. +- Baselines: native XGBoost quantile with 100 rounds and 50 quantiles; CatBoost + MultiQuantile with 1000 rounds and 99 quantiles. Both use two threads. +- Exactly 15 finite, error-free result rows are required. +- CRPS is primary. Log score, CRLS, CDE, and DPD are excluded from decisions + because support and density-grid differences make them unsuitable here. + +## Frozen V2 candidate + +The benchmark model name is `openboost_histogram_cpu_v2`: + +```python +OpenBoostHistogramWrapper( + n_distribution_bins=50, + n_trees=100, + learning_rate=0.05, + max_depth=6, + n_feature_bins=254, + curvature_scale=1.0, + temperature_grid=(0.5, 0.7, 0.85, 1.0, 1.2), + calibration_fraction=0.2, + calibration_seed=42, + evaluation_subdivisions=2, +) +``` + +All `HistogramBoost` constructor values not shown remain at committed defaults: +split and leaf L2 regularization both resolve to 1, and total base prior weight +is 1. Each outer fold performs these steps: + +1. split only the outer training rows into 80% inner-train and 20% calibration; +2. fit the candidate on inner-train and choose temperature by mean exact CRPS + on calibration rows; +3. refit a fresh candidate on every outer training row; +4. apply the frozen selected temperature to outer-test probabilities; +5. divide every uniform training bin into two equal-density evaluation bins. + +Outer-test targets never select rounds, bins, temperature, support, or any +other hyperparameter. Subdivision preserves the represented density, exact +CRPS, mean, and variance; it only reduces whole-bin quantile-envelope error. +The additional inner fit is included in reported training time. + +## Development acceptance + +Let `B` be whichever baseline has lower five-fold mean CRPS, selected once for +the dataset rather than per fold. V2 passes only if every condition holds: + +1. all 15 expected rows are present, finite, and error-free; +2. `mean_CRPS(V2) / mean_CRPS(B) <= 1.02`; +3. V2 CRPS is no greater than `B` on at least three of five folds; +4. V2 mean absolute 90% coverage error is at most 0.05 and at least 0.02 lower + than `B`; +5. V2 mean 90% interval score is at most `1.05 * B`; +6. V2 mean RMSE is at most `1.05 * B`. + +Passing this development gate allows exactly one frozen confirmation run. It +does not establish an overall win. On `537_houses`, a dataset-level CRPS win +requires ratio below 1.00 and at least four of five fold wins, plus every +guardrail above. Full-suite paired results remain necessary for the product +goal. diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py index 26ddc88..313bf34 100644 --- a/benchmarks/scoringbench/run.py +++ b/benchmarks/scoringbench/run.py @@ -30,6 +30,16 @@ def _csv(value: str) -> list[str]: return [item.strip() for item in value.split(",") if item.strip()] +def _float_csv(value: str) -> tuple[float, ...]: + try: + result = tuple(float(item) for item in _csv(value)) + except ValueError as exc: + raise argparse.ArgumentTypeError("expected comma-separated numbers") from exc + if not result: + raise argparse.ArgumentTypeError("expected at least one number") + return result + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -75,9 +85,7 @@ def _verify_dataset_files(datasets: list[dict], ensure_cached) -> list[dict]: continue expected = str(expected).lower() if len(expected) != 64 or any(char not in "0123456789abcdef" for char in expected): - raise ValueError( - f"invalid raw_sha256 for dataset {dataset['name']!r}: {expected!r}" - ) + raise ValueError(f"invalid raw_sha256 for dataset {dataset['name']!r}: {expected!r}") if dataset.get("source") != "pmlb" or not dataset.get("url"): raise ValueError( "raw_sha256 verification currently requires a PMLB URL; " @@ -295,9 +303,7 @@ def _audit_records( "model": key[1], "fold": key[2], "error_type": ( - str(row["error_type"]) - if _is_present_text(row.get("error_type")) - else None + str(row["error_type"]) if _is_present_text(row.get("error_type")) else None ), "error": str(error), } @@ -386,8 +392,8 @@ def _build_parser() -> argparse.ArgumentParser: default=["openboost_cpu", "ngboost"], help=( "Comma-separated models: openboost_cpu, openboost_cuda, " - "openboost_histogram_cpu, ngboost, xgboost_quantile, xgblss, " - "catboost_quantile" + "openboost_histogram_cpu, openboost_histogram_cpu_v2, ngboost, " + "xgboost_quantile, xgblss, catboost_quantile" ), ) parser.add_argument("--n-trees", type=int, default=500) @@ -410,6 +416,14 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--histogram-learning-rate", type=float, default=0.05) parser.add_argument("--histogram-max-depth", type=int, default=6) parser.add_argument("--histogram-curvature-scale", type=float, default=1.0) + parser.add_argument( + "--histogram-v2-temperature-grid", + type=_float_csv, + default=(0.5, 0.7, 0.85, 1.0, 1.2), + ) + parser.add_argument("--histogram-v2-calibration-fraction", type=float, default=0.2) + parser.add_argument("--histogram-v2-calibration-seed", type=int, default=42) + parser.add_argument("--histogram-v2-evaluation-subdivisions", type=int, default=2) parser.add_argument( "--xgboost-rounds", type=int, @@ -578,6 +592,18 @@ def _model_parameters(args) -> dict[str, dict]: "n_feature_bins": 254, "curvature_scale": args.histogram_curvature_scale, }, + "openboost_histogram_cpu_v2": { + "n_distribution_bins": args.histogram_bins, + "n_trees": args.histogram_rounds, + "learning_rate": args.histogram_learning_rate, + "max_depth": args.histogram_max_depth, + "n_feature_bins": 254, + "curvature_scale": args.histogram_curvature_scale, + "temperature_grid": args.histogram_v2_temperature_grid, + "calibration_fraction": args.histogram_v2_calibration_fraction, + "calibration_seed": args.histogram_v2_calibration_seed, + "evaluation_subdivisions": args.histogram_v2_evaluation_subdivisions, + }, "ngboost": { "dist": "normal", "n_estimators": args.n_trees, @@ -622,6 +648,9 @@ def _model_factories(args): "openboost_histogram_cpu": lambda: OpenBoostHistogramWrapper( **parameters["openboost_histogram_cpu"] ), + "openboost_histogram_cpu_v2": lambda: OpenBoostHistogramWrapper( + **parameters["openboost_histogram_cpu_v2"] + ), } if "ngboost" in args.models: @@ -655,6 +684,7 @@ def _model_factories(args): "openboost_cpu", "openboost_cuda", "openboost_histogram_cpu", + "openboost_histogram_cpu_v2", "ngboost", "xgboost_quantile", "xgblss", @@ -677,10 +707,7 @@ def _write_provenance( import openboost as ob official_shape = ( - not args.smoke - and args.sample_size == 3000 - and args.n_folds == 5 - and args.n_repeats == 1 + not args.smoke and args.sample_size == 3000 and args.n_folds == 5 and args.n_repeats == 1 ) official_protocol_compatible = official_shape and not args.development_run if args.smoke: @@ -719,9 +746,7 @@ def _write_provenance( "scoringbench_git": _git_state(scoringbench_dir), "ci": _ci_state(), "arguments": vars(args), - "model_parameters": { - name: _model_parameters(args)[name] for name in args.models - }, + "model_parameters": {name: _model_parameters(args)[name] for name in args.models}, "datasets": [ { "name": dataset["name"], diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py index ec4e6d0..84d4148 100644 --- a/tests/test_scoringbench_provenance.py +++ b/tests/test_scoringbench_provenance.py @@ -145,9 +145,7 @@ def test_load_dataset_registry_accepts_frozen_scoringbench_list(tmp_path): ] -def test_registry_path_is_resolved_before_artifact_working_directory( - tmp_path, monkeypatch -): +def test_registry_path_is_resolved_before_artifact_working_directory(tmp_path, monkeypatch): registry = tmp_path / "datasets.json" registry.write_text('[{"name": "alpha"}]') monkeypatch.chdir(tmp_path) @@ -260,6 +258,10 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): assert args.histogram_learning_rate == 0.05 assert args.histogram_max_depth == 6 assert args.histogram_curvature_scale == 1.0 + assert args.histogram_v2_temperature_grid == (0.5, 0.7, 0.85, 1.0, 1.2) + assert args.histogram_v2_calibration_fraction == 0.2 + assert args.histogram_v2_calibration_seed == 42 + assert args.histogram_v2_evaluation_subdivisions == 2 assert args.xgboost_rounds == 100 assert args.xgboost_quantiles == 50 assert args.xgblss_rounds == 100 @@ -283,6 +285,18 @@ def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): "n_feature_bins": 254, "curvature_scale": 1.0, } + assert parameters["openboost_histogram_cpu_v2"] == { + "n_distribution_bins": 50, + "n_trees": 100, + "learning_rate": 0.05, + "max_depth": 6, + "n_feature_bins": 254, + "curvature_scale": 1.0, + "temperature_grid": (0.5, 0.7, 0.85, 1.0, 1.2), + "calibration_fraction": 0.2, + "calibration_seed": 42, + "evaluation_subdivisions": 2, + } assert parameters["xgboost_quantile"]["num_boost_round"] == 100 assert parameters["xgblss"]["num_boost_round"] == 100 assert parameters["catboost_quantile"]["iterations"] == 1000 From 2476dd79a3a8e9e182e613827ae897813047895e Mon Sep 17 00:00:00 2001 From: J Xu Date: Sun, 16 Aug 2026 00:38:46 -0700 Subject: [PATCH 48/49] bench: record HistogramBoost V2 development win --- .../README.md | 65 +++++++ .../benchmark_outcome.json | 21 +++ .../candidate_evaluation.json | 50 +++++ .../datasets.json | 20 ++ .../openboost_manifest.json | 176 ++++++++++++++++++ .../raw/catboost_quantile/197_cpu_act.parquet | Bin 0 -> 30022 bytes .../197_cpu_act.parquet | Bin 0 -> 30066 bytes .../raw/xgboost_quantile/197_cpu_act.parquet | Bin 0 -> 30030 bytes .../summary.json | 87 +++++++++ learnings/2026-08-15-histogram-crps-boost.md | 55 +++++- 10 files changed, 471 insertions(+), 3 deletions(-) create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/catboost_quantile/197_cpu_act.parquet create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/openboost_histogram_cpu_v2/197_cpu_act.parquet create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/xgboost_quantile/197_cpu_act.parquet create mode 100644 benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/summary.json diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md new file mode 100644 index 0000000..daf1358 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md @@ -0,0 +1,65 @@ +# 197_cpu_act HistogramBoost V2 development result + +This is consumed development/tuning evidence, not held-out confirmation or +leaderboard evidence. It evaluates the frozen HistogramBoost V2 candidate on +the preregistered `197_cpu_act` development dataset. The run used five folds, +seed 42, a 3000-row cap, clean OpenBoost source `39bdb63`, and clean +ScoringBench source `a938a667`. + +V2 used 50 distribution bins, 100 shared-vector trees, learning rate 0.05, +depth 6, and PSD Gauss--Newton curvature scale 1. It trained against the exact +piecewise-uniform histogram CRPS, interpreted `base_smoothing=1` as a total +Dirichlet concentration, selected a temperature from +`[0.5, 0.7, 0.85, 1.0, 1.2]` on an inner 80/20 split of each outer training +fold, refit on the full outer training fold, and losslessly subdivided every +output bin into two equal-density bins for evaluation. The extra inner fit is +included in training time. Outer-test targets were not used for model or +temperature selection. + +| Model | CRPS | RMSE | Official 90% coverage | Official 90% interval score | Sharpness | Mean fit time (s) | +|---|---:|---:|---:|---:|---:|---:| +| HistogramBoost V2 | **1.287204** | **2.551506** | **94.80%** | **11.281116** | 3.719920 | 161.41 | +| CatBoost quantile | 1.326236 | 2.582497 | 71.73% | 13.329529 | **1.556934** | 141.00 | +| XGBoost quantile | 1.620136 | 3.698760 | 67.47% | 17.979673 | 1.619687 | **14.80** | + +HistogramBoost V2 passed the frozen development gate. Relative to the +once-per-dataset selected strong baseline, CatBoost, it lowered mean CRPS by +2.943%, won four of five folds, lowered RMSE by 1.200%, and lowered the +official 90% interval score by 15.367%. Its mean absolute nominal-90% coverage +error was 4.80 percentage points, within the frozen five-point guardrail. + +Against the frozen native XGBoost quantile baseline, HistogramBoost V2 won all +five folds and lowered mean CRPS by 20.550%, RMSE by 31.017%, and the official +90% interval score by 37.256%. This is a single consumed development dataset, +so it does not establish an overall, full-suite, or SOTA win. + +The calibration metrics need careful interpretation. ScoringBench extracts +intervals using whole bin edges, so its coverage and interval scores are +representation-sensitive. V2's two-way subdivision preserves the represented +density, physical CRPS, mean, and continuous variance, while reducing the +coarse-bin envelope error. The official numbers above are therefore pinned +protocol metrics, not standalone calibration claims. Sharpness is diagnostic +only and is not part of the acceptance gate. + +Within this one four-core Actions run, V2 averaged 161.41 seconds per fold, +including the inner calibration fit. It was 1.145x as slow as CatBoost and +10.907x as slow as XGBoost. These are implementation diagnostics, not general +speed claims. They make vector-tree GPU acceleration a concrete next systems +target now that the quality hypothesis has a positive signal. + +The frozen evaluator returned `development_pass=true`, but an independent +protocol audit found that this commit's workflow did not execute that evaluator +inside CI and that the confirmation path was not yet phase-bound. Therefore +the untouched `537_houses` confirmation dataset remains locked until the gate +and provenance fixes are committed. The evaluator's +`confirmation_dataset_win=true` field is phase-agnostic and must not be read as +a confirmation result for this development run. + +Run: [31932958804](https://github.com/jxucoder/openboost/actions/runs/31932958804), +job `95130441713`, artifact `9260176320`, digest +`sha256:572672e818cf60a295826f7b057bad0269f5121f5bfc31a258188eea12234adc`. +The artifact contains 15/15 valid rows with no errors, missing rows, +duplicates, unexpected rows, or non-finite metrics. The compressed dataset +matched the frozen SHA-256 +`d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc`. + diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json new file mode 100644 index 0000000..9c8629e --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "197_cpu_act", + "expected_rows": 15, + "observed_rows": 15, + "status": "complete", + "valid_rows": 15 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 15, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 15, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 15 +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json new file mode 100644 index 0000000..20b1f3a --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json @@ -0,0 +1,50 @@ +{ + "accepted": true, + "baselines": [ + "xgboost_quantile", + "catboost_quantile" + ], + "candidate": "openboost_histogram_cpu_v2", + "comparisons": { + "candidate_fold_wins": 4, + "coverage_error_improvement": 0.13466669321060185, + "crps_ratio": 0.9705694692862287, + "interval_score_ratio": 0.8463251301218064, + "rmse_ratio": 0.9879996039440585 + }, + "confirmation_dataset_win": true, + "dataset": "197_cpu_act", + "development_pass": true, + "guardrails": { + "at_least_3_of_5_fold_wins": true, + "complete_15_rows": true, + "coverage_error_at_most_0_05": true, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": true, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "phase": "development", + "schema_version": 1, + "selected_strong_baseline": "catboost_quantile", + "summaries": { + "catboost_quantile": { + "mean_abs_coverage_90_error": 0.1826666831970215, + "mean_crps": 1.3262356982847625, + "mean_interval_score_90": 13.329529331871402, + "mean_rmse": 2.5824970354629198 + }, + "openboost_histogram_cpu_v2": { + "mean_abs_coverage_90_error": 0.047999989986419654, + "mean_crps": 1.2872038778326929, + "mean_interval_score_90": 11.2811156462585, + "mean_rmse": 2.55150604822407 + }, + "xgboost_quantile": { + "mean_abs_coverage_90_error": 0.22533334493637086, + "mean_crps": 1.6201360866516095, + "mean_interval_score_90": 17.979672910724513, + "mean_rmse": 3.698760308978361 + } + } +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json new file mode 100644 index 0000000..54674d0 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json @@ -0,0 +1,20 @@ +[ + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "target_col": "target", + "abbr": "1CA", + "openboost_role": "development", + "raw_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/537_houses/537_houses.tsv.gz", + "target_col": "target", + "abbr": "5H", + "openboost_role": "untouched_confirmation", + "raw_sha256": "d383fd58bb79760ffaf0b615942fd47d926a19e75a8f980b79fbe9ad479d9cbf" + } +] diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json new file mode 100644 index 0000000..ba8aeea --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json @@ -0,0 +1,176 @@ +{ + "arguments": { + "allow_confirmation": false, + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "197_cpu_act" + ], + "dataset_registry": "benchmarks/scoringbench/protocols/crps_distribution_v1.json", + "development_run": true, + "histogram_bins": 50, + "histogram_curvature_scale": 1.0, + "histogram_learning_rate": 0.05, + "histogram_max_depth": 6, + "histogram_rounds": 100, + "histogram_v2_calibration_fraction": 0.2, + "histogram_v2_calibration_seed": 42, + "histogram_v2_evaluation_subdivisions": 2, + "histogram_v2_temperature_grid": [ + 0.5, + 0.7, + 0.85, + 1.0, + 1.2 + ], + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_histogram_cpu_v2", + "xgboost_quantile", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "nll", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31932958804", + "source_sha": "39bdb632c3cc374161e12e2e6230d649a933b576", + "tested_sha": "39bdb632c3cc374161e12e2e6230d649a933b576" + }, + "created_at": "2026-08-16T07:35:14+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "frozen_file", + "resolved_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "source_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1" + }, + "datasets": [ + { + "id": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "name": "197_cpu_act", + "source": "pmlb" + } + ], + "expected_result_rows": 15, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "openboost_histogram_cpu_v2": { + "calibration_fraction": 0.2, + "calibration_seed": 42, + "curvature_scale": 1.0, + "evaluation_subdivisions": 2, + "learning_rate": 0.05, + "max_depth": 6, + "n_distribution_bins": 50, + "n_feature_bins": 254, + "n_trees": 100, + "temperature_grid": [ + 0.5, + 0.7, + 0.85, + 1.0, + 1.2 + ] + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "39bdb632c3cc374161e12e2e6230d649a933b576", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 15 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 15, + "schema_version": 3, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "verified_dataset_files": [ + { + "name": "197_cpu_act", + "sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc", + "size_bytes": 381809, + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz" + } + ], + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/catboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/catboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0c5803a6eb8158d16f7921e042191c1a96c4c67c GIT binary patch literal 30022 zcmd5_3w#q*)=wYQP(YxN5)hHCGV-!CX-iuKW+rKyq_icJHff`5&8yEOjY-p7 z`rsUS>V*ZWk**)f4Mkfw)aP`QGKnl*HfAc=4U?tFQpU=qyro>q+oVe-ib#hKmq;Xo zCMLqaQxlSsRuoIZIs9_Mc(rw^l+uoym$`U;GW8U-RGacuB9%J8HFocWBr5x}m-R`n zBvUvC-cl~*Z4d`s8l8i(<8MDDe0PGJ^YxIV%WpYJPCin$p&cEX><-14M*GW@BV>QKe2lLy1ehn2X8p@$h?HpMy$A>1QZD6f5C>cuor7c7#@Xh(6R0m|7tTqWnn2knZ`%9j+Y)N&+`;Xi z{#HUQ*mUdT3$`VSv;|)xm-4!Z11^oup)yJK?jtA8lK=i{)dZ*P9Qo78Z_^YtXUTsi zwf*5u+gWmJX5t>njB_E(mx&_F^pQQcwyzrhGEo(PsiMH{g~k*y@9usOL-f_0hfw$pk!33Ov=nqc6P-MMlEIIGPqzrQEHSD~Um~>xK z__WkM@-ElD_H=6U*1sQV|K~_*^nmfFzq)$_H8kDWo0Afo?`9nRMe}@gRWSVKxX%1$<8k zl|~>S2*NE7-#Pirv1_Qu*L-*TZK+pNx3|u#KfLQ#)L%cn`PrujXHeQ1$2-o|T`dv> z-cl~*Z4d`s8i7MeAP8t2tR3R}0>(kI=FkAC6r6_;96WL23lx{mad#Yz5CT-fOa+WN zAA(S61b9IRtlO{rX7|eLs0RkW_%F|%$yDW@ZC_^ocOsR%yluGl_(ZCr>4nGkrA`qE z0dFan^16rvE*0e*x0FkH zT|@zwiaH;%QZr3@`l}krP|Eq?t#`h9$6(4Yd-&-$k_S-_JaYE^j@rT0U7hbp2c!%U z;lNwUrMwN|fJ>uu`0t!09ZR!TlZsKBni~f!-}>nLqZH3Sv4&hZ;>k?2eAU)}CLCP# zz~45DbVR;HF6DI*2V5GR!@A$RnD+kfUL)IO@4h!XcMCc3u~$a@NwbCAzoO)cjH=hj z&dcS$&fNWm2nW7IF6C_z2V5GR!?Z2>|NC^=saXrV_MP-RdGG88cW%G_`f00XZ~CZY z@eji#v+HI*ajfPy=2`fPmA8;fc^$+7mqzEX{__QkJ^BOW_KoF-_qXpOhuwe0FNyc= zC9lfe)xAAA6=8BG0XS`HqUq z4v}MiZkX}l_Xo)*k3V{U)!BojZQ!l%DCCDkIPjKoDQ|-~;L_+EUMf!e+({oHQ@3<~ zxai-9$;tC>vwX1pF!{;3n=U)J`Y<_S>Bfo=PK$jfc}uyJw?Q0mX><;RcFotWKOZB1 zI^}*Y^^K$CW@qcBkFPsQ()P`tjj|mjhuxi9KH(R!4;61Im-04<11^ou;a_(?y7$Yk zPmqt!dSlS8+ONsBN%q+aM0OG&%=< z_jf6GlXV~cX6!qUog;4_cgNTbnzQ82o$sxBYx57}%$dJ46wEwB9$Qg#dFB0LJ2~D` zF6C`d9B^s$IJA9oaNm7%6R9)b*OFCJB$WBwl3lOA^%L20W>WrpD}N?;p4lhYHc3Rj zo4lo5%G)3gxHLM4eD$CdS8^&f<=AH*t^8#GW$Y?ByYI^+>dCc3Uu^ig4g9d2nXI$F6C_y2V5GR!)1FaW!tA-Nhus^=ePIDs85!69jM-)Mp;)qXf3Q6 zLCIHbd}xayU4#Q~DVOp#hyyN-&Y?e?vgACTJ#@|1ft|?{{&4H9SEP3C{jKrY{nw{< zuDSc@_MIzIJG=g&%`qI4cH(Ooz69ndP8Y=imqw4nH?EAQ%!dwdUE&-5yW4U;-`X-- zJ4$}w$ksiX*FJGZ`Nh`f{y6#PRqGFJ#X0aLaw)HiIN;Lg94^_KwluJ}LmQj2Woyio zM6%?KCvgP)$Fshw0M2BH@BKyzD9># z1YkI(B*#TK*y!$Qxe|Gb*V1CwD4gX zAo{G%KJQbOdc`7pV zd*I3WFxcj{)pB9vs0&5h7!is%GC@(m=Tx7^+~}(HH9Db*lVRW`*R?oo64_vQyv%0y znZ0)31W746GGk-|oGmuHgOjAEjvhO@JCl%%m8Fi(m81+C&5T=FVtQ)&lEm~Ci9|ag zJ-__c=&3OKNMwmcX|Xf}s$%TQ^| zc;Q|KQNhb?R}y>10xF)rL8!PiX4IgU1pzOAP9R>tir7Dn^D;DH!+RMjjTtZ8%OEOv z`S!1fz2gBDPthP$TpBZ~z{^}_9$?4it5*}BWw2g`C1yA;!=y3eVP3MR?6}-Ef!K2m zpyIh0go;aJMrGEkf@yBYwZwh`fbfJ10>Y&+0|i#Y06oUo-;v+xHM)&!9GC%*ENmk&IC9-J%ZqH zY0PlKU5Ehhw1QZs06aVug79!@%y=Pt906c=HgQ)r0OENM1c*yx1{B$uzz#})ylgsg zUoIfy*$#w^OM8c$EkM32pLiq>kntP`LdK=NLl(|D_st;I6#z1x*+9s+w0FpP0-qn5 zNj!5sAmjNAgp5mjhg={)UN@WAGz*aNECxcxrM*FhJ)!{l8H#v?1mr&7l8E$e*mVn# zH>rqiG$7*{3*<8{?VZoUzWqueu|o~Wc%A|w(C3;XuAVq#AbAmiBygp5mjhb-*d zJG8`p4Itw=351MGdxxAaXy-j8#1S1JqmmqrB|%mMMIdkofwg!F&F9?3*3GmV|zbE*J4tw?u`7M{`Z^|L31 zOzpc9()T71%WfoIx&hUWCk4dw1US&@Jp(ci8DLS@Xm{9Z!-OnM zt*^t)sMa}J%)aS4NZr-yba#jqIN`h-9i&3lu2zTRmZ{@|P$;#5WAW0y22+YaF|s}e zLWh|+6ff;#ut53;M+_SNeoG7HBIE$d@oLDCFI4lC?qXe>L26A)GD zV;TV`bFy#(VTFAR9p;`G0a2AcrV(&*dQTBx3;P&4%snvzqAGn%BM^c7o+7{&_Azvr zdtwAcRr;7lAOZzFMSv~rW9TsV#0ZG0^f8TK1af&VZ4wm*Yxai#&oJB;nIMB(CjSiz02Rb6No>+7)#sn;elNVcb7_~S=ryw>ujM_z~ z14lHm(BahLKt~Z99Y!qWO_1Mk677v_AFUoKb)No-U^8dRkr2$W*G3az7Fl1H&Y8!uA^x`4R-JsRiZ8`1~I(Tw6+TL zRe3t6qt&D_ztg-QQ)>XD;#!>VWF<6rQWE>^6Jzj zO%`Rl!eY#2W!c_Fi?i5gny;G#aZP|^O*WAMnhJOv7*7IDMGwK zzuj90-0dADtu;mkYvU^MS(Op_bXcm(9lA#F*QIIzKO3xy9`IZSpH=IqvuPcE&R3^J zQ_x^5tjS{Pa4H-&O>uKgbwvZ>YjGL87A?d}8xo(6vPQ-~r`c%mnu-eqTy-6B=F((! zsTQ)_O>UF1y}4SM+hBqAL+w$MUE;TPFkJmL#$52Lya(TN6_#pMZV&D0GeKOS&9iDy zKhc1`roEt4jrw5_6XNNr&Mt4VWS6k*;>3AjI`M0}g=S50hgFm7ux6J<>f@MB-1@q+ z!fnk~IjpX7w?&x~iC?q1y4-K7E?HWI(p{nXtik!9X4mNm+!ZifHRDjKwPt8i|f4p z5Z4~Kip4eVoP}{!#unF_{t(w5xQfL!?wo~j&513p_5C5PJ#ZC^Yuq^tawBsUjcdF)H(8zSJ*_{z@m_z{C@UP*%J#M% z)^|;?XR3oW=)!n&vuYe!)k+`inaZ0*=QmeZ)S0u(>#W(ZH)7XVaa+F!IjWR3 z#&!kllh9rcU*ow{Zd-AI2liWjw05#_d&an}m0dPw?*nb^Fu5YeGMm{0DGIzMqlw+) z#c!{{V|SR1wid1(MdHZ$kzL|8DH|>o$0n08H_KG5i_|B1UqbZa*49o#E9`e%aF)QW z=L;IqUbK24-C7o@e@3^7+n9pb!k)YRvkXH=W`^<#$x_Hw3d zN1YYUnXFDjU5yjYZ4CMT(x!;#GEr^%L*FT9&qnYu*+bvqZ2Cjr$&GaG(?j23Z7zM^ zX)kS}Bh59D`cA;+()XRB3z&BTHkZEd%)fwnhqvh$edoOA9q2o(O@HV+=RfZhwf38N zXMRr3dCxn%O~2?n%@;85ur~4QI~9J&D+Jd#5zkpg_2M_bXjPDJ2&`q;+=s|{ze!9l zetn_Y3OTG8)-0lWm!{3(TM<*62lV3B<~D6fgV~r{XVW-PzA(bxumaBh9j%Z{a-&=n z$|c4xj;1z?CgOh333<=v$mcq6rV_{n;`uP}`}}g3kdK6Q{z6may*`s`v?>je_kAuy ztJzrIW`Z0miX-GC!*iHzHBPsqCi^^cs!j#eXYfXv$1TQ!W|Pq$vE6LOTsP!Znz{BD z$@3O8nw^FwZqF;4H)M0v#aYmw+%+x(>^+&>bsoz9Lhic5WGv3IWb05~81jqlpi|Od zbu}}2O~~QGb4i>xiFMtZqg}WZ3*kM639&o`yJ$0xV#$5jg~d$bd)xgHP$it z_8Lg4tGF>#2C4QOcrVORLs=``3>kQT36xvW`yBMHp9eYlLRt-4O(-{CjB@m}6}{)s zR#5V5QGF`N*Xz+cWb8q|4DJMgHDDF{H6W`;@A-5UO_%!Nu7Fyng8Hk_y9&J*wxC?U z7hMR@qPqgLAHC<(z^@d*K?$x0z#F{l5f!x`QL!Q_47EZ(GW8-;Z9SO6Kk%+b@8wVt zt!9WJ&rxmgLl|_H-U9a-;9*`BXwx(z8oasywIHd6?LVkQsYe$env~O~<-=Q1F;fY0 zoU0E#AXZ=*=Z)Kt*_+jD=K7y1Wv<`d1RhY!$UknziVtb@&f+QEmNRvkRlh4 z=>DiVf7GQpyg$|YCa!&0f0!Vmj>P?n(W55P`AkvkTvT417pTwK+xif0i6>B>qL=lR zqfnMx0`(Q?UA^*`X-idUE+4^>TB$6eW*Qjo6hEyeP@k`t^)c;RY6;Yr-`n~SKYcKMrTVsz z_6VO}nfXY|pQV>On7b3qOhG%Ru=A;!))eNk?ZM4Q*!=DVP!)eh%$|4CDr3ODB$|DT z(4IM~hnZdzurCX<$L7N!-n|7qok94_IfS^<9HWX{ei937?WS_%x!L>Q(c? z>|tRR(w-IQ9j)2H9_6wE=ht(YPOqBF*+YDr;M2>jU)b-hc>fmWkaNmt)s%pJX_$TN z?W1HC6{QvqUq|QbIsamApBg_sH)yZZ!%#)>;p}5?pLtCCXxljZdR5Id7&=Ty#n)5J zcoDWwImERxV6WDN*~i>IWlgB%)EdrSO_vqI&^@pA=_sSs%7DE-nmsfBgxsrxMX@tr zuP%g{uqS-vno4+Y5IJB3BNRV1T^y4=+>NzyeH1NdVf^#gVyJQ{s0>b_yxx+bG2Bj>j0YO0A6_|XV1^~qIk^RTc)})*fVf=Knlpi z`n(?Pd6+K6EcO{any-(M3;3f^phEYKk(!3Hhri5^o$YCQmP9_@ZT8&UYWH1db-S-) iE^&|NE@_WvuW!SDr5XMc&i&*c$>{|Wi5~tRRR0GTj>lL4 literal 0 HcmV?d00001 diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/openboost_histogram_cpu_v2/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/openboost_histogram_cpu_v2/197_cpu_act.parquet new file mode 100644 index 0000000000000000000000000000000000000000..247bab30c49ec11d284cdc691df2a654fbc48d67 GIT binary patch literal 30066 zcmd5_4SZA8*-y(?DDc`MDFK;4XD=wwCTY_)ir#yZwn=JR+5$}*(UPyWX_GW2Y10;+ z)xpF~9aH8+eYd_79dpX7I45IDYv(3@Fe~^`8FL7}z_eO)3OWVf=iHCvHs|I;S_1hw zd+&3e=luWAb3X1l=Q-P|=o+PLqO3}`s$7;PlgyMz7C&_5&+lGaaKmZx?ic4TQ-1I@ z*?YKi!>=>GA@4lA_pL7$eNEn(zy8(J+rB1c5?Q8f?ozNzmra&Uo+p>`mU1btn<-f= zBF)Hu1r^3I}QYK8;SSpF+u-j!(|NGfV)OLeQU-I+>>aL$X`sZ|r`O{ebr@rQr>=#g}a zNZae(dSD7Q{Lm)DMULeIz$z7At*w; zKbU<4`Mcn&zZ`~0^cYaVk;_ZEz)DIrmf2v`LjC%RiqMCJ|%0&7BuPK-Ex`+cVjlv-!2z8$w z;~M=q1fiel36eXr8R3GAjDLv-fhl2v!02*8h>%7hAPB-!?^ge0)3YfSG~$+Y<~jALdyyxp0938cYe}H>1n7eT0<3OuLL-prlTyN{w8f3>&6( ze|cmHHTdJz2fv-Pl$x-6<3)L0OQ>h_Gg_B@a|N|ZamM}Ak!2!o^CfaAuZt+)Qc<@@ ztH8_BoMZ zz?aCSd>x1bE{)GYde{5G{`dPx$&c??c-M0el5aoplfj2~+(uS8JNozc^pZ;_96A2_ zuHRgP$APz$OL-l{0hh+-@WH*60|m?fPCodH_l)+dqxuy8S9sP&MXU+FK{>N7j zk()2M{R-VPhsakm4WGW!cSwW-uPK-Ex`+cVjn5(N;oVE!*+TmP^d6ax$V?pDV*N&2h z9>45{-g(E!-9I1r{^r0@(){724-f1)D#C%+luLPC!~vJa=P>g1EqAV4Gfd`GZ@aZo zIZVzkUNGgEH-^c5#YbNJ;{K1w-Wyyy_T~57U;)QTH^^-<5Srcf`9+%fINpH8ISeg1~e2UkxLSwHZa zaw)HiIN;Lw96nut>P(F~ohlq5mLKxYpx*rEo5z0nSvs{>q2AryHl6xxQQ`l+sh=Uj zf!CBvd0oT-m&WJdd;axbe7Wsn>d~jRTvghbL0wv~{D;?e&7$tjXuBeNe>yd=_xjVC zWnya>UQ;gRbrA<#8lS^RuE+6QhMd}e$NG2Y{dyMl{51!knE9TZ+MfQDLwfpRYVFl3 z^0}LVw_N5eHlvp96nBaz4*kw%zA(pUh0_mwoT6=U)1JYX1+H)*2VQIkkTz z@YGd*cyDTdR!8*;e+HCe=YG7VT*}vh;($xz$Kk2N>Q%p&9qYSm=$9zqkGBvt%Jx!(rDo{Q2J*+n`elz^U{Li?K@Mi&JEmqrP~^3dK|BAJmQ zNtq#)q|T5op8zb-ZzB6gf(M^3VfWNX4dLQaQQS*qS>>`+W-oV9(KN7CY6=@{-u4EI zEdWq6WtrhET!CcqG(oXk_`*~^bpg9+OG!g#%FAUGU-hMdxlnbR*Wt0YwfO@L%}#%y zt;uI`Gf*8W5u~}%wdFC&a2|8M$yH0_ll``4hub1sBax+Lr^+X|Ee`f+qR;K;9#efP zPG<9Y{p{mZS6fqq-`3{ibf)^7Ek3Wu;pfWJ>`uSe;qyD&JgmVqTU&?2XK8XYs1&Ru z!|4e)d>s}STTe(H*36Hh$;-o<1yMA4d04Y3iY6})Yigos@^XQuTHGTIC!c8ZxwyDU zZFWb4tBnuSG_Nzz(CTmS2P^@$jv1W93xR3pf}0jZfp2g*8abtDtib1NYUU)9G*`?aJV#KxbUp2 zQt)_ZQiw+{BKrRe@Sw*gnpR4zE8<@%9u>yf_6%ZoI-ueyBjhD6Jqs`MLtfsNN<4Wn z@zP=Z_Lb4Yy^1(92!3iNQ?P z%ZL;Z=VgR6VZ2B$!>Hipz#L+q9HC-K9|jeZCX5>PvMA(b58z!w9GJ^`8Ik1Syo``0 zj2G!;7!|yH<$J_|d4P(ibr32pO&B%orAFxGEAxrNmvU{1(mu=x?Twk-IlFNgMg=eT zEhG*v08~72gS^D0XXB-snFroW$vm1u?D{_OVHWFUMAC+9%Lr-0c#&lCJZKA zYYXs(mJr9U06aXyg79!@!g!-s@&d#|%ZL+80TIuqAVgf6Frr|eAb>lbOMICFaCqtj z!Qs+`;e@*o0p1BE(W3-BJWYb|aB0GLqxLugz%TQNjd=iw=R^=7E=?FvWM=|9C;@U$ z0kKI9$ap3MA>-1qA?FE@Hx?217XmV#|3JvNbZp4NS!dIgM6U*r@vH|z#-(FJE)@8D z|8ip6Re+4=IuJ509UHPnfZTfx@#NKjjAu9yGA-<^v()(y<{6 z`}UJsqMrt2Jez@#ap~BQg?)QxF|kJn$aoF|A>-1qAq)FDZ8q1U+o9j5uTjWIRiOka6kQkcFdasGK-{Eg<8$351MG6Gjdn zxF|*qrbCs)i3$M3_tppym!1`%@Z{on74hW?0K~Hn2oRT^6`-&KpQs^vY5)*VF(5!( zdKQ4A=cg}M5gS(lAigd`fVebaK*0$a-&tV;{GVE4Q!T*Z*#iWJOXI>#OBZCZuMJF@BSOl&uFLXR+zwFBf?PxvSTs5vE_*|S zkcDXoba@%oMpv69P>_$*J?(C9mso)tPQcO8Dpc)hce!p}x*!aNk{vh}FCA|%l?W6g z8)qPNyop2c(s2e0C4_LqQAOhn8#?R6VMi5>Gi*4$h2xGc8)sla8VyH|C>&>KEMJEc z5LFpx8X+h16*z&2!f}RoAtP|B7P?3L@azJx&|kkVPh=&wtyjEe=}%$zvfh zw`*~{ka+B9=;+%L2>6`V_5gG2y$;Twvur+xCD7)B6X|tXE~nLJ@pYktZ*P~y=W7ch zLU3mu-JwTPl{yD*A7mjrj*qY~M+IM}n%+lp*CRkR5({Swbr_o!6u`dlesirtGmYK(ARbwTiTjTN`>F3 zD{HZ;I+a$Fnw91GomO{gz`W8}0A+!C(`skAj&_>eE|*nb(SimnA8L>Kyt1IJi{ToqH>ts|${4<@mDW0~I!1d2%n%o7b45Ms zCwkD=cWTOYs2_$gA)cPPyvi19UK!giZk!jUlf1U;w&+W{Y+9|s zuPsmOvUw`KR#kp9eyx_e%AmQftlMJJv?sg2S@X;;o4eFwt}W`SV8=N=4q)!FyKB4Z zO?FpZw=tMxj$W&$24ZM#ww7wT>s6Yr*j!!hc71J^-CgSE`|Ww)4C5sfS4CoRt-cWA z8iT7?T$9dO7*|zdajm});u?diSX`6NSs2&+#Nyg?A;dKXSFyMzowG2m>cryOdLhI$ z23N7TCY|%BxK^%=vVI%&-D*`{0N*WjbgrUtO*ZEio4YgC`qQ86^=G}R##N{4?1-_x zYk@seBdkHYlg-VhcPZ*r0oXHDDqx=!ZG9KC)YUXv@+uo`d9XKP*H}qgzlS+$RrRJ$ zCG3;XUJhU5d9+@8sm2HUtsq)E*||Mq($>lzJG1wJwsx64QDd3M?17XTzu9DF_jt+M zYw*}z7L&b=Ye$hda(?8MdCjWk^Tn~nY*H)Cb;f9YlJ{klUeemyUE2=(9S@u(aO-)E z6YWLoy6N_cX#F$3P142`#1{5UcGz39`{FuP89HZZtkC60ojc>&B(ER4tgx3e=eZhf zaL#0N*EZI>;oPRSC|KSS^;{;d&4ti+D%rCUd`!mZJDkmh(0A0)&V6F^9oFXj_npr2 z7CPEo6Rq!rY|ek*DLIRICuDQ}`_9U%f^xC>Mz5!@%$Fmb-*}B&_qh&C&PzOs>(Us*S$y^VGIm zOqCsG$g!e0LQXO=huKl@_PXlxzC%vct%Uk&{n6%et4Y&pHU*=$o86@LLSChnYk!eE zuf}O{*S2tbUeUZEo1-pOK!5Vqdun0t$>gpJQT`Wl*Ii~)slu9PM0sJzFLr`XS+mX4 z%H%a6hYQbT9d>s8t_tPCP>#99sD!m8tk23IH!19QkYC~QYA82a;mq$UZ>exLGWqs; zNUCeOF;oGm_I!9R&euU%JKYKy_+S~7+tB;9=)I~Ma`MHr4zyZOZoU-d=xG~z&!=sm z6f~gvw2-f_Lhq2V2mK1T69CqLRT|WTtO>mr(W_~?JP3CMbVe=IzZ$)3(R*w(yp@zPl_1Bt z`p^Sn1(u<{j_Lz8jA~g`BZp^X@K7K};h|SOGp5lKaughHLf*5gNC+W=`fVCuf*$Bj z^v(tz#fJARSNmmhJ579)Gg?@lb)1$s+EAtsgX`?ld_1{PbL;h5V z`on};Kl-~}$!O{dID8#lwTi2+q?id3I1NAMk$FN55AhuRU1P&h81jd1TLNu^6uEfB z_eaP1qbtwn{i&;J;o67whY2F;NZhX!J?f*K&y=*UK;@-{q59lotqw_ z3T34=R9{JzXH5PwZK*9+^YL3X#`-D{zmmdGea^AghuXvF3)R;-*7~42y22W&kFN5K zA$~O5U+D4zK7LhWtdD8m;=)jUjI;mqKBj%kt)cpg##$fZR~3$5c~!@# z_K2Kcnfb`TpQTs2n7b3qOhLPsu=A;o))yDD?ZM4Q*!=DXP%VE(%$|4CT2sirES`Ov z(4IM~hnZdf`4_^idMd@8#${oc~~Q zG;~48o^H(@J)cI}GocmapHOwQzN4Fsf0K4~ggx`=6?|Wq`5p2VejD>u(ds#U9c?Tw z8Wn%E5D|V4w8&H|cJD>a%twYo&fZA3S8;r*+4&q6Ch%kGW1e6Y0i=cZcUqP7wPE{$ z2zyuD;8W58pGH(uleRj- z9u{V!+Or0|qcuC&qg+<#{CWk`>9s33dx&ofe0rJn3;Vql@87~4a()G^T@tb{kFZa? zeN@b%qTI^i8|k7d&cB4)r#?up2-_R0V5p+_aQ2C}Pc_p%h7QiYNn4)_Lx%~e_+3SoE_6fI7MGI;Nz!eH1NdVf^#<#z=c+f)=a?(V){Z`A~R( z?^8KgBECl2ht}tU{s0>b_=U$SbG6uD>;jsW5MJdf&Yqv|Me&%sw@h^vuxH@#fE192 z^?4K8^DteCS?n`}duW!TuZZrIkI$t3FNG9JPkyOF|+v@)V;)%`grh?m8nnAcCmF=Ypc5fNUc0A)CV@GCJqJC3($#X#y>w`8oEz zbI-Z||2cPg_ug}VP8D6Pkfq3~WOK@8X)?)hiDcqq>&|U3-JOv{b-uNFf#K*r6Z|( zs-9ZA<-$M;=fG>orMxZTfJl@pf zjt!^oUVPoN9}XHpt(`sYg6}UfYP#)c;p?}K5a|uPhFr?qA`ZAT3Wv%rQCL_x=0nEc z^}3HzrP4bd*a1OUQ$P75B$WK-<`2Y!kO_)x5EvaU2ocgK1O!3&-*ZnM-M(uSbw2;D zKkj<)25N|7`>!rOd;|6UnF%ld?uXHoKXt?2jQux=1cBF(OL<$w0hfw$pk$hInUtBP zGf`InA3j6=E?m0wb8w$3D%#D&BrEF+d5q2*SS3kq3|z`oezkK7bV2J^({$UuYn7l}2G8 z=mP^^dw%z%>{}^&%DlF~gFmCzZMb;fsuv0B-ErE3e|dQl6)+F~u5k3vM54fJ$fdju ziUKZ;!XZ2m^1x5YRM#QoFE!>!s&w$0XP!BnG&Co3W8GmU2&-0oEgl4<$PENk0oM~E zq)`Y6g0T2D(`P5<=TiB%KQ#BTECqE~mv1O&R8R+(Hs5j2@;vI%W6D*{Ph^V(f!B~r zc^$+7mqy_b4gwkobJu*#_)DdI_6>A}eS7vCO_fe|^gD`zAo=X6BjQ0odTbDw3b-Id zNTU!C1Yz}kOCHZyGM)Ok&{?G+)YQ~_a>k6eT-!G;p9vVjZ7QZq0`I|;i8=gCPgK?bLw9J>t zrMwN|fJwnYVy*)S$U%c{~aw%U2;($wIbEx`uz=QJT`^dHJ^H!(s`HCFxnY$1N>H0K^z;oeXF_R_}kKhM}t zn*FywpoFi}8R<7?-Wax@+>`t79sN{dUn|~HF6DI*2V5GP!_GA$SAMJdntb7rK)Z9} zA@av3zB611!k1&@TQkqTe~4Vsxie+{ibEoC;4S4+-Ue~NrLj3|efZHIc0GBF{9*r? zy<68FC0+kA8NZrwl>F89q)R7f9V5T}X!Jj`+($(?@Ro8ZZ-Y4C(%2kIp8nlEM>VI( zpO>V)y|4Nd*;;zuY94or^dH}q{`Sn%WMRY9k4NR765+sW$fdju;($wIb9msyhP`*) zbe>%H&T4PI=`8stqkNf9ah9as-29ihyUvpbud}I~6=y{_@EUR{Z-Y4C(%2mO!p8E@ z!A~u|K@#fRkEfk$U%Eu@Tpwt;>DfzUdhNG!^Gr$9nITE{KDzgkNE~<#xin-0aloaq zY4w< z^QT9P#DUk4OL<$w0hh++u=V1Z^-q)#)Gc?WzWDgsvDC{K)FanDkx8XJum3V_o059& z)Ismue`bns;5Fn@-WGAdrLj5mg##DLbH|ZdrF7GgR~>V8InqterNoFWtphhrANR`f z0c8_5Em*wu$G6f3Zo=0vyoFrK>!LW|(%5m>U6o3o4IJ8-(f*tJ58U$gM*Ft67XCEt z_{P!^r*=*Et=u$En?Cu(^T#*h9C%B)l-ETZaA|A~*KAGOb+23a_CIfpx$it1JrO?diYEu zUovr!pja+^VJaV($(~{)rJ)n%iL%Lj)g#M?Le=e!mey9Ezy7X(+3j~Y?IQILjnt82 z5*%sNr70?jyuZ)dWOtcmcSvMuHz&*cxy*LynPPU$qJgk`) zMU$6@HS?or^761|VH8bX9@bPx(d6XCx= z*y_13a@2()Zj1;;9GS2vvt<)UQbl@Y}D;jEVt=^%s$zDG#o z#*6eaj0#@W935JJVJaifOYGN1MGL{j>{l8BdXAU;iJy^P5Ea9&18mT%uke3}8M zcp?XRiA%4*OLfRg=&|qIMC={QdKroJUoel@Nj9| zcwJZW0>q2A5zB4^L_B4J5OHbTh=P5B0B%tZv2qH);n@)chfCvz6YfF;c*~T;qYA*o zGa(2Mm&T3PWsf5OT$x8anG1k;+5-XN(zpRdb|$cc5+FaCPduXnWIWk{ka20xkaGpd zPZkm{6aX@w;y}o_v}ee|S?8Il#A|9m#uFO|8JG49xj^9a3)6_rKL=zyoq>>XY0r?= z0_4|jC*GS5$aoS1A>-1XAj2L}fV`O^c94LKZ$1z*F6|kzuy4PoA-5RPd^}JTpBlW_`oHn%V4@#Ni3@bKzwhF z0CDLR0SZqp7R@47RskTMSU`Zd^ojt59eCMn;?ZgV#Ipzp5SLy7VAuI+WexG<900`E zWe5%tMY?>Z65YN;C9Bg$_mLZ<3LHC=c6z%|xIe+7UP9+gbjKumn0KyXj z$S+(P)33oC5P!PIV69I|ean>Y-fYOhc($ffO`kFi8AWz=kV|LrjKKa?~d07s(&E8&bZFL4* zZeP~a%wOC&IWvoWt!K*Y5mM%LUY6VJvV(dS!^5aHI9tvB z{5+)Y4!ArWVg)WZ?MBC_P_;YYbS{{j8HPfM4IGP?_BNPO1d5UMG7vh-#G!a;FN1}W zKR9BSqF#m#oonK-U5a`cHk``Bal4lFGO!?Jh9gH5_A)e3Lu!X%09pRoB0a2A+rV)yO zy1NLlg}n?N;hq=)QI%e%5srYWn|VOZ6!tQ7qwmi00)x{ZoM^3n(} z*#3we$A-k+9maBZU2!ZyK?GdD?XbdaSY*=m{I}K8Y`6N6ygNkZCM}K^689Vp9etbq zey_t4@H5BW^WgkB%j&h8{jFX&k)D_3bXdG*ZwET~_H>xN-qs)@1oz|7eR(8RsTS+F^1zofchngIJkWR|422oprg@DyyyraFKp{o3U<|vQn!@ z7y+9>5wzE;bL@2*C&JddOg@XUB&V`@7E`~)P@7{kR5#gl#fUfPxBD7^yS<|Sq$`z>#k~#HCN-bx+^^vWnMIX zE#|t)psB8`(`--&65Zb{xhAL8RpK_)7Isvy;~XCcF!$J8wH-!-%~{u}433y#}0NyoBPK6JK0s^@X^0!&NM<3Fj<~t1`a0 z8v8ZYyQPZGRWz=N=G<&`wRgAv^d)-zX;fA_>y+(n-K_7LVb9b6YtYU_bF=E4 zIdw`u?3pTaV4oCieHS#>RX3P(D;uo2us34YSP5IdhdFALMnk&-_DN_jhp+M68jr0+ z?S=hT5UrhT+@3LEYh|~M+513SJ5BDWvCL)mKnk_bWH7ONyu|G_c&H$D?Bz_k&IT)-Gg)1=4MrE7+td~Y z%bTN~%fz(l3w@`OJsZKtWH)_>v*`iLQO{XL^%6I~lxiT~5L(Nyxet-^ev_D9;`&016>?Z{tXV|$u1%Z6 zw<4}K59uYW&26P+O=g3t!KQPfd|{NmVKtonI|GnQ@}OK4$|WW)j;1z?F6w^J1$obw z=;u0crV`2p;`uP}`|@&^kdK6Qey1t=UZ2S|T9viY_kHf#fZ0&lW`Z0miX-GCBXgK- zMwiEF%)N}9s!IX&)%v2%;}(Ou#bgLZZ8w`i<$=6P3)lW4d0w@{?5b_%_PnBbLpDcU zk^}w8V|3TT-jm5)7ohwvXoVxMqrADo(ZJ-}jgVB=aAT+fQtf&0UX-VW zvH;xz8Teoslv~mJ4D?<#3v%*Bv=+3QQEt8j<>+ZEde5V+pcE`c^=TkqUxnTwV-NZj za3=t)0jnga16d<_FQjW|x;zMX1+;n%)L(<%HR!#l73KPU=t4j#x+_2j(R(2c{K^3w zl;C;*yuo`FqM{8VDpo{=p;i<`raoj^+6bob54>y9dnHsvYZ+q5b5tAr5C)x7Wr6z) z@Gz?ev}qa<4PRY=T98!7_8(NDtU?zcnw3+g6vA6^2~!DjoU0E#AXZ=*>g%XJV8f`E zRW)#UdIk>#vI`!1)iGllJt0TI;YQ>=tBQmWGN|9G1}5l%Zb$EI;8D!DAKc5J=`QmZ zAl9^T_>J0%2z!Vosx0&aT$=9kEt{RkI7;g+xvc+sIvDb&Lew87)Vi+U?Mg;do6q5E z>8d$geZ@sgkicp9F^|j>YIule*Y6q|j)IUsbn7H&8>Gm^Bep+U&L3@g9`8?GRWsK< ztUpW;QAgr_CFoHX?R=&zmQa1gRqh`7%e1AY zT*b$4P7mv=K>UgeLiIU%S|4f;y*E@}dr#|w>gWnfs6M*N+k^PgY=5E4^ZEEy^{_ss zeTxc0_1Sw`ABtb8H&ma$hxIY-TW$%}SJ>0~5WlK${K~7^y0k~+{L0KnrTkfXrIWck z!ORr2YZ5!3YH3|j0oxwje1y&KJ^ z737~#b+oRnlZ}6)rY6Fk`Sc3DFUQ zqGINw(gMz2PY0?vKC{^Q92O?!D z|l>_ zS)udmnM|kG%;fALzRmFIW!5k3_g1`r3vI&>%>2{kUL7ooT_JmI5zK_$;Um{H!h3_r z0RtGJ_-W~qxa{F>tc~lVXh93(pSRaX+A|ZhU_FQiorcMW!UKGt%E1!x)ze4g-=-x3>({c9jm-(@?Jxx!S$TK}=?_B}A|2C_~a~pGsJCnPloylI`hW|!0 U{0E%-$Ul-ZcS Date: Sun, 16 Aug 2026 00:42:13 -0700 Subject: [PATCH 49/49] fix: ignore zero-weight histogram samples --- src/openboost/_models/_histogram_boost.py | 8 ++++ src/openboost/_validation.py | 15 +++++-- tests/test_histogram_boost.py | 55 +++++++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py index d893f18..608b995 100644 --- a/src/openboost/_models/_histogram_boost.py +++ b/src/openboost/_models/_histogram_boost.py @@ -427,6 +427,14 @@ def fit( weights = validate_sample_weight(sample_weight, X_valid.shape[0]) if weights is not None and float(np.sum(weights)) <= 0.0: raise ValueError("sample_weight must contain positive total weight") + if weights is not None and np.any(weights == 0.0): + # A zero-weight observation must be equivalent to removing it. In + # particular, it must not influence feature bins or the target + # support learned below. + positive_weight = weights > 0.0 + X_valid = X_valid[positive_weight] + y_valid = y_valid[positive_weight] + weights = weights[positive_weight] self.X_binned_ = array(X_valid, n_bins=self.n_feature_bins, device="cpu") if self.X_binned_.any_categorical: diff --git a/src/openboost/_validation.py b/src/openboost/_validation.py index 7983de6..dcbb540 100644 --- a/src/openboost/_validation.py +++ b/src/openboost/_validation.py @@ -252,8 +252,10 @@ def validate_sample_weight( if sample_weight is None: return None - if not isinstance(sample_weight, np.ndarray): - sample_weight = np.asarray(sample_weight, dtype=np.float32) + try: + sample_weight = np.asarray(sample_weight, dtype=np.float64) + except (TypeError, ValueError) as exc: + raise ValueError("sample_weight must contain numeric values.") from exc if sample_weight.ndim != 1: raise ValueError( @@ -275,7 +277,14 @@ def validate_sample_weight( if not np.all(np.isfinite(sample_weight)): raise ValueError("sample_weight must contain only finite values.") - return sample_weight.astype(np.float32) + with np.errstate(over="ignore", invalid="ignore"): + sample_weight_float32 = sample_weight.astype(np.float32) + if not np.all(np.isfinite(sample_weight_float32)): + raise ValueError( + "sample_weight must remain finite when converted to float32." + ) + + return sample_weight_float32 def validate_eval_set( diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py index 4644771..fb3062d 100644 --- a/tests/test_histogram_boost.py +++ b/tests/test_histogram_boost.py @@ -293,6 +293,61 @@ def test_histogram_boost_sample_weight_controls_base_distribution(): weighted.fit(X, y, sample_weight=np.zeros(len(y))) with pytest.raises(ValueError, match="only finite"): weighted.fit(X, y, sample_weight=np.full(len(y), np.inf)) + with pytest.raises(ValueError, match="finite when converted to float32"): + weighted.fit(X, y, sample_weight=np.full(len(y), 1e300, dtype=np.float64)) + + +@pytest.mark.parametrize( + ("outlier_X", "outlier_y"), + [ + (np.array([1.5, -0.5], dtype=np.float32), 1_000.0), + (np.array([1e6, -1e6], dtype=np.float32), 1.5), + ], + ids=("target-outlier", "feature-outlier"), +) +def test_histogram_boost_zero_weight_outlier_matches_dropped_row(outlier_X, outlier_y): + X = np.array( + [ + [-2.0, 1.0], + [-1.0, 0.5], + [0.0, 0.0], + [1.0, -0.5], + [2.0, -1.0], + [3.0, -1.5], + ], + dtype=np.float32, + ) + y = np.array([-1.0, -0.5, 0.0, 1.0, 2.0, 2.5], dtype=np.float32) + X_with_outlier = np.vstack([X, outlier_X]) + y_with_outlier = np.append(y, np.float32(outlier_y)) + weights = np.append(np.ones(len(y), dtype=np.float32), np.float32(0.0)) + params = { + "n_distribution_bins": 6, + "n_trees": 3, + "max_depth": 2, + "n_feature_bins": 8, + } + + dropped = ob.HistogramBoost(**params).fit(X, y) + weighted = ob.HistogramBoost(**params).fit( + X_with_outlier, + y_with_outlier, + sample_weight=weights, + ) + + np.testing.assert_array_equal(weighted.target_bin_edges_, dropped.target_bin_edges_) + for weighted_edges, dropped_edges in zip( + weighted.X_binned_.bin_edges, + dropped.X_binned_.bin_edges, + strict=True, + ): + np.testing.assert_array_equal(weighted_edges, dropped_edges) + np.testing.assert_allclose( + weighted.predict_distribution(X).probas, + dropped.predict_distribution(X).probas, + rtol=0.0, + atol=0.0, + ) def test_histogram_boost_base_smoothing_is_total_prior_weight():