diff --git a/.github/workflows/scoringbench.yml b/.github/workflows/scoringbench.yml new file mode 100644 index 0000000..ebdfd2a --- /dev/null +++ b/.github/workflows/scoringbench.yml @@ -0,0 +1,299 @@ +name: ScoringBench + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/scoringbench.yml" + - "benchmarks/scoringbench/**" + - "src/openboost/**" + - "pyproject.toml" + workflow_dispatch: + inputs: + mode: + description: Benchmark scope + required: true + default: smoke + type: choice + options: + - smoke + - quality_shard + - strong_shard + - development_shard + - crps_distribution_shard + - crps_distribution_v2_shard + dataset_name: + description: Exact ScoringBench dataset name for a quality/strong shard + required: true + default: 1027_ESL + type: string + n_trees: + description: Boosting rounds for quality_shard + required: true + default: "500" + type: string + learning_rate: + description: OpenBoost learning rate for development_shard + required: true + default: "0.01" + type: string + training_objective: + description: OpenBoost objective for development_shard + required: true + default: nll + type: choice + options: + - nll + - crps + max_depth: + description: OpenBoost tree depth for development_shard + required: true + default: "3" + type: string + reg_lambda: + description: OpenBoost L2 leaf regularization for development_shard + required: true + default: "1.0" + type: string + min_child_weight: + description: OpenBoost minimum child Hessian for development_shard + required: true + default: "1.0" + type: string + allow_confirmation: + description: Unlock the preregistered confirmation dataset + required: true + default: false + type: boolean + +concurrency: + group: scoringbench-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + scoringbench-cpu: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + OPENBOOST_BACKEND: cpu + OPENBOOST_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PYTHONHASHSEED: "0" + NUMBA_NUM_THREADS: "2" + OMP_NUM_THREADS: "2" + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + benchmarks/scoringbench/requirements.txt + benchmarks/scoringbench/requirements-strong-baselines.txt + + - name: Check out pinned ScoringBench + run: | + git clone --filter=blob:none https://github.com/jonaslandsgesell/ScoringBench .repos/ScoringBench + git -C .repos/ScoringBench checkout --detach "$(cat benchmarks/scoringbench/SCORINGBENCH_COMMIT)" + + - name: Install benchmark environment + run: | + python -m pip install --upgrade pip + python -m pip install -r benchmarks/scoringbench/requirements.txt + python -m pip install -e . + + - name: Install strong comparison models + if: github.event_name == 'workflow_dispatch' && (inputs.mode == 'strong_shard' || inputs.mode == 'crps_distribution_shard' || inputs.mode == 'crps_distribution_v2_shard') + run: | + python -m pip install -r benchmarks/scoringbench/requirements-strong-baselines.txt + + - name: Validate wrapper contract + env: + PYTHONPATH: .repos/ScoringBench + run: | + python -m pytest -o addopts="" \ + benchmarks/scoringbench/test_openboost_wrapper.py -q + + - name: Run smoke benchmark + if: github.event_name == 'pull_request' || inputs.mode == 'smoke' + run: | + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --smoke \ + --n-trees 20 \ + --output-dir "${RUNNER_TEMP}/scoringbench-smoke" + + - name: Run official quality sentinel + if: github.event_name == 'pull_request' + run: | + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --dataset-name 1027_ESL \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees 500 \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Run official quality shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'quality_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + N_TREES: ${{ inputs.n_trees }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + if [[ ! "${N_TREES}" =~ ^[1-9][0-9]*$ ]]; then + echo "n_trees must be a positive integer" >&2 + exit 2 + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees "${N_TREES}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Run strong-baseline diagnostic shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'strong_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + N_TREES: ${{ inputs.n_trees }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + if [[ ! "${N_TREES}" =~ ^[1-9][0-9]*$ ]]; then + echo "n_trees must be a positive integer" >&2 + exit 2 + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost,xgboost_quantile,xgblss,catboost_quantile \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees "${N_TREES}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Run OpenBoost development shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'development_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + N_TREES: ${{ inputs.n_trees }} + LEARNING_RATE: ${{ inputs.learning_rate }} + TRAINING_OBJECTIVE: ${{ inputs.training_objective }} + MAX_DEPTH: ${{ inputs.max_depth }} + REG_LAMBDA: ${{ inputs.reg_lambda }} + MIN_CHILD_WEIGHT: ${{ inputs.min_child_weight }} + run: | + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --n-trees "${N_TREES}" \ + --learning-rate "${LEARNING_RATE}" \ + --training-objective "${TRAINING_OBJECTIVE}" \ + --max-depth "${MAX_DEPTH}" \ + --reg-lambda "${REG_LAMBDA}" \ + --min-child-weight "${MIN_CHILD_WEIGHT}" \ + --development-run \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Run preregistered CRPS distribution shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'crps_distribution_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + ALLOW_CONFIRMATION: ${{ inputs.allow_confirmation }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + CONFIRMATION_FLAG=() + if [[ "${ALLOW_CONFIRMATION}" == "true" ]]; then + CONFIRMATION_FLAG=(--allow-confirmation) + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_histogram_cpu,xgboost_quantile,catboost_quantile \ + --dataset-registry \ + benchmarks/scoringbench/protocols/crps_distribution_v1.json \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --development-run \ + "${CONFIRMATION_FLAG[@]}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Run preregistered CRPS distribution V2 shard + if: github.event_name == 'workflow_dispatch' && inputs.mode == 'crps_distribution_v2_shard' + env: + DATASET_NAME: ${{ inputs.dataset_name }} + ALLOW_CONFIRMATION: ${{ inputs.allow_confirmation }} + run: | + if [[ -z "${DATASET_NAME}" ]]; then + echo "dataset_name must not be empty" >&2 + exit 2 + fi + CONFIRMATION_FLAG=() + if [[ "${ALLOW_CONFIRMATION}" == "true" ]]; then + CONFIRMATION_FLAG=(--allow-confirmation) + fi + python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_histogram_cpu_v2,xgboost_quantile,catboost_quantile \ + --dataset-registry \ + benchmarks/scoringbench/protocols/crps_distribution_v1.json \ + --dataset-name "${DATASET_NAME}" \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --development-run \ + "${CONFIRMATION_FLAG[@]}" \ + --output-dir "${RUNNER_TEMP}/scoringbench-quality" + + - name: Verify smoke artifact + if: github.event_name == 'pull_request' || inputs.mode == 'smoke' + run: | + MANIFEST="${RUNNER_TEMP}/scoringbench-smoke/openboost_manifest.json" + test -s "${MANIFEST}" + test -s "${RUNNER_TEMP}/scoringbench-smoke/benchmark_outcome.json" + test -n "$(find "${RUNNER_TEMP}/scoringbench-smoke/raw" -name '*.parquet' -print -quit)" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); assert m["protocol_mode"] == "smoke" and m["result_rows"] == 4 and m["outcome"]["status"] == "complete"' "${MANIFEST}" + + - name: Verify quality-shard artifact + if: github.event_name == 'pull_request' || inputs.mode == 'quality_shard' || inputs.mode == 'strong_shard' || inputs.mode == 'development_shard' || inputs.mode == 'crps_distribution_shard' || inputs.mode == 'crps_distribution_v2_shard' + run: | + MANIFEST="${RUNNER_TEMP}/scoringbench-quality/openboost_manifest.json" + test -s "${MANIFEST}" + test -s "${RUNNER_TEMP}/scoringbench-quality/benchmark_outcome.json" + test -s "${RUNNER_TEMP}/scoringbench-quality/datasets.json" + test -n "$(find "${RUNNER_TEMP}/scoringbench-quality/raw" -name '*.parquet' -print -quit)" + python -c 'import json, sys; m=json.load(open(sys.argv[1])); mode=sys.argv[2]; distribution=mode in {"crps_distribution_shard", "crps_distribution_v2_shard"}; dev=mode == "development_shard" or distribution; expected=25 if mode == "strong_shard" else (15 if distribution else (5 if mode == "development_shard" else 10)); protocol="development_tuning" if dev else "official_quality_shard"; assert m["protocol_mode"] == protocol and m["official_protocol_compatible"] is (not dev) and m["result_rows"] == expected and m["expected_result_rows"] == expected and m["outcome"]["status"] == "complete" and m["openboost_git"]["dirty"] is False and (not distribution or len(m["verified_dataset_files"]) == 1)' "${MANIFEST}" "${{ inputs.mode }}" + + - name: Upload benchmark artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: scoringbench-${{ github.event_name }}-${{ github.sha }} + path: | + ${{ runner.temp }}/scoringbench-smoke + ${{ runner.temp }}/scoringbench-quality + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8b1a1e1..dbf0b3d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -6,23 +6,36 @@ on: pull_request: branches: [main] +concurrency: + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - # Fast tests: <3 min, runs on every PR + # Core tests: runs on every PR across the supported OS/Python matrix. fast-tests: runs-on: ${{ matrix.os }} + timeout-minutes: 25 strategy: matrix: os: [ubuntu-latest, macos-latest] python-version: ["3.10", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + - name: Restore compiled Numba kernels + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/numba-cache + key: numba-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('src/openboost/**/*.py') }} + restore-keys: | + numba-${{ runner.os }}-py${{ matrix.python-version }}- + - name: Install OpenMP runtime (macOS) if: runner.os == 'macOS' run: | @@ -42,51 +55,74 @@ jobs: - name: Run fast tests (CPU backend) env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | - pytest tests/ -v --tb=short -m "not slow and not benchmark" + pytest tests/ -v --tb=short -m "not slow and not benchmark and not jax" - # Full tests: includes slow tests, runs after fast tests pass + - name: Run JAX distribution tests serially (Linux, Python 3.12) + if: runner.os == 'Linux' && matrix.python-version == '3.12' + env: + OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache + run: | + pytest tests/test_distribution_gradients.py -v --tb=short -n 0 -m jax + + # Slow tests and docs run after the core matrix; do not duplicate core tests. full-tests: runs-on: ubuntu-latest + timeout-minutes: 20 needs: fast-tests steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" + - name: Restore compiled Numba kernels + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/numba-cache + key: numba-${{ runner.os }}-py3.12-${{ hashFiles('src/openboost/**/*.py') }} + restore-keys: | + numba-${{ runner.os }}-py3.12- + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -e ".[test,sklearn]" pip install "xgboost>=2.0" - - name: Run all tests (CPU backend) + - name: Run slow tests (CPU backend) env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | - pytest tests/ -v --tb=short + pytest tests/ -v --tb=short -m "slow and not benchmark and not jax" - name: Run documentation examples (CPU backend) env: OPENBOOST_BACKEND: "cpu" + NUMBA_CACHE_DIR: ${{ runner.temp }}/numba-cache run: | python scripts/run_doc_examples.py - # Performance regression check: main branch only + # Compare the pushed main range against the previous remote main on one runner. performance-check: runs-on: ubuntu-latest + timeout-minutes: 15 if: github.ref == 'refs/heads/main' needs: full-tests steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" @@ -98,6 +134,26 @@ jobs: - name: Performance regression check env: + BASELINE_SHA: ${{ github.event.before }} OPENBOOST_BACKEND: "cpu" run: | - python benchmarks/check_performance.py + BASELINE_ROOT="${RUNNER_TEMP}/openboost-baseline" + git worktree add --detach "${BASELINE_ROOT}" "${BASELINE_SHA}" + NUMBA_CACHE_DIR="${RUNNER_TEMP}/numba-baseline" \ + python benchmarks/check_performance.py \ + --source-root "${BASELINE_ROOT}" \ + --benchmark-only \ + --output "${RUNNER_TEMP}/performance-baseline.json" + NUMBA_CACHE_DIR="${RUNNER_TEMP}/numba-current" \ + python benchmarks/check_performance.py \ + --baseline "${RUNNER_TEMP}/performance-baseline.json" \ + --output "${RUNNER_TEMP}/performance-current.json" + + - name: Upload performance results + if: always() + uses: actions/upload-artifact@v4 + with: + name: performance-check-${{ github.sha }} + path: | + ${{ runner.temp }}/performance-baseline.json + ${{ runner.temp }}/performance-current.json diff --git a/.gitignore b/.gitignore index 8294445..7b1b665 100644 --- a/.gitignore +++ b/.gitignore @@ -221,4 +221,5 @@ logs/ # Benchmark results (generated) benchmarks/results/*.json +benchmarks/results/scoringbench*/ tasks/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5ca99ab --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,156 @@ +# OpenBoost Agent Guide + +This is the canonical repository guidance for coding agents and automated +contributors. Tool-specific instruction files should point here instead of +duplicating policy. + +## Mission + +OpenBoost is a readable Python gradient-boosting research platform. Its current +product focus is **calibration-first distributional boosting for tabular risk**: +NaturalBoost, proper scoring, calibration, exposure-aware targets, custom +distributions, and verified CPU/CUDA execution. + +Do not position the repository as a drop-in replacement for XGBoost, LightGBM, +or CatBoost. Standard GBDT, GAM, DART, linear leaves, Ray, multi-GPU, and +train-many are supporting or experimental capabilities unless a committed, +reproducible artifact proves otherwise. + +## Start Here + +Before a non-trivial change: + +1. Read this file and the relevant entries in `learnings/`. +2. Check `git status --short --branch`; preserve unrelated user changes. +3. Read the implementation, its tests, and the public documentation together. +4. Write a short plan for work spanning three or more meaningful steps. +5. Identify the smallest test that can fail before editing. + +Do not trust phase comments, docstrings, README claims, or green CI as proof by +themselves. Verify the actual call path and the tests that exercise it. + +## Current Priority Order + +1. Silent correctness and persistence bugs. +2. Deterministic CPU behavior and reference parity. +3. One verified single-GPU NaturalBoost path, including end-to-end quality. +4. Third-party evidence through ScoringBench and real domain case studies. +5. Stable packaging, versioned persistence, and a smaller public API. +6. New features only after the above gates are satisfied. + +Treat Ray, multi-GPU, out-of-core training, GOSS speedups, and fused train-many +as experimental. Do not expand or market them until exact correctness and +scaling artifacts exist. The repository audit in +`learnings/2026-08-15-repository-audit.md` records the current evidence gaps. + +## Architecture + +```text +Models (`src/openboost/_models/`) + -> tree core (`src/openboost/_core/`) + -> CPU/CUDA backends (`src/openboost/_backends/`) + -> distributions (`src/openboost/_distributions.py`) + -> validation and persistence +``` + +- `BinnedArray` is feature-major: `(n_features, n_samples)`. +- Bin 255 is reserved for missing values; use at most 254 regular bins. +- The backend is process-global, not thread-local. Use `backend_context` for a + scoped switch and do not run mixed-backend fits concurrently in one process. +- NaturalBoost fits one tree per distribution parameter per round. +- CUDA eligibility is narrower than the public model surface. A fallback must + be visible, tested, and represented honestly in benchmark provenance. + +## Correctness Rules + +- A serialization change requires prediction round trips for numeric, + categorical, missing-value, and specialized leaf/tree state that it touches. +- A CUDA change requires CPU/CUDA parity for gradients, splits, leaves, + predictions, and final task metrics—not just matching array shapes. +- Never silently ignore `sample_weight`, exposure, callbacks, evaluation sets, + constraints, or sampling parameters. Support them or reject them explicitly. +- Randomized behavior must be driven by the model's declared seed; do not use + unscoped global `numpy.random` state. +- Distributed child histograms must be derived from routed samples. Scaling a + parent histogram is not an exact substitute. +- Public examples are tests of product behavior. If an example cannot run, fix + it or label the feature experimental before documenting it. + +## Evidence and Benchmark Rules + +- Every performance or quality claim must link to a committed raw artifact. +- Record git SHA and dirty state, dataset/version/hash, split seed, package + versions, OS, CPU/RAM/thread count, GPU/driver/CUDA, and exact CLI arguments. +- Compare end-to-end fit and prediction, including distribution gradients, + transfers, compilation policy, and fallbacks. Kernel microbenchmarks cannot + support an end-to-end product claim. +- Use repeated folds/seeds and publish failures. Compare at matched predictive + quality; do not declare a speed win when CRPS/NLL/calibration regresses. +- Keep official ScoringBench results separate from OpenBoost's large-sample + extension. See `benchmarks/scoringbench/README.md`. +- Synthetic experiments generate hypotheses. Real third-party datasets and + upstream-accepted results generate evidence. + +## Commands + +Use `uv`; do not mutate the project environment with ad-hoc `pip` or Conda +commands. + +```bash +# Install +uv sync --extra dev +uv sync --extra cuda + +# Focused test while iterating +OPENBOOST_BACKEND=cpu uv run pytest tests/test_file.py -n 0 -q + +# CPU regression suite +OPENBOOST_BACKEND=cpu uv run pytest tests/ -m "not gpu and not benchmark" --tb=short + +# Lint production code and changed support files +uv run ruff check src/openboost/ path/to/changed_file.py + +# Documentation and packaging +uv run mkdocs build +uv build +``` + +Run CUDA tests only on real CUDA hardware. A skipped GPU job is not a passing +GPU validation. ScoringBench has a separate Linux environment documented under +`benchmarks/scoringbench/`. + +## Working and Commit Discipline + +- Keep changes small and cohesive. Prefer root-cause fixes over compatibility + shims that conceal invalid state. +- Commit after each independently verified slice: test/benchmark harness, + correctness fix, documentation/learning update, or infrastructure change. +- Do not bundle unrelated cleanup into a fix. Do not amend or rewrite existing + commits unless the user explicitly asks. +- Do not push, publish, create a release, or update an external leaderboard + unless the user asks for that external action. +- Before every commit: inspect the staged diff, run the narrowest meaningful + tests, and include the verification in the relevant learning entry. + +## Learning Log + +`learnings/` is the durable project memory for decisions, failed attempts, +experiments, and non-obvious operational facts. + +- Add or update an entry for every non-trivial change. +- Use `learnings/TEMPLATE.md`. +- Record evidence and falsified hypotheses, not a diary of shell commands. +- Link files and commits. State what was not verified. +- Never include credentials, tokens, private URLs, or user-specific secrets. +- Release notes describe user-facing changes; learning entries explain why the + implementation and evidence changed. + +## Definition of Done + +A change is done only when: + +1. The intended behavior is covered by a focused test or reproducible artifact. +2. Relevant regression tests and lint pass. +3. Documentation and capability claims match the implemented boundary. +4. A learning entry captures important decisions, failures, and follow-ups. +5. The change is committed as a cohesive unit and the remaining work is stated. diff --git a/CLAUDE.md b/CLAUDE.md index b8e57b9..b7405a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,148 +1,14 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -OpenBoost is a GPU-native, all-Python gradient boosting library (~20K lines). It uses Numba JIT for CPU kernels and CuPy/numba-cuda for GPU acceleration. Designed as a research-friendly alternative to XGBoost/LightGBM with full Python source. - -## Commands - -```bash -# Environment (always use uv, never pip/conda/poetry) -uv sync # Install/sync dependencies -uv sync --extra cuda # With GPU support -uv sync --extra dev # With dev tools (test + bench + sklearn + ruff) - -# Testing (parallelized with pytest-xdist, -n auto is in addopts) -uv run pytest tests/ -v --tb=short # All tests (CPU, parallel) -uv run pytest tests/test_core.py -v # Single test file -uv run pytest tests/test_core.py::test_name -v # Single test -uv run pytest tests/ -n 0 # Force serial (debugging) -OPENBOOST_BACKEND=cuda uv run pytest tests/ # GPU tests -OPENBOOST_BACKEND=cpu uv run pytest tests/ # Force CPU - -# Profiling -uv run python benchmarks/profile_loop.py # Profile training (50K samples default) -uv run python benchmarks/profile_loop.py --summarize # Machine-readable bottleneck summary -OPENBOOST_PROFILE=1 uv run python script.py # Profile any training run via env var - -# Linting -uv run ruff check src/openboost/ # Lint -uv run ruff check src/openboost/ --fix # Autofix - -# Docs -uv run mkdocs serve # Local docs server -uv run mkdocs build # Build docs - -# Build -uv build # Build wheel/sdist -``` - -## Architecture - -### Layer Overview - -``` -Models (_models/) → Core (_core/) → Backends (_backends/) - ↓ ↓ ↓ -GradientBoosting fit_tree() _cpu.py (Numba JIT) -NaturalBoost histograms _cuda.py (CuPy kernels) -OpenBoostGAM split finding -DART, LinearLeaf growth strategies -``` - -### Data Layer (`_array.py`) -`BinnedArray` is the fundamental data structure — quantile-bins continuous features into uint8 (max 255 bins). Missing values encode as `MISSING_BIN = 255`. Native categorical feature support with auto-detection of string/object columns. All tree-building operates on binned data. - -### Core (`_core/`) -- **`_tree.py`** — `fit_tree()`, `fit_tree_gpu_native()`, `fit_tree_symmetric()`: the main tree-fitting entry points -- **`_primitives.py`** — Low-level histogram building, split finding, sample partitioning -- **`_growth.py`** — Three growth strategies: `LevelWiseGrowth` (XGBoost-style), `LeafWiseGrowth` (LightGBM-style), `SymmetricGrowth` (CatBoost-style) - -### Backend Dispatch (`_backends/`) -`get_backend()` / `set_backend()` switch between CPU and CUDA implementations. Same interface, different kernels. Control via `OPENBOOST_BACKEND` env var or `set_backend('cuda')`. Use `backend_context('cpu')` context manager for temporary switches. - -### Models (`_models/`) -- **`_boosting.py`** — `GradientBoosting`, `MultiClassGradientBoosting`: main model classes with full callback/eval_set support -- **`_sklearn.py`** — sklearn-compatible wrappers (`OpenBoostRegressor`, `OpenBoostClassifier`, `OpenBoostDARTRegressor`, `OpenBoostGAMRegressor`, `OpenBoostDistributionalRegressor`, `OpenBoostLinearLeafRegressor`) -- **`_distributional.py`** — `NaturalBoost`: distributional GBDT with callbacks/eval_set support -- **`_dart.py`** — `DART`: dropout boosting with callbacks/eval_set support -- **`_linear_leaf.py`**, **`_gam.py`** — Specialized model variants (both support callbacks/eval_set/early stopping; GAM also supports pairwise interactions, smoothing, and monotone shape constraints) - -### Persistence (`_persistence.py`) -`PersistenceMixin` provides `save()`/`load()` on all models. Generic `ob.load(path)` auto-detects model class from saved state. - -### Profiling (`_profiler.py`) -`ProfilingCallback` instruments training by wrapping core primitives (`build_node_histograms`, `find_node_splits`, `partition_samples`, `compute_leaf_values`, `fit_tree`) with timers. Outputs JSON reports to `logs/` with per-phase breakdown, bottleneck identification, and run-over-run comparison. CLI runner: `benchmarks/profile_loop.py`. - -### Loss Functions (`_loss.py`) -9 objectives, each with CPU/GPU/dispatcher implementations returning `(gradient, hessian)`. Custom losses are callables with signature `fn(pred, y) -> (grad, hess)`; register by name via `register_loss` (optional `loss_value_fn` for true loss reporting), and mark GPU-capable losses with `@ob.device_loss`. `register_distribution` / `register_growth_strategy` extend the other factories. - -### Distributions (`_distributions.py`) -8 distributional families for NaturalBoost (Normal, LogNormal, Gamma, Poisson, StudentT, Tweedie, NegativeBinomial). Each implements `nll_grad_hess()` for natural gradient computation. - -## Key Conventions - -- **Python 3.10+** target. Ruff rules: E, F, I, UP, B, SIM (line length 100; E501, E402, F821 ignored). -- **uv only** for package management — never `pip install` or `conda`. -- All Numba-jitted functions use `@njit` or `@cuda.jit`. CPU kernels are in `_backends/_cpu.py`, CUDA in `_backends/_cuda.py`. -- Test environment variable `OPENBOOST_BACKEND=cpu` forces CPU backend in CI. -- Tests use `pytest-xdist` (`-n auto --dist loadfile`) for parallel execution. Shared fixtures are in `tests/conftest.py` (session-scoped datasets, function-scoped gradients). -- **GPU-native builder** (`fit_tree_gpu_native`) does not support missing values or categorical features. The training loop in `_boosting.py` auto-falls back to `fit_tree()` with a warning when the data has NaN or categorical columns. -- **Callbacks**: all models — `GradientBoosting`, `MultiClassGradientBoosting`, `DART`, `NaturalBoost`/`DistributionalGBDT`, `LinearLeafGBDT`, and `OpenBoostGAM` — support `callbacks` and `eval_set` in `fit()`. -- **`random_state`**: `GradientBoosting` and `DART` accept `random_state` for reproducibility. Sklearn wrappers pass it through. DART also accepts `seed` (alias). -- **`suggest_params()`**: Returns sklearn-style names by default (`n_estimators`). Pass `style='core'` to get core API names (`n_trees`). -- **Profiling**: `ProfilingCallback` wraps core primitives with timers. Enable via callback or `OPENBOOST_PROFILE=1` env var. Reports go to `logs/` as JSON. - -## Working Style - -### 1. Plan Mode Default -- Enter plan mode for ANY non-trivial task (3+ steps or architectural decisions) -- If something goes sideways, STOP and re-plan immediately -- don't keep pushing -- Use plan mode for verification steps, not just building -- Write detailed specs upfront to reduce ambiguity - -### 2. Subagent Strategy -- Use subagents liberally to keep main context window clean -- Offload research, exploration, and parallel analysis to subagents -- For complex problems, throw more compute at it via subagents -- One task per subagent for focused execution - -### 3. Self-Improvement Loop -- After ANY correction from the user: update `tasks/lessons.md` with the pattern -- Write rules for yourself that prevent the same mistake -- Ruthlessly iterate on these lessons until mistake rate drops -- Review lessons at session start for relevant project - -### 4. Verification Before Done -- Never mark a task complete without proving it works -- Diff behavior between main and your changes when relevant -- Ask yourself: "Would a staff engineer approve this?" -- Run tests, check logs, demonstrate correctness - -### 5. Demand Elegance (Balanced) -- For non-trivial changes: pause and ask "is there a more elegant way?" -- If a fix feels hacky: "Knowing everything I know now, implement the elegant solution" -- Skip this for simple, obvious fixes -- don't over-engineer -- Challenge your own work before presenting it - -### 6. Autonomous Bug Fixing -- When given a bug report: just fix it. Don't ask for hand-holding -- Point at logs, errors, failing tests -- then resolve them -- Zero context switching required from the user -- Go fix failing CI tests without being told how - -## Task Management - -1. **Plan First**: Write plan to `tasks/todo.md` with checkable items -2. **Verify Plan**: Check in before starting implementation -3. **Track Progress**: Mark items complete as you go -4. **Explain Changes**: High-level summary at each step -5. **Document Results**: Add review section to `tasks/todo.md` -6. **Capture Lessons**: Update `tasks/lessons.md` after corrections - -## Core Principles - -- **Simplicity First**: Make every change as simple as possible. Impact minimal code. -- **No Laziness**: Find root causes. No temporary fixes. Senior developer standards. +The canonical repository guidance is [`AGENTS.md`](./AGENTS.md). Read it in full +before changing code, benchmarks, documentation, CI, packaging, or release +state. + +Claude-specific compatibility notes: + +- Use `uv` for environments and commands. +- Store durable decisions, failed experiments, and user corrections in + `learnings/`, not the ignored `tasks/` directory. +- Keep commits small and verified. Do not push or publish unless requested. +- Treat GPU, distributed, out-of-core, and performance claims according to the + evidence gates in `AGENTS.md`; comments and green-but-skipped CI are not proof. diff --git a/benchmarks/check_performance.py b/benchmarks/check_performance.py index bc8aa5c..7fd4e90 100644 --- a/benchmarks/check_performance.py +++ b/benchmarks/check_performance.py @@ -1,10 +1,11 @@ -"""Performance regression check for CI. +"""Performance regression check for CI or a local baseline. Runs a fixed, small benchmark and compares against stored baselines. Fails if any metric degrades by more than 20%. Usage: - uv run python benchmarks/check_performance.py + uv run python benchmarks/check_performance.py --baseline baseline.json + uv run python benchmarks/check_performance.py --benchmark-only --output result.json uv run python benchmarks/check_performance.py --update-baselines """ @@ -12,6 +13,9 @@ import argparse import json +import os +import platform +import subprocess import sys import time import tracemalloc @@ -20,7 +24,6 @@ import numpy as np PROJECT_ROOT = Path(__file__).parent.parent -sys.path.insert(0, str(PROJECT_ROOT / "src")) BASELINE_FILE = Path(__file__).parent / "results" / "performance_baselines.json" @@ -90,17 +93,43 @@ def run_fixed_benchmark(): } -def save_baselines(results): - """Save results as new baselines.""" - BASELINE_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(BASELINE_FILE, "w") as f: +def collect_provenance(source_root: Path) -> dict[str, str]: + """Collect enough environment data to interpret a raw CI result.""" + import openboost as ob + + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=source_root, + check=False, + capture_output=True, + text=True, + ) + git_commit = commit.stdout.strip() if commit.returncode == 0 else "unknown" + + return { + "git_commit": git_commit, + "openboost_version": ob.__version__, + "python_version": platform.python_version(), + "numpy_version": np.__version__, + "platform": platform.platform(), + "processor": platform.processor() or "unknown", + "openboost_backend": os.environ.get("OPENBOOST_BACKEND", "auto"), + "numba_num_threads": os.environ.get("NUMBA_NUM_THREADS", "default"), + "numba_cache_dir": os.environ.get("NUMBA_CACHE_DIR", "default"), + } + + +def save_results(results, path: Path): + """Save benchmark results.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: json.dump(results, f, indent=2) - print(f"Baselines saved to {BASELINE_FILE}") + print(f"Results saved to {path}") -def load_baselines(): +def load_baselines(path: Path): """Load stored baselines.""" - with open(BASELINE_FILE) as f: + with open(path) as f: return json.load(f) @@ -140,10 +169,52 @@ def main(): "--update-baselines", action="store_true", help="Update baselines with current results" ) + parser.add_argument( + "--baseline", + type=Path, + default=BASELINE_FILE, + help="Baseline JSON to compare against", + ) + parser.add_argument( + "--output", + type=Path, + help="Write the current raw benchmark result to this JSON file", + ) + parser.add_argument( + "--benchmark-only", + action="store_true", + help="Run and save the benchmark without comparing it", + ) + parser.add_argument( + "--source-root", + type=Path, + default=PROJECT_ROOT, + help="Repository root whose src/openboost implementation should run", + ) args = parser.parse_args() + if ( + not args.update_baselines + and not args.benchmark_only + and not args.baseline.exists() + ): + print(f"No baseline found at {args.baseline}", file=sys.stderr) + print( + "Pass --baseline, use --benchmark-only, or explicitly create a " + "local baseline with --update-baselines.", + file=sys.stderr, + ) + sys.exit(2) + + source_dir = args.source_root.resolve() / "src" + if not source_dir.is_dir(): + parser.error(f"source root has no src directory: {args.source_root}") + sys.path.insert(0, str(source_dir)) + print("Running fixed benchmark...") results = run_fixed_benchmark() + results["benchmark_schema_version"] = 1 + results["provenance"] = collect_provenance(args.source_root.resolve()) print(f" fit_time: {results['fit_time_median']:.4f}s") print(f" predict_time: {results['predict_time_median']:.4f}s") @@ -151,17 +222,17 @@ def main(): print(f" mse: {results['mse']:.6f}") print(f" r2: {results['r2']:.4f}") - if args.update_baselines: - save_baselines(results) + if args.output: + save_results(results, args.output) + + if args.benchmark_only: return - if not BASELINE_FILE.exists(): - print(f"\nNo baselines found at {BASELINE_FILE}") - print("Run with --update-baselines to create them.") - save_baselines(results) + if args.update_baselines: + save_results(results, args.baseline) return - baselines = load_baselines() + baselines = load_baselines(args.baseline) regressions = check_regression(results, baselines) if regressions: diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/README.md b/benchmarks/evidence/scoringbench/1027_esl_20260816/README.md new file mode 100644 index 0000000..302c6dc --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/README.md @@ -0,0 +1,48 @@ +# ScoringBench `1027_ESL` sentinel — 2026-08-16 + +This directory freezes OpenBoost's first successful real-dataset shard under +ScoringBench's official five-fold, 3,000-row-cap protocol. It is an integration +and direction-finding result, not a full-suite leaderboard claim. + +## Provenance + +- GitHub Actions run: [31922702524](https://github.com/jxucoder/openboost/actions/runs/31922702524) +- Actions artifact: `9256929555` +- Actions artifact digest: + `sha256:0c5176c4a28cd44444f6a086643a01f636b0dcb5e165a4a30d3d992f3e96da97` +- OpenBoost source head: `921c024b23fc4549b251222f35ce64a4b6846505` +- Tested PR merge: `71dd86b7c95245593030ca488981bc2607f02eb5` +- ScoringBench: `a938a667b7839b41e9272929010573410301c0b4` +- Models: OpenBoost NaturalBoost Normal CPU and NGBoost Normal +- Shared parameters: 500 rounds, learning rate 0.01, depth 3, 99 quantiles, + seed 42 + +See `openboost_manifest.json` for the complete environment and arguments. +`datasets.json` is ScoringBench's resolved registry. The two Parquet files under +`raw/` contain all fold-level metrics. + +## Descriptive result + +| Metric (lower is better unless noted) | OpenBoost mean | NGBoost mean | OpenBoost relative | Fold count | +| --- | ---: | ---: | ---: | ---: | +| CRPS | 0.300527 | 0.307714 | -2.34% | 2/5 lower | +| Log score | 0.710078 | 0.675967 | +5.05% | 1/5 lower | +| RMSE | 0.545923 | 0.554927 | -1.62% | 3/5 lower | +| PIT KS statistic | 0.089366 | 0.105493 | -15.29% | 3/5 lower | +| 90% interval score | 2.436058 | 2.737215 | -11.00% | 4/5 lower | +| Fit time (seconds) | 2.111374 | 3.298695 | -35.99% | 5/5 lower | +| 90% coverage (closer to 0.90 is better) | 0.854723 | 0.813718 | — | 5/5 closer | + +Mean fit time is 1.56× lower for OpenBoost in this run. The first OpenBoost fold +includes cold Numba compilation, but this artifact does not separately report +cold and warm timing, so it cannot support a general speed claim. + +## Interpretation limits + +- This is one small real dataset and five correlated CV folds, with no repeated + seeds or uncertainty interval over the paired differences. +- OpenBoost improves several metrics here but loses log score. CRPS also improves + on only two individual folds despite its better mean. +- The result says nothing about CUDA or large-data scaling. +- A defensible value claim requires the complete ScoringBench suite, upstream + review, and a separate multi-size CPU/CUDA extension. diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json b/benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json new file mode 100644 index 0000000..a15f942 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/openboost_manifest.json @@ -0,0 +1,83 @@ +{ + "arguments": { + "dataset_index": null, + "dataset_name": [ + "1027_ESL" + ], + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "models": [ + "openboost_cpu", + "ngboost" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "smoke": false + }, + "ci": { + "event_name": "pull_request", + "head_ref": "codex/scoringbench-release-hardening", + "provider": "github_actions", + "ref": "refs/pull/19/merge", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31922702524", + "source_sha": "921c024b23fc4549b251222f35ce64a4b6846505", + "tested_sha": "71dd86b7c95245593030ca488981bc2607f02eb5" + }, + "created_at": "2026-08-16T02:54:19+00:00", + "datasets": [ + { + "id": null, + "name": "1027_ESL", + "source": "pmlb" + } + ], + "official_protocol_compatible": true, + "openboost_git": { + "commit": "71dd86b7c95245593030ca488981bc2607f02eb5", + "dirty": false + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "official_quality_shard", + "result_rows": 10, + "schema_version": 1, + "scoringbench_git": { + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": null +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/ngboost/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/ngboost/1027_ESL.parquet new file mode 100644 index 0000000..89c2217 Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/ngboost/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/openboost_cpu/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/openboost_cpu/1027_ESL.parquet new file mode 100644 index 0000000..ce8c2ce Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_20260816/raw/openboost_cpu/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json b/benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json new file mode 100644 index 0000000..af83cd4 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_20260816/summary.json @@ -0,0 +1,62 @@ +{ + "artifact_digest": "sha256:0c5176c4a28cd44444f6a086643a01f636b0dcb5e165a4a30d3d992f3e96da97", + "artifact_id": 9256929555, + "dataset": "1027_ESL", + "files": { + "openboost_manifest.json": "sha256:e1e825d99c35c8f41eabbacfad388923ae6b6271267f827814d1644d46b87121", + "raw/ngboost/1027_ESL.parquet": "sha256:d9cd14271f2a6471bd01976f66363550f14eaaf5273694c9de6fe80a70a0c679", + "raw/openboost_cpu/1027_ESL.parquet": "sha256:58c0f434c97b8578a3d283b5d37ecf9f7103cfdbd6cc85246cb4a713722eb3e0" + }, + "folds": 5, + "metrics": { + "coverage_90": { + "ngboost_abs_error": 0.0862823605537415, + "ngboost_mean": 0.8137176394462585, + "openboost_abs_error": 0.04527667760848997, + "openboost_closer_folds": 5, + "openboost_mean": 0.85472332239151 + }, + "crps": { + "ngboost_mean": 0.3077139963362946, + "openboost_fold_wins": 2, + "openboost_mean": 0.3005270428812241, + "openboost_relative_percent": -2.3355952412434267 + }, + "interval_score_90": { + "ngboost_mean": 2.737215196639375, + "openboost_fold_wins": 4, + "openboost_mean": 2.436057677703067, + "openboost_relative_percent": -11.002332564354278 + }, + "log_score": { + "ngboost_mean": 0.67596661214236, + "openboost_fold_wins": 1, + "openboost_mean": 0.7100775390578736, + "openboost_relative_percent": 5.04624434147789 + }, + "pit_ks_stat": { + "ngboost_mean": 0.10549263800016417, + "openboost_fold_wins": 3, + "openboost_mean": 0.08936585621575437, + "openboost_relative_percent": -15.287115850098198 + }, + "rmse": { + "ngboost_mean": 0.5549269313560135, + "openboost_fold_wins": 3, + "openboost_mean": 0.5459231513016759, + "openboost_relative_percent": -1.6225163252279073 + }, + "train_time": { + "ngboost_mean": 3.298694705963135, + "ngboost_over_openboost_speedup": 1.5623446940621515, + "openboost_fold_wins": 5, + "openboost_mean": 2.1113744735717774, + "openboost_relative_percent": -35.99363803643326 + } + }, + "openboost_source_sha": "921c024b23fc4549b251222f35ce64a4b6846505", + "protocol_mode": "official_quality_shard", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4", + "tested_merge_sha": "71dd86b7c95245593030ca488981bc2607f02eb5", + "workflow_run_id": 31922702524 +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md new file mode 100644 index 0000000..c0968cb --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/README.md @@ -0,0 +1,53 @@ +# 1027_ESL strong-baseline diagnostic + +This is a clean, ScoringBench-shaped five-fold diagnostic shard. It +compares OpenBoost CPU against NGBoost, native XGBoost multi-quantile, +Gaussian XGBoostLSS, and CatBoost MultiQuantile. All 25 expected +dataset/model/fold rows completed; the OpenBoost and ScoringBench checkouts were +clean. The exact model constructors, package versions, platform, source SHAs, +and CI identity are in `openboost_manifest.json`. + +Lower is better for every displayed score except raw coverage, whose target is +0.90. `Coverage error` is the fold-level mean of `abs(coverage_90 - 0.90)`. +The CRPS values are ScoringBench's histogram-based scores after every model is +converted to its common `DistributionPrediction` representation. + +| Model | CRPS | RMSE | 90% coverage | Coverage error | 90% interval score | PIT KS | Train seconds | +|---|---:|---:|---:|---:|---:|---:|---:| +| OpenBoost CPU | **0.3005** | 0.5459 | **0.8547** | **0.0510** | **2.4361** | **0.0894** | 1.3845 | +| XGBoostLSS | 0.3043 | **0.5384** | 0.7952 | 0.1048 | 2.8794 | 0.0991 | **0.1519** | +| NGBoost | 0.3077 | 0.5550 | 0.8137 | 0.0863 | 2.7371 | 0.1055 | 2.4431 | +| CatBoost quantile | 0.3249 | 0.5901 | 0.7172 | 0.1828 | 4.0657 | 0.1466 | 2.4481 | +| XGBoost quantile | 0.3257 | 0.6178 | 0.6127 | 0.2873 | 3.9807 | 0.2204 | 0.3869 | + +On this shard, OpenBoost has the best mean CRPS, interval score, coverage +error, and PIT KS statistic. Relative to native XGBoost quantile, its CRPS is +7.7% lower, interval score is 38.8% lower, coverage error is 82.2% lower, and +RMSE is 11.6% lower. It wins four of five folds on CRPS and all five folds on +RMSE, interval score, coverage error, and PIT KS. + +This is not an overall win. XGBoostLSS has 1.4% better RMSE and is about 9.1 +times faster in these fit-only timings; OpenBoost has 1.2% better CRPS and +substantially better interval calibration. The timing is a cold/warm mixture +from one persistent process, is not a scale claim, and differed materially +across otherwise equivalent Actions runs. + +The raw artifact also contains ScoringBench's reconstructed-density log score +and CRLS, but they are intentionally excluded from the comparison table. +Out-of-support targets are clamped to a quantile model's boundary density, and +CRLS is integrated over model-specific support; the upstream implementation +therefore does not make those two metrics comparable across these parametric +and finite-quantile predictions. Gaussian analytic NLL requires a separate +audit, and any cross-model density score requires a common support/tail rule. + +The result is one small dataset, one seed, five correlated folds, unequal +model-specific default budgets, CPU only, and no paired confidence interval. +It is a diagnostic signal, not a library-level marketing claim. The next +quality decision must come from multiple untouched datasets and ultimately the +complete official suite. Hyperparameter changes prompted by this shard must be +developed elsewhere and not re-labelled as held-out evidence. + +Source artifact: [GitHub Actions run 31925701435](https://github.com/jxucoder/openboost/actions/runs/31925701435), artifact `9257853524`, digest +`sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765`. +`summary.json` hashes every frozen raw/result/provenance input except this +README and the summary itself, and records the unrounded diagnostic means. diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json new file mode 100644 index 0000000..177475a --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1027_ESL", + "expected_rows": 25, + "observed_rows": 25, + "status": "complete", + "valid_rows": 25 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 25, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 25, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 25 +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json new file mode 100644 index 0000000..90d47df --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/openboost_manifest.json @@ -0,0 +1,158 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1027_ESL" + ], + "dataset_registry": null, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "models": [ + "openboost_cpu", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31925701435", + "source_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b", + "tested_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b" + }, + "created_at": "2026-08-16T04:12:18+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "name": "1027_ESL", + "source": "pmlb" + } + ], + "expected_result_rows": 25, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "ngboost": { + "dist": "normal", + "learning_rate": 0.01, + "n_estimators": 500, + "n_quantiles": 99, + "ngb_params": { + "random_state": 42 + } + }, + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "n_quantiles": 99, + "n_trees": 500 + }, + "xgblss": { + "distribution": "Gaussian", + "n_quantiles": 99, + "num_boost_round": 100, + "xgblss_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": true, + "openboost_git": { + "changes": [], + "commit": "cea891a8cb964c655c30092ab0a223f4f140c03b", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 25 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "official_quality_shard", + "result_rows": 25, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": null +} diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/catboost_quantile/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/catboost_quantile/1027_ESL.parquet new file mode 100644 index 0000000..989af95 Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/catboost_quantile/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/ngboost/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/ngboost/1027_ESL.parquet new file mode 100644 index 0000000..2e781bc Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/ngboost/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/openboost_cpu/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/openboost_cpu/1027_ESL.parquet new file mode 100644 index 0000000..64267db Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/openboost_cpu/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgblss/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgblss/1027_ESL.parquet new file mode 100644 index 0000000..583ad0f Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgblss/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgboost_quantile/1027_ESL.parquet b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgboost_quantile/1027_ESL.parquet new file mode 100644 index 0000000..1420ed1 Binary files /dev/null and b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/raw/xgboost_quantile/1027_ESL.parquet differ diff --git a/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json new file mode 100644 index 0000000..7f6bb8b --- /dev/null +++ b/benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json @@ -0,0 +1,130 @@ +{ + "artifact": { + "digest": "sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765", + "id": 9257853524, + "workflow_run_id": 31925701435, + "workflow_url": "https://github.com/jxucoder/openboost/actions/runs/31925701435" + }, + "dataset": "1027_ESL", + "file_sha256": { + "benchmark_outcome.json": "d0640e3d74a7d14c615b61963121b873d8500d9664d07d78fb98234b7a14f3e4", + "datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "openboost_manifest.json": "70a1cc6c05fa5109d59bd319e3a0a3d8d76307664f40c7b0147f5283b5cae3c5", + "raw/catboost_quantile/1027_ESL.parquet": "461258ad37aa5d7fb4e3bb4a7659832d0bca7b6f2e714776746d8539466696b9", + "raw/ngboost/1027_ESL.parquet": "a87222737ce8bea41532f91af7f1fb0e826dae7aac0430f57078de199fc25194", + "raw/openboost_cpu/1027_ESL.parquet": "737edc08cab22cb3f28b1b6a237b718ce698f38aa03046f792b63ea11a1e42ae", + "raw/xgblss/1027_ESL.parquet": "fd24636edaa89d83217f26788da15e0d5a005b80cd1ba13940e7ea3af585d1c3", + "raw/xgboost_quantile/1027_ESL.parquet": "8e09f343e54ba328094954c6477c2b4a2e62a67466d3f071d0d5ba627ce50692" + }, + "folds": 5, + "metric_comparability": { + "cross_model_primary": [ + "crps", + "rmse", + "coverage_90", + "coverage_90_abs_error", + "interval_score_90", + "pit_ks_stat" + ], + "diagnostic_not_cross_model_comparable": { + "crls": "Integrated on model-specific support; upstream implementation documents that values are not directly comparable across differing bin grids.", + "log_score": "Finite quantile support clamps out-of-support targets to a boundary-bin density, creating a tail-boundary artifact." + }, + "timing": "Fit-only cold/warm mixture from one persistent process; not a scale or steady-state benchmark." + }, + "means": { + "catboost_quantile": { + "coverage_90": 0.7172312140464783, + "coverage_90_abs_error": 0.18276878595352175, + "crls": 0.5825405189879156, + "crps": 0.32486018716094617, + "interval_score_90": 4.06573659420003, + "log_score": -1.3190115168420253, + "pit_ks_stat": 0.14662502054983878, + "rmse": 0.5901363119168295, + "train_time": 2.4480895519256594 + }, + "ngboost": { + "coverage_90": 0.8137176394462585, + "coverage_90_abs_error": 0.08628236055374147, + "crls": 0.8919766693054967, + "crps": 0.30774525268048586, + "interval_score_90": 2.7370853096534686, + "log_score": 0.676160510947884, + "pit_ks_stat": 0.10549263800016417, + "rmse": 0.5549747191688402, + "train_time": 2.443098306655884 + }, + "openboost_cpu": { + "coverage_90": 0.85472332239151, + "coverage_90_abs_error": 0.05104986906051636, + "crls": 0.9305498802189568, + "crps": 0.30052704288122417, + "interval_score_90": 2.436057677703067, + "log_score": 0.7100775390578737, + "pit_ks_stat": 0.08936585621575435, + "rmse": 0.545923151301676, + "train_time": 1.3844860553741456 + }, + "xgblss": { + "coverage_90": 0.7951819777488709, + "coverage_90_abs_error": 0.10481802225112917, + "crls": 0.8545918044143812, + "crps": 0.30426339893126575, + "interval_score_90": 2.879351361511471, + "log_score": 0.6375427933712055, + "pit_ks_stat": 0.09914356858963747, + "rmse": 0.538414204542853, + "train_time": 0.15186233520507814 + }, + "xgboost_quantile": { + "coverage_90": 0.6127077579498291, + "coverage_90_abs_error": 0.2872922420501709, + "crls": 0.5757246011831073, + "crps": 0.32567075673952595, + "interval_score_90": 3.980709384171024, + "log_score": -2.2200916983373404, + "pit_ks_stat": 0.22044577530811366, + "rmse": 0.6177832060042735, + "train_time": 0.38692564964294435 + } + }, + "openboost_source_sha": "cea891a8cb964c655c30092ab0a223f4f140c03b", + "relative_percent_of_fold_means": { + "catboost_quantile": { + "coverage_90_abs_error": -72.06860635737965, + "crps": -7.490343612856022, + "interval_score_90": -40.08323900819789, + "pit_ks_stat": -39.05142800278187, + "rmse": -7.492025100361004, + "train_time": -43.4462659143693 + }, + "ngboost": { + "coverage_90_abs_error": -40.83394481457232, + "crps": -2.3455145892229403, + "interval_score_90": -10.998109225485326, + "pit_ks_stat": -15.287115850098193, + "rmse": -1.6309874224037508, + "train_time": -43.33072674143711 + }, + "xgblss": { + "coverage_90_abs_error": -51.296668297930594, + "crps": -1.2280004966636278, + "interval_score_90": -15.395609224144986, + "pit_ks_stat": -9.862175139522938, + "rmse": 1.394641280907218, + "train_time": 811.6717805665942 + }, + "xgboost_quantile": { + "coverage_90_abs_error": -82.23068305074477, + "crps": -7.720593064612176, + "interval_score_90": -38.80342816810849, + "pit_ks_stat": -59.46129786753722, + "rmse": -11.631921037053978, + "train_time": 257.81707846242597 + } + }, + "protocol_mode": "official_quality_shard", + "result_rows": 25, + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md new file mode 100644 index 0000000..fd3d758 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/README.md @@ -0,0 +1,46 @@ +# 1028_SWD corrected numeric-binning diagnostic + +This is development/tuning evidence, not held-out or leaderboard evidence. It +reruns the frozen OpenBoost NLL configuration after commit `236c2df` corrected +numeric binning so that `m` cut edges retain all `m + 1` intervals. The prior +implementation silently merged the highest interval into its predecessor. + +The candidate used 500 rounds, learning rate 0.01, depth 3, seed 42, five +folds, sample cap 3000, 99 quantiles, and pinned ScoringBench commit +`a938a667`. Strong-baseline rows come from the already-frozen, same-dataset, +same-split artifact in `../1028_swd_lr_sweep_20260816/baseline_lr_001/`. + +| Model | CRPS | RMSE | 90% coverage error | 90% interval score | PIT KS | Sharpness | +|---|---:|---:|---:|---:|---:|---:| +| OpenBoost before fix | 0.355075 | 0.626043 | 0.0310 | 2.559809 | **0.087349** | 0.580131 | +| **OpenBoost after fix** | **0.347845** | **0.615779** | **0.0290** | **2.509361** | 0.098027 | 0.560487 | +| XGBoostLSS Gaussian | 0.349289 | 0.617750 | 0.0410 | 2.670105 | 0.100551 | 0.526127 | +| NGBoost Gaussian | 0.348185 | 0.616315 | 0.0320 | 2.572281 | 0.099395 | 0.542749 | +| XGBoost quantile | 0.338359 | 0.636753 | 0.2670 | 2.788130 | 0.302062 | 0.505502 | +| CatBoost quantile | 0.336377 | 0.635947 | 0.2380 | 2.825541 | 0.172461 | **0.491907** | + +Relative to the pre-fix OpenBoost artifact, corrected binning improves CRPS by +2.04% (four of five folds), RMSE by 1.64%, interval score by 1.97%, coverage +error by 6.45%, and sharpness by 3.39%. PIT KS worsens by 12.22%, so the change +is not uniformly better on every diagnostic, although it fixes an unambiguous +representation bug. + +On this consumed development dataset, corrected OpenBoost has 0.41% lower mean +CRPS than XGBoostLSS and wins four of five paired folds. It also improves RMSE, +coverage error, interval score, and PIT KS. Against NGBoost, mean CRPS is only +0.10% lower and OpenBoost wins two of five folds; treat that as parity, not a +win. + +The stated goal is still unmet. Corrected OpenBoost CRPS remains 2.80% worse +than native XGBoost quantile and 3.41% worse than CatBoost quantile, winning +only one of five folds against each. OpenBoost is much better calibrated on +this dataset and has lower interval score and RMSE, but those guardrails do not +erase the pre-registered CRPS loss. The next candidate must target the +remaining distribution-shape/quantile gap and then be evaluated on new data. + +Run: [31928677396](https://github.com/jxucoder/openboost/actions/runs/31928677396), +artifact `9258714239`, digest +`sha256:c75eea2d93bf10d5de1ae48a8ab37eed66c1dc1c326a9810ab8dc808c4656836`. +The run completed 5/5 rows from clean source `236c2df`; the manifest records a +clean pinned ScoringBench checkout and the full Linux environment. Fit times +from separate Actions processes are not used as speed evidence. diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json new file mode 100644 index 0000000..598c1da --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 5, + "observed_rows": 5, + "status": "complete", + "valid_rows": 5 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 5, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 5, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 5 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json new file mode 100644 index 0000000..c66acd1 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/openboost_manifest.json @@ -0,0 +1,126 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "development_run": true, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_cpu" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "nll", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31928677396", + "source_sha": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510", + "tested_sha": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510" + }, + "created_at": "2026-08-16T05:25:28+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 5, + "model_parameters": { + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "model_params": { + "min_child_weight": 1.0, + "reg_lambda": 1.0, + "training_objective": "nll" + }, + "n_quantiles": 99, + "n_trees": 500 + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 5 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 5, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000..9f07642 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/raw/openboost_cpu/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json new file mode 100644 index 0000000..1fc1f72 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/summary.json @@ -0,0 +1,74 @@ +{ + "artifact": { + "digest": "sha256:c75eea2d93bf10d5de1ae48a8ab37eed66c1dc1c326a9810ab8dc808c4656836", + "id": 9258714239, + "source_sha": "236c2dfad7c6ea6ea8d406077a59eec3ac32e510", + "workflow_run_id": 31928677396 + }, + "baseline": "../1028_swd_lr_sweep_20260816/baseline_lr_001", + "candidate_metrics": { + "coverage_90_abs_error": 0.028999996185302756, + "crps": 0.34784456793765095, + "interval_score_90": 2.509361067814866, + "pit_ks_stat": 0.09802668231196801, + "rmse": 0.6157794340555057, + "sharpness": 0.5604867750552687 + }, + "comparisons": { + "catboost_quantile": { + "baseline_crps": 0.3363766926503152, + "candidate_crps_relative_percent": 3.4092359958058482, + "candidate_crps_wins": 1, + "candidate_coverage_error_relative_percent": -87.81512762882153, + "candidate_interval_score_relative_percent": -11.19005295818769, + "candidate_pit_ks_relative_percent": -43.160047784863856, + "candidate_rmse_relative_percent": -3.1713396938279903 + }, + "ngboost": { + "baseline_crps": 0.3481847240643794, + "candidate_crps_relative_percent": -0.09769415577965956, + "candidate_crps_wins": 2, + "candidate_coverage_error_relative_percent": -9.374993015079236, + "candidate_interval_score_relative_percent": -2.4460776539851126, + "candidate_pit_ks_relative_percent": -1.3762692644908037, + "candidate_rmse_relative_percent": -0.08693406957233085 + }, + "openboost_before_fix": { + "baseline_crps": 0.35507492380808053, + "candidate_crps_relative_percent": -2.0362901983857373, + "candidate_crps_wins": 4, + "candidate_coverage_error_relative_percent": -6.4516223308077265, + "candidate_interval_score_relative_percent": -1.9707568344182858, + "candidate_pit_ks_relative_percent": 12.224803295387954, + "candidate_rmse_relative_percent": -1.6393938069868441 + }, + "xgblss": { + "baseline_crps": 0.3492890583364678, + "candidate_crps_relative_percent": -0.41355157407345633, + "candidate_crps_wins": 4, + "candidate_coverage_error_relative_percent": -29.26827566315279, + "candidate_interval_score_relative_percent": -6.020129964847132, + "candidate_pit_ks_relative_percent": -2.510826416548262, + "candidate_rmse_relative_percent": -0.31891960877288295 + }, + "xgboost_quantile": { + "baseline_crps": 0.3383589409871867, + "candidate_crps_relative_percent": 2.8034213970492994, + "candidate_crps_wins": 1, + "candidate_coverage_error_relative_percent": -89.13857803317438, + "candidate_interval_score_relative_percent": -9.99843321488677, + "candidate_pit_ks_relative_percent": -67.54753242799605, + "candidate_rmse_relative_percent": -3.293809336382647 + } + }, + "decision": "correctness_fix_accepted_quality_goal_not_met", + "file_sha256": { + "benchmark_outcome.json": "cefba05ad79fdb6056f55cba5afda606d79984537856a41868b7e4a88c12e4a1", + "datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "openboost_manifest.json": "3f8b2678899ebb79cee25a0b1f9401cc8d5377f09698294907e9bdbf4bb5c9f1", + "raw/openboost_cpu/1028_SWD.parquet": "5c810080aafcc6f561e86ca517ec20988fb2bb9f1a59a1aa6b97c2908efb334d" + }, + "folds": 5, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md new file mode 100644 index 0000000..9ed3969 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/README.md @@ -0,0 +1,37 @@ +# 1028_SWD Gaussian CRPS-objective diagnostic + +This is development/tuning evidence, not held-out or leaderboard evidence. It +compares the frozen NLL-trained OpenBoost baseline in +`../1028_swd_lr_sweep_20260816/baseline_lr_001/` with one candidate that +changed only `training_objective` from `nll` to `crps`. Both used 500 rounds, +learning rate 0.01, depth 3, identical regularization, seed 42, five folds, +sample cap 3000, 99 quantiles, and pinned ScoringBench commit `a938a667`. + +| OpenBoost objective | CRPS | RMSE | 90% coverage error | 90% interval score | PIT KS | Sharpness | +|---|---:|---:|---:|---:|---:|---:| +| NLL | 0.355075 | 0.626043 | 0.0310 | **2.559809** | 0.087349 | **0.580131** | +| Gaussian CRPS | **0.353996** | **0.625552** | **0.0280** | 2.660561 | **0.080873** | 0.595394 | + +The CRPS objective improved mean CRPS by 0.30% and won four of five paired +folds. It also improved PIT KS by 7.41%, absolute 90% coverage error by 9.68%, +and RMSE by 0.08%. However, it widened the predictive distribution by 2.63% +and worsened 90% interval score by 3.94% on every fold. This is a real but small +primary-metric improvement with a material guardrail regression. + +It does not meet the stated benchmark goal. Candidate CRPS remains 4.62% worse +than native XGBoost quantile, 1.35% worse than Gaussian XGBoostLSS, 1.67% worse +than NGBoost, and 5.24% worse than CatBoost quantile on the already-frozen +five-model baseline. Therefore the objective is retained as a mathematically +tested development capability, not selected as the final ScoringBench model. +The next diagnostic should tune scale without changing the mean model, using +training-fold-only calibration; test-fold scale selection would be leakage. + +Run: [31927426636](https://github.com/jxucoder/openboost/actions/runs/31927426636), +artifact `9258329504`, digest +`sha256:02f3827ee1787a6559f841cd277407d7faa1f2af2dd8031caf39794d8173594e`. +The run completed 5/5 expected rows from clean OpenBoost source `f092613` and a +clean pinned ScoringBench checkout. `summary.json` records unrounded effects +and SHA-256 hashes for every copied file. Fit-time differences from separate +Actions processes are not treated as speed evidence; reconstructed log score +and CRLS remain excluded from the decision for the documented evaluator +comparability reasons. diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json new file mode 100644 index 0000000..598c1da --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 5, + "observed_rows": 5, + "status": "complete", + "valid_rows": 5 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 5, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 5, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 5 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json new file mode 100644 index 0000000..f887dee --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/openboost_manifest.json @@ -0,0 +1,126 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "development_run": true, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_cpu" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "crps", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31927426636", + "source_sha": "f092613f5d3b77c54b100c8a07be865ec3173fe3", + "tested_sha": "f092613f5d3b77c54b100c8a07be865ec3173fe3" + }, + "created_at": "2026-08-16T04:53:35+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 5, + "model_parameters": { + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "model_params": { + "min_child_weight": 1.0, + "reg_lambda": 1.0, + "training_objective": "crps" + }, + "n_quantiles": 99, + "n_trees": 500 + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "f092613f5d3b77c54b100c8a07be865ec3173fe3", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 5 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 5, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000..964f849 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/raw/openboost_cpu/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json new file mode 100644 index 0000000..d5543af --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_crps_20260816/summary.json @@ -0,0 +1,57 @@ +{ + "artifact": { + "digest": "sha256:02f3827ee1787a6559f841cd277407d7faa1f2af2dd8031caf39794d8173594e", + "id": 9258329504, + "source_sha": "f092613f5d3b77c54b100c8a07be865ec3173fe3", + "workflow_run_id": 31927426636 + }, + "baseline": "../1028_swd_lr_sweep_20260816/baseline_lr_001", + "candidate_vs_nll": { + "coverage_90_abs_error": { + "baseline_mean": 0.030999999046325687, + "candidate_lower_wins": 2, + "candidate_mean": 0.027999999523162854, + "relative_percent": -9.677418114367365 + }, + "crps": { + "baseline_mean": 0.35507492380808053, + "candidate_lower_wins": 4, + "candidate_mean": 0.353996062777474, + "relative_percent": -0.30384038924407 + }, + "interval_score_90": { + "baseline_mean": 2.559808672169682, + "candidate_lower_wins": 0, + "candidate_mean": 2.6605614850848283, + "relative_percent": 3.9359509173687135 + }, + "pit_ks_stat": { + "baseline_mean": 0.08734850000489737, + "candidate_lower_wins": 4, + "candidate_mean": 0.08087311043727916, + "relative_percent": -7.4132807858808825 + }, + "rmse": { + "baseline_mean": 0.6260427399635591, + "candidate_lower_wins": 3, + "candidate_mean": 0.6255523498852755, + "relative_percent": -0.07833172513302955 + }, + "sharpness": { + "baseline_mean": 0.5801309335762859, + "candidate_lower_wins": 0, + "candidate_mean": 0.5953941579557143, + "relative_percent": 2.630996469251601 + } + }, + "decision": "keep_objective_reject_candidate_configuration", + "file_sha256": { + "benchmark_outcome.json": "cefba05ad79fdb6056f55cba5afda606d79984537856a41868b7e4a88c12e4a1", + "datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "openboost_manifest.json": "2ff8fd620d6ec1cc1f6c142790eebf3249532ca11f6aa0281efd138bd0367c96", + "raw/openboost_cpu/1028_SWD.parquet": "f1f199eb9c48efdcd00e4acdc55132db2e54eedc4817b4a3b90a41ce8614fb3f" + }, + "folds": 5, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md new file mode 100644 index 0000000..177bad3 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/README.md @@ -0,0 +1,42 @@ +# 1028_SWD learning-rate development sweep + +This is tuning evidence, not held-out or leaderboard evidence. The clean +five-model baseline used 500 OpenBoost/NGBoost rounds at learning rate 0.01. +After inspecting that result, one OpenBoost-only candidate changed only the +learning rate to 0.03. Both runs used the same ScoringBench commit, dataset, +seed, five folds, sample cap, Normal distribution, depth, regularization, and +99-quantile representation. + +The baseline disproved a broad win: OpenBoost had the best mean 90% interval +score, absolute coverage error, and PIT KS, but its mean CRPS was 4.9% worse +than native XGBoost quantile, 1.7% worse than XGBoostLSS, and 2.0% worse than +NGBoost. Its CRPS lost to native XGBoost quantile on all five folds. + +| OpenBoost setting | CRPS | RMSE | 90% coverage | Coverage error | 90% interval score | PIT KS | Sharpness | +|---|---:|---:|---:|---:|---:|---:|---:| +| lr=0.01, 500 rounds | **0.3551** | **0.6260** | **0.8910** | **0.0310** | **2.5598** | **0.0873** | 0.5801 | +| lr=0.03, 500 rounds | 0.3566 | 0.6278 | 0.8680 | 0.0320 | 2.6811 | 0.0959 | **0.5490** | + +Increasing the learning rate sharpened the distribution by 5.4%, but mean +CRPS worsened by 0.44%, interval score by 4.74%, PIT KS by 9.82%, and coverage +moved farther below 90%. The candidate improved CRPS on only two of five +paired folds and interval score on one. This falsifies the hypothesis that the +current CRPS gap is primarily caused by an update budget that is too small; the +0.03 default used by an older NGBoost comparison should not be copied into the +ScoringBench wrapper. + +The apparent 11.6% fit-time change is from separate Actions processes and is +not a speed result. ScoringBench reconstructed log score and CRLS are retained +in the raw Parquet files but excluded from the decision because of the known +finite-support and model-specific-grid comparability problems. + +Baseline: [run 31926255124](https://github.com/jxucoder/openboost/actions/runs/31926255124), artifact `9258026048`, digest +`sha256:495ac3c8c02c0846423131c4636dfde05f0da872b637bca2be9493b3c02aa8b4`. + +Candidate: [run 31926664340](https://github.com/jxucoder/openboost/actions/runs/31926664340), artifact `9258115955`, digest +`sha256:b3b77cbf691bef574b0a27c897b8a92a22e90a4c158f0f7e3ae556400d60e599`. + +`summary.json` records exact unrounded effects and SHA-256 hashes for every +copied raw/result/provenance file. The next diagnostic must separate point-mean +error from scale calibration, then test post-fit scale calibration or a +different scale objective without changing the consumed `1027_ESL` shard. diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json new file mode 100644 index 0000000..27af05f --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 25, + "observed_rows": 25, + "status": "complete", + "valid_rows": 25 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 25, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 25, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 25 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json new file mode 100644 index 0000000..2b79b92 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/openboost_manifest.json @@ -0,0 +1,158 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "models": [ + "openboost_cpu", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31926255124", + "source_sha": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36", + "tested_sha": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36" + }, + "created_at": "2026-08-16T04:26:25+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 25, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "ngboost": { + "dist": "normal", + "learning_rate": 0.01, + "n_estimators": 500, + "n_quantiles": 99, + "ngb_params": { + "random_state": 42 + } + }, + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.01, + "max_depth": 3, + "n_quantiles": 99, + "n_trees": 500 + }, + "xgblss": { + "distribution": "Gaussian", + "n_quantiles": 99, + "num_boost_round": 100, + "xgblss_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": true, + "openboost_git": { + "changes": [], + "commit": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 25 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "official_quality_shard", + "result_rows": 25, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": null +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet new file mode 100644 index 0000000..31d1266 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/ngboost/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/ngboost/1028_SWD.parquet new file mode 100644 index 0000000..52a1eb6 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/ngboost/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000..0b5adbc Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgblss/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgblss/1028_SWD.parquet new file mode 100644 index 0000000..d2ff166 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgblss/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet new file mode 100644 index 0000000..62598b1 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json new file mode 100644 index 0000000..598c1da --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "1028_SWD", + "expected_rows": 5, + "observed_rows": 5, + "status": "complete", + "valid_rows": 5 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 5, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 5, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 5 +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json new file mode 100644 index 0000000..2307b84 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/datasets.json @@ -0,0 +1,482 @@ +[ + { + "name": "Abalone", + "source": "openml", + "id": 183, + "abbr": "A" + }, + { + "name": "Student_Performance", + "source": "openml", + "id": 42352, + "abbr": "SP" + }, + { + "name": "Infrared_Thermography_Temperature", + "source": "openml", + "id": 46613, + "abbr": "ITT" + }, + { + "name": "Parkinsons_Telemonitoring", + "source": "openml", + "id": 4531, + "abbr": "PT" + }, + { + "name": "Energy_Efficiency", + "source": "openml", + "id": 44960, + "abbr": "EE" + }, + { + "name": "QsarFishToxicity", + "source": "openml", + "id": 44970, + "abbr": "Q" + }, + { + "name": "concrete_compressive_strength", + "source": "openml", + "id": 44959, + "abbr": "CCS" + }, + { + "name": "PRODUCTIVITY", + "source": "openml", + "id": 42989, + "abbr": "P" + }, + { + "name": "AIRFOIL", + "source": "openml", + "id": 44957, + "abbr": "A" + }, + { + "name": "BIAS_CORRECTION", + "source": "openml", + "id": 42897, + "abbr": "BC" + }, + { + "name": "1027_ESL", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1027_ESL/1027_ESL.tsv.gz", + "abbr": "1E" + }, + { + "name": "1028_SWD", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "abbr": "1S" + }, + { + "name": "1029_LEV", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1029_LEV/1029_LEV.tsv.gz", + "abbr": "1L" + }, + { + "name": "1030_ERA", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1030_ERA/1030_ERA.tsv.gz", + "abbr": "1E" + }, + { + "name": "1199_BNG_echoMonths", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1199_BNG_echoMonths/1199_BNG_echoMonths.tsv.gz", + "abbr": "1BE" + }, + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "abbr": "1CA" + }, + { + "name": "225_puma8NH", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/225_puma8NH/225_puma8NH.tsv.gz", + "abbr": "2P" + }, + { + "name": "227_cpu_small", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/227_cpu_small/227_cpu_small.tsv.gz", + "abbr": "2CS" + }, + { + "name": "294_satellite_image", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/294_satellite_image/294_satellite_image.tsv.gz", + "abbr": "2SI" + }, + { + "name": "344_mv", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/344_mv/344_mv.tsv.gz", + "abbr": "3M" + }, + { + "name": "503_wind", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/503_wind/503_wind.tsv.gz", + "abbr": "5W" + }, + { + "name": "529_pollen", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/529_pollen/529_pollen.tsv.gz", + "abbr": "5P" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/537_houses/537_houses.tsv.gz", + "abbr": "5H" + }, + { + "name": "547_no2", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/547_no2/547_no2.tsv.gz", + "abbr": "5N" + }, + { + "name": "564_fried", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/564_fried/564_fried.tsv.gz", + "abbr": "5F" + }, + { + "name": "595_fri_c0_1000_10", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/595_fri_c0_1000_10/595_fri_c0_1000_10.tsv.gz", + "abbr": "5FC" + }, + { + "name": "1193_BNG_lowbwt", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1193_BNG_lowbwt/1193_BNG_lowbwt.tsv.gz", + "abbr": "1BL" + }, + { + "name": "1201_BNG_breastTumor", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1201_BNG_breastTumor/1201_BNG_breastTumor.tsv.gz", + "abbr": "1BB" + }, + { + "name": "1203_BNG_pwLinear", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1203_BNG_pwLinear/1203_BNG_pwLinear.tsv.gz", + "abbr": "1BP" + }, + { + "name": "215_2dplanes", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/215_2dplanes/215_2dplanes.tsv.gz", + "abbr": "22" + }, + { + "name": "218_house_8L", + "source": "pmlb", + "url": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/218_house_8L/218_house_8L.tsv.gz", + "abbr": "2H8" + }, + { + "name": "Wizmir", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/wizmir.zip", + "abbr": "W" + }, + { + "name": "Ele2", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/ele-2.zip", + "abbr": "E" + }, + { + "name": "Treasury", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/treasury.zip", + "abbr": "T" + }, + { + "name": "Mortgage", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/mortgage.zip", + "abbr": "M" + }, + { + "name": "Laser", + "source": "keel", + "url": "https://sci2s.ugr.es/keel/dataset/data/regression/laser.zip", + "abbr": "L" + }, + { + "name": "1000-Cameras-Dataset", + "source": "openml", + "id": 43714, + "abbr": "1" + }, + { + "name": "3D_Estimation_using_RSSI_of_WLAN_dataset_complete_1_target", + "source": "openml", + "id": 45720, + "abbr": "3EU" + }, + { + "name": "Ailerons", + "source": "openml", + "id": 296, + "abbr": "A" + }, + { + "name": "Another-Dataset-on-used-Fiat-500-(1538-rows)", + "source": "openml", + "id": 43828, + "abbr": "A" + }, + { + "name": "BNG(mv)", + "source": "openml", + "id": 1213, + "abbr": "B" + }, + { + "name": "BNG(stock)", + "source": "openml", + "id": 1200, + "abbr": "B" + }, + { + "name": "CPMP-2015-regression", + "source": "openml", + "id": 41700, + "abbr": "C" + }, + { + "name": "CPS1988", + "source": "openml", + "id": 43963, + "abbr": "C" + }, + { + "name": "CookbookReviews", + "source": "openml", + "id": 45744, + "abbr": "C" + }, + { + "name": "Goodreads-Computer-Books", + "source": "openml", + "id": 43785, + "abbr": "G" + }, + { + "name": "IEEE80211aa-GATS", + "source": "openml", + "id": 43180, + "abbr": "I" + }, + { + "name": "Job_Profitability", + "source": "openml", + "id": 44311, + "abbr": "JP" + }, + { + "name": "Kaggle_bike_sharing_demand_challange", + "source": "openml", + "id": 1414, + "abbr": "KBS" + }, + { + "name": "MiamiHousing2016", + "source": "openml", + "id": 43093, + "abbr": "M" + }, + { + "name": "Moneyball", + "source": "openml", + "id": 41021, + "abbr": "M" + }, + { + "name": "NASA_PHM2008", + "source": "openml", + "id": 42821, + "abbr": "NP" + }, + { + "name": "OnlineNewsPopularity", + "source": "openml", + "id": 4545, + "abbr": "O" + }, + { + "name": "SAT11-HAND-runtime-regression", + "source": "openml", + "id": 41980, + "abbr": "S" + }, + { + "name": "analcatdata_supreme", + "source": "openml", + "id": 504, + "abbr": "AS" + }, + { + "name": "avocado_sales", + "source": "openml", + "id": 43927, + "abbr": "AS" + }, + { + "name": "bank32nh", + "source": "openml", + "id": 558, + "abbr": "B" + }, + { + "name": "bank8FM", + "source": "openml", + "id": 572, + "abbr": "B" + }, + { + "name": "boston", + "source": "openml", + "id": 531, + "abbr": "B" + }, + { + "name": "chscase_foot", + "source": "openml", + "id": 703, + "abbr": "CF" + }, + { + "name": "colleges", + "source": "openml", + "id": 42727, + "abbr": "C" + }, + { + "name": "dataset_sales", + "source": "openml", + "id": 42183, + "abbr": "DS" + }, + { + "name": "debutanizer", + "source": "openml", + "id": 23516, + "abbr": "D" + }, + { + "name": "delta_elevators", + "source": "openml", + "id": 198, + "abbr": "DE" + }, + { + "name": "fifa", + "source": "openml", + "id": 44026, + "abbr": "F" + }, + { + "name": "house_16H_reg", + "source": "openml", + "id": 574, + "abbr": "H1R" + }, + { + "name": "house_prices_nominal", + "source": "openml", + "id": 42563, + "abbr": "HPN" + }, + { + "name": "kin8nm", + "source": "openml", + "id": 189, + "abbr": "K" + }, + { + "name": "mauna-loa-atmospheric-co2", + "source": "openml", + "id": 41187, + "abbr": "M" + }, + { + "name": "pol_reg", + "source": "openml", + "id": 201, + "abbr": "PR" + }, + { + "name": "puma32H", + "source": "openml", + "id": 308, + "abbr": "P" + }, + { + "name": "sensory", + "source": "openml", + "id": 546, + "abbr": "S" + }, + { + "name": "socmob", + "source": "openml", + "id": 541, + "abbr": "S" + }, + { + "name": "space_ga", + "source": "openml", + "id": 507, + "abbr": "SG" + }, + { + "name": "stock_fardamento02", + "source": "openml", + "id": 42545, + "abbr": "SF" + }, + { + "name": "sulfur", + "source": "openml", + "id": 23515, + "abbr": "S" + }, + { + "name": "topo_2_1", + "source": "openml", + "id": 422, + "abbr": "T21" + }, + { + "name": "us_crime", + "source": "openml", + "id": 42730, + "abbr": "UC" + }, + { + "name": "weather_izmir", + "source": "openml", + "id": 42369, + "abbr": "WI" + }, + { + "name": "yprop_4_1", + "source": "openml", + "id": 416, + "abbr": "Y41" + } +] \ No newline at end of file diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json new file mode 100644 index 0000000..1f835b3 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/openboost_manifest.json @@ -0,0 +1,124 @@ +{ + "arguments": { + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "1028_SWD" + ], + "dataset_registry": null, + "development_run": true, + "learning_rate": 0.03, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_cpu" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31926664340", + "source_sha": "db7d430649e779287d06539a0527848ffbf815b6", + "tested_sha": "db7d430649e779287d06539a0527848ffbf815b6" + }, + "created_at": "2026-08-16T04:34:19+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "scoringbench_dynamic", + "resolved_sha256": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "source_sha256": null + }, + "datasets": [ + { + "id": "https://github.com/EpistasisLab/penn-ml-benchmarks/raw/master/datasets/1028_SWD/1028_SWD.tsv.gz", + "name": "1028_SWD", + "source": "pmlb" + } + ], + "expected_result_rows": 5, + "model_parameters": { + "openboost_cpu": { + "backend": "cpu", + "learning_rate": 0.03, + "max_depth": 3, + "model_params": { + "min_child_weight": 1.0, + "reg_lambda": 1.0 + }, + "n_quantiles": 99, + "n_trees": 500 + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "db7d430649e779287d06539a0527848ffbf815b6", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 5 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 5, + "schema_version": 2, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "versions": { + "catboost": null, + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.3.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.18.0", + "torch": "2.13.0", + "xgboost": null, + "xgboostlss": null + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet new file mode 100644 index 0000000..8567250 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json new file mode 100644 index 0000000..5cd1987 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/summary.json @@ -0,0 +1,79 @@ +{ + "artifacts": { + "baseline_lr_001": { + "digest": "sha256:495ac3c8c02c0846423131c4636dfde05f0da872b637bca2be9493b3c02aa8b4", + "id": 9258026048, + "source_sha": "24e9873a0eff442a33ab0f970624bb8f9c4a6f36", + "workflow_run_id": 31926255124 + }, + "candidate_lr_003": { + "digest": "sha256:b3b77cbf691bef574b0a27c897b8a92a22e90a4c158f0f7e3ae556400d60e599", + "id": 9258115955, + "source_sha": "db7d430649e779287d06539a0527848ffbf815b6", + "workflow_run_id": 31926664340 + } + }, + "candidate_vs_baseline": { + "coverage_90_abs_error": { + "baseline_mean": 0.030999999046325687, + "candidate_lower_wins": 3, + "candidate_mean": 0.03200000524520876, + "candidate_minus_baseline": 0.0010000061988830744, + "relative_percent": 3.2258265472482375 + }, + "crps": { + "baseline_mean": 0.35507492380808053, + "candidate_lower_wins": 2, + "candidate_mean": 0.3566396681314354, + "candidate_minus_baseline": 0.0015647443233548497, + "relative_percent": 0.4406800419960381 + }, + "interval_score_90": { + "baseline_mean": 2.559808672169682, + "candidate_lower_wins": 1, + "candidate_mean": 2.6811327630567057, + "candidate_minus_baseline": 0.12132409088702367, + "relative_percent": 4.739576524060681 + }, + "pit_ks_stat": { + "baseline_mean": 0.08734850000489737, + "candidate_lower_wins": 3, + "candidate_mean": 0.09592410150802438, + "candidate_minus_baseline": 0.008575601503127014, + "relative_percent": 9.817686053734416 + }, + "rmse": { + "baseline_mean": 0.6260427399635591, + "candidate_lower_wins": 3, + "candidate_mean": 0.6278397715490043, + "candidate_minus_baseline": 0.0017970315854451968, + "relative_percent": 0.28704615048324 + }, + "sharpness": { + "baseline_mean": 0.5801309335762859, + "candidate_lower_wins": 5, + "candidate_mean": 0.5489976021750633, + "candidate_minus_baseline": -0.03113333140122254, + "relative_percent": -5.366604261092824 + } + }, + "dataset": "1028_SWD", + "decision": "reject_lr_003", + "file_sha256": { + "baseline_lr_001/benchmark_outcome.json": "55f203900d837ce575c17eb55e165590c9e4b7cc33a29e3f4b363bb006934048", + "baseline_lr_001/datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "baseline_lr_001/openboost_manifest.json": "43c1a91fb8b6c76ece5a9557389a8f1fc4f9bda403c6cd1c604a0ddb62db5c26", + "baseline_lr_001/raw/catboost_quantile/1028_SWD.parquet": "1be2fab8e079e9203c23c7a50b23d83b7798bce3a35e7cf6ee8374a35ac18a63", + "baseline_lr_001/raw/ngboost/1028_SWD.parquet": "4cd7c3de7e764ab296b4c3712335626537dbb0c961a6ea448907ba68c9d3b1d4", + "baseline_lr_001/raw/openboost_cpu/1028_SWD.parquet": "20f5e2fd14d086cf2f14f78447c0816d3155ca5e9b9fe0228366678fe7e6051e", + "baseline_lr_001/raw/xgblss/1028_SWD.parquet": "b525e10262bc178e961b8cb1761f7b5aa1b397430e8bb9be8fee69b03036a3d2", + "baseline_lr_001/raw/xgboost_quantile/1028_SWD.parquet": "fabfacadaef81cdf354f5fe7a571391c4d6168f73766a92a444b9adc7c38b21f", + "candidate_lr_003/benchmark_outcome.json": "cefba05ad79fdb6056f55cba5afda606d79984537856a41868b7e4a88c12e4a1", + "candidate_lr_003/datasets.json": "c81d1dbc444de36dff5e02975664fc34c06a057e9de012a1fa998534b461526c", + "candidate_lr_003/openboost_manifest.json": "b0e660512c9762fd7e1109b1d9f77d8b0e552424863f0197db2aaafab39d0182", + "candidate_lr_003/raw/openboost_cpu/1028_SWD.parquet": "7672d89594c4cad05f413a0181fdf15b838534aa60a5ba4eab4998be343c7296" + }, + "folds": 5, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4" +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md new file mode 100644 index 0000000..d78397a --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/README.md @@ -0,0 +1,51 @@ +# 197_cpu_act HistogramBoost preregistered development result + +This is development/tuning evidence, not held-out, confirmation, or +leaderboard evidence. It is the first run of the frozen HistogramBoost +candidate on the preregistered `197_cpu_act` development dataset. The candidate +used 50 distribution bins, 100 shared-vector trees, learning rate 0.05, depth +6, and PSD Gauss--Newton curvature scale 1. The benchmark used five folds, +seed 42, a 3000-row cap, and pinned ScoringBench commit `a938a667`. + +The strong baseline was selected once per dataset as the lower-mean-CRPS model +between native XGBoost quantile and CatBoost MultiQuantile. CatBoost was the +selected baseline. + +| Model | CRPS | RMSE | 90% coverage | 90% interval score | Sharpness | Mean fit time (s) | +|---|---:|---:|---:|---:|---:|---:| +| CatBoost quantile | **1.326236** | **2.582497** | 71.73% | **13.329529** | **1.556934** | 144.43 | +| HistogramBoost | 1.354233 | 2.591390 | **98.23%** | 13.496925 | 6.476879 | 84.34 | +| XGBoost quantile | 1.620136 | 3.698760 | 67.47% | 17.979673 | 1.619687 | **13.59** | + +HistogramBoost did **not** pass the frozen development gate. Its CRPS was +2.111% worse than CatBoost, just beyond the 2% non-inferiority threshold, and +it won only two of five folds rather than the required three. Its absolute +90% coverage error was 8.23 percentage points rather than at most five. The +interval-score and RMSE guardrails passed. Therefore the untouched +`537_houses` confirmation dataset remains locked. + +There is a useful but limited positive signal: HistogramBoost beat the native +XGBoost quantile baseline on all five folds, lowering mean CRPS by 16.41%, +90% interval score by 24.93%, and RMSE by 29.94%. This is one development +dataset and cannot support an overall win claim. + +The failure shape is informative. HistogramBoost's mean prediction RMSE was +within 0.34% of CatBoost, but its distribution was much wider: sharpness +6.48 versus 1.56 and 98.23% empirical coverage at the nominal 90% interval. +The next development work should diagnose why the finite-bin PMF retains too +much tail mass before changing model capacity. It should not use the untouched +confirmation dataset. + +Within this single four-core Actions run, HistogramBoost fit in 84.34 seconds +per fold on average: 1.71x faster than CatBoost but 6.21x slower than XGBoost. +These timings are implementation diagnostics, not a general speed claim. + +Run: [31930502694](https://github.com/jxucoder/openboost/actions/runs/31930502694), +artifact `9259424361`, digest +`sha256:9b8d5676727cbece7a61ac1067c6705089cb69b9e8cc3718bd0e79e34b056f6a`. +The artifact contains 15/15 valid rows with no errors, missing rows, duplicates, +or non-finite metrics. OpenBoost source `835335a` and ScoringBench were clean; +the compressed dataset matched the preregistered SHA-256 +`d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc`. +The fail-closed evaluator was committed as `3cbd763` before this result was +observed. diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json new file mode 100644 index 0000000..9c8629e --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "197_cpu_act", + "expected_rows": 15, + "observed_rows": 15, + "status": "complete", + "valid_rows": 15 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 15, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 15, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 15 +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json new file mode 100644 index 0000000..ae5b6de --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/candidate_evaluation.json @@ -0,0 +1,50 @@ +{ + "accepted": false, + "baselines": [ + "xgboost_quantile", + "catboost_quantile" + ], + "candidate": "openboost_histogram_cpu", + "comparisons": { + "candidate_fold_wins": 2, + "coverage_error_improvement": 0.10033333301544194, + "crps_ratio": 1.02111053670206, + "interval_score_ratio": 1.0125582707407659, + "rmse_ratio": 1.0034435540465978 + }, + "confirmation_dataset_win": false, + "dataset": "197_cpu_act", + "development_pass": false, + "guardrails": { + "at_least_3_of_5_fold_wins": false, + "complete_15_rows": true, + "coverage_error_at_most_0_05": false, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": false, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "phase": "development", + "schema_version": 1, + "selected_strong_baseline": "catboost_quantile", + "summaries": { + "catboost_quantile": { + "mean_abs_coverage_90_error": 0.1826666831970215, + "mean_crps": 1.3262356982847625, + "mean_interval_score_90": 13.329529331871402, + "mean_rmse": 2.5824970354629198 + }, + "openboost_histogram_cpu": { + "mean_abs_coverage_90_error": 0.08233335018157957, + "mean_crps": 1.3542332456689852, + "mean_interval_score_90": 13.496925170068025, + "mean_rmse": 2.591390003579715 + }, + "xgboost_quantile": { + "mean_abs_coverage_90_error": 0.22533334493637086, + "mean_crps": 1.6201360866516095, + "mean_interval_score_90": 17.979672910724513, + "mean_rmse": 3.698760308978361 + } + } +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json new file mode 100644 index 0000000..54674d0 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/datasets.json @@ -0,0 +1,20 @@ +[ + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "target_col": "target", + "abbr": "1CA", + "openboost_role": "development", + "raw_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/537_houses/537_houses.tsv.gz", + "target_col": "target", + "abbr": "5H", + "openboost_role": "untouched_confirmation", + "raw_sha256": "d383fd58bb79760ffaf0b615942fd47d926a19e75a8f980b79fbe9ad479d9cbf" + } +] diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json new file mode 100644 index 0000000..3d9527b --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/openboost_manifest.json @@ -0,0 +1,156 @@ +{ + "arguments": { + "allow_confirmation": false, + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "197_cpu_act" + ], + "dataset_registry": "benchmarks/scoringbench/protocols/crps_distribution_v1.json", + "development_run": true, + "histogram_bins": 50, + "histogram_curvature_scale": 1.0, + "histogram_learning_rate": 0.05, + "histogram_max_depth": 6, + "histogram_rounds": 100, + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_histogram_cpu", + "xgboost_quantile", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "nll", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31930502694", + "source_sha": "835335aa9b5fe89708eb757aef16c4fc60472574", + "tested_sha": "835335aa9b5fe89708eb757aef16c4fc60472574" + }, + "created_at": "2026-08-16T06:28:35+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "frozen_file", + "resolved_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "source_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1" + }, + "datasets": [ + { + "id": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "name": "197_cpu_act", + "source": "pmlb" + } + ], + "expected_result_rows": 15, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "openboost_histogram_cpu": { + "curvature_scale": 1.0, + "learning_rate": 0.05, + "max_depth": 6, + "n_distribution_bins": 50, + "n_feature_bins": 254, + "n_trees": 100 + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "835335aa9b5fe89708eb757aef16c4fc60472574", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 15 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 15, + "schema_version": 3, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "verified_dataset_files": [ + { + "name": "197_cpu_act", + "sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc", + "size_bytes": 381809, + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz" + } + ], + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/catboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/catboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000..6fff9b2 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/catboost_quantile/197_cpu_act.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/openboost_histogram_cpu/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/openboost_histogram_cpu/197_cpu_act.parquet new file mode 100644 index 0000000..5b90333 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/openboost_histogram_cpu/197_cpu_act.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/xgboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/xgboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000..d0d04f5 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/raw/xgboost_quantile/197_cpu_act.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json new file mode 100644 index 0000000..79ae949 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_20260816/summary.json @@ -0,0 +1,79 @@ +{ + "artifact": { + "digest": "sha256:9b8d5676727cbece7a61ac1067c6705089cb69b9e8cc3718bd0e79e34b056f6a", + "id": 9259424361, + "source_sha": "835335aa9b5fe89708eb757aef16c4fc60472574", + "workflow_run_id": 31930502694 + }, + "comparisons": { + "catboost_quantile": { + "crps_fold_wins": 2, + "crps_relative_percent": 2.111053670206009, + "interval_score_90_relative_percent": 1.2558270740765876, + "rmse_relative_percent": 0.34435540465977965, + "train_time_ratio": 0.583997739143586 + }, + "xgboost_quantile": { + "crps_fold_wins": 5, + "crps_relative_percent": -16.41237690916291, + "interval_score_90_relative_percent": -24.93230974175631, + "rmse_relative_percent": -29.938958269629378, + "train_time_ratio": 6.205556146272291 + } + }, + "dataset": "197_cpu_act", + "decision": "reject_candidate_keep_confirmation_locked", + "evaluator": { + "commit": "3cbd76393043654429b8edb792d688b37a9de46f", + "development_pass": false, + "selected_strong_baseline": "catboost_quantile" + }, + "file_sha256": { + "benchmark_outcome.json": "36da3bc6bac10df15a1860f2b2c0dfecca6031db1e3dbd6c0a8e69a628cbd06e", + "candidate_evaluation.json": "8f2b2ab2b548ec580a49af2854d105b261fb8f733a0851cb8984544e9bb7727c", + "datasets.json": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "openboost_manifest.json": "dad19f279b8ad37c64313d2dbc4f30c4dcf46cb5135a8e53ce5063677f046dbe", + "raw/catboost_quantile/197_cpu_act.parquet": "e9ced85f09ce1125a3ff7a18dd1053ee091b3496c4433330df6c5aecbf6c8e51", + "raw/openboost_histogram_cpu/197_cpu_act.parquet": "a9e7b244b4ed7b1f10a3a28123ea05939f06d8c375f01b747dd0e33c74a300cc", + "raw/xgboost_quantile/197_cpu_act.parquet": "43af95887d4706021fc626f80af9b1030c45bb0962c1e5f0b6fab7c1f1185158" + }, + "folds": 5, + "guardrails": { + "at_least_3_of_5_fold_wins": false, + "complete_15_rows": true, + "coverage_error_at_most_0_05": false, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": false, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "protocol": "development_tuning", + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4", + "summaries": { + "catboost_quantile": { + "coverage_90": 0.7173333168029785, + "crps": 1.3262356982847625, + "interval_score_90": 13.329529331871402, + "rmse": 2.5824970354629198, + "sharpness": 1.5569338635111154, + "train_time": 144.42591681480408 + }, + "openboost_histogram_cpu": { + "coverage_90": 0.9823333501815796, + "crps": 1.3542332456689852, + "interval_score_90": 13.496925170068025, + "rmse": 2.591390003579715, + "sharpness": 6.476878909743213, + "train_time": 84.34440889358521 + }, + "xgboost_quantile": { + "coverage_90": 0.6746666550636291, + "crps": 1.6201360866516095, + "interval_score_90": 17.979672910724513, + "rmse": 3.698760308978361, + "sharpness": 1.6196870388235536, + "train_time": 13.591756629943848 + } + }, + "verified_dataset_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md new file mode 100644 index 0000000..daf1358 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/README.md @@ -0,0 +1,65 @@ +# 197_cpu_act HistogramBoost V2 development result + +This is consumed development/tuning evidence, not held-out confirmation or +leaderboard evidence. It evaluates the frozen HistogramBoost V2 candidate on +the preregistered `197_cpu_act` development dataset. The run used five folds, +seed 42, a 3000-row cap, clean OpenBoost source `39bdb63`, and clean +ScoringBench source `a938a667`. + +V2 used 50 distribution bins, 100 shared-vector trees, learning rate 0.05, +depth 6, and PSD Gauss--Newton curvature scale 1. It trained against the exact +piecewise-uniform histogram CRPS, interpreted `base_smoothing=1` as a total +Dirichlet concentration, selected a temperature from +`[0.5, 0.7, 0.85, 1.0, 1.2]` on an inner 80/20 split of each outer training +fold, refit on the full outer training fold, and losslessly subdivided every +output bin into two equal-density bins for evaluation. The extra inner fit is +included in training time. Outer-test targets were not used for model or +temperature selection. + +| Model | CRPS | RMSE | Official 90% coverage | Official 90% interval score | Sharpness | Mean fit time (s) | +|---|---:|---:|---:|---:|---:|---:| +| HistogramBoost V2 | **1.287204** | **2.551506** | **94.80%** | **11.281116** | 3.719920 | 161.41 | +| CatBoost quantile | 1.326236 | 2.582497 | 71.73% | 13.329529 | **1.556934** | 141.00 | +| XGBoost quantile | 1.620136 | 3.698760 | 67.47% | 17.979673 | 1.619687 | **14.80** | + +HistogramBoost V2 passed the frozen development gate. Relative to the +once-per-dataset selected strong baseline, CatBoost, it lowered mean CRPS by +2.943%, won four of five folds, lowered RMSE by 1.200%, and lowered the +official 90% interval score by 15.367%. Its mean absolute nominal-90% coverage +error was 4.80 percentage points, within the frozen five-point guardrail. + +Against the frozen native XGBoost quantile baseline, HistogramBoost V2 won all +five folds and lowered mean CRPS by 20.550%, RMSE by 31.017%, and the official +90% interval score by 37.256%. This is a single consumed development dataset, +so it does not establish an overall, full-suite, or SOTA win. + +The calibration metrics need careful interpretation. ScoringBench extracts +intervals using whole bin edges, so its coverage and interval scores are +representation-sensitive. V2's two-way subdivision preserves the represented +density, physical CRPS, mean, and continuous variance, while reducing the +coarse-bin envelope error. The official numbers above are therefore pinned +protocol metrics, not standalone calibration claims. Sharpness is diagnostic +only and is not part of the acceptance gate. + +Within this one four-core Actions run, V2 averaged 161.41 seconds per fold, +including the inner calibration fit. It was 1.145x as slow as CatBoost and +10.907x as slow as XGBoost. These are implementation diagnostics, not general +speed claims. They make vector-tree GPU acceleration a concrete next systems +target now that the quality hypothesis has a positive signal. + +The frozen evaluator returned `development_pass=true`, but an independent +protocol audit found that this commit's workflow did not execute that evaluator +inside CI and that the confirmation path was not yet phase-bound. Therefore +the untouched `537_houses` confirmation dataset remains locked until the gate +and provenance fixes are committed. The evaluator's +`confirmation_dataset_win=true` field is phase-agnostic and must not be read as +a confirmation result for this development run. + +Run: [31932958804](https://github.com/jxucoder/openboost/actions/runs/31932958804), +job `95130441713`, artifact `9260176320`, digest +`sha256:572672e818cf60a295826f7b057bad0269f5121f5bfc31a258188eea12234adc`. +The artifact contains 15/15 valid rows with no errors, missing rows, +duplicates, unexpected rows, or non-finite metrics. The compressed dataset +matched the frozen SHA-256 +`d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc`. + diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json new file mode 100644 index 0000000..9c8629e --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/benchmark_outcome.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "dataset": "197_cpu_act", + "expected_rows": 15, + "observed_rows": 15, + "status": "complete", + "valid_rows": 15 + } + ], + "duplicate_rows": [], + "error_rows": [], + "expected_rows": 15, + "invalid_metric_rows": [], + "missing_rows": [], + "observed_rows": 15, + "schema_version": 1, + "status": "complete", + "unexpected_rows": [], + "valid_rows": 15 +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json new file mode 100644 index 0000000..20b1f3a --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/candidate_evaluation.json @@ -0,0 +1,50 @@ +{ + "accepted": true, + "baselines": [ + "xgboost_quantile", + "catboost_quantile" + ], + "candidate": "openboost_histogram_cpu_v2", + "comparisons": { + "candidate_fold_wins": 4, + "coverage_error_improvement": 0.13466669321060185, + "crps_ratio": 0.9705694692862287, + "interval_score_ratio": 0.8463251301218064, + "rmse_ratio": 0.9879996039440585 + }, + "confirmation_dataset_win": true, + "dataset": "197_cpu_act", + "development_pass": true, + "guardrails": { + "at_least_3_of_5_fold_wins": true, + "complete_15_rows": true, + "coverage_error_at_most_0_05": true, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": true, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "phase": "development", + "schema_version": 1, + "selected_strong_baseline": "catboost_quantile", + "summaries": { + "catboost_quantile": { + "mean_abs_coverage_90_error": 0.1826666831970215, + "mean_crps": 1.3262356982847625, + "mean_interval_score_90": 13.329529331871402, + "mean_rmse": 2.5824970354629198 + }, + "openboost_histogram_cpu_v2": { + "mean_abs_coverage_90_error": 0.047999989986419654, + "mean_crps": 1.2872038778326929, + "mean_interval_score_90": 11.2811156462585, + "mean_rmse": 2.55150604822407 + }, + "xgboost_quantile": { + "mean_abs_coverage_90_error": 0.22533334493637086, + "mean_crps": 1.6201360866516095, + "mean_interval_score_90": 17.979672910724513, + "mean_rmse": 3.698760308978361 + } + } +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json new file mode 100644 index 0000000..54674d0 --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/datasets.json @@ -0,0 +1,20 @@ +[ + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "target_col": "target", + "abbr": "1CA", + "openboost_role": "development", + "raw_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/537_houses/537_houses.tsv.gz", + "target_col": "target", + "abbr": "5H", + "openboost_role": "untouched_confirmation", + "raw_sha256": "d383fd58bb79760ffaf0b615942fd47d926a19e75a8f980b79fbe9ad479d9cbf" + } +] diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json new file mode 100644 index 0000000..ba8aeea --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/openboost_manifest.json @@ -0,0 +1,176 @@ +{ + "arguments": { + "allow_confirmation": false, + "catboost_rounds": 1000, + "dataset_index": null, + "dataset_name": [ + "197_cpu_act" + ], + "dataset_registry": "benchmarks/scoringbench/protocols/crps_distribution_v1.json", + "development_run": true, + "histogram_bins": 50, + "histogram_curvature_scale": 1.0, + "histogram_learning_rate": 0.05, + "histogram_max_depth": 6, + "histogram_rounds": 100, + "histogram_v2_calibration_fraction": 0.2, + "histogram_v2_calibration_seed": 42, + "histogram_v2_evaluation_subdivisions": 2, + "histogram_v2_temperature_grid": [ + 0.5, + 0.7, + 0.85, + 1.0, + 1.2 + ], + "learning_rate": 0.01, + "list_datasets": false, + "lite": false, + "max_depth": 3, + "min_child_weight": 1.0, + "models": [ + "openboost_histogram_cpu_v2", + "xgboost_quantile", + "catboost_quantile" + ], + "n_folds": 5, + "n_quantiles": 99, + "n_repeats": 1, + "n_trees": 500, + "output_dir": "/home/runner/work/_temp/scoringbench-quality", + "reg_lambda": 1.0, + "sample_size": 3000, + "scoringbench_dir": ".repos/ScoringBench", + "seed": 42, + "shard_count": null, + "shard_index": null, + "smoke": false, + "training_objective": "nll", + "xgblss_rounds": 100, + "xgboost_quantiles": 50, + "xgboost_rounds": 100 + }, + "ci": { + "event_name": "workflow_dispatch", + "head_ref": "", + "provider": "github_actions", + "ref": "refs/heads/codex/scoringbench-release-hardening", + "repository": "jxucoder/openboost", + "run_attempt": "1", + "run_id": "31932958804", + "source_sha": "39bdb632c3cc374161e12e2e6230d649a933b576", + "tested_sha": "39bdb632c3cc374161e12e2e6230d649a933b576" + }, + "created_at": "2026-08-16T07:35:14+00:00", + "dataset_registry": { + "file": "datasets.json", + "mode": "frozen_file", + "resolved_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "source_sha256": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1" + }, + "datasets": [ + { + "id": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "name": "197_cpu_act", + "source": "pmlb" + } + ], + "expected_result_rows": 15, + "model_parameters": { + "catboost_quantile": { + "catboost_params": { + "allow_writing_files": false, + "random_seed": 42, + "thread_count": 2 + }, + "iterations": 1000, + "n_quantiles": 99 + }, + "openboost_histogram_cpu_v2": { + "calibration_fraction": 0.2, + "calibration_seed": 42, + "curvature_scale": 1.0, + "evaluation_subdivisions": 2, + "learning_rate": 0.05, + "max_depth": 6, + "n_distribution_bins": 50, + "n_feature_bins": 254, + "n_trees": 100, + "temperature_grid": [ + 0.5, + 0.7, + 0.85, + 1.0, + 1.2 + ] + }, + "xgboost_quantile": { + "n_bins": 50, + "num_boost_round": 100, + "xgb_params": { + "device": "cpu", + "nthread": 2, + "seed": 42 + } + } + }, + "official_protocol_compatible": false, + "openboost_git": { + "changes": [], + "commit": "39bdb632c3cc374161e12e2e6230d649a933b576", + "dirty": false + }, + "outcome": { + "duplicate_rows": 0, + "error_rows": 0, + "file": "benchmark_outcome.json", + "invalid_metric_rows": 0, + "missing_rows": 0, + "status": "complete", + "unexpected_rows": 0, + "valid_rows": 15 + }, + "platform": { + "cpu_count": 4, + "gpu": null, + "machine": "x86_64", + "processor": "x86_64", + "python": "3.12.13", + "release": "6.17.0-1022-azure", + "system": "Linux" + }, + "protocol": "ScoringBench", + "protocol_mode": "development_tuning", + "result_rows": 15, + "schema_version": 3, + "scoringbench_git": { + "changes": [], + "commit": "a938a667b7839b41e9272929010573410301c0b4", + "dirty": false + }, + "verified_dataset_files": [ + { + "name": "197_cpu_act", + "sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc", + "size_bytes": 381809, + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz" + } + ], + "versions": { + "catboost": "1.2.10", + "cupy-cuda12x": null, + "ngboost": "0.5.11", + "numba": "0.67.0", + "numba-cuda": null, + "numpy": "2.2.6", + "openboost": "1.0.0rc1", + "pandas": "2.2.3", + "pyarrow": "25.0.1", + "scikit-learn": "1.9.0", + "scipy": "1.16.3", + "torch": "2.9.1", + "xgboost": "3.3.0", + "xgboostlss": "0.6.1" + }, + "warning": "This is a development/tuning run and must not be represented as held-out leaderboard evidence." +} diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/catboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/catboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000..0c5803a Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/catboost_quantile/197_cpu_act.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/openboost_histogram_cpu_v2/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/openboost_histogram_cpu_v2/197_cpu_act.parquet new file mode 100644 index 0000000..247bab3 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/openboost_histogram_cpu_v2/197_cpu_act.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/xgboost_quantile/197_cpu_act.parquet b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/xgboost_quantile/197_cpu_act.parquet new file mode 100644 index 0000000..a5cce79 Binary files /dev/null and b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/raw/xgboost_quantile/197_cpu_act.parquet differ diff --git a/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/summary.json b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/summary.json new file mode 100644 index 0000000..07472fc --- /dev/null +++ b/benchmarks/evidence/scoringbench/development/197_cpu_act_histogram_crps_v2_20260816/summary.json @@ -0,0 +1,87 @@ +{ + "artifact": { + "digest": "sha256:572672e818cf60a295826f7b057bad0269f5121f5bfc31a258188eea12234adc", + "id": 9260176320, + "job_id": 95130441713, + "run_attempt": 1, + "source_sha": "39bdb632c3cc374161e12e2e6230d649a933b576", + "workflow_run_id": 31932958804 + }, + "comparisons": { + "catboost_quantile": { + "crps_fold_wins": 4, + "crps_relative_percent": -2.943053071377133, + "interval_score_90_relative_percent": -15.367486987819357, + "rmse_relative_percent": -1.2000396055941498, + "train_time_ratio": 1.1447814739687 + }, + "xgboost_quantile": { + "crps_fold_wins": 5, + "crps_relative_percent": -20.549644660221045, + "interval_score_90_relative_percent": -37.256279898565104, + "rmse_relative_percent": -31.017264297160573, + "train_time_ratio": 10.907460281929694 + } + }, + "dataset": "197_cpu_act", + "decision": "development_gate_passed_confirmation_locked_pending_protocol_hardening", + "evaluator": { + "candidate": "openboost_histogram_cpu_v2", + "development_pass": true, + "file_sha256": "839cfe1ed01d7e878196b095d61a041374cdfeda3b35f2cb1a89441cf7ca2e81", + "selected_strong_baseline": "catboost_quantile" + }, + "file_sha256": { + "benchmark_outcome.json": "36da3bc6bac10df15a1860f2b2c0dfecca6031db1e3dbd6c0a8e69a628cbd06e", + "candidate_evaluation.json": "0afa20cb09e8b4f2ddeab1d4e8f176ca4ad5d16102a373a86656f44f835d0f99", + "datasets.json": "29d4ca6fdd1281452e598b85fe4151092b82e9d2bfc1cfc7ceb9ee1a10df82e1", + "openboost_manifest.json": "ef0e72dc84a641652999df908f4ddb3c768ea52150a9ad2a62afbf5f121d8d25", + "raw/catboost_quantile/197_cpu_act.parquet": "ba72c6d8f859962bdb91f23839da3daf5db65db4976f169f793fb4310385dfa2", + "raw/openboost_histogram_cpu_v2/197_cpu_act.parquet": "0492e7f966ccfb681dbd44aa470253eb4449380e6b3390a6531494bfb5bc0c65", + "raw/xgboost_quantile/197_cpu_act.parquet": "41f0c99cb410e2f6e5d18f1996745ae7e9d27dc1dfbdc4d10037fd0cfffcebbc" + }, + "folds": 5, + "guardrails": { + "at_least_3_of_5_fold_wins": true, + "complete_15_rows": true, + "coverage_error_at_most_0_05": true, + "coverage_error_improves_by_0_02": true, + "crps_ratio_at_most_1_02": true, + "interval_score_ratio_at_most_1_05": true, + "rmse_ratio_at_most_1_05": true + }, + "protocol": { + "file": "benchmarks/scoringbench/protocols/crps_distribution_v2.md", + "file_sha256": "0cb82e07b5c37b5f2ff5635c7e1104462b275e9d9e36dde8eafb7389b32a74d9", + "mode": "development_tuning" + }, + "scoringbench_sha": "a938a667b7839b41e9272929010573410301c0b4", + "summaries": { + "catboost_quantile": { + "coverage_90": 0.7173333168029785, + "crps": 1.3262356982847625, + "interval_score_90": 13.329529331871402, + "rmse": 2.5824970354629198, + "sharpness": 1.5569338635111154, + "train_time": 140.99584250450135 + }, + "openboost_histogram_cpu_v2": { + "coverage_90": 0.9479999899864197, + "crps": 1.2872038778326929, + "interval_score_90": 11.2811156462585, + "rmse": 2.55150604822407, + "sharpness": 3.719920012731918, + "train_time": 161.40942840576173 + }, + "xgboost_quantile": { + "coverage_90": 0.6746666550636291, + "crps": 1.6201360866516095, + "interval_score_90": 17.979672910724513, + "rmse": 3.698760308978361, + "sharpness": 1.6196870388235536, + "train_time": 14.79807620048523 + } + }, + "verified_dataset_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc", + "workflow_sha256": "34245371c90b9651b4e6d14f4caf7cf21a4cd892e0c867dfe3a6866917a641bb" +} diff --git a/benchmarks/scoringbench/README.md b/benchmarks/scoringbench/README.md new file mode 100644 index 0000000..55c2b52 --- /dev/null +++ b/benchmarks/scoringbench/README.md @@ -0,0 +1,306 @@ +# OpenBoost on ScoringBench + +[ScoringBench](https://github.com/jonaslandsgesell/ScoringBench) is an external +benchmark for probabilistic regression. It evaluates complete predictive +distributions with proper scoring rules and publishes accepted results at +[scoringbench.com](https://scoringbench.com/). This integration uses its dataset +loader, folds, metrics and Parquet schema without modifying the checkout. + +This is the primary third-party value benchmark for OpenBoost. It answers two +different questions with two deliberately separate protocols: + +1. **Official quality track**: ScoringBench's default 3,000-row cap and full + dataset suite. These results can be proposed for its public leaderboard. +2. **Scale extension**: selected ScoringBench datasets with a larger or removed + row cap. This measures OpenBoost's CPU/CUDA scaling but must not be presented + as an official ScoringBench leaderboard result. + +## What counts as a good model + +NGBoost is a canonical natural-gradient reference, not the product bar. The +quality target is the strongest practical boosting alternative in the same +ScoringBench protocol. The acceptance comparison set is: + +- ScoringBench's XGBoost multi-quantile wrapper (`xgboost_quantile`); +- Gaussian XGBoostLSS (`xgblss`), which is the closest full-distribution + XGBoost-family competitor; +- CatBoost MultiQuantile (`catboost_quantile`); +- NGBoost as the method/reference baseline. + +OpenBoost is a **good ScoringBench model** only when the completed full suite +places it first or statistically tied for first on primary proper scores, it +beats the strongest XGBoost-family baseline on a majority of paired datasets, +and the result is not purchased with a material regression in interval score, +calibration, point RMSE, or failure coverage. CRPS is the cross-model +optimization target; interval score/coverage, RMSE, failures, and training time +remain explicit guardrails. A single shard decides what to debug, not whether +this target has been achieved. + +ScoringBench's current reconstructed log score and CRLS are not acceptance +metrics for this comparison. Finite quantile support clamps out-of-support +targets to a boundary-bin density, while CRLS is integrated over each model's +own support. Compare Gaussian models with a separately audited analytic NLL; +only restore a cross-model density score after validating one common grid, +resolution, support, and tail rule. + +The diagnostic deliberately uses the budgets registered by ScoringBench +rather than forcing every implementation to share one arbitrary tree count: +100 rounds/50 quantiles for XGBoost quantile, 100 Gaussian rounds for +XGBoostLSS, 1,000 iterations/99 quantiles for CatBoost, and 500 rounds for +OpenBoost/NGBoost. These choices and resolved package versions are recorded in +the manifest. Its timing is descriptive; a later speed claim requires a +quality-matched compute sweep. + +The preregistered `crps_distribution_v1` development experiment adds +`openboost_histogram_cpu`: 50 target bins, 100 shared-vector trees, learning +rate 0.05, depth 6, and positive-semidefinite Gauss--Newton curvature scale 1. +It is a development candidate, not an official leaderboard row. Its immutable +dataset roles, raw-file hashes, acceptance thresholds, and confirmation lock +are recorded under `benchmarks/scoringbench/protocols/`. + +## Environment + +Use a separate Linux environment because ScoringBench currently constrains +NumPy to `>=2,<2.3` and imports PyTorch for its metrics. Intel macOS is not +supported by the complete launcher: the available PyTorch wheel uses the NumPy +1.x ABI and can crash with ScoringBench's NumPy 2.x requirement. Published CPU +and CUDA measurements should come from Linux in any case. The smaller wrapper +contract test remains useful for local adapter development. + +```bash +git clone https://github.com/jonaslandsgesell/ScoringBench .repos/ScoringBench +git -C .repos/ScoringBench checkout "$(cat benchmarks/scoringbench/SCORINGBENCH_COMMIT)" + +uv venv .venv-scoringbench --python 3.12 +uv pip install --python .venv-scoringbench/bin/python \ + -r benchmarks/scoringbench/requirements.txt +uv pip install --python .venv-scoringbench/bin/python -e . +``` + +`SCORINGBENCH_COMMIT` freezes the upstream protocol used for committed results. +Test newer upstream revisions separately before updating that file. The manifest +records the checked-out revision and dirty state. + +On Intel macOS, the latest Numba release may not publish a compatible wheel. +The full benchmark remains unsupported there, but the wrapper contract can be +checked with the last compatible wheel instead of compiling llvmlite locally: + +```bash +uv pip install --python .venv-scoringbench/bin/python 'numba==0.63.1' +uv pip install --python .venv-scoringbench/bin/python --no-deps -e . +``` + +For CUDA, install OpenBoost's CUDA extra using the package versions appropriate +for the benchmark machine: + +```bash +uv pip install --python .venv-scoringbench/bin/python -e '.[cuda]' +``` + +Optional comparison models: + +```bash +uv pip install --python .venv-scoringbench/bin/python xgboostlss catboost +``` + +For the frozen strong-baseline environment used by CI diagnostics: + +```bash +uv pip install --python .venv-scoringbench/bin/python \ + -r benchmarks/scoringbench/requirements-strong-baselines.txt +``` + +## Validate the adapter + +This uses one existing sklearn dataset and the complete ScoringBench metrics, +but is only an integration smoke test: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --smoke \ + --n-trees 20 \ + --output-dir /tmp/openboost-scoringbench-smoke +``` + +The smaller wrapper contract test can also be run directly: + +```bash +PYTHONPATH=.repos/ScoringBench \ + .venv-scoringbench/bin/python -m pytest \ + benchmarks/scoringbench/test_openboost_wrapper.py -q +``` + +The `ScoringBench` GitHub workflow runs this contract, the two-fold smoke, and +a `1027_ESL` quality sentinel on relevant pull requests, then uploads the raw +Parquet files and manifests. The PMLB/GitHub-backed sentinel avoids making the +basic quality gate depend on OpenML dataset uptime. It uses the official +five-fold, 3,000-row protocol and default 500 rounds; one dataset is still not a +quality claim. Manual `quality_shard` mode accepts an exact dataset name so the +suite can be sharded without downloading every dataset just to resolve an +index. Only a completed full suite belongs in a leaderboard submission. + +Every launcher invocation also writes `benchmark_outcome.json`. ScoringBench's +upstream runner deliberately catches dataset/model exceptions so the remaining +campaign can continue; therefore a zero return from upstream is not proof of a +complete shard. OpenBoost audits every expected dataset/model/fold row, rejects +duplicates, captured model errors, missing rows, and non-finite core metrics, +then exits non-zero when that audit is incomplete. The failure report remains +in the uploaded artifact and is evidence, not disposable CI noise. + +The manifest's `official_protocol_compatible` field means only that the run has +ScoringBench's 3,000-row, five-fold, one-repeat shape and was not marked as +development tuning. It does not certify leaderboard acceptance, immutable +dataset bytes, equal compute budgets, or statistical sufficiency. + +## Official quality track + +Run the official default: five folds, one repeat, at most 3,000 rows per +dataset. Start with OpenBoost and the existing NGBoost wrapper: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --output-dir benchmarks/results/scoringbench-quality +``` + +Run one strong-baseline diagnostic before changing model behavior: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost,xgboost_quantile,xgblss,catboost_quantile \ + --dataset-name 1027_ESL \ + --sample-size 3000 \ + --n-folds 5 \ + --n-repeats 1 \ + --output-dir benchmarks/results/scoringbench-strong-diagnostic +``` + +After freezing that baseline, test candidate OpenBoost settings on a different +development dataset. `--development-run` deliberately makes the manifest +ineligible for official evidence even though it retains the same five folds and +metrics: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu \ + --dataset-name 1028_SWD \ + --sample-size 3000 \ + --n-folds 5 \ + --n-trees 250 \ + --learning-rate 0.04 \ + --training-objective crps \ + --max-depth 2 \ + --reg-lambda 1.0 \ + --min-child-weight 1.0 \ + --development-run \ + --output-dir benchmarks/results/scoringbench-development +``` + +Record every tried configuration, including failures. Select one configuration +using only development datasets; do not repeatedly inspect the held-out suite. +`--training-objective crps` is an explicit development candidate for Gaussian +CRPS. It does not change `eval_metric`, is not enabled by default, and must not +be described as an official result until it is frozen and rerun on untouched +data. + +Use `--dataset-index N` or `--dataset-name NAME` for resumable shards. Use +`--list-datasets` to display the validated list. Do not tune OpenBoost on the +test folds. If hyperparameters are changed, apply the same declared search +budget to every comparison model. + +For a parallel campaign, freeze the resolved upstream registry and partition +that exact ordered list into stable strided shards. The first sentinel's +registry is committed as evidence and can seed the first full campaign: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,ngboost \ + --dataset-registry \ + benchmarks/evidence/scoringbench/1027_esl_20260816/datasets.json \ + --shard-index 0 \ + --shard-count 10 \ + --output-dir benchmarks/results/scoringbench-quality/shard-0 +``` + +Run every index from zero through `shard-count - 1`. The union covers each +registry entry exactly once. Both the source-registry and copied artifact +hashes are recorded, so a campaign cannot silently mix changing OpenML suite +membership across jobs. + +After all shards complete, run ScoringBench's own aggregation and autoranking: + +```bash +cd .repos/ScoringBench +python aggregate_datasets.py \ + --raw_dir ../../benchmarks/results/scoringbench-quality/raw \ + --out_dir ../../benchmarks/results/scoringbench-quality +python autorank_leaderboard.py --output_dir ../../benchmarks/results/scoringbench-quality +``` + +For an upstream submission, copy `openboost_wrapper.py` into +`scoringbench/wrappers/`, register `OpenBoostWrapper` in the upstream wrapper +exports and add a zero-argument factory to its `MODELS` dictionary. Submit the +wrapper, raw/aggregated Parquet artifacts and leaderboard JSON for independent +review. + +## ScoringBench scale extension + +First identify large datasets from the official list, then run the same folds +and metrics without the 3,000-row cap. CPU and CUDA are separate model names so +their results cannot be confused: + +```bash +.venv-scoringbench/bin/python benchmarks/scoringbench/run.py \ + --scoringbench-dir .repos/ScoringBench \ + --models openboost_cpu,openboost_cuda,ngboost \ + --dataset-name '' \ + --sample-size 0 \ + --n-folds 5 \ + --output-dir benchmarks/results/scoringbench-scale +``` + +Run each CUDA measurement in a fresh process. Report cold and repeated runs +separately, and include failures/OOMs. The generated `openboost_manifest.json` +records both git commits, dirty state, arguments, package versions, platform and +GPU identity. In GitHub Actions it also distinguishes the tested merge commit +from the pull request's source-head commit. Runs whose manifest says +`scoringbench_scale_extension` are not official leaderboard runs. +ScoringBench's resolved `datasets.json` is redirected into the output directory +so the exact registry travels with official artifacts without dirtying the +OpenBoost checkout. + +Generated ScoringBench directories are gitignored. Publish accepted evidence in +ScoringBench's designated output/LFS repository or intentionally force-add a +frozen artifact; do not commit an arbitrary local smoke run. + +The first frozen official-protocol sentinel is under +`benchmarks/evidence/scoringbench/1027_esl_20260816/`. It preserves the raw +Parquet rows and documents both favorable and unfavorable metrics. Do not +generalize that single-dataset result into a library-level claim. + +## Evidence gate + +OpenBoost should claim value only after all of the following are true: + +- the wrapper and results are accepted upstream by ScoringBench; +- quality is reported across the full suite, not a selected winning subset; +- paired fold-level CRPS/interval-score differences include + uncertainty intervals or the upstream statistical ranking; +- CPU and CUDA predictions pass a separate parity gate; +- a scale curve uses at least three real datasets and multiple data sizes; +- a speed claim is made only at matched predictive quality, with raw Parquet + files and `openboost_manifest.json` published. + +The benchmark is allowed to disprove the product hypothesis. If OpenBoost is +not competitive on proper scoring rules or does not accelerate at larger row +counts, the result should be published and the implementation fixed before the +README makes a performance claim. diff --git a/benchmarks/scoringbench/SCORINGBENCH_COMMIT b/benchmarks/scoringbench/SCORINGBENCH_COMMIT new file mode 100644 index 0000000..b823077 --- /dev/null +++ b/benchmarks/scoringbench/SCORINGBENCH_COMMIT @@ -0,0 +1 @@ +a938a667b7839b41e9272929010573410301c0b4 diff --git a/benchmarks/scoringbench/__init__.py b/benchmarks/scoringbench/__init__.py new file mode 100644 index 0000000..ee2d4bb --- /dev/null +++ b/benchmarks/scoringbench/__init__.py @@ -0,0 +1,6 @@ +"""OpenBoost integration for the external ScoringBench benchmark. + +The adapter is not imported here because ScoringBench is an optional external +checkout. Import ``benchmarks.scoringbench.openboost_wrapper`` explicitly after +putting that checkout on ``PYTHONPATH``. +""" diff --git a/benchmarks/scoringbench/evaluate_crps_candidate.py b/benchmarks/scoringbench/evaluate_crps_candidate.py new file mode 100644 index 0000000..739323c --- /dev/null +++ b/benchmarks/scoringbench/evaluate_crps_candidate.py @@ -0,0 +1,177 @@ +"""Evaluate a preregistered ScoringBench CRPS candidate from raw Parquet rows.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path + +METRICS = ("crps", "coverage_90", "interval_score_90", "rmse") + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) + + +def _finite_number(value) -> bool: + if value is None or isinstance(value, bool): + return False + try: + return math.isfinite(float(value)) + except (TypeError, ValueError): + return False + + +def evaluate_records( + records: list[dict], + *, + candidate: str, + baselines: tuple[str, str], + n_folds: int = 5, + phase: str = "development", +) -> dict: + """Apply the frozen CRPS/coverage/interval/RMSE acceptance rules.""" + if phase not in {"development", "confirmation"}: + raise ValueError("phase must be 'development' or 'confirmation'") + models = (candidate, *baselines) + datasets = {str(record.get("dataset")) for record in records} + if len(datasets) != 1: + raise ValueError(f"expected exactly one dataset, got {sorted(datasets)}") + dataset = datasets.pop() + + indexed: dict[tuple[str, int], dict] = {} + for record in records: + model = str(record.get("model")) + if model not in models: + continue + try: + fold = int(record["fold"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"invalid fold identity: {record.get('fold')!r}") from exc + key = (model, fold) + if key in indexed: + raise ValueError(f"duplicate result row: model={model}, fold={fold}") + error = record.get("error") + if error is not None and str(error).strip() and str(error).lower() != "nan": + raise ValueError(f"captured model error: model={model}, fold={fold}: {error}") + invalid = [metric for metric in METRICS if not _finite_number(record.get(metric))] + if invalid: + raise ValueError(f"non-finite metrics for model={model}, fold={fold}: {invalid}") + indexed[key] = record + + expected = {(model, fold) for model in models for fold in range(n_folds)} + missing = sorted(expected - set(indexed)) + unexpected = sorted(set(indexed) - expected) + if missing or unexpected or len(records) != len(expected): + raise ValueError( + f"incomplete rows: expected={len(expected)}, observed={len(records)}, " + f"missing={missing}, unexpected={unexpected}" + ) + + summaries = {} + for model in models: + rows = [indexed[(model, fold)] for fold in range(n_folds)] + summaries[model] = { + "mean_crps": _mean([float(row["crps"]) for row in rows]), + "mean_abs_coverage_90_error": _mean( + [abs(float(row["coverage_90"]) - 0.9) for row in rows] + ), + "mean_interval_score_90": _mean([float(row["interval_score_90"]) for row in rows]), + "mean_rmse": _mean([float(row["rmse"]) for row in rows]), + } + + baseline = min(baselines, key=lambda name: summaries[name]["mean_crps"]) + candidate_summary = summaries[candidate] + baseline_summary = summaries[baseline] + crps_ratio = candidate_summary["mean_crps"] / baseline_summary["mean_crps"] + fold_wins = sum( + float(indexed[(candidate, fold)]["crps"]) <= float(indexed[(baseline, fold)]["crps"]) + for fold in range(n_folds) + ) + coverage_improvement = ( + baseline_summary["mean_abs_coverage_90_error"] + - candidate_summary["mean_abs_coverage_90_error"] + ) + interval_ratio = ( + candidate_summary["mean_interval_score_90"] / baseline_summary["mean_interval_score_90"] + ) + rmse_ratio = candidate_summary["mean_rmse"] / baseline_summary["mean_rmse"] + + guardrails = { + "complete_15_rows": len(indexed) == 3 * n_folds, + "crps_ratio_at_most_1_02": crps_ratio <= 1.02, + "at_least_3_of_5_fold_wins": fold_wins >= 3, + "coverage_error_at_most_0_05": (candidate_summary["mean_abs_coverage_90_error"] <= 0.05), + "coverage_error_improves_by_0_02": coverage_improvement >= 0.02, + "interval_score_ratio_at_most_1_05": interval_ratio <= 1.05, + "rmse_ratio_at_most_1_05": rmse_ratio <= 1.05, + } + development_pass = all(guardrails.values()) + confirmation_win = development_pass and crps_ratio < 1.0 and fold_wins >= 4 + + return { + "schema_version": 1, + "dataset": dataset, + "phase": phase, + "candidate": candidate, + "baselines": list(baselines), + "selected_strong_baseline": baseline, + "summaries": summaries, + "comparisons": { + "crps_ratio": crps_ratio, + "candidate_fold_wins": fold_wins, + "coverage_error_improvement": coverage_improvement, + "interval_score_ratio": interval_ratio, + "rmse_ratio": rmse_ratio, + }, + "guardrails": guardrails, + "development_pass": development_pass, + "confirmation_dataset_win": confirmation_win, + "accepted": development_pass if phase == "development" else confirmation_win, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("result_dir", type=Path) + parser.add_argument("--candidate", default="openboost_histogram_cpu") + parser.add_argument( + "--baselines", + default="xgboost_quantile,catboost_quantile", + help="Exactly two comma-separated baseline model names", + ) + parser.add_argument("--phase", choices=("development", "confirmation"), default="development") + parser.add_argument("--output", type=Path) + return parser + + +def main() -> int: + args = _parser().parse_args() + baselines = tuple(part.strip() for part in args.baselines.split(",") if part.strip()) + if len(baselines) != 2: + raise SystemExit("--baselines must contain exactly two model names") + + import pandas as pd + + parquet_files = sorted((args.result_dir / "raw").glob("*/*.parquet")) + if not parquet_files: + raise SystemExit(f"no raw Parquet files found under {args.result_dir / 'raw'}") + records = [] + for path in parquet_files: + records.extend(pd.read_parquet(path).to_dict(orient="records")) + result = evaluate_records( + records, + candidate=args.candidate, + baselines=baselines, + phase=args.phase, + ) + payload = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(payload) + print(payload, end="") + return 0 if result["accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/scoringbench/openboost_wrapper.py b/benchmarks/scoringbench/openboost_wrapper.py new file mode 100644 index 0000000..e38691e --- /dev/null +++ b/benchmarks/scoringbench/openboost_wrapper.py @@ -0,0 +1,298 @@ +"""ScoringBench wrappers for OpenBoost distributional models. + +This module intentionally lives in OpenBoost's repository while the integration +is being validated. It is also shaped as an upstream-ready ScoringBench wrapper: +copy it to ``scoringbench/wrappers/openboost_wrapper.py`` and update the upstream +registry when submitting benchmark results. +""" + +from __future__ import annotations + +from contextlib import nullcontext + +import numpy as np + +try: + from scoringbench.wrappers.base import DistributionPrediction, ProbabilisticWrapper + from scoringbench.wrappers.quantile_based import quantiles_to_distribution +except ImportError as exc: # pragma: no cover - depends on the external checkout + raise ImportError( + "OpenBoostWrapper requires a ScoringBench checkout on PYTHONPATH. " + "See benchmarks/scoringbench/README.md." + ) from exc + + +class OpenBoostWrapper(ProbabilisticWrapper): + """OpenBoost NaturalBoost with a Gaussian predictive distribution. + + Parameters mirror ScoringBench's NGBoost Gaussian entry by default: 500 + boosting rounds, learning rate 0.01, depth-3 trees and 99 quantile levels. + The backend is explicit so CPU and CUDA results cannot be accidentally + conflated on a leaderboard. + + Parameters + ---------- + backend: + ``"cpu"``, ``"cuda"`` or ``"auto"``. ``"auto"`` uses OpenBoost's + normal backend detection; reproducible benchmark runs should use an + explicit backend. + n_trees: + Number of NaturalBoost rounds. + learning_rate: + Boosting shrinkage. + max_depth: + Maximum depth of each parameter tree. + n_bins: + Histogram bins. OpenBoost reserves bin 255 for missing values, so 254 + is the largest non-warning value. + n_quantiles: + Number of probability levels used to convert the analytic Normal + distribution into ScoringBench's common PMF representation. + model_params: + Additional keyword arguments forwarded to ``NaturalBoostNormal``. + """ + + _VALID_BACKENDS = {"auto", "cpu", "cuda"} + + def __init__( + self, + *, + backend: str = "cpu", + n_trees: int = 500, + learning_rate: float = 0.01, + max_depth: int = 3, + n_bins: int = 254, + n_quantiles: int = 99, + model_params: dict | None = None, + ) -> None: + backend = backend.lower() + if backend not in self._VALID_BACKENDS: + raise ValueError( + f"backend must be one of {sorted(self._VALID_BACKENDS)}, got {backend!r}" + ) + if n_quantiles < 2: + raise ValueError("n_quantiles must be at least 2") + + self.backend = backend + self.n_trees = n_trees + self.learning_rate = learning_rate + self.max_depth = max_depth + self.n_bins = n_bins + self.n_quantiles = n_quantiles + self.model_params = dict(model_params or {}) + + self._alphas = np.linspace( + 1 / (n_quantiles + 1), + n_quantiles / (n_quantiles + 1), + n_quantiles, + dtype=np.float64, + ) + self._model = None + self._resolved_backend: str | None = None + self._y_range = (0.0, 1.0) + + @staticmethod + def _sanitize_X(X) -> np.ndarray: + X = np.asarray(X, dtype=np.float32) + if X.ndim != 2: + raise ValueError(f"X must be 2-dimensional, got shape {X.shape}") + return np.nan_to_num(X, nan=0.0, posinf=1e7, neginf=-1e7) + + def _backend_context(self): + import openboost as ob + + if self._resolved_backend is None: + return nullcontext() + return ob.backend_context(self._resolved_backend) + + def _require_fitted(self) -> None: + if self._model is None: + raise RuntimeError("Model not fitted. Call fit() first.") + + def fit(self, X, y) -> OpenBoostWrapper: + import openboost as ob + + X = self._sanitize_X(X) + y = np.asarray(y, dtype=np.float32).reshape(-1) + valid = np.isfinite(y) + X, y = X[valid], y[valid] + if len(y) == 0: + raise ValueError("No valid finite training samples") + + lo, hi = float(y.min()), float(y.max()) + if lo == hi: + pad = max(abs(lo) * 1e-6, 1e-7) + lo, hi = lo - pad, hi + pad + self._y_range = (lo, hi) + + self._resolved_backend = ob.get_backend() if self.backend == "auto" else self.backend + params = { + "n_trees": self.n_trees, + "learning_rate": self.learning_rate, + "max_depth": self.max_depth, + "n_bins": self.n_bins, + **self.model_params, + } + self._model = ob.NaturalBoostNormal(**params) + with self._backend_context(): + self._model.fit(X, y) + return self + + def predict(self, X) -> np.ndarray: + self._require_fitted() + X = self._sanitize_X(X) + with self._backend_context(): + pred = self._model.predict(X) + return np.asarray(pred, dtype=np.float64).reshape(-1) + + def predict_distribution(self, X) -> DistributionPrediction: + self._require_fitted() + X = self._sanitize_X(X) + with self._backend_context(): + output = self._model.predict_distribution(X) + mean = np.asarray(output.mean(), dtype=np.float64).reshape(-1) + quantiles = np.column_stack( + [ + np.asarray(output.quantile(float(alpha)), dtype=np.float64) + for alpha in self._alphas + ] + ) + + return quantiles_to_distribution( + quantiles, + self._alphas, + mean=mean, + y_range=self._y_range, + ) + + +class OpenBoostHistogramWrapper(ProbabilisticWrapper): + """OpenBoost shared-tree histogram distribution trained by CRPS. + + The defaults are the candidate frozen before the + ``crps_distribution_v1`` development run: 50 target bins, 100 trees, + learning rate 0.05, depth 6, and Gauss--Newton curvature scale 1. + """ + + def __init__( + self, + *, + n_distribution_bins: int = 50, + n_trees: int = 100, + learning_rate: float = 0.05, + max_depth: int = 6, + n_feature_bins: int = 254, + curvature_scale: float = 1.0, + temperature_grid: tuple[float, ...] = (1.0,), + calibration_fraction: float = 0.2, + calibration_seed: int = 42, + evaluation_subdivisions: int = 1, + model_params: dict | None = None, + ) -> None: + self.n_distribution_bins = n_distribution_bins + self.n_trees = n_trees + self.learning_rate = learning_rate + self.max_depth = max_depth + self.n_feature_bins = n_feature_bins + self.curvature_scale = curvature_scale + self.temperature_grid = tuple(float(value) for value in temperature_grid) + self.calibration_fraction = calibration_fraction + self.calibration_seed = calibration_seed + self.evaluation_subdivisions = evaluation_subdivisions + self.model_params = dict(model_params or {}) + self._model = None + self._selected_temperature = 1.0 + self._temperature_scores: dict[float, float] = {} + + @staticmethod + def _sanitize_X(X) -> np.ndarray: + X = np.asarray(X, dtype=np.float32) + if X.ndim != 2: + raise ValueError(f"X must be 2-dimensional, got shape {X.shape}") + # HistogramBoost handles NaN explicitly; only infinities need a finite + # sentinel to match the other ScoringBench wrappers. + return np.nan_to_num(X, nan=np.nan, posinf=1e7, neginf=-1e7) + + def _require_fitted(self) -> None: + if self._model is None: + raise RuntimeError("Model not fitted. Call fit() first.") + + def fit(self, X, y) -> OpenBoostHistogramWrapper: + import openboost as ob + + X = self._sanitize_X(X) + y = np.asarray(y, dtype=np.float32).reshape(-1) + valid = np.isfinite(y) + X, y = X[valid], y[valid] + if len(y) == 0: + raise ValueError("No valid finite training samples") + if not self.temperature_grid or any( + not np.isfinite(value) or value <= 0.0 for value in self.temperature_grid + ): + raise ValueError("temperature_grid must contain positive finite values") + if not 0.0 < self.calibration_fraction < 1.0: + raise ValueError("calibration_fraction must lie in (0, 1)") + if ( + isinstance(self.evaluation_subdivisions, bool) + or not isinstance(self.evaluation_subdivisions, (int, np.integer)) + or self.evaluation_subdivisions < 1 + ): + raise ValueError("evaluation_subdivisions must be a positive integer") + + params = { + "n_distribution_bins": self.n_distribution_bins, + "n_trees": self.n_trees, + "learning_rate": self.learning_rate, + "max_depth": self.max_depth, + "n_feature_bins": self.n_feature_bins, + "curvature_scale": self.curvature_scale, + **self.model_params, + } + + self._selected_temperature = 1.0 + self._temperature_scores = {} + if len(self.temperature_grid) > 1 and len(y) >= 4: + rng = np.random.default_rng(self.calibration_seed) + indices = rng.permutation(len(y)) + n_calibration = min( + max(1, int(round(self.calibration_fraction * len(y)))), + len(y) - 2, + ) + calibration_idx = indices[:n_calibration] + inner_train_idx = indices[n_calibration:] + calibration_model = ob.HistogramBoost(**params).fit( + X[inner_train_idx], + y[inner_train_idx], + ) + calibration_output = calibration_model.predict_distribution(X[calibration_idx]) + for temperature in self.temperature_grid: + score = np.mean(calibration_output.tempered(temperature).crps(y[calibration_idx])) + self._temperature_scores[temperature] = float(score) + self._selected_temperature = min( + self.temperature_grid, + key=lambda value: ( + self._temperature_scores[value], + abs(value - 1.0), + value, + ), + ) + + self._model = ob.HistogramBoost(**params).fit(X, y) + return self + + def predict(self, X) -> np.ndarray: + return np.asarray(self.predict_distribution(X).mean, dtype=np.float64).reshape(-1) + + def predict_distribution(self, X) -> DistributionPrediction: + self._require_fitted() + output = self._model.predict_distribution(self._sanitize_X(X)).tempered( + self._selected_temperature + ) + output = output.subdivide(self.evaluation_subdivisions) + return DistributionPrediction( + probas=np.asarray(output.probas, dtype=np.float64), + bin_edges=np.asarray(output.bin_edges, dtype=np.float64), + bin_midpoints=np.asarray(output.bin_midpoints, dtype=np.float64), + mean=np.asarray(output.mean(), dtype=np.float64), + is_natively_gridded_model=True, + ) diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v1.json b/benchmarks/scoringbench/protocols/crps_distribution_v1.json new file mode 100644 index 0000000..54674d0 --- /dev/null +++ b/benchmarks/scoringbench/protocols/crps_distribution_v1.json @@ -0,0 +1,20 @@ +[ + { + "name": "197_cpu_act", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/197_cpu_act/197_cpu_act.tsv.gz", + "target_col": "target", + "abbr": "1CA", + "openboost_role": "development", + "raw_sha256": "d00fd6bac2eda0821a04ff10277663d211127745a19a474c0feee41a16914fdc" + }, + { + "name": "537_houses", + "source": "pmlb", + "url": "https://media.githubusercontent.com/media/EpistasisLab/pmlb/9cc9017958f2d8284e62d8bc54b77cb6fa1e9592/datasets/537_houses/537_houses.tsv.gz", + "target_col": "target", + "abbr": "5H", + "openboost_role": "untouched_confirmation", + "raw_sha256": "d383fd58bb79760ffaf0b615942fd47d926a19e75a8f980b79fbe9ad479d9cbf" + } +] diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v1.md b/benchmarks/scoringbench/protocols/crps_distribution_v1.md new file mode 100644 index 0000000..535f819 --- /dev/null +++ b/benchmarks/scoringbench/protocols/crps_distribution_v1.md @@ -0,0 +1,83 @@ +# CRPS distribution experiment v1 + +This protocol was frozen before loading either dataset or observing any result +on them. It separates architecture development from confirmation after the +`1027_ESL` and `1028_SWD` diagnostics were consumed. + +## Frozen data roles + +- Development: `197_cpu_act`, a continuous regression dataset with 8,192 rows + and 21 machine-performance features. +- Untouched confirmation: `537_houses`, a continuous housing/population + regression dataset with 8 numeric features. Do not load, validate, or run + this entry until one candidate implementation and configuration have been + frozen after the development result. The launcher enforces this registry + role unless the confirmation run explicitly supplies `--allow-confirmation`. + +Both URLs in `crps_distribution_v1.json` point to PMLB commit +`9cc9017958f2d8284e62d8bc54b77cb6fa1e9592`. The registry also records the +expected SHA-256 of each compressed source file. A run must fail closed if the +downloaded bytes do not match. The mutable PMLB `master` URLs are ineligible +for this experiment. + +## Protocol + +- ScoringBench commit: + `a938a667b7839b41e9272929010573410301c0b4`. +- Five folds, one repeat, seed 42, sample cap 3,000, CPU execution. +- Compare the frozen OpenBoost candidate with ScoringBench's native XGBoost + multi-quantile model and CatBoost MultiQuantile model. +- XGBoost: 100 rounds and 50 quantiles, seed 42, two threads. +- CatBoost: 1,000 rounds and 99 quantiles, seed 42, two threads. +- Use an empty output directory. Exactly 15 finite result rows must exist. +- CRPS is primary. Log score and CRLS are excluded because the pinned evaluator + does not put different finite-support predictions on a common support. + +The candidate family is a non-parametric histogram distribution trained by +direct CRPS gradients with one shared tree and vector leaves per round. Only a +positive-semidefinite Gauss--Newton curvature is eligible; the earlier +absolute-value transform of an indefinite exact diagonal is rejected. The +candidate's exact public API and hyperparameters must be committed before the +development dataset is loaded. + +The frozen candidate is `openboost_histogram_cpu`, backed by: + +```python +openboost.HistogramBoost( + n_distribution_bins=50, + n_trees=100, + learning_rate=0.05, + max_depth=6, + n_feature_bins=254, + curvature_scale=1.0, +) +``` + +All other constructor values remain at the committed defaults. The wrapper +passes its regular shared grid and PMF directly to ScoringBench as a natively +gridded prediction; it does not derive or regrid quantiles. + +## Development acceptance + +Let `B` be whichever of XGBoost quantile and CatBoost MultiQuantile has the +lower five-fold mean CRPS. Choose `B` once per dataset, never separately for +each fold. A candidate passes only if every condition holds: + +1. all 15 expected rows are present, finite, and error-free; +2. `mean_CRPS(candidate) / mean_CRPS(B) <= 1.02`; +3. candidate CRPS is no greater than `B` on at least three of five folds; +4. candidate mean absolute 90% coverage error is at most 0.05 and at least + 0.02 lower than `B`; +5. candidate mean 90% interval score is at most `1.05 * B`; +6. candidate mean RMSE is at most `1.05 * B`. + +Passing development allows one frozen run on `537_houses`; it is not evidence +of a general win. + +## Confirmation language + +On `537_houses`, OpenBoost may be described as having lower CRPS on that +dataset only when its mean CRPS ratio to `B` is below 1.00 and it wins at least +four of five folds. A ratio at or below 1.02 that also satisfies the guardrails +is only calibrated non-inferiority. Neither outcome is an overall or SOTA +claim. The full ScoringBench suite remains the product acceptance test. diff --git a/benchmarks/scoringbench/protocols/crps_distribution_v2.md b/benchmarks/scoringbench/protocols/crps_distribution_v2.md new file mode 100644 index 0000000..b1d972f --- /dev/null +++ b/benchmarks/scoringbench/protocols/crps_distribution_v2.md @@ -0,0 +1,92 @@ +# CRPS distribution experiment V2 + +This protocol freezes the second HistogramBoost candidate before its full +five-fold `197_cpu_act` run. The V1 five-fold artifact and V2 fold-0 diagnostics +are consumed development evidence. `537_houses` remains untouched and locked. + +## Why V2 exists + +V1 beat the frozen native XGBoost quantile baseline on all five folds, but was +2.111% worse than CatBoost on mean CRPS, won only two of five CatBoost folds, +and retained an over-wide PMF. Audits identified three correctable causes: + +1. training used a midpoint ranked-probability approximation while evaluation + used exact piecewise-uniform histogram CRPS; +2. additive smoothing was applied once per bin, so total prior strength grew + with output resolution; +3. whole-bin interval extraction made the 50-bin representation coarser than + the CatBoost representation. + +V2 aligns the training score with evaluation, treats smoothing as total +Dirichlet concentration, selects probability temperature using only an inner +split of each outer training fold, and losslessly subdivides bins for evaluation. + +## Frozen data and benchmark + +- Development: `197_cpu_act` from the commit-pinned registry + `crps_distribution_v1.json`. +- Untouched confirmation: `537_houses` from the same registry. The launcher + must reject it unless `--allow-confirmation` is explicit. +- Dataset bytes and SHA-256 values are unchanged from V1. +- ScoringBench commit: + `a938a667b7839b41e9272929010573410301c0b4`. +- Five folds, one repeat, seed 42, sample cap 3000, CPU execution. +- Baselines: native XGBoost quantile with 100 rounds and 50 quantiles; CatBoost + MultiQuantile with 1000 rounds and 99 quantiles. Both use two threads. +- Exactly 15 finite, error-free result rows are required. +- CRPS is primary. Log score, CRLS, CDE, and DPD are excluded from decisions + because support and density-grid differences make them unsuitable here. + +## Frozen V2 candidate + +The benchmark model name is `openboost_histogram_cpu_v2`: + +```python +OpenBoostHistogramWrapper( + n_distribution_bins=50, + n_trees=100, + learning_rate=0.05, + max_depth=6, + n_feature_bins=254, + curvature_scale=1.0, + temperature_grid=(0.5, 0.7, 0.85, 1.0, 1.2), + calibration_fraction=0.2, + calibration_seed=42, + evaluation_subdivisions=2, +) +``` + +All `HistogramBoost` constructor values not shown remain at committed defaults: +split and leaf L2 regularization both resolve to 1, and total base prior weight +is 1. Each outer fold performs these steps: + +1. split only the outer training rows into 80% inner-train and 20% calibration; +2. fit the candidate on inner-train and choose temperature by mean exact CRPS + on calibration rows; +3. refit a fresh candidate on every outer training row; +4. apply the frozen selected temperature to outer-test probabilities; +5. divide every uniform training bin into two equal-density evaluation bins. + +Outer-test targets never select rounds, bins, temperature, support, or any +other hyperparameter. Subdivision preserves the represented density, exact +CRPS, mean, and variance; it only reduces whole-bin quantile-envelope error. +The additional inner fit is included in reported training time. + +## Development acceptance + +Let `B` be whichever baseline has lower five-fold mean CRPS, selected once for +the dataset rather than per fold. V2 passes only if every condition holds: + +1. all 15 expected rows are present, finite, and error-free; +2. `mean_CRPS(V2) / mean_CRPS(B) <= 1.02`; +3. V2 CRPS is no greater than `B` on at least three of five folds; +4. V2 mean absolute 90% coverage error is at most 0.05 and at least 0.02 lower + than `B`; +5. V2 mean 90% interval score is at most `1.05 * B`; +6. V2 mean RMSE is at most `1.05 * B`. + +Passing this development gate allows exactly one frozen confirmation run. It +does not establish an overall win. On `537_houses`, a dataset-level CRPS win +requires ratio below 1.00 and at least four of five fold wins, plus every +guardrail above. Full-suite paired results remain necessary for the product +goal. diff --git a/benchmarks/scoringbench/requirements-strong-baselines.txt b/benchmarks/scoringbench/requirements-strong-baselines.txt new file mode 100644 index 0000000..f29da26 --- /dev/null +++ b/benchmarks/scoringbench/requirements-strong-baselines.txt @@ -0,0 +1,5 @@ +# Strong practical baselines for the manually dispatched diagnostic/full suite. +# Keep these exact so a campaign cannot silently change model implementations. +xgboost==3.3.0 +xgboostlss==0.6.1 +catboost==1.2.10 diff --git a/benchmarks/scoringbench/requirements.txt b/benchmarks/scoringbench/requirements.txt new file mode 100644 index 0000000..9e7ac5a --- /dev/null +++ b/benchmarks/scoringbench/requirements.txt @@ -0,0 +1,12 @@ +# Base ScoringBench runtime, deliberately excluding its many heavyweight model +# extras. Install only the additional baselines you plan to run. +numpy>=2.0,<2.3 +scikit-learn>=1.3 +pandas>=2.0 +torch>=2.0 +pyarrow>=15 +autorank>=1.2 +openml>=0.15 +pytest>=7 +pytest-xdist>=3 +ngboost>=0.5 diff --git a/benchmarks/scoringbench/run.py b/benchmarks/scoringbench/run.py new file mode 100644 index 0000000..313bf34 --- /dev/null +++ b/benchmarks/scoringbench/run.py @@ -0,0 +1,938 @@ +"""Run OpenBoost through an unmodified ScoringBench checkout. + +The official suite owns datasets, folds, metrics and Parquet output. This +launcher only registers OpenBoost (plus selected existing baselines), records +provenance and exposes a small smoke mode for integration testing. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import math +import os +import platform +import subprocess +import sys +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = PROJECT_ROOT / "src" + + +def _csv(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +def _float_csv(value: str) -> tuple[float, ...]: + try: + result = tuple(float(item) for item in _csv(value)) + except ValueError as exc: + raise argparse.ArgumentTypeError("expected comma-separated numbers") from exc + if not result: + raise argparse.ArgumentTypeError("expected at least one number") + return result + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_dataset_registry(path: Path) -> list[dict]: + """Load and minimally validate a frozen ScoringBench dataset registry.""" + payload = json.loads(path.read_text()) + datasets = payload.get("datasets") if isinstance(payload, dict) else payload + if not isinstance(datasets, list) or not datasets: + raise ValueError(f"dataset registry must contain a non-empty list: {path}") + if any(not isinstance(dataset, dict) or not dataset.get("name") for dataset in datasets): + raise ValueError(f"every dataset registry entry must be an object with a name: {path}") + names = [dataset["name"].casefold() for dataset in datasets] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"duplicate case-insensitive dataset names in {path}: {duplicates}") + return datasets + + +def _resolve_dataset_registry_path(value: str | None) -> Path | None: + """Resolve a CLI registry path before changing into the artifact directory.""" + if value is None: + return None + return Path(value).expanduser().resolve() + + +def _verify_dataset_files(datasets: list[dict], ensure_cached) -> list[dict]: + """Materialize and verify dataset files pinned by a frozen registry. + + ScoringBench's processed cache is intentionally fast but cannot prove which + raw bytes produced an entry. A registry may therefore provide + ``raw_sha256``. Those entries are downloaded through ScoringBench's own raw + cache and checked before validation or fold construction. + """ + verified = [] + for dataset in datasets: + expected = dataset.get("raw_sha256") + if expected is None: + continue + expected = str(expected).lower() + if len(expected) != 64 or any(char not in "0123456789abcdef" for char in expected): + raise ValueError(f"invalid raw_sha256 for dataset {dataset['name']!r}: {expected!r}") + if dataset.get("source") != "pmlb" or not dataset.get("url"): + raise ValueError( + "raw_sha256 verification currently requires a PMLB URL; " + f"dataset {dataset['name']!r} has source={dataset.get('source')!r}" + ) + + filename = f"{dataset['name']}.tsv.gz" + path = Path(ensure_cached(dataset["name"], dataset["url"], filename)) + actual = _sha256(path) + if actual != expected: + raise ValueError( + f"raw dataset hash mismatch for {dataset['name']!r}: " + f"expected {expected}, got {actual} at {path}" + ) + verified.append( + { + "name": dataset["name"], + "url": dataset["url"], + "sha256": actual, + "size_bytes": path.stat().st_size, + } + ) + return verified + + +def _enforce_dataset_role_lock(datasets: list[dict], *, allow_confirmation: bool) -> None: + """Prevent accidental observation of preregistered confirmation data.""" + locked = [ + dataset["name"] + for dataset in datasets + if dataset.get("openboost_role") == "untouched_confirmation" + ] + if locked and not allow_confirmation: + raise ValueError( + "confirmation dataset is still locked; freeze the candidate first, " + "then rerun with --allow-confirmation: " + ", ".join(locked) + ) + + +def _git_state(path: Path) -> dict: + def run(*args: str) -> str | None: + try: + result = subprocess.run( + ["git", "-C", str(path), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + status = run("status", "--porcelain") + return { + "commit": run("rev-parse", "HEAD"), + "dirty": bool(status) if status is not None else None, + "changes": status.splitlines() if status else [], + } + + +def _package_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _gpu_info() -> dict | None: + try: + from numba import cuda + + if not cuda.is_available(): + return None + device = cuda.get_current_device() + name = device.name.decode() if isinstance(device.name, bytes) else str(device.name) + return { + "name": name, + "compute_capability": list(device.compute_capability), + } + except Exception as exc: # provenance should never fail the benchmark + return {"error": f"{type(exc).__name__}: {exc}"} + + +def _ci_state() -> dict | None: + """Return non-secret GitHub Actions identity for artifact provenance.""" + if os.environ.get("GITHUB_ACTIONS") != "true": + return None + + names = { + "event_name": "GITHUB_EVENT_NAME", + "repository": "GITHUB_REPOSITORY", + "ref": "GITHUB_REF", + "tested_sha": "GITHUB_SHA", + "source_sha": "OPENBOOST_SOURCE_SHA", + "head_ref": "GITHUB_HEAD_REF", + "run_id": "GITHUB_RUN_ID", + "run_attempt": "GITHUB_RUN_ATTEMPT", + } + return { + "provider": "github_actions", + **{key: os.environ.get(env_name) for key, env_name in names.items()}, + } + + +@contextmanager +def _working_directory(path: Path): + """Temporarily direct upstream relative outputs into an artifact directory.""" + original = Path.cwd() + path.mkdir(parents=True, exist_ok=True) + os.chdir(path) + try: + yield + finally: + os.chdir(original) + + +_REQUIRED_DISTRIBUTIONAL_METRICS = ( + "crps", + "log_score", + "rmse", + "coverage_90", + "interval_score_90", + "train_time", +) + + +def _is_present_finite(value) -> bool: + """Return whether a benchmark value is present and numerically finite.""" + if value is None or isinstance(value, bool): + return False + try: + return math.isfinite(float(value)) + except (TypeError, ValueError): + return False + + +def _is_present_text(value) -> bool: + return value is not None and bool(str(value).strip()) and str(value).lower() != "nan" + + +def _audit_records( + records: list[dict], + datasets: list[dict], + model_names: list[str], + *, + n_folds: int, + n_repeats: int, +) -> dict: + """Audit exact dataset/model/fold coverage after the upstream runner returns. + + ScoringBench intentionally catches dataset and model exceptions so a long + campaign can continue. That behavior is useful for throughput, but its + return code cannot be used as a completeness signal. This audit turns + missing, duplicate, error, and non-finite metric rows into explicit data. + """ + expected_keys = { + (dataset["name"], model_name, fold) + for dataset in datasets + for model_name in model_names + for fold in range(n_folds * n_repeats) + } + rows_by_key: dict[tuple[str, str, int], list[dict]] = {} + unexpected_rows = [] + for row in records: + try: + key = (str(row["dataset"]), str(row["model"]), int(row["fold"])) + except (KeyError, TypeError, ValueError): + unexpected_rows.append( + { + "reason": "invalid_identity", + "dataset": repr(row.get("dataset")), + "model": repr(row.get("model")), + "fold": repr(row.get("fold")), + } + ) + continue + if key not in expected_keys: + unexpected_rows.append( + { + "reason": "unexpected_identity", + "dataset": key[0], + "model": key[1], + "fold": key[2], + } + ) + continue + rows_by_key.setdefault(key, []).append(row) + + missing_rows = [ + {"dataset": dataset, "model": model, "fold": fold} + for dataset, model, fold in sorted(expected_keys - rows_by_key.keys()) + ] + duplicate_rows = [ + { + "dataset": key[0], + "model": key[1], + "fold": key[2], + "count": len(rows), + } + for key, rows in sorted(rows_by_key.items()) + if len(rows) != 1 + ] + error_rows = [] + invalid_metric_rows = [] + valid_keys = set() + for key, rows in rows_by_key.items(): + if len(rows) != 1: + continue + row = rows[0] + error = row.get("error") + if _is_present_text(error): + error_rows.append( + { + "dataset": key[0], + "model": key[1], + "fold": key[2], + "error_type": ( + str(row["error_type"]) if _is_present_text(row.get("error_type")) else None + ), + "error": str(error), + } + ) + continue + invalid_metrics = [ + metric + for metric in _REQUIRED_DISTRIBUTIONAL_METRICS + if not _is_present_finite(row.get(metric)) + ] + if invalid_metrics: + invalid_metric_rows.append( + { + "dataset": key[0], + "model": key[1], + "fold": key[2], + "metrics": invalid_metrics, + } + ) + continue + valid_keys.add(key) + + dataset_outcomes = [] + expected_per_dataset = len(model_names) * n_folds * n_repeats + for dataset in datasets: + name = dataset["name"] + observed = sum(key[0] == name for key in rows_by_key) + valid = sum(key[0] == name for key in valid_keys) + dataset_outcomes.append( + { + "dataset": name, + "expected_rows": expected_per_dataset, + "observed_rows": observed, + "valid_rows": valid, + "status": "complete" if valid == expected_per_dataset else "incomplete", + } + ) + + complete = ( + len(valid_keys) == len(expected_keys) + and not missing_rows + and not duplicate_rows + and not error_rows + and not invalid_metric_rows + and not unexpected_rows + ) + return { + "schema_version": 1, + "status": "complete" if complete else "incomplete", + "expected_rows": len(expected_keys), + "observed_rows": len(records), + "valid_rows": len(valid_keys), + "missing_rows": missing_rows, + "duplicate_rows": duplicate_rows, + "error_rows": error_rows, + "invalid_metric_rows": invalid_metric_rows, + "unexpected_rows": unexpected_rows, + "datasets": dataset_outcomes, + } + + +def _write_outcome(output_dir: Path, outcome: dict) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / "benchmark_outcome.json" + path.write_text(json.dumps(outcome, indent=2, sort_keys=True) + "\n") + return path + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run OpenBoost on the external ScoringBench protocol" + ) + parser.add_argument( + "--scoringbench-dir", + default=os.environ.get("SCORINGBENCH_DIR", ".repos/ScoringBench"), + help="Path to a ScoringBench git checkout", + ) + parser.add_argument( + "--output-dir", + default="benchmarks/results/scoringbench", + help="ScoringBench Parquet and OpenBoost manifest output", + ) + parser.add_argument( + "--models", + type=_csv, + default=["openboost_cpu", "ngboost"], + help=( + "Comma-separated models: openboost_cpu, openboost_cuda, " + "openboost_histogram_cpu, openboost_histogram_cpu_v2, ngboost, " + "xgboost_quantile, xgblss, catboost_quantile" + ), + ) + parser.add_argument("--n-trees", type=int, default=500) + parser.add_argument("--learning-rate", type=float, default=0.01) + parser.add_argument("--max-depth", type=int, default=3) + parser.add_argument( + "--training-objective", + choices=("nll", "crps"), + default="nll", + help=( + "OpenBoost training objective. CRPS is a development candidate; " + "use --development-run while evaluating it." + ), + ) + parser.add_argument("--reg-lambda", type=float, default=1.0) + parser.add_argument("--min-child-weight", type=float, default=1.0) + parser.add_argument("--n-quantiles", type=int, default=99) + parser.add_argument("--histogram-rounds", type=int, default=100) + parser.add_argument("--histogram-bins", type=int, default=50) + parser.add_argument("--histogram-learning-rate", type=float, default=0.05) + parser.add_argument("--histogram-max-depth", type=int, default=6) + parser.add_argument("--histogram-curvature-scale", type=float, default=1.0) + parser.add_argument( + "--histogram-v2-temperature-grid", + type=_float_csv, + default=(0.5, 0.7, 0.85, 1.0, 1.2), + ) + parser.add_argument("--histogram-v2-calibration-fraction", type=float, default=0.2) + parser.add_argument("--histogram-v2-calibration-seed", type=int, default=42) + parser.add_argument("--histogram-v2-evaluation-subdivisions", type=int, default=2) + parser.add_argument( + "--xgboost-rounds", + type=int, + default=100, + help="Boosting rounds for the ScoringBench XGBoost quantile baseline", + ) + parser.add_argument( + "--xgboost-quantiles", + type=int, + default=50, + help="Quantile outputs for the ScoringBench XGBoost quantile baseline", + ) + parser.add_argument( + "--xgblss-rounds", + type=int, + default=100, + help="Boosting rounds for the ScoringBench Gaussian XGBoostLSS baseline", + ) + parser.add_argument( + "--catboost-rounds", + type=int, + default=1000, + help="Iterations for the ScoringBench CatBoost MultiQuantile baseline", + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--n-folds", type=int, default=5) + parser.add_argument("--n-repeats", type=int, default=1) + parser.add_argument( + "--sample-size", + type=int, + default=3000, + help="Official ScoringBench default is 3000; use 0 only for a scale extension", + ) + selection = parser.add_mutually_exclusive_group() + selection.add_argument( + "--dataset-index", + type=int, + action="append", + help="Run selected index from ScoringBench's validated dataset list (repeatable)", + ) + selection.add_argument( + "--dataset-name", + action="append", + help="Run exact case-insensitive dataset name from the validated list (repeatable)", + ) + selection.add_argument( + "--shard-index", + type=int, + help="Run one zero-based strided shard from the dataset registry", + ) + parser.add_argument( + "--shard-count", + type=int, + help="Total number of stable registry shards; requires --shard-index", + ) + parser.add_argument( + "--dataset-registry", + help=( + "Frozen ScoringBench datasets.json to use instead of rebuilding the " + "dynamic upstream registry" + ), + ) + parser.add_argument( + "--lite", + action="store_true", + help="Use two folds while retaining ScoringBench datasets and metrics", + ) + parser.add_argument( + "--smoke", + action="store_true", + help="Use sklearn diabetes with 2 folds; validates integration, not leaderboard evidence", + ) + parser.add_argument( + "--development-run", + action="store_true", + help=( + "Mark this run as tuning-only evidence that must not be submitted or " + "reported as a held-out leaderboard result" + ), + ) + parser.add_argument( + "--allow-confirmation", + action="store_true", + help=( + "Unlock a registry entry marked untouched_confirmation. Use only " + "after the candidate implementation and configuration are committed." + ), + ) + parser.add_argument( + "--list-datasets", + action="store_true", + help="Print ScoringBench's validated dataset names and exit", + ) + return parser + + +def _select_datasets(all_datasets: list[dict], args) -> list[dict]: + if args.dataset_index: + invalid = [i for i in args.dataset_index if i < 0 or i >= len(all_datasets)] + if invalid: + raise ValueError( + f"dataset indices out of range: {invalid}; valid range is 0..{len(all_datasets) - 1}" + ) + return [all_datasets[i] for i in args.dataset_index] + + if args.dataset_name: + lookup = {dataset["name"].casefold(): dataset for dataset in all_datasets} + missing = [name for name in args.dataset_name if name.casefold() not in lookup] + if missing: + raise ValueError(f"unknown dataset names: {missing}; use --list-datasets") + return [lookup[name.casefold()] for name in args.dataset_name] + + if args.shard_index is not None: + if args.shard_count is None or args.shard_count <= 0: + raise ValueError("--shard-count must be a positive integer with --shard-index") + if args.shard_index < 0 or args.shard_index >= args.shard_count: + raise ValueError( + f"--shard-index must be in 0..{args.shard_count - 1}, got {args.shard_index}" + ) + selected = [ + dataset + for position, dataset in enumerate(all_datasets) + if position % args.shard_count == args.shard_index + ] + if not selected: + raise ValueError( + f"shard {args.shard_index}/{args.shard_count} selects no datasets " + f"from a registry of size {len(all_datasets)}" + ) + return selected + + if args.shard_count is not None: + raise ValueError("--shard-count requires --shard-index") + + return all_datasets + + +def _validate_selected_datasets(all_datasets: list[dict], args, validate) -> list[dict]: + """Validate only named shards; indexed shards retain validated-list semantics.""" + if args.dataset_name or args.shard_index is not None: + return validate(_select_datasets(all_datasets, args)) + return _select_datasets(validate(all_datasets), args) + + +def _model_parameters(args) -> dict[str, dict]: + """Return the exact benchmark constructor parameters for every model.""" + openboost_common = { + "n_trees": args.n_trees, + "learning_rate": args.learning_rate, + "max_depth": args.max_depth, + "n_quantiles": args.n_quantiles, + "model_params": { + "reg_lambda": args.reg_lambda, + "min_child_weight": args.min_child_weight, + "training_objective": args.training_objective, + }, + } + return { + "openboost_cpu": {"backend": "cpu", **openboost_common}, + "openboost_cuda": {"backend": "cuda", **openboost_common}, + "openboost_histogram_cpu": { + "n_distribution_bins": args.histogram_bins, + "n_trees": args.histogram_rounds, + "learning_rate": args.histogram_learning_rate, + "max_depth": args.histogram_max_depth, + "n_feature_bins": 254, + "curvature_scale": args.histogram_curvature_scale, + }, + "openboost_histogram_cpu_v2": { + "n_distribution_bins": args.histogram_bins, + "n_trees": args.histogram_rounds, + "learning_rate": args.histogram_learning_rate, + "max_depth": args.histogram_max_depth, + "n_feature_bins": 254, + "curvature_scale": args.histogram_curvature_scale, + "temperature_grid": args.histogram_v2_temperature_grid, + "calibration_fraction": args.histogram_v2_calibration_fraction, + "calibration_seed": args.histogram_v2_calibration_seed, + "evaluation_subdivisions": args.histogram_v2_evaluation_subdivisions, + }, + "ngboost": { + "dist": "normal", + "n_estimators": args.n_trees, + "learning_rate": args.learning_rate, + "n_quantiles": args.n_quantiles, + "ngb_params": {"random_state": args.seed}, + }, + "xgboost_quantile": { + "n_bins": args.xgboost_quantiles, + "num_boost_round": args.xgboost_rounds, + "xgb_params": {"device": "cpu", "seed": args.seed, "nthread": 2}, + }, + "xgblss": { + "n_quantiles": args.n_quantiles, + "num_boost_round": args.xgblss_rounds, + "distribution": "Gaussian", + "xgblss_params": {"device": "cpu", "seed": args.seed, "nthread": 2}, + }, + "catboost_quantile": { + "n_quantiles": args.n_quantiles, + "iterations": args.catboost_rounds, + "catboost_params": { + "allow_writing_files": False, + "random_seed": args.seed, + "thread_count": 2, + }, + }, + } + + +def _model_factories(args): + from benchmarks.scoringbench.openboost_wrapper import ( + OpenBoostHistogramWrapper, + OpenBoostWrapper, + ) + + parameters = _model_parameters(args) + + factories = { + "openboost_cpu": lambda: OpenBoostWrapper(**parameters["openboost_cpu"]), + "openboost_cuda": lambda: OpenBoostWrapper(**parameters["openboost_cuda"]), + "openboost_histogram_cpu": lambda: OpenBoostHistogramWrapper( + **parameters["openboost_histogram_cpu"] + ), + "openboost_histogram_cpu_v2": lambda: OpenBoostHistogramWrapper( + **parameters["openboost_histogram_cpu_v2"] + ), + } + + if "ngboost" in args.models: + from scoringbench.wrappers.ngboost_wrapper import NGBoostWrapper + + factories["ngboost"] = lambda: NGBoostWrapper(**parameters["ngboost"]) + + if "xgboost_quantile" in args.models: + from scoringbench.wrappers.xgb_vector import XGBQuantileVectorWrapper + + factories["xgboost_quantile"] = lambda: XGBQuantileVectorWrapper( + **parameters["xgboost_quantile"] + ) + + if "xgblss" in args.models: + from scoringbench.wrappers.xgblss_wrapper import XGBLSSWrapper + + factories["xgblss"] = lambda: XGBLSSWrapper(**parameters["xgblss"]) + + if "catboost_quantile" in args.models: + from scoringbench.wrappers.catboost_wrapper import CatBoostQuantileWrapper + + factories["catboost_quantile"] = lambda: CatBoostQuantileWrapper( + **parameters["catboost_quantile"] + ) + + valid = set(factories) + unknown = [name for name in args.models if name not in valid] + if unknown: + allowed = [ + "openboost_cpu", + "openboost_cuda", + "openboost_histogram_cpu", + "openboost_histogram_cpu_v2", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile", + ] + raise ValueError(f"unknown models {unknown}; allowed values: {allowed}") + + return {name: factories[name] for name in args.models} + + +def _write_provenance( + output_dir: Path, + scoringbench_dir: Path, + args, + datasets: list[dict], + result_rows: int, + outcome: dict, + verified_dataset_files: list[dict] | None = None, +) -> Path: + import openboost as ob + + official_shape = ( + not args.smoke and args.sample_size == 3000 and args.n_folds == 5 and args.n_repeats == 1 + ) + official_protocol_compatible = official_shape and not args.development_run + if args.smoke: + protocol_mode = "smoke" + elif args.development_run: + protocol_mode = "development_tuning" + elif args.sample_size != 3000: + protocol_mode = "scoringbench_scale_extension" + elif official_shape and ( + args.dataset_index or args.dataset_name or args.shard_index is not None + ): + protocol_mode = "official_quality_shard" + elif official_shape: + protocol_mode = "official_quality" + else: + protocol_mode = "scoringbench_protocol_deviation" + + registry_path = output_dir / "datasets.json" + manifest = { + "schema_version": 3, + "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "protocol": "ScoringBench", + "protocol_mode": protocol_mode, + "official_protocol_compatible": official_protocol_compatible, + "warning": ( + None + if official_protocol_compatible + else ( + "This is a development/tuning run and must not be represented as " + "held-out leaderboard evidence." + if args.development_run + else "This run is not directly comparable to the official 5-fold, sample_size=3000 leaderboard." + ) + ), + "openboost_git": _git_state(PROJECT_ROOT), + "scoringbench_git": _git_state(scoringbench_dir), + "ci": _ci_state(), + "arguments": vars(args), + "model_parameters": {name: _model_parameters(args)[name] for name in args.models}, + "datasets": [ + { + "name": dataset["name"], + "source": dataset.get("source", "openml"), + "id": dataset.get("id", dataset.get("url", dataset.get("loader"))), + } + for dataset in datasets + ], + "dataset_registry": { + "mode": "frozen_file" if args.dataset_registry else "scoringbench_dynamic", + "resolved_sha256": _sha256(registry_path) if registry_path.exists() else None, + "source_sha256": ( + _sha256(Path(args.dataset_registry).expanduser().resolve()) + if args.dataset_registry + else None + ), + "file": "datasets.json" if registry_path.exists() else None, + }, + "verified_dataset_files": verified_dataset_files or [], + "result_rows": result_rows, + "expected_result_rows": outcome["expected_rows"], + "outcome": { + "status": outcome["status"], + "valid_rows": outcome["valid_rows"], + "missing_rows": len(outcome["missing_rows"]), + "duplicate_rows": len(outcome["duplicate_rows"]), + "error_rows": len(outcome["error_rows"]), + "invalid_metric_rows": len(outcome["invalid_metric_rows"]), + "unexpected_rows": len(outcome["unexpected_rows"]), + "file": "benchmark_outcome.json", + }, + "platform": { + "python": platform.python_version(), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "processor": platform.processor(), + "cpu_count": os.cpu_count(), + "gpu": _gpu_info(), + }, + "versions": { + "openboost": ob.__version__, + "numpy": np.__version__, + **{ + name: _package_version(name) + for name in ( + "scipy", + "scikit-learn", + "pandas", + "pyarrow", + "torch", + "numba", + "numba-cuda", + "cupy-cuda12x", + "ngboost", + "xgboost", + "xgboostlss", + "catboost", + ) + }, + }, + } + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / "openboost_manifest.json" + path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + return path + + +def main() -> int: + args = _build_parser().parse_args() + dataset_registry_path = _resolve_dataset_registry_path(args.dataset_registry) + if sys.platform == "darwin" and platform.machine() == "x86_64": + raise SystemExit( + "The complete ScoringBench runner is unsupported on Intel macOS: " + "the available PyTorch wheel uses the NumPy 1.x ABI while " + "ScoringBench requires NumPy 2.x. Run the benchmark on Linux " + "(the target environment for published CPU/CUDA results). The " + "wrapper contract test can still be run separately." + ) + scoringbench_dir = Path(args.scoringbench_dir).expanduser().resolve() + if not (scoringbench_dir / "scoringbench" / "runner.py").exists(): + raise SystemExit( + f"No ScoringBench checkout found at {scoringbench_dir}. " + "Clone https://github.com/jonaslandsgesell/ScoringBench first." + ) + + sys.path.insert(0, str(SRC_ROOT)) + sys.path.insert(0, str(PROJECT_ROOT)) + sys.path.insert(0, str(scoringbench_dir)) + + from scoringbench.datasets import ( + _ensure_cached, + get_DATASETS_CONFIG, + validate_datasets, + ) + from scoringbench.runner import run_benchmark + from scoringbench.utils import set_seed + + set_seed(args.seed) + output_dir = Path(args.output_dir).expanduser().resolve() + verified_dataset_files = [] + if args.smoke: + datasets = [ + { + "name": "diabetes_smoke", + "source": "sklearn", + "loader": "load_diabetes", + "abbr": "DBS", + "sample_size": min(args.sample_size or 442, 442), + } + ] + args.n_folds = 2 + else: + # ScoringBench exports its resolved dataset registry to Path.cwd(). Keep + # that reproducibility artifact with the benchmark instead of dirtying + # the OpenBoost checkout. + with _working_directory(output_dir): + if args.dataset_registry: + all_datasets = _load_dataset_registry(dataset_registry_path) + Path("datasets.json").write_text( + json.dumps(all_datasets, indent=2, ensure_ascii=False) + "\n" + ) + else: + all_datasets = get_DATASETS_CONFIG() + if args.list_datasets: + datasets = validate_datasets(all_datasets) + for index, dataset in enumerate(datasets): + print(f"{index:3d} {dataset['name']}") + return 0 + + def validate_with_raw_verification(selected): + nonlocal verified_dataset_files + _enforce_dataset_role_lock( + selected, + allow_confirmation=args.allow_confirmation, + ) + verified_dataset_files = _verify_dataset_files( + selected, + _ensure_cached, + ) + if verified_dataset_files: + # Force the pinned raw file through preprocessing instead + # of accepting an opaque processed cache entry. + os.environ["SCORINGBENCH_NO_CACHE"] = "1" + return validate_datasets(selected) + + datasets = _validate_selected_datasets( + all_datasets, + args, + validate_with_raw_verification, + ) + + if args.lite: + args.n_folds = 2 + + model_factories = _model_factories(args) + result = run_benchmark( + datasets_config=datasets, + model_factories=model_factories, + output_dir=output_dir, + n_folds=args.n_folds, + n_repeats_cv=args.n_repeats, + seed=args.seed, + sample_size=args.sample_size, + ) + outcome = _audit_records( + result.to_dict(orient="records"), + datasets, + list(model_factories), + n_folds=args.n_folds, + n_repeats=args.n_repeats, + ) + outcome_path = _write_outcome(output_dir, outcome) + manifest = _write_provenance( + output_dir, + scoringbench_dir, + args, + datasets, + result_rows=len(result), + outcome=outcome, + verified_dataset_files=verified_dataset_files, + ) + print(f"OpenBoost outcome: {outcome_path} ({outcome['status']})") + print(f"OpenBoost provenance: {manifest}") + return 0 if outcome["status"] == "complete" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/scoringbench/test_openboost_wrapper.py b/benchmarks/scoringbench/test_openboost_wrapper.py new file mode 100644 index 0000000..c9a186b --- /dev/null +++ b/benchmarks/scoringbench/test_openboost_wrapper.py @@ -0,0 +1,99 @@ +"""Upstream-style smoke test for the OpenBoost ScoringBench wrapper.""" + +import numpy as np +from scoringbench.wrappers.base import DistributionPrediction + +from benchmarks.scoringbench.openboost_wrapper import ( + OpenBoostHistogramWrapper, + OpenBoostWrapper, +) + + +def test_openboost_wrapper_distribution_contract(): + rng = np.random.default_rng(42) + X = rng.normal(size=(160, 4)).astype(np.float32) + sigma = 0.25 + np.abs(X[:, 1]) + y = (2 * X[:, 0] - X[:, 2] + rng.normal(scale=sigma)).astype(np.float32) + + model = OpenBoostWrapper( + backend="cpu", + n_trees=12, + learning_rate=0.05, + max_depth=2, + n_quantiles=15, + ) + returned = model.fit(X[:120], y[:120]) + distribution = model.predict_distribution(X[120:]) + + assert returned is model + assert isinstance(distribution, DistributionPrediction) + assert distribution.probas.shape == (40, 14) + assert distribution.bin_edges.shape == (40, 15) + assert distribution.mean.shape == (40,) + assert np.all(np.isfinite(distribution.probas)) + assert np.all(np.isfinite(distribution.bin_edges)) + assert np.allclose(distribution.probas.sum(axis=1), 1.0) + assert np.all(np.diff(distribution.bin_edges, axis=1) > 0) + + +def test_openboost_wrapper_forwards_crps_training_objective(): + rng = np.random.default_rng(7) + X = rng.normal(size=(100, 3)).astype(np.float32) + y = (X[:, 0] + rng.normal(scale=0.4, size=100)).astype(np.float32) + model = OpenBoostWrapper( + backend="cpu", + n_trees=5, + learning_rate=0.05, + max_depth=2, + n_quantiles=9, + model_params={"training_objective": "crps"}, + ) + + model.fit(X[:80], y[:80]) + + assert model._model.training_objective == "crps" + assert np.all(np.isfinite(model.predict(X[80:]))) + + +def test_openboost_histogram_wrapper_preserves_native_distribution_grid(): + rng = np.random.default_rng(11) + X = rng.normal(size=(100, 3)).astype(np.float32) + y = (X[:, 0] + rng.normal(scale=0.5, size=100)).astype(np.float32) + model = OpenBoostHistogramWrapper( + n_distribution_bins=8, + n_trees=3, + max_depth=2, + n_feature_bins=12, + ).fit(X[:80], y[:80]) + + distribution = model.predict_distribution(X[80:]) + + assert isinstance(distribution, DistributionPrediction) + assert distribution.is_natively_gridded_model is True + assert distribution.probas.shape == (20, 8) + assert distribution.bin_edges.shape == (9,) + np.testing.assert_allclose(distribution.probas.sum(axis=1), 1.0) + np.testing.assert_allclose(distribution.mean, model.predict(X[80:])) + + +def test_openboost_histogram_wrapper_selects_temperature_on_inner_validation(): + rng = np.random.default_rng(19) + X = rng.normal(size=(80, 3)).astype(np.float32) + y = (X[:, 0] + rng.normal(scale=0.3, size=80)).astype(np.float32) + model = OpenBoostHistogramWrapper( + n_distribution_bins=6, + n_trees=2, + max_depth=1, + n_feature_bins=10, + temperature_grid=(0.7, 1.0, 1.2), + calibration_fraction=0.2, + calibration_seed=3, + evaluation_subdivisions=2, + ).fit(X[:60], y[:60]) + + assert model._selected_temperature in model.temperature_grid + assert set(model._temperature_scores) == set(model.temperature_grid) + assert all(np.isfinite(list(model._temperature_scores.values()))) + distribution = model.predict_distribution(X[60:]) + assert distribution.probas.shape == (20, 12) + np.testing.assert_allclose(distribution.mean, model.predict(X[60:])) diff --git a/docs/api/models.md b/docs/api/models.md index 6a1958b..db707c1 100644 --- a/docs/api/models.md +++ b/docs/api/models.md @@ -63,6 +63,20 @@ All OpenBoost model classes. ## Probabilistic Models (NaturalBoost) +### HistogramBoost + +::: openboost.HistogramBoost + options: + show_root_heading: true + show_source: true + +### HistogramDistributionOutput + +::: openboost.HistogramDistributionOutput + options: + show_root_heading: true + show_source: true + ### NaturalBoost ::: openboost.NaturalBoost diff --git a/docs/getting-started/gpu-setup.md b/docs/getting-started/gpu-setup.md index b748347..da17210 100644 --- a/docs/getting-started/gpu-setup.md +++ b/docs/getting-started/gpu-setup.md @@ -28,19 +28,20 @@ ob.set_backend("cuda") ## GPU Performance -GPU acceleration provides significant speedups for larger datasets: +GPU benefit depends on dataset shape, tree parameters, distribution, CUDA +stack, transfer policy, and JIT warm-up. OpenBoost does not publish a universal +speedup table without a checked-in benchmark artifact. -| Dataset Size | Typical Speedup | -|--------------|-----------------| -| <5K samples | ~1x (CPU overhead dominates) | -| 5K-10K | 2-7x | -| 25K+ | 2-3x | -| 100K+ | 5-10x | +For a defensible comparison: -!!! tip "Best practices for GPU" - - Ensure data is `float32` (not `float64`) - - Use larger datasets (GPU overhead not worth it for <5K samples) - - GPU shows best speedup at 10K+ samples +1. force the backend with `ob.set_backend("cpu")` or `"cuda"`; +2. run a warm-up that is excluded from timed repetitions; +3. compare predictions and task metrics before comparing runtime; +4. report repeated fit/predict timings, peak memory, failures, and hardware; +5. save raw results with the OpenBoost commit and dependency versions. + +The ScoringBench integration under `benchmarks/scoringbench/` defines separate +official-quality and scale-extension protocols for probabilistic models. ## Multi-GPU Training @@ -67,8 +68,9 @@ model.fit(X, y) ### Training seems slow on GPU - Ensure data is `float32` (not `float64`) -- Use larger datasets (GPU overhead not worth it for <5K samples) -- GPU shows best speedup at 10K+ samples +- Exclude first-use JIT compilation only when the benchmark protocol says so +- Check that `ob.get_backend()` reports `"cuda"` +- Measure fit and prediction separately; do not assume a crossover dataset size ### Model trained on GPU, loading on CPU machine diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 6a81837..eb16016 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -43,7 +43,7 @@ For CUDA GPU acceleration: |-------|-----------------|---------| | `cuda` | CuPy for GPU acceleration | `pip install "openboost[cuda]"` | | `sklearn` | scikit-learn integration | `pip install "openboost[sklearn]"` | -| `distributed` | Ray for multi-GPU training | `pip install "openboost[distributed]"` | +| `distributed` | Ray for experimental multi-GPU work | `pip install "openboost[distributed]"` | | `all` | Everything | `pip install "openboost[all]"` | ## Requirements diff --git a/docs/user-guide/model-persistence.md b/docs/user-guide/model-persistence.md index 30579ec..a278f76 100644 --- a/docs/user-guide/model-persistence.md +++ b/docs/user-guide/model-persistence.md @@ -28,6 +28,7 @@ All models support save/load: - `DART` - `OpenBoostGAM` - `NaturalBoostNormal`, `NaturalBoostGamma`, etc. +- `HistogramBoost` - `LinearLeafGBDT` ## Using joblib/pickle Directly diff --git a/docs/user-guide/models/gradient-boosting.md b/docs/user-guide/models/gradient-boosting.md index aa9b280..d789def 100644 --- a/docs/user-guide/models/gradient-boosting.md +++ b/docs/user-guide/models/gradient-boosting.md @@ -44,6 +44,7 @@ predictions = model.predict(X_test) | `n_bins` | int | 254 | Number of histogram bins | | `growth` | str | `'levelwise'` | Tree growth strategy: `'levelwise'`, `'leafwise'`, or `'symmetric'` | | `max_leaves` | int/None | None | Max leaves per tree for `'leafwise'` growth (defaults to `2**max_depth`) | +| `batch_size` | int/None | None | Reserved; non-None values raise `NotImplementedError` | | `random_state` | int/None | None | Seed for reproducible training | ## Loss Functions @@ -92,6 +93,11 @@ training path. CUDA, distributed, and multi-GPU training raise `NotImplementedError` when weights are supplied, so weighted observations are never silently treated as unweighted. +High-level mini-batch and out-of-core fitting are not implemented. The +`batch_size` parameter fails fast when set; low-level memmap and mini-batch +histogram helpers are experimental building blocks rather than a `model.fit` +path. + ## Feature Importance ```python diff --git a/docs/user-guide/models/histogram-boost.md b/docs/user-guide/models/histogram-boost.md new file mode 100644 index 0000000..a914b88 --- /dev/null +++ b/docs/user-guide/models/histogram-boost.md @@ -0,0 +1,71 @@ +# HistogramBoost + +`HistogramBoost` predicts a flexible probability histogram instead of assuming +a Normal, Gamma, or other parametric family. Each bin represents uniform +density, and the model trains the complete CDF with that histogram's exact +continuous ranked probability score (CRPS). + +Each boosting round learns one tree structure. Every leaf stores a vector of +updates for the ordered histogram logits. A softmax converts those logits into +non-negative probabilities that sum to one, so predicted CDFs are monotone and +quantiles cannot cross. + +```python +import openboost as ob + +model = ob.HistogramBoost( + n_distribution_bins=50, + n_trees=100, + learning_rate=0.05, + max_depth=6, +) +model.fit(X_train, y_train) + +distribution = model.predict_distribution(X_test) +mean = distribution.mean() +lower, upper = distribution.interval(alpha=0.1) +samples = distribution.sample(n_samples=100, seed=42) +``` + +## When to use it + +Use `HistogramBoost` when a two-parameter distribution is too restrictive—for +example when conditional outcomes may be skewed or have more than one mode—and +CRPS is the primary quality target. Use NaturalBoost when a named distribution, +analytic likelihood, exposure offset, or distribution-specific interpretation +is important. + +## Current boundary + +This is a CPU-only development model. It currently supports numeric features, +including missing values, and sample weights. CUDA, categorical splits, +callbacks, evaluation sets, early stopping, and row/column sampling are not yet +implemented. + +The target grid is learned from the training target range. Predictions cannot +put mass outside that finite range plus its half-bin padding, so held-out +extremes may be clipped. Always evaluate CRPS together with coverage, interval +score, RMSE, and failure rate. + +The default configuration is frozen for the repository's preregistered +ScoringBench development experiment. It is not yet an overall leaderboard or +state-of-the-art claim. + +## Parameters + +| Parameter | Default | Meaning | +|---|---:|---| +| `n_distribution_bins` | 50 | Number of ordered target bins | +| `n_trees` | 100 | Shared-vector boosting rounds | +| `learning_rate` | 0.05 | Shrinkage applied to each tree | +| `max_depth` | 6 | Maximum routing depth | +| `n_feature_bins` | 254 | Numeric feature histogram bins | +| `curvature_scale` | 1.0 | Scale of the PSD Gauss–Newton diagonal | +| `reg_lambda` | 1.0 | L2 regularization used to select split structures | +| `leaf_reg_lambda` | `None` | L2 regularization for vector leaves; `None` reuses `reg_lambda` | +| `base_smoothing` | 1.0 | Total Dirichlet prior weight, spread evenly across target bins | +| `reg_alpha` | 0.0 | L1 regularization for vector leaf values | + +`predict_distribution()` returns `HistogramDistributionOutput`, which provides +`mean()`, `variance()`, `std()`, exact `crps()`, `tempered()`, density-preserving +`subdivide()`, `quantile()`, `interval()`, and `sample()`. diff --git a/docs/user-guide/training/large-scale.md b/docs/user-guide/training/large-scale.md index 40fdab7..b87ddba 100644 --- a/docs/user-guide/training/large-scale.md +++ b/docs/user-guide/training/large-scale.md @@ -1,71 +1,69 @@ -# Large-Scale Training +# Scaling Training -Train on datasets that don't fit in memory or need faster training. +OpenBoost has a supported full-dataset CPU/CUDA path, a supported sampling +option, and several experimental scaling primitives. Keep those categories +separate when choosing a training path or reporting a benchmark. + +| Capability | Status | Important boundary | +|------------|--------|--------------------| +| Single-device full-data training | Supported | Dataset and histograms must fit in memory | +| GOSS sampling | Supported | Speed and quality are data-dependent | +| `fit_trees_batch` | Reference implementation | Shares bins; configurations fit sequentially | +| Mini-batch histogram helpers | Low-level primitive | Not integrated with `model.fit` | +| Memory-mapped binned arrays | Storage primitive | Not an out-of-core `model.fit` path | +| Distributed/multi-GPU | Experimental | No published parity or scaling artifact yet | ## GOSS Sampling -Gradient-based One-Side Sampling (from LightGBM) - train 3x faster with minimal accuracy loss. +Gradient-based One-Side Sampling keeps high-gradient observations and samples +from the remainder on every boosting round: ```python import openboost as ob model = ob.GradientBoosting( n_trees=100, - subsample_strategy='goss', - goss_top_rate=0.2, # Keep top 20% high-gradient samples - goss_other_rate=0.1, # Sample 10% of the rest + subsample_strategy="goss", + goss_top_rate=0.2, + goss_other_rate=0.1, + random_state=42, ) model.fit(X_train, y_train) ``` -### How GOSS Works - -1. Sort samples by gradient magnitude -2. Keep top `goss_top_rate` samples (most informative) -3. Randomly sample `goss_other_rate` from the rest -4. Weight the random samples to maintain unbiased gradients - -### GOSS Parameters - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `goss_top_rate` | 0.2 | Fraction of high-gradient samples to keep | -| `goss_other_rate` | 0.1 | Fraction of remaining samples to keep | +With these rates, approximately 28% of observations participate in a round. +That arithmetic is not a speed or accuracy guarantee: compare GOSS with the +full-data path on fixed folds and seeds before using it. -**Result**: Train on ~28% of samples with similar accuracy. +## Single-GPU Scaling -## Memory-Mapped Arrays - -For datasets larger than RAM: +Select CUDA explicitly so an unavailable GPU cannot silently turn a benchmark +into a CPU run: ```python import openboost as ob -# Create memory-mapped binned array (saves to disk) -X_mmap = ob.create_memmap_binned('large_data.npy', X_large) - -# Load for training (no copy, uses disk) -X_mmap = ob.load_memmap_binned('large_data.npy', n_features, n_samples) - -# Train as normal -model = ob.GradientBoosting(n_trees=100) -model.fit(X_mmap, y_train) +ob.set_backend("cuda") +model = ob.GradientBoosting(n_trees=100, random_state=42) +model.fit(X_train, y_train) ``` -## Mini-Batch Training +Report the OpenBoost commit, environment, GPU model, driver/CUDA versions, +warm-up policy, seeds, fit time, prediction time, peak memory, and CPU/CUDA +prediction parity. Use the scale-extension protocol in +`benchmarks/scoringbench/` for probabilistic benchmark work. -Accumulate histograms in batches: +## Mini-Batch and Memory-Mapped Primitives -```python -from openboost import MiniBatchIterator, accumulate_histograms_minibatch +`MiniBatchIterator`, `accumulate_histograms_minibatch`, +`create_memmap_binned`, and `load_memmap_binned` are low-level building blocks. +The memmap layout is feature-major `(n_features, n_samples)`; it is not a raw +sample-major matrix accepted by high-level `model.fit`. -# Process 100k samples at a time -hist_grad, hist_hess = accumulate_histograms_minibatch( - X_mmap, grad, hess, - batch_size=100_000, - n_features=n_features, -) -``` +The `batch_size` model parameter is reserved. Passing a non-`None` value raises +`NotImplementedError` rather than pretending to perform mini-batch training. +Do not claim datasets larger than memory are supported end to end until a +training loop integrates these primitives and has correctness tests. ## Train Many Configurations @@ -90,65 +88,32 @@ trees_by_config = ob.fit_trees_batch( ) ``` -The current implementation is a correctness reference: it shares binned input -data but fits configurations sequentially. GPU kernel fusion is planned without -changing this API's results. +The current implementation shares the binned input but fits configurations +sequentially. Treat it as a correctness reference, not fused GPU training. -## Multi-GPU Training +## Experimental Multi-GPU Path -Distribute training across multiple GPUs: +The Ray multi-GPU path is available for development experiments: ```python import openboost as ob -# Automatic multi-GPU with Ray -model = ob.GradientBoosting(n_trees=100, n_gpus=4) -model.fit(X, y) - -# Or specify exact devices -model = ob.GradientBoosting(n_trees=100, devices=[0, 2]) -model.fit(X, y) -``` - -### Requirements - -```bash -pip install "openboost[distributed]" # Installs Ray +model = ob.GradientBoosting(n_trees=100, devices=[0, 1]) +model.fit(X_train, y_train) ``` -## Scaling Guidelines +It does not support `sample_weight`, and the repository does not yet contain a +validated two-/four-GPU parity and scaling artifact. Do not use it for release +claims until exact single-device parity, repeated timings, peak memory, and +failure cases are published on real multi-GPU hardware. -| Dataset Size | Recommendation | -|--------------|----------------| -| <100K samples | Standard training | -| 100K-1M | GOSS sampling | -| 1M-10M | GOSS + memory-mapped | -| >10M | Multi-GPU + GOSS | +## Evidence Gate -## Example: Large Dataset +A scaling claim is ready only when the checked-in artifact records: -```python -import numpy as np -import openboost as ob - -# Simulate large dataset (10M samples) -n_samples = 10_000_000 -n_features = 100 - -# Create memory-mapped data -X_mmap = ob.create_memmap_binned( - 'large_X.npy', - np.random.randn(n_samples, n_features).astype(np.float32) -) - -y = np.random.randn(n_samples).astype(np.float32) - -# Train with GOSS -model = ob.GradientBoosting( - n_trees=100, - subsample_strategy='goss', - goss_top_rate=0.1, - goss_other_rate=0.05, -) -model.fit(X_mmap, y) -``` +1. a frozen OpenBoost commit and dependency lock; +2. real datasets plus at least one controlled synthetic scaling curve; +3. CPU/CUDA prediction parity and task-quality metrics; +4. repeated fit/predict timings after an explicit warm-up policy; +5. hardware, drivers, thread counts, peak memory, seeds, and failures; +6. comparisons against maintained baselines under the same protocol. diff --git a/examples/gpu_training.py b/examples/gpu_training.py index 949d32a..622318a 100644 --- a/examples/gpu_training.py +++ b/examples/gpu_training.py @@ -226,23 +226,17 @@ def main(): """ print(best_practices) - # --- Scaling Guide --- - print("\n8. Expected GPU speedups by dataset size...") + # --- Benchmark Evidence --- + print("\n8. How to measure GPU value...") scaling_info = """ - | Dataset Size | Features | Trees | Expected Speedup | - |--------------|----------|-------|------------------| - | 5K samples | 10 | 100 | ~1-2x | - | 10K samples | 20 | 100 | ~2-5x | - | 50K samples | 20 | 100 | ~3-7x | - | 100K samples | 50 | 200 | ~5-10x | - | 500K samples | 100 | 500 | ~10-20x | - - Factors affecting speedup: - - More features = better GPU utilization - - More bins = better GPU utilization - - GAM shows best speedups (parallel feature updates) - - First run includes JIT compilation overhead + GPU speedup is workload- and hardware-dependent. For a publishable result: + - Force CPU and CUDA backends explicitly + - Verify prediction and task-metric parity first + - State whether JIT warm-up is excluded + - Run repeated fit and predict timings + - Record peak memory, hardware, CUDA/driver, seeds, and failures + - Store raw results with the exact OpenBoost commit """ print(scaling_info) @@ -250,8 +244,7 @@ def main(): print("\n9. Multi-GPU training...") print(""" - For datasets that don't fit on a single GPU or to speed up training further, - OpenBoost supports multi-GPU training via Ray: + OpenBoost includes an experimental multi-GPU path via Ray: # Install Ray pip install ray[default] @@ -273,7 +266,10 @@ def main(): Multi-GPU training uses data parallelism: - Each GPU processes a subset of samples - Histograms are aggregated across GPUs - - Near-linear scaling with number of GPUs + - sample_weight is not supported + + Do not infer scaling from this example. The repository still needs a + checked-in two-/four-GPU parity and repeated-timing artifact. """) # --- Summary --- diff --git a/learnings/2026-08-15-categorical-cardinality.md b/learnings/2026-08-15-categorical-cardinality.md new file mode 100644 index 0000000..219af63 --- /dev/null +++ b/learnings/2026-08-15-categorical-cardinality.md @@ -0,0 +1,60 @@ +# 2026-08-15: Categorical Tree Cardinality + +## Context + +Categorical values are encoded into `uint8`, so `BinnedArray` can represent up +to 254 non-missing categories. Tree nodes, however, represent the categories +sent left with one `uint64` bitset. The split search previously evaluated all +categories but silently omitted category codes 64 and above from that set. +Reported gain and actual routing could therefore disagree. + +## Decision or Result + +Keep the encoding limit at 254 because non-tree consumers can use those bins, +but reject categorical tree training above 64 categories in the shared split +dispatch before either the CPU or CUDA implementation runs. This preserves the +useful `BinnedArray` capability while preventing a tree from silently learning +an unrepresentable split. + +Supporting more than 64 categories correctly requires a multiword bitset (or a +different category-set representation) across split search, partitioning, CPU +prediction, CUDA prediction, tree storage, and persistence. It is a feature, +not a safe one-line limit increase. + +## Changes + +- `src/openboost/_core/_split.py`: require category counts when categorical + features are present and fail before dispatch when any count exceeds 64. +- `src/openboost/_array.py`: document the distinction between the 254-category + encoding limit and the 64-category tree-split limit. +- `tests/test_categorical.py`: prove 65 categories can be binned but cannot be + passed into categorical tree training. + +## Verification + +- Before the guard, the new 65-category training case did not raise. +- `uv run pytest -q tests/test_categorical.py tests/test_growth.py`: 46 passed. +- A focused 64-category probe separated category 63 correctly. +- `uv run ruff check src/openboost/_array.py src/openboost/_core/_split.py`: + passed. +- `git diff --check`: passed. + +## Failed Attempts + +- The first design rejected more than 64 categories inside `ob.array()`. That + conflated safe bin encoding with tree routing and would unnecessarily block + consumers such as GAM. The guard was moved to the shared tree split path. +- Linting the entire legacy categorical test file surfaced pre-existing style + issues unrelated to this change. The source files were used as the scoped + lint gate; the complete behavior files were still executed with pytest. + +## Risks and Follow-ups + +- Multiword bitsets are required before advertising native high-cardinality + categorical tree support. +- Real GPU verification should include category code 63 and CPU/CUDA parity; + this machine has no CUDA environment. + +## Commits + +- `356103b` — `fix: reject unrepresentable categorical tree splits` diff --git a/learnings/2026-08-15-categorical-persistence.md b/learnings/2026-08-15-categorical-persistence.md new file mode 100644 index 0000000..36df21a --- /dev/null +++ b/learnings/2026-08-15-categorical-persistence.md @@ -0,0 +1,61 @@ +# 2026-08-15: Categorical Tree Persistence + +## Context + +`GradientBoosting.save()` serialized categorical trees with the draft field +names `is_categorical` and `category_masks`. `TreeStructure` actually stores +the routing state as `is_categorical_split` and `cat_bitsets`, so those arrays +were omitted and categorical predictions could change after loading a model. + +## Decision or Result + +Tree persistence now writes the canonical routing fields and reconstructs them +through the `TreeStructure` constructor. The loader also recognizes the draft +key names in case an external state contains them. + +Serialization version 2 identifies files written with the corrected schema. +When a version 1 file advertises categorical input metadata, loading emits a +warning because a file produced by the broken serializer cannot reconstruct +the category bitsets that were never written. Such a model must be retrained +and resaved before production use. + +## Changes + +- `src/openboost/_persistence.py`: persist categorical bitsets and split flags, + restore missing/categorical arrays in the constructor, retain draft-key read + compatibility, and bump the serialization version to 2. +- `tests/test_persistence.py`: add an exact prediction round trip containing + categorical group splits and missing values; assert schema version and tree + arrays. + +## Verification + +- Before the fix, the new test failed because loaded + `is_categorical_split` was `None`. +- `uv run pytest -q tests/test_persistence.py tests/test_categorical.py`: + 34 passed. +- Focused categorical round-trip test: 1 passed. +- `uv run ruff check src/openboost/_persistence.py`: passed. +- `git diff --check`: passed. + +## Failed Attempts + +- A combined sandboxed `uv` verification could not read the shared uv cache; + rerunning with the already approved uv cache permission completed normally. +- The first lint pass found a local `numpy` import that shadowed the module + import inside `_from_state_dict`; removing the redundant local import fixed + the scope error. + +## Risks and Follow-ups + +- Version 1 categorical files created by the broken serializer are not + repairable because their bitsets are absent; the warning is detection, not a + migration. +- Category routing uses one `uint64` bitset. Inputs with more than 64 category + codes are currently accepted elsewhere but cannot be represented correctly; + add a fail-fast guard or implement multiword bitsets before claiming support + above 64 categories. + +## Commits + +- `aecf33f` — `fix: preserve categorical tree state on save` diff --git a/learnings/2026-08-15-histogram-crps-boost.md b/learnings/2026-08-15-histogram-crps-boost.md new file mode 100644 index 0000000..0ee7bbc --- /dev/null +++ b/learnings/2026-08-15-histogram-crps-boost.md @@ -0,0 +1,204 @@ +# 2026-08-15: Histogram CRPS Boosting + +## Context + +The corrected Gaussian NaturalBoost model reached parity with Gaussian +XGBoostLSS on the consumed `1028_SWD` ScoringBench dataset but remained 2.80% +behind native XGBoost multi-quantile on mean CRPS. CRPS training, early +stopping, scale calibration, Student-t, empirical residual shapes, and +independent quantile trees did not close that gap. The remaining limitation was +distribution shape, not only Gaussian scale optimization. + +## Decision or Result + +Add a separate non-parametric estimator, `HistogramBoost`, rather than forcing +many histogram logits through NaturalBoost's one-tree-per-parameter loop. Each +round learns one shared routing structure with a vector of logit updates in +every leaf. Softmax probabilities imply a valid monotone CDF, so quantile +crossing is impossible. + +The training loss is a discretized CRPS (ranked probability score) on an +ordered target grid. Its curvature is the positive-semidefinite diagonal of the +Gauss--Newton matrix, `2 * diag(J.T W J)`. An earlier prototype that took the +absolute value of an indefinite exact Hessian diagonal is mathematically +invalid and was not carried into OpenBoost. + +The frozen development defaults are 50 distribution bins, 100 shared-vector +trees, learning rate 0.05, depth 6, and curvature scale 1. These defaults were +chosen using only the already-consumed `1028_SWD` diagnostics. They must not be +tuned again before the preregistered `197_cpu_act` run. + +## Changes + +- `src/openboost/_core/_vector_tree.py`: CPU-only level-wise vector histogram + tree with shared splits, vector Newton leaves, missing-value routing, and + output-count-invariant gain/child-weight aggregation. +- `src/openboost/_models/_histogram_boost.py`: sklearn-cloneable + `HistogramBoost`, train-only target support, empirical smoothed base logits, + PSD CRPS gradients/curvature, sample weights, and full histogram distribution + output (`mean`, `variance`, `quantile`, `interval`, and `sample`). +- Persistence reconstructs `VectorLeaves` with its output dimension and can + auto-load `HistogramBoost`. +- Shared sample-weight validation now rejects all non-finite values. +- The model is exported from the public package, while the vector-tree builder + remains internal until its CPU/CUDA contract is complete. + +## Verification + +- Focused model, persistence, growth, and validation suite: 82 passed. +- Ruff lint passed for every changed production/test file; the three new files + pass Ruff formatting. +- Finite differences validate the CRPS logit gradient. +- An explicit Jacobian validates the PSD Gauss--Newton diagonal. +- Brute-force split/leaf checks cover vector gain and missing routing. +- Fit/predict tests cover monotone CDFs, train-only support, moments, quantiles, + deterministic sampling, sample weights, sklearn cloning, and persistence. +- A local, non-artifact `1028_SWD` fold-0 smoke produced CRPS 0.327624 versus + 0.334891 for the frozen native XGBoost quantile row. This is consumed-data + implementation evidence only and cannot support a product claim. +- The ScoringBench adapter exposes the frozen model as + `openboost_histogram_cpu` and passes the model's regular PMF grid through the + benchmark's native-grid path. The three wrapper contracts and 13 provenance + tests pass against the pinned local ScoringBench checkout. +- The complete non-GPU/non-benchmark CPU suite passed with 764 tests and 32 + expected skips. Public documentation marks the estimator CPU/numeric only, + explains its finite-support risk, and makes no benchmark-win claim. +- The normal MkDocs build passes. Strict mode still aborts on 29 pre-existing + Griffe warnings in older callbacks, distributions, losses, models, array, + tree, and importance docstrings; none originate from `HistogramBoost`. +- `evaluate_crps_candidate.py` turns the preregistered thresholds into a + fail-closed machine-readable decision: exact rows, one globally selected + strong baseline, paired fold wins, CRPS ratio, coverage-error improvement, + interval-score ratio, and RMSE ratio. This evaluator was committed before + observing the new development outcome. + +## Failed Attempts + +- The first preregistered Actions run (`31930232251`) failed before dataset + download because the launcher resolved a relative frozen-registry path after + changing into its artifact directory. Registry inputs are now resolved + before that directory change and covered by a regression test. The failed + run observed no benchmark outcome and remains part of the audit trail. +- Absolute-value exact Hessian: improved the prototype but has no valid PSD or + majorization interpretation; reject it. +- Gaussian residual-shape calibration and independent scalar quantile models: + remained materially behind the native quantile baseline; do not productize + them as the winning path. +- Reusing NaturalBoost's parameter loop: would fit 50 independent structures + per round and lose the shared-tree scaling and non-crossing design. + +## Risks and Follow-ups + +- The implementation is intentionally CPU/numeric only. CUDA histograms and + prediction kernels are scalar today; a GPU path needs output tiling and exact + CPU/CUDA parity before it is enabled. +- Categorical splits, callbacks, evaluation sets, early stopping, subsampling, + and column sampling are not implemented. +- Fixed train-range support can clip held-out extremes. The preregistered run + must publish failures and all interval/calibration guardrails. +- Public `sample_weight` still has a release blocker: rows with zero weight are + removed from gradient aggregation but not from feature binning or target + support. A zero-weight feature or target outlier can therefore change the + fitted model. Delete zero-weight rows before both binning steps and add + deletion-equivalence tests. +- The V2 workflow verifies result completeness and provenance but does not yet + execute the acceptance evaluator in CI. The confirmation path is also not + phase-bound. Keep `537_houses` locked until both are machine-enforced. + +## First preregistered development result + +The frozen `197_cpu_act` run completed 15/15 rows from clean source with the +preregistered dataset hash, but the candidate failed the development gate. +CatBoost was the strong baseline. HistogramBoost was 2.111% worse on mean CRPS +and won only two of five folds; its 98.23% coverage produced 8.23 percentage +points of absolute 90% coverage error. The interval-score and RMSE guardrails +passed. The confirmation dataset therefore remains locked. + +HistogramBoost did beat native XGBoost quantile on all five folds, with 16.41% +lower mean CRPS, 24.93% lower 90% interval score, and 29.94% lower RMSE. This is +a one-dataset development signal, not an overall win. The candidate was badly +over-dispersed: mean sharpness 6.48 versus CatBoost's 1.56, while RMSE was only +0.34% worse. Diagnose retained tail mass on this consumed dataset before +changing capacity or touching confirmation data. + +The follow-up audit found two semantic problems for V2. First, V1 trained a +midpoint ranked-probability approximation while ScoringBench evaluated the +exact piecewise-uniform histogram CRPS; the within-bin term is not a constant. +Second, `base_smoothing=1` added one pseudocount per bin, so prior strength grew +with output resolution. V2 uses the exact energy-form CRPS gradient and PSD +simplex-tangent curvature, and treats `base_smoothing` as a total Dirichlet +concentration spread evenly across bins. Both changes have analytic and +finite-difference tests; neither is yet benchmark evidence. + +The training audit also showed that one absolute `reg_lambda` controlled both +split structure and leaf updates. Lowering it from 1 to 0.1 on a consumed fold +reduced sharpness strongly but changed the learned structure enough to worsen +RMSE. V2 therefore adds an optional `leaf_reg_lambda`: split regularization +stays at 1, while leaf shrinkage can be studied independently. `None` preserves +the original coupled behavior. + +On consumed `197_cpu_act` fold 0, the exact-objective model at 100 rounds +improved CRPS from 1.3614 to 1.3187 and RMSE from 2.8446 to 2.7647, but official +90% coverage remained 97.67%. A diagnostic temperature sweep found that 0.7 +improved CRPS again to 1.2957, RMSE to 2.7271, coverage to 94.0%, and interval +score to 10.9102. Because that temperature used an already-consumed outer fold, +it cannot be frozen directly. The ScoringBench wrapper now supports selecting +temperature by exact CRPS on an inner training-only split, then refitting the +base model on the complete outer training fold. Its default grid remains +`(1.0,)`, so V1 behavior does not silently change. + +The inner split selected temperature 0.85 on consumed fold 0. Outer metrics +were CRPS 1.2986, RMSE 2.7148, coverage 95.67%, and interval score 11.7174. +Exact within-bin coverage was 93.17%, confirming that the remaining official +coverage excess came partly from whole-bin interval envelopes. The distribution +output now supports density-preserving subdivision, and the wrapper can refine +50 training bins to 100 evaluation bins. Subdivision leaves exact CRPS, mean, +and variance unchanged; it only reduces evaluator quantile-grid error. + +## Frozen V2 development result + +The frozen V2 Actions run `31932958804` completed 15/15 rows from clean +OpenBoost source `39bdb63` and pinned ScoringBench source `a938a667`. The +downloaded artifact had GitHub digest +`sha256:572672e818cf60a295826f7b057bad0269f5121f5bfc31a258188eea12234adc`, +and the compressed `197_cpu_act` input matched its preregistered SHA-256. The +fail-closed evaluator returned `development_pass=true`. + +HistogramBoost V2 achieved mean CRPS 1.287204. That was 2.943% lower than +CatBoost MultiQuantile's 1.326236, with four strict fold wins out of five, and +20.550% lower than native XGBoost quantile's 1.620136, with five of five fold +wins. V2 also lowered RMSE by 1.200% relative to CatBoost and by 31.017% +relative to XGBoost. Its official 90% interval score was 11.281116 versus +13.329529 and 17.979673, respectively. + +Official 90% coverage was 94.80%, so the frozen absolute-error guardrail passed +at 4.80 percentage points. This value is not a standalone calibration claim: +ScoringBench uses whole-bin interval envelopes and V2's lossless two-way +subdivision reduces that representation error without changing the density, +physical CRPS, mean, or continuous variance. Exact interpolated multi-level +coverage should become the future calibration gate. + +The quality improvement did not make the CPU implementation fast. Mean fit +time, including the inner calibration fit, was 161.41 seconds per fold: 1.145x +CatBoost and 10.907x XGBoost in this one four-core Actions run. This converts +GPU vector histograms from a speculative feature into a concrete systems +target: preserve the frozen quality behavior while reducing the ten-fold CPU +gap to native XGBoost. + +This remains consumed one-dataset development evidence. It does not establish +an overall ScoringBench, full-suite, or SOTA win. An independent audit also +found that the current gate mixes CatBoost with the XGBoost-family goal, omits +XGBoostLSS from the V2 run, and is not executed inside the workflow. The next +protocol slice must separate the strict XGBoost-family win, CatBoost +competitiveness, calibration/RMSE guardrails, and confirmation decision before +touching `537_houses`. + +## Commits + +- `b9db276` — `feat: add histogram CRPS boosting` +- `3cbd763` — `bench: automate CRPS candidate acceptance` +- `8ddd276` — `fix: align histogram training with continuous CRPS` +- `35ae55d` — `feat: separate vector split and leaf regularization` +- `ec810fd` — `feat: select histogram temperature on inner validation` +- `5f376e4` — `feat: refine histogram evaluation grids losslessly` +- `39bdb63` — `bench: freeze HistogramBoost V2 protocol` diff --git a/learnings/2026-08-15-linux-jax-ci.md b/learnings/2026-08-15-linux-jax-ci.md new file mode 100644 index 0000000..7312cb5 --- /dev/null +++ b/learnings/2026-08-15-linux-jax-ci.md @@ -0,0 +1,75 @@ +# 2026-08-15: Linux JAX CI Isolation + +## Context + +PR #19's fast-test matrix passed on macOS/Python 3.10 and 3.12, while both +Ubuntu jobs remained in the pytest step for more than 15 minutes. Lint and +dependency installation had already passed. The test extra installs JAX only +on Linux, and two JAX compilation tests were running inside the repository-wide +`pytest-xdist -n auto` process pool. + +## Decision or Result + +Mark the two JAX-dependent tests explicitly and run them once, serially, on +Linux/Python 3.12. The regular fast and full suites exclude that marker. This +separates JAX compilation/runtime behavior from xdist worker behavior and avoids +duplicating the same optional-backend test across two Python versions. + +Add job-level timeouts and workflow concurrency cancellation so a future hang +cannot consume the default six-hour GitHub Actions limit or leave superseded PR +runs executing indefinitely. + +The replacement Ubuntu/Python 3.12 job falsified the original performance +hypothesis: the non-JAX core suite passed 729 tests in 937.02 seconds, while the +two isolated JAX tests passed in 3.59 seconds. JAX was not the source of the long +step. Keep the isolation because it makes the optional path explicit, but treat +the 15-minute cold core suite as the CI cost to optimize. + +The downstream `full-tests` job was also rerunning the entire core suite after +all four matrix jobs had passed. It now runs only tests marked `slow` plus the +documentation examples. Numba's on-disk cache is restored per OS and Python +version so later commits can reuse compiled kernels; the first run for a new +source hash remains a cold run. + +## Changes + +- `tests/test_distribution_gradients.py`: mark the two true JAX tests. +- `pyproject.toml`: register the `jax` pytest marker. +- `.github/workflows/unit-tests.yml`: exclude JAX from parallel suites, add one + serial Linux/Python 3.12 step, add bounded job timeouts, and cancel superseded + runs on the same ref. A follow-up avoids the duplicate core suite, restores a + versioned Numba cache, and moves JavaScript actions off deprecated Node 20 + releases. + +## Verification + +- Non-JAX distribution gradient selection: 49 passed, 2 deselected. +- JAX selection on macOS Intel: 2 selected and correctly skipped because no JAX + wheel is installed on that platform. +- Workflow YAML parsing and `git diff --check`: passed. +- The decisive verification is the replacement PR workflow on Ubuntu; local + macOS cannot reproduce the Linux-only JAX installation. +- Replacement Ubuntu/Python 3.12 job: 729 passed, 32 skipped in 937.02 seconds; + isolated JAX step: 2 passed in 3.59 seconds. +- Revised local downstream selection: 2 slow tests passed, 762 deselected in + 10.91 seconds; all 6 executable documentation files passed. + +## Failed Attempts + +- GitHub does not publish downloadable logs for an in-progress job; the log + endpoint returned 404, so there was no responsible way to name a specific + test from partial output. +- Repeated status polling confirmed the Ubuntu pair was symmetric and isolated + to pytest but could not distinguish JAX compilation from another Linux-only + stall. The new split is designed to make the next run diagnostic as well as + faster. + +## Risks and Follow-ups + +- Measure one warm-cache PR run before claiming the cache improved latency. +- The two slow tests and documentation examples still need to pass in the + revised downstream job before the duplicate-suite removal is accepted. + +## Commits + +- `7e26d0e` — `ci: isolate JAX tests from xdist` diff --git a/learnings/2026-08-15-performance-gate.md b/learnings/2026-08-15-performance-gate.md new file mode 100644 index 0000000..5fcdf81 --- /dev/null +++ b/learnings/2026-08-15-performance-gate.md @@ -0,0 +1,67 @@ +# 2026-08-15: Performance Regression Gate + +## Context + +The performance CI expected `benchmarks/results/performance_baselines.json`, +but that file was ignored and absent from fresh checkouts. On a missing file, +the script benchmarked the current commit, saved that same result as its own +baseline, and exited successfully. The advertised regression gate was therefore +a no-op on every fresh GitHub runner. + +## Decision or Result + +Compare the code before a push with the code after the push on the same GitHub +runner instead of committing an absolute timing baseline from unrelated +hardware. The baseline revision is `github.event.before`, not `HEAD^`, so one +workflow covers every commit in a multi-commit push to main. + +Both revisions run through the current fixed harness with separate Numba cache +directories. Raw parent/current JSON files are uploaded and include commit, +runtime versions, platform, backend, and relevant Numba environment fields. +Missing baselines now exit with status 2 before doing expensive work; creating a +local baseline requires an explicit flag. + +## Changes + +- `benchmarks/check_performance.py`: add explicit baseline/output/source-root + options, benchmark-only mode, provenance, and fail-closed missing-baseline + behavior. +- `.github/workflows/unit-tests.yml`: fetch history, create a detached worktree + at the previous remote main, benchmark old/current code on one runner with + isolated caches, and upload both artifacts. +- `tests/test_performance_check.py`: cover equal results, runtime/quality + regressions, explicit baseline loading, and provenance. + +## Verification + +- Missing-baseline CLI check exited 2 in 0.2 seconds and created no baseline. +- Unit suite: 4 passed. +- Ruff and workflow YAML parsing: passed. +- End-to-end temporary-worktree simulation: + - baseline commit `6440e31143e4cbd56e9d523a25a8f48bca302670`; + - current commit `077211066af1956f38e2a7183cd77bdd0ace140c`; + - both artifacts contained provenance and comparison returned no regressions. +- The temporary worktree was removed after verification. + +## Failed Attempts + +- A committed baseline generated on this Intel macOS host was rejected as a CI + design because absolute timings are not portable to GitHub's Linux runners. +- Comparing only `HEAD^` was rejected because a push containing several commits + would test only the final commit. `github.event.before` represents the actual + remote-main baseline for the pushed range. + +## Risks and Follow-ups + +- Shared hosted runners remain noisy. The current median-of-three and 20% + threshold are a regression alarm, not publication-quality performance proof. +- Parent and current code use dependencies installed from the current checkout; + this isolates source regressions but does not detect dependency-only speed + changes. +- The workflow must run on GitHub once to validate hosted-runner behavior and + artifact upload. External ScoringBench results remain the value proof; this CI + microbenchmark is maintenance infrastructure. + +## Commits + +- `30b9ab5` — `ci: compare performance across pushed revisions` diff --git a/learnings/2026-08-15-repository-audit.md b/learnings/2026-08-15-repository-audit.md new file mode 100644 index 0000000..6c6ad81 --- /dev/null +++ b/learnings/2026-08-15-repository-audit.md @@ -0,0 +1,67 @@ +# 2026-08-15: Repository Audit and Product Focus + +## Context + +A parallel code, benchmark, release, and ecosystem audit evaluated whether +OpenBoost had a credible path to a niche comparable in clarity—not impact—to +XGBoost. The audit was read-only and used local tests plus current primary +sources for competing libraries and publication routes. + +## Decision or Result + +OpenBoost has a substantive CPU tree core and a strong distributional subsystem, +but it is not yet a trustworthy general-purpose boosting library. The product +focus is now **calibration-first distributional boosting for tabular risk**: +NaturalBoost, exposure-aware count/severity models, proper scoring, calibration, +custom distributions, and verified single-GPU acceleration. + +Generic GBDT, GAM, DART, linear leaves, Ray, multi-GPU, out-of-core, GOSS, and +train-many must not share equal product priority. GPU remains strategically +important, but the next milestone is one correct and evidenced NaturalBoost CUDA +path—not broader unverified GPU surface area. + +## Evidence + +- CPU suite at audit time: 721 passed, 32 skipped, 3 deselected. +- Total coverage: 53%; `_models/_distributional.py` 95%, distributions 79%, + CUDA backend 0%, multi-GPU 16%, distributed tree 17%. +- The only committed third-party artifact was a three-dataset, one-seed CPU + NaturalBoost/NGBoost comparison showing approximate parity, not dominance. +- The strongest external validation opportunity was ScoringBench, which accepts + probabilistic model wrappers and publishes proper-scoring leaderboards. + +## Release-Blocking Findings + +- Categorical persistence used field names different from `TreeStructure`, so a + save/load round trip could change predictions. +- Categorical binning accepted up to 254 values while routing used one 64-bit + bitset. +- GPU GAM training dropped the base score after its first prediction update. +- Ray/multi-GPU workers initialized predictions inconsistently with final model + inference, and multi-GPU child histograms were approximate. +- The documented memmap out-of-core example passed a feature-major array to a + sample-major high-level API; `batch_size` was not connected to model training. +- The performance CI baseline was absent and regenerated on fresh runners, so + the check could succeed without detecting regressions. + +These findings must be re-verified against current code before fixing; this +entry records the audit state, not permanent truth. + +## Product and Evidence Gates + +1. Remove silent correctness failures. +2. Establish deterministic CPU reference behavior. +3. Verify end-to-end CPU/CUDA NaturalBoost parity. +4. Submit full-suite ScoringBench results with raw artifacts. +5. Add a real exposure-aware insurance case study such as freMTPL2. +6. Seek external users and contributions before JOSS/JMLR software submission. + +## Risks and Follow-ups + +- Distributional boosting mostly models aleatoric uncertainty; it does not by + itself solve epistemic/OOD uncertainty. +- PGBM, XGBoostLSS, LightGBMLSS, NGBoost, CatBoost uncertainty, and Py-Boost + already occupy adjacent positions. GPU probabilistic boosting alone is not a + unique claim. +- A benchmark is allowed to falsify the product hypothesis. Quality regressions + cannot be traded for speed without an explicit decision metric. diff --git a/learnings/2026-08-15-scaling-boundaries.md b/learnings/2026-08-15-scaling-boundaries.md new file mode 100644 index 0000000..2b52a3c --- /dev/null +++ b/learnings/2026-08-15-scaling-boundaries.md @@ -0,0 +1,68 @@ +# 2026-08-15: Scaling Boundaries + +## Context + +The high-level `GradientBoosting` and `MultiClassGradientBoosting` APIs exposed +`batch_size`, but neither training loop read it. The large-scale guide also +passed a feature-major binned memmap directly to a high-level `fit` method that +validates sample-major input. GPU and multi-GPU pages quoted speedups without a +checked-in reproducible artifact. + +## Decision or Result + +Unsupported scale features now fail or read as experimental instead of looking +production-ready: + +- a non-`None` high-level `batch_size` raises `NotImplementedError` before fit; +- memmap and mini-batch histogram utilities remain available as low-level + building blocks, not an out-of-core model API; +- GOSS is described by its sampling behavior, without a universal speed/quality + promise; +- multi-GPU is explicitly experimental until parity and repeated two-/four-GPU + measurements exist; +- numeric GPU speedup tables were removed in favor of an evidence checklist. + +## Changes + +- `src/openboost/_models/_boosting.py`: validate the reserved batch parameter in + single-output and multiclass fits; remove unsupported performance wording. +- `tests/test_large_scale.py`: cover both high-level model families. +- `docs/user-guide/training/large-scale.md`: replace the broken out-of-core + recipe with a capability/status matrix and evidence gate. +- GPU setup, installation, sklearn docstrings, model guide, and GPU example: + align public wording with the actual support boundaries. + +## Verification + +- Before the fix, both new high-level tests failed because no exception was + raised and training proceeded while ignoring `batch_size`. +- Focused batch-size tests: 2 passed. +- `uv run pytest -q tests/test_large_scale.py tests/test_core.py + tests/test_losses.py`: 77 passed, 3 expected bin-count warnings. +- `uv run mkdocs build`: passed with the repository's 29 existing griffe + warnings. +- Focused source Ruff checks, example compilation, and `git diff --check`: + passed. + +## Failed Attempts + +- `uv run mkdocs build --strict` stopped on 29 existing griffe warnings in API + docstrings across callbacks, distributions, losses, models, arrays, trees, + and importance helpers. The current docs workflow is non-strict, so the + matching build was used for this change. Strict docs cleanliness remains a + separate maintenance task. + +## Risks and Follow-ups + +- The low-level memmap and mini-batch helpers are not proof of end-to-end + out-of-core training. Implement and test a real loop before reintroducing that + claim. +- Multi-GPU correctness is not established by documentation. Require exact + single-device parity and real two-/four-GPU artifacts before promotion. +- Run single-GPU scale-extension benchmarks on Linux/CUDA with full provenance; + this Intel macOS environment cannot provide those results. + +## Commits + +- `ee555cb` — `fix: fail fast for unsupported model batching` +- `6440e31` — `docs: mark experimental scaling boundaries` diff --git a/learnings/2026-08-15-scoringbench-integration.md b/learnings/2026-08-15-scoringbench-integration.md new file mode 100644 index 0000000..26e5a77 --- /dev/null +++ b/learnings/2026-08-15-scoringbench-integration.md @@ -0,0 +1,299 @@ +# 2026-08-15: ScoringBench Integration + +## Context + +OpenBoost needed an existing third-party benchmark or competition to demonstrate +value. ScoringBench was selected because it evaluates full probabilistic +regression distributions with proper scoring rules and accepts upstream model +wrappers and result artifacts. + +## Decision or Result + +The integration has two explicitly separated protocols: + +1. `official_quality`: ScoringBench's five-fold, 3,000-row protocol for an + upstream leaderboard submission. +2. `scoringbench_scale_extension`: the same datasets/folds/metrics with a larger + sample cap to compare OpenBoost CPU, OpenBoost CUDA, and existing baselines. + +Scale-extension results must never be represented as official leaderboard +results. ScoringBench proves general probabilistic quality; it does not exercise +OpenBoost's exposure-aware API, which still needs a domain benchmark. + +## Changes + +- `benchmarks/scoringbench/openboost_wrapper.py`: upstream-shaped NaturalBoost + Gaussian wrapper using ScoringBench's shared quantile-to-PMF conversion. +- `benchmarks/scoringbench/run.py`: launcher for an unmodified ScoringBench + checkout, baseline registration, protocol labeling, and provenance manifest. +- `benchmarks/scoringbench/README.md`: environment, official track, scale track, + upstream submission, and evidence gates. +- `.gitignore`: ignore arbitrary local ScoringBench result directories. +- `.github/workflows/scoringbench.yml`: pinned Linux contract/smoke validation, + artifact upload, and a manually dispatched official-quality shard. +- `benchmark_outcome.json`: an exact dataset/model/fold completeness audit that + makes upstream-captured failures and non-finite distributional metrics + machine-readable and changes the launcher exit status to failure. +- Frozen-registry sharding: `--dataset-registry`, `--shard-index`, and + `--shard-count` split one ordered registry across workers without rebuilding + a potentially changing OpenML suite in each job; source and copied-registry + hashes are recorded in schema-v2 manifests. +- Strong-baseline mode adds the ScoringBench native XGBoost quantile, Gaussian + XGBoostLSS, and CatBoost MultiQuantile wrappers with frozen package versions + and their registered model-specific budgets. This makes NGBoost a reference, + not the acceptance bar. +- Development mode exposes OpenBoost tree count, learning rate, depth, L2 leaf + regularization, and minimum child Hessian while forcing the manifest protocol + label to `development_tuning`. It preserves ScoringBench folds and metrics but + is intentionally ineligible for held-out or leaderboard evidence. + +## Verification + +- Wrapper contract: 1 passed against ScoringBench commit + `a938a667b7839b41e9272929010573410301c0b4`. +- Fresh isolated-environment contract after adding xdist: 1 passed on Intel + macOS with Numba 0.63.1; this validates the adapter only, not benchmark scores. +- OpenBoost distributional regression tests: 47 passed. +- ScoringBench provenance/outcome/sharding tests: 8 passed after adding frozen + registry loading, exact-completion and mixed missing/error/non-finite cases, + and proof that strided shards cover each entry exactly once. +- Strong-baseline parser/provenance suite: 9 passed. The frozen Linux target + dependency contract resolved 87 packages including NumPy 2.2.6, Pandas + 2.2.3, Torch 2.9.1, XGBoost 3.3.0, XGBoostLSS 0.6.1, and CatBoost 1.2.10. +- `ruff check benchmarks/scoringbench`: passed. +- Python compilation and manifest protocol classification: passed. +- GitHub workflow YAML parsed locally; the pinned wrapper contract passed before + the workflow was added, then the complete Linux smoke passed in run #1. +- Linux ScoringBench run #1 passed the contract, two-fold OpenBoost/NGBoost + smoke, artifact verification, and upload. Artifact `9256565927` contains the + manifest and both raw Parquet files with 4 result rows; its digest is + `sha256:3c98f640af58291f7cb648ac38bfa04ff0a44bd198698fb47a2b4f22dfb98862`. + This proves the integration path only, not comparative model value. +- CI/source provenance, artifact-working-directory, and named-shard selection + tests: 4 passed. +- Linux ScoringBench run #4 confirmed `source_sha`, tested PR merge SHA, clean + checkouts, pinned upstream SHA, and 4 smoke rows in artifact `9256692529`. +- Linux ScoringBench run #6 completed the `1027_ESL` official-quality sentinel: + 10 fold/model rows, clean provenance, and artifact verification all passed. + Artifact `9256929555` has digest + `sha256:0c5176c4a28cd44444f6a086643a01f636b0dcb5e165a4a30d3d992f3e96da97`. +- The frozen evidence under + `benchmarks/evidence/scoringbench/1027_esl_20260816/` preserves the manifest, + resolved dataset registry, both raw Parquet files, and a descriptive summary. + OpenBoost's mean CRPS/RMSE/90% interval score and time were better on this + shard, but mean log score was worse and CRPS won only 2/5 folds. +- Clean strong-baseline run `31925701435` completed all 25 expected rows for + `1027_ESL` at source commit `cea891a` and pinned ScoringBench commit + `a938a667`. Artifact `9257853524` has digest + `sha256:4044cc803958036d16c55aefed98c3142486e7ddba4bdfca61f364d5e7310765`. + The source checkout had no porcelain changes, and every frozen input/result + file is checksummed in + `benchmarks/evidence/scoringbench/1027_esl_strong_20260816/summary.json`. +- `1028_SWD` clean baseline run `31926255124` completed 25/25 rows in artifact + `9258026048` (digest `sha256:495ac3c8c02c0846423131c4636dfde05f0da872b637bca2be9493b3c02aa8b4`). + Development run `31926664340` completed 5/5 rows with the expected + `development_tuning` label in artifact `9258115955` (digest + `sha256:b3b77cbf691bef574b0a27c897b8a92a22e90a4c158f0f7e3ae556400d60e599`). + Both source checkouts were clean; every copied artifact file is checksummed + under `benchmarks/evidence/scoringbench/development/1028_swd_lr_sweep_20260816/`. +- On that one diagnostic shard, OpenBoost ranked first on mean CRPS, 90% + interval score, absolute 90% coverage error, and PIT KS. Against native + XGBoost quantile it reduced those metrics by 7.7%, 38.8%, 82.2%, and 59.5% + respectively and reduced RMSE by 11.6%. Against Gaussian XGBoostLSS it had + 1.2% lower CRPS and 51.3% lower coverage error, but 1.4% worse RMSE and about + 9.1 times its fit-only per-fold training time. +- Those results define a useful hypothesis, not a win: the shard has one small + dataset, one seed, correlated folds, unequal model-specific budgets, CPU + only, and no confidence interval. Timing also varied materially between two + otherwise equivalent Actions runs, so it cannot support a speed claim. +- A post-run metric audit found that the current ScoringBench reconstructed + log score clamps targets outside finite quantile support to the boundary-bin + density. CRLS is integrated over each model's own support, and the upstream + implementation explicitly warns that values are not comparable across + different bin grids. Both metrics remain in the raw artifact but are excluded + from cross-model conclusions. Use a separately audited analytic Gaussian NLL + for parametric-only density comparison; do not optimize OpenBoost against the + current quantile log-score artifact. +- Untouched development dataset `1028_SWD` showed the opposite CRPS ranking + from `1027_ESL`: OpenBoost had the best interval score, coverage error, and + PIT KS, but mean CRPS was 4.9% worse than native XGBoost quantile, 1.7% worse + than XGBoostLSS, and 2.0% worse than NGBoost. It lost CRPS to native XGBoost + on all five folds. This disproves a general quality-win claim and identifies + a reproducible sharpness/calibration trade-off. +- A development-only `0.03 × 500` OpenBoost run narrowed mean sharpness by 5.4% + relative to `0.01 × 500`, but worsened CRPS by 0.44%, interval score by 4.74%, + PIT KS by 9.82%, and absolute 90% coverage error by 3.23%. It improved paired + CRPS in only two of five folds. Reject the larger fixed learning rate; the + next experiment must separate mean accuracy from post-fit scale calibration + or test a different scale objective. +- Gaussian CRPS training is not a new algorithmic claim: NGBoost already + publishes a Normal CRPS score and generalized natural-gradient metric. A + direct local prototype of that metric was unstable at larger learning rates + on heteroscedastic synthetic data (mean predicted scale exploded), so it was + rejected as OpenBoost's implementation path rather than copied blindly. +- The exact Gaussian CRPS Hessian is indefinite in the tails and cannot be fed + to OpenBoost's positive-Hessian tree solver. The implemented explicit + `training_objective='crps'` instead uses the strictly positive expected CRPS + curvature under the current Normal prediction. This keeps the default NLL + path unchanged, keeps training objective independent from `eval_metric`, and + fails early for non-Normal distributions. +- In a local heteroscedastic synthetic diagnostic, expected-curvature CRPS + training reduced held-out CRPS relative to NLL training at each tested fixed + learning rate (`0.003`, `0.01`, `0.03`, and `0.1`) for 300 depth-3 rounds. + This is a development hypothesis only, not benchmark evidence. The core + mathematical/API slice passed 191 tests (2 skipped), including finite + differences, default-NLL identity, objective logging, sklearn cloning, and + persistence. It must still win on the frozen `1028_SWD` development folds + before being promoted into the ScoringBench wrapper experiment. +- The benchmark launcher and manual Actions workflow now record and forward + `training_objective`; the default remains `nll`, while CRPS candidates must + carry the `development_tuning` protocol label. The provenance suite passed + 10 tests and both NLL/CRPS wrapper contracts passed against the pinned + ScoringBench checkout. +- Clean CRPS-objective run `31927426636` completed 5/5 `1028_SWD` rows in + artifact `9258329504` (digest + `sha256:02f3827ee1787a6559f841cd277407d7faa1f2af2dd8031caf39794d8173594e`). + Against the identical NLL configuration it improved mean CRPS by 0.30% + (four of five folds), PIT KS by 7.41%, coverage error by 9.68%, and RMSE by + 0.08%, but worsened 90% interval score by 3.94% on all five folds and widened + sharpness by 2.63%. It still trails every frozen strong baseline on CRPS. + Keep the objective implementation, reject this configuration as the final + wrapper, and next isolate post-fit scale calibration on training folds only. +- The apparent mean-model gap was traced to numeric binning, not histogram + subtraction. `np.searchsorted` over `m` cut edges legitimately returns + indices `0..m`, but both fit and transform clipped the result to `m - 1`. + That silently merged the highest numeric interval into its predecessor; a + binary feature with `n_bins=2` could become constant. The corrected binning + keeps the top index, tests the NaN and out-of-range paths, and records a + binning-semantics version in persistence. Models saved before serialization + version 4 retain the legacy routing so loading them does not silently change + predictions. Histogram subtraction was independently checked against direct + child histograms and selected the same splits; do not replace that optimized + path as part of this fix. +- The binning/persistence slice passed 83 focused tests across array handling, + tree growth, core fitting, and model round trips. Production files and the + changed binning test pass Ruff; `tests/test_persistence.py` still has its + pre-existing import-order/unused-import findings, which are unrelated to this + correctness change. A fresh ScoringBench artifact is still required before + interpreting the quality impact. +- The full non-GPU/non-benchmark CPU suite subsequently passed with 752 tests + and 32 expected skips. Clean Linux development run `31928677396` then + completed all five `1028_SWD` folds from source `236c2df`; artifact + `9258714239` has digest + `sha256:c75eea2d93bf10d5de1ae48a8ab37eed66c1dc1c326a9810ab8dc808c4656836`. + The committed raw artifact is under + `benchmarks/evidence/scoringbench/development/1028_swd_binning_fix_20260816/`. +- Correct binning reduces OpenBoost mean CRPS from 0.355075 to 0.347845 + (2.04%, four of five folds) and RMSE from 0.626043 to 0.615779. On this + consumed development dataset it beats XGBoostLSS CRPS by 0.41% in four of + five folds, while NGBoost remains effective parity (0.10% lower mean but only + two OpenBoost fold wins). The primary goal remains unmet: OpenBoost is 2.80% + behind native XGBoost quantile and 3.41% behind CatBoost quantile, winning + only one fold against each. Its coverage error, interval score, PIT KS, and + RMSE are substantially better than those two quantile rows here; report this + as a Pareto trade-off, not a CRPS win. +- Two local, non-artifact follow-ups were rejected before implementation. + Replacing the Normal shape with the training residual empirical shape made + negligible CRPS difference. Independent OpenBoost quantile models improved + with more rounds but did not close the native-XGBoost gap, while a prototype + exact residual-quantile leaf refit was worse. Do not productize either path + without a stronger multi-dataset hypothesis and a clean artifact. +- A shared-tree vector-PMF prototype identified a different ceiling: directly + optimizing discretized CRPS can close the shape gap that a two-parameter + Gaussian cannot. The first prototype used an invalid absolute-value + transform of an indefinite exact diagonal Hessian and is rejected. Replacing + it with the positive-semidefinite Gauss--Newton diagonal, 50 bins, 100 rounds, + and learning rate 0.05 produced mean CRPS 0.333588 on the already-consumed + `1028_SWD` folds, versus 0.338359 for the frozen native XGBoost quantile row. + This selects an architecture for implementation; it is not new-dataset or + OpenBoost-core evidence. +- Before loading more data, protocol `crps_distribution_v1` froze + `197_cpu_act` for development and `537_houses` for one untouched confirmation + run. Their PMLB source commit, compressed-file hashes, model budgets, metric + guardrails, fold-win thresholds, and permitted claim language are committed + under `benchmarks/scoringbench/protocols/`. The candidate API and exact + hyperparameters must be committed before loading the development entry. +- Frozen registries may now include `raw_sha256`. The launcher materializes + those entries through ScoringBench's raw cache, rejects a mismatch before + validation, bypasses the opaque processed cache for the run, and records the + verified URL, digest, and byte size in manifest schema 3. This makes the + protocol's immutable-data requirement executable rather than documentary. +- A registry entry marked `untouched_confirmation` is rejected before download + unless the launcher receives `--allow-confirmation`. This is a deliberate, + manifest-recorded unlock after the selected candidate is committed. +- Integration commit: `a4555bc` (`bench: add ScoringBench integration`). + +## Failed Attempts + +- A first local dependency resolution appeared to make XGBoostLSS 0.6.1 select + Torch 2.2.2 beside NumPy 2.2.6. PyPI metadata disproved the suspected hard + pin: XGBoostLSS declares `torch>=2.1,<2.10`. The old selection came from + resolving for Intel macOS, where 2.2.2 is the final available Torch wheel; + benchmark dependencies must be resolved for the Linux target platform. +- Strong-baseline run `31925230473` completed all 25 expected rows, but its + OpenBoost checkout was dirty after training. CatBoost writes `catboost_info/` + in the process working directory unless configured otherwise. The factory + now sets `allow_writing_files=False`, manifests include porcelain change + paths, and the artifact gate rejects a dirty OpenBoost checkout. + +- The composer-swarm Cursor scout repeatedly failed with macOS Keychain error + `SecItemCopyMatching failed -50`. Use local inspection until its CLI + authentication is repaired; do not repeatedly retry it during one task. +- The complete ScoringBench runner is not viable on Intel macOS. ScoringBench + requires NumPy 2.x, while the available PyTorch wheel uses the NumPy 1.x ABI. + One run crashed and later attempts entered an uninterruptible kernel exit + state. The launcher now refuses this platform before importing ScoringBench. +- A fresh isolated environment could not collect the wrapper test because the + repository-wide pytest configuration enables xdist while the benchmark + requirements omitted `pytest-xdist`. The isolated requirements now include + it. +- Installing unconstrained `numba>=0.60` on Intel macOS selected Numba 0.67, + for which no compatible wheel was available; llvmlite then tried to build + against LLVM 20 although that release requires LLVM 22. Installing the last + available Intel wheel (`numba==0.63.1`) and the editable project with + `--no-deps` is sufficient for the wrapper-only contract. This workaround is + not a supported full benchmark environment. Published CPU/CUDA runs must use + Linux. +- Building ScoringBench's official dataset registry writes `datasets.json` to + `Path.cwd()`. A dataset-list probe therefore polluted the OpenBoost root and + would make a later manifest report a dirty checkout. The launcher now builds + and validates that registry from inside the output directory and restores the + original working directory even after an exception. +- ScoringBench validates datasets by loading them. Selecting a shard by index + therefore validates the entire registry before the index exists. Exact-name + shards now select first and validate only the requested datasets; index and + list modes retain their validated-list semantics. +- The first Abalone sentinel never reached model fitting: all three OpenML suite + requests returned server errors and dataset 183 then exhausted retries with + HTTP 504. ScoringBench caught the dataset exception and returned an empty + result while exiting successfully; OpenBoost's `result_rows == 10` artifact + gate correctly failed the job. The PR sentinel now uses ScoringBench's + `1027_ESL` PMLB/GitHub source so this small gate does not depend on the OpenML + data endpoint. Full-suite runs still must report OpenML failures rather than + silently treating them as model results. +- ScoringBench's outer runner catches dataset exceptions and its fold runner + converts model exceptions into rows; neither condition necessarily produces + a failing process. The OpenBoost launcher now audits the returned records + after the entire shard finishes, preserves all omissions/errors, and only + then exits non-zero for an incomplete outcome. + +## Risks and Follow-ups + +- Run the official full suite on Linux and submit the wrapper/results upstream. +- Treat `1027_ESL` as consumed diagnostic evidence. Do not tune on it and then + relabel the result as held out. Use separate development datasets to test + Normal scale/log-score hypotheses, preserve CRPS/calibration guardrails, and + make the final decision on untouched datasets or the complete suite. +- The acceptance bar is stronger than NGBoost parity: OpenBoost should rank + first or statistically tied on the primary proper scores, beat the strongest + XGBoost-family baseline on a majority of paired datasets, and avoid material + regressions in interval score, calibration, RMSE, failure rate, or resource + use. CRPS is the current cross-model primary metric. Density scoring becomes + a guardrail only after common-support handling is validated; report + per-dataset paired effects and uncertainty, not only macro means. +- Run a separate large-sample curve on at least three real ScoringBench datasets. +- Add CPU/CUDA prediction parity before interpreting a CUDA timing result. +- The first artifact identified the PR merge commit but not the source-head SHA. + The manifest now records both; confirm the mapping in the next CI artifact. +- Add freMTPL2 or another real exposure-aware case study after the third-party + quality result exists. diff --git a/learnings/README.md b/learnings/README.md new file mode 100644 index 0000000..5b9f1db --- /dev/null +++ b/learnings/README.md @@ -0,0 +1,40 @@ +# OpenBoost Learnings + +This directory is the repository's durable engineering memory. It records why a +change was made, what evidence supports it, which attempts failed, and what +remains unknown. It is deliberately separate from release notes and generated +benchmark output. + +## When to Write an Entry + +Create or update an entry when work includes any of the following: + +- a non-obvious correctness fix; +- a benchmark, experiment, or falsified hypothesis; +- an architecture or product-scope decision; +- a dependency, platform, CI, packaging, or release incident; +- a user correction that should change future agent behavior. + +Use `YYYY-MM-DD-short-topic.md`. Prefer one entry per coherent investigation; +append follow-up evidence rather than creating many tiny diary files. + +## Required Content + +Start from `TEMPLATE.md` and include: + +- context and the question being answered; +- decision or result; +- files/behavior changed; +- verification and artifact locations; +- failed attempts and why they failed; +- remaining risks and next action. + +Keep entries factual and concise. Do not include credentials, secrets, private +URLs, or copied raw logs. Link the relevant commit after it exists. + +## Current Entries + +- `2026-08-15-repository-audit.md` — product focus, correctness risks, and + evidence gaps found in the deep audit. +- `2026-08-15-scoringbench-integration.md` — third-party benchmark integration, + validation, and Intel macOS runtime limitation. diff --git a/learnings/TEMPLATE.md b/learnings/TEMPLATE.md new file mode 100644 index 0000000..a8efe46 --- /dev/null +++ b/learnings/TEMPLATE.md @@ -0,0 +1,32 @@ +# YYYY-MM-DD: Topic + +## Context + +What question, bug, or decision triggered this work? + +## Decision or Result + +What did we decide or learn? Separate measured facts from hypotheses. + +## Changes + +- File or subsystem: behavioral change and reason. + +## Verification + +- Exact focused test or benchmark. +- Result and artifact location. +- Environment limitations that affect interpretation. + +## Failed Attempts + +- Attempt: failure mode and the rule learned from it. + +## Risks and Follow-ups + +- What remains unverified? +- What is the next concrete gate? + +## Commits + +- `SHA` — subject diff --git a/mkdocs.yml b/mkdocs.yml index 11e3b6f..a741c89 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - DART: user-guide/models/dart.md - OpenBoostGAM: user-guide/models/gam.md - Linear Leaf GBDT: user-guide/models/linear-leaf.md + - HistogramBoost: user-guide/models/histogram-boost.md - NaturalBoost: - Overview: user-guide/naturalboost/overview.md - Distributions: user-guide/naturalboost/distributions.md diff --git a/pyproject.toml b/pyproject.toml index ae350e9..27913aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ markers = [ "numerical: marks numerical agreement tests against reference implementations", "parity: marks CPU/GPU parity tests", "benchmark: marks performance benchmark tests (not run in CI)", + "jax: marks tests that import and compile JAX (run serially in Linux CI)", ] [dependency-groups] diff --git a/src/openboost/__init__.py b/src/openboost/__init__.py index 950a945..fa57103 100644 --- a/src/openboost/__init__.py +++ b/src/openboost/__init__.py @@ -184,6 +184,8 @@ # Phase 15/16: Distributional GBDT (NaturalBoost) DistributionalGBDT, GradientBoosting, + HistogramBoost, + HistogramDistributionOutput, # Phase 15: Linear Leaf GBDT LinearLeafGBDT, MultiClassGradientBoosting, @@ -333,6 +335,8 @@ def __getattr__(name: str): "MISSING_BIN", # High-level API (recommended) "GradientBoosting", + "HistogramBoost", + "HistogramDistributionOutput", "MultiClassGradientBoosting", "OpenBoostGAM", "DART", diff --git a/src/openboost/_array.py b/src/openboost/_array.py index 24bea82..191d224 100644 --- a/src/openboost/_array.py +++ b/src/openboost/_array.py @@ -23,6 +23,8 @@ # Reserved bin index for missing values (NaN) MISSING_BIN: int = 255 +_LEGACY_BINNING_VERSION: int = 1 +_CURRENT_BINNING_VERSION: int = 2 @dataclass @@ -51,6 +53,11 @@ class BinnedArray: is_categorical: NDArray[np.bool_] = field(default_factory=lambda: np.array([], dtype=np.bool_)) category_maps: list[dict | None] = field(default_factory=list) n_categories: NDArray[np.int32] = field(default_factory=lambda: np.array([], dtype=np.int32)) + # Version 1 clipped searchsorted outputs to ``len(edges) - 1`` and + # accidentally merged the top numeric bin into its predecessor. Keep the + # version on fitted bin metadata so models saved before the correction + # retain their original prediction routing after loading. + binning_version: int = _CURRENT_BINNING_VERSION def __repr__(self) -> str: n_missing = int(np.sum(self.has_missing)) if len(self.has_missing) > 0 else 0 @@ -91,6 +98,12 @@ def transform(self, X: ArrayLike) -> BinnedArray: >>> X_test_binned = X_train_binned.transform(X_test) >>> predictions = model.predict(X_test_binned) """ + # Direct pickle/joblib payloads from before the version field existed + # bypass PersistenceMixin. Treat their missing attribute as legacy. + binning_version = vars(self).get( + 'binning_version', _LEGACY_BINNING_VERSION + ) + # Convert to numpy X_np = _to_numpy(X) if X_np.ndim == 1: @@ -141,10 +154,13 @@ def transform(self, X: ArrayLike) -> BinnedArray: # No bin edges (constant feature) binned[:, j] = 0 else: - # searchsorted finds the bin index + # ``m`` cut edges define ``m + 1`` bins, indexed 0..m. + # Version 1 incorrectly clipped the upper index to m - 1; + # preserve that behavior only for models trained before + # the top-bin correctness fix. bin_idx = np.searchsorted(edges, col[~nan_mask], side='right') - # Clip to valid range (in case test values exceed training range) - bin_idx = np.clip(bin_idx, 0, len(edges) - 1) + max_bin = _numeric_max_bin(edges, binning_version) + bin_idx = np.clip(bin_idx, 0, max_bin) binned[~nan_mask, j] = bin_idx.astype(np.uint8) # Handle missing values @@ -169,6 +185,7 @@ def transform(self, X: ArrayLike) -> BinnedArray: is_categorical=self.is_categorical, category_maps=self.category_maps, n_categories=self.n_categories, + binning_version=binning_version, ) @@ -198,6 +215,7 @@ def array( categorical_features: List of column indices that are categorical. These use category encoding instead of quantile binning. Max 254 unique categories per feature (255 reserved for NaN). + Tree models currently support at most 64 categories. device: Target device ("cuda" or "cpu"). Auto-detected if None. Returns: @@ -287,6 +305,7 @@ def array( is_categorical=is_categorical, category_maps=category_maps, n_categories=n_categories, + binning_version=_CURRENT_BINNING_VERSION, ) @@ -404,21 +423,42 @@ def _bin_numeric_feature( else: edges = np.nanpercentile(valid_col, percentiles) edges = np.unique(edges) - # searchsorted is what digitize calls internally, minus overhead + # ``m`` cut edges create ``m + 1`` bins. The searchsorted output + # already lies in 0..m; allowing m is essential for preserving the + # top interval, especially for low-cardinality integer features. bin_idx = np.searchsorted(edges, valid_col, side='right') - np.clip(bin_idx, 0, len(edges) - 1, out=bin_idx) + np.clip( + bin_idx, + 0, + _numeric_max_bin(edges, _CURRENT_BINNING_VERSION), + out=bin_idx, + ) binned_col[~nan_mask] = bin_idx.astype(np.uint8) binned_col[nan_mask] = MISSING_BIN else: edges = np.percentile(col, percentiles) edges = np.unique(edges) bin_idx = np.searchsorted(edges, col, side='right') - np.clip(bin_idx, 0, len(edges) - 1, out=bin_idx) + np.clip( + bin_idx, + 0, + _numeric_max_bin(edges, _CURRENT_BINNING_VERSION), + out=bin_idx, + ) binned_col = bin_idx.astype(np.uint8) return binned_col, edges, has_nan, False, None, 0 +def _numeric_max_bin(edges: NDArray, binning_version: int) -> int: + """Return the highest valid numeric bin for persisted binning semantics.""" + if binning_version <= _LEGACY_BINNING_VERSION: + return max(len(edges) - 1, 0) + # ``array`` creates at most 253 cut edges, so the current top bin is at + # most 253 and remains safely below MISSING_BIN (255). + return min(len(edges), MISSING_BIN - 1) + + def _bin_categorical_feature( col: NDArray, ) -> tuple[NDArray[np.uint8], NDArray[np.float64], bool, bool, dict, int]: diff --git a/src/openboost/_core/_split.py b/src/openboost/_core/_split.py index 029e0cd..b589e4f 100644 --- a/src/openboost/_core/_split.py +++ b/src/openboost/_core/_split.py @@ -16,6 +16,9 @@ from numpy.typing import NDArray +_MAX_CATEGORICAL_SPLIT_CARDINALITY = 64 + + class SplitInfo(NamedTuple): """Information about a split. @@ -257,7 +260,9 @@ def find_best_split_with_categorical( min_gain: Minimum gain to make a split has_missing: Boolean array (n_features,) for features with NaN is_categorical: Boolean array (n_features,) for categorical features - n_categories: Number of categories per feature (0 for numeric) + n_categories: Number of categories per feature (0 for numeric). Categorical + tree splits support at most 64 categories because the left + category set is represented by one uint64 bitset. Returns: SplitInfo with best feature, threshold/bitset, gain, etc. @@ -273,6 +278,20 @@ def find_best_split_with_categorical( # Check if we have categorical features any_categorical = is_categorical is not None and np.any(is_categorical) any_missing = has_missing is not None and np.any(has_missing) + + if any_categorical: + if n_categories is None: + raise ValueError( + "n_categories is required when is_categorical contains True" + ) + categorical_counts = np.asarray(n_categories)[np.asarray(is_categorical)] + if np.any(categorical_counts > _MAX_CATEGORICAL_SPLIT_CARDINALITY): + observed = int(np.max(categorical_counts)) + raise ValueError( + f"Categorical feature has {observed} categories; maximum " + f"supported is {_MAX_CATEGORICAL_SPLIT_CARDINALITY} because " + "categorical tree routing uses a single uint64 bitset" + ) # If no categorical and no missing, use standard split if not any_categorical and not any_missing: @@ -333,4 +352,3 @@ def find_best_split_with_categorical( cat_bitset=cat_bitset, cat_threshold=cat_threshold, ) - diff --git a/src/openboost/_core/_vector_tree.py b/src/openboost/_core/_vector_tree.py new file mode 100644 index 0000000..a7497d2 --- /dev/null +++ b/src/openboost/_core/_vector_tree.py @@ -0,0 +1,286 @@ +"""CPU tree fitting for shared-structure, vector-valued boosting trees. + +This module is intentionally separate from the scalar tree builder. A vector +tree chooses one split structure for all outputs, sums the per-output Newton +gain when comparing splits, and stores one vector in every leaf. The first +consumer is :class:`openboost.HistogramBoost`. + +The implementation is CPU-only. Keeping that boundary explicit avoids +silently routing vector gradients through CUDA kernels whose histogram and +prediction layouts are scalar. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +from numpy.typing import NDArray + +from .._array import MISSING_BIN, BinnedArray +from ._growth import TreeStructure, VectorLeaves + + +@dataclass(frozen=True) +class VectorSplit: + """Best split found for one vector-gradient node.""" + + feature: int = -1 + threshold: int = -1 + gain: float = 0.0 + missing_go_left: bool = True + + +def _soft_threshold(values: NDArray, reg_alpha: float) -> NDArray: + if reg_alpha <= 0.0: + return values + return np.sign(values) * np.maximum(np.abs(values) - reg_alpha, 0.0) + + +def _node_score( + sum_grad: NDArray, + sum_hess: NDArray, + reg_lambda: float, + reg_alpha: float, +) -> NDArray: + """Per-candidate vector Newton score, averaged across outputs.""" + shrunk = _soft_threshold(sum_grad, reg_alpha) + return np.mean(shrunk * shrunk / (sum_hess + reg_lambda), axis=-1) + + +def _leaf_value( + sum_grad: NDArray, + sum_hess: NDArray, + reg_lambda: float, + reg_alpha: float, +) -> NDArray: + shrunk = _soft_threshold(sum_grad, reg_alpha) + return -shrunk / (sum_hess + reg_lambda) + + +def _feature_histogram( + bins: NDArray, + grad: NDArray, + hess: NDArray, +) -> tuple[NDArray, NDArray]: + """Aggregate ``(n_samples, n_outputs)`` statistics into 256 bins.""" + n_outputs = grad.shape[1] + hist_grad = np.zeros((256, n_outputs), dtype=np.float64) + hist_hess = np.zeros((256, n_outputs), dtype=np.float64) + np.add.at(hist_grad, bins, grad) + np.add.at(hist_hess, bins, hess) + return hist_grad, hist_hess + + +def _find_best_vector_split( + binned: NDArray, + grad: NDArray, + hess: NDArray, + *, + min_child_weight: float, + reg_lambda: float, + reg_alpha: float, + min_gain: float, +) -> VectorSplit: + """Find the best numeric split, including both missing-value directions.""" + total_grad = np.sum(grad, axis=0, dtype=np.float64) + total_hess = np.sum(hess, axis=0, dtype=np.float64) + parent_score = float(_node_score(total_grad, total_hess, reg_lambda, reg_alpha)) + best = VectorSplit() + + for feature in range(binned.shape[0]): + hist_grad, hist_hess = _feature_histogram(binned[feature], grad, hess) + missing_grad = hist_grad[MISSING_BIN] + missing_hess = hist_hess[MISSING_BIN] + nonmissing_grad = total_grad - missing_grad + nonmissing_hess = total_hess - missing_hess + + # Threshold 254 cannot leave a non-missing sample on the right, so the + # useful numeric thresholds are 0..253. Empty sides are rejected by + # the child-weight check below. + left_grad = np.cumsum(hist_grad[:MISSING_BIN], axis=0)[:-1] + left_hess = np.cumsum(hist_hess[:MISSING_BIN], axis=0)[:-1] + right_grad = nonmissing_grad - left_grad + right_hess = nonmissing_hess - left_hess + + for missing_go_left in (True, False): + candidate_left_grad = left_grad + (missing_grad if missing_go_left else 0.0) + candidate_left_hess = left_hess + (missing_hess if missing_go_left else 0.0) + candidate_right_grad = right_grad + (0.0 if missing_go_left else missing_grad) + candidate_right_hess = right_hess + (0.0 if missing_go_left else missing_hess) + + # Mean curvature keeps min_child_weight invariant to n_outputs. + valid = (np.mean(candidate_left_hess, axis=1) >= min_child_weight) & ( + np.mean(candidate_right_hess, axis=1) >= min_child_weight + ) + if not np.any(valid): + continue + + gains = ( + _node_score( + candidate_left_grad, + candidate_left_hess, + reg_lambda, + reg_alpha, + ) + + _node_score( + candidate_right_grad, + candidate_right_hess, + reg_lambda, + reg_alpha, + ) + - parent_score + ) + gains = np.where(valid, gains, -np.inf) + threshold = int(np.argmax(gains)) + gain = float(gains[threshold]) + if gain > best.gain and gain > min_gain: + best = VectorSplit(feature, threshold, gain, missing_go_left) + + return best + + +def fit_vector_tree( + X: BinnedArray | NDArray, + grad: NDArray, + hess: NDArray, + *, + max_depth: int = 6, + min_child_weight: float = 1e-3, + reg_lambda: float = 1.0, + leaf_reg_lambda: float | None = None, + reg_alpha: float = 0.0, + min_gain: float = 0.0, +) -> TreeStructure: + """Fit one CPU, level-wise tree with shared structure and vector leaves. + + Parameters + ---------- + X: + CPU ``BinnedArray`` or feature-major uint8 matrix. + grad, hess: + Arrays with shape ``(n_samples, n_outputs)``. + reg_lambda: + L2 regularization used to compare split structures. + leaf_reg_lambda: + L2 regularization for the final vector leaf update. ``None`` reuses + ``reg_lambda`` for backward-compatible behavior. + """ + if isinstance(X, BinnedArray): + if X.device != "cpu" or hasattr(X.data, "__cuda_array_interface__"): + raise NotImplementedError("fit_vector_tree currently supports CPU data only") + if X.any_categorical: + raise NotImplementedError( + "fit_vector_tree does not yet support categorical feature splits" + ) + binned = np.asarray(X.data, dtype=np.uint8) + n_features = X.n_features + n_samples = X.n_samples + else: + binned = np.asarray(X, dtype=np.uint8) + if binned.ndim != 2: + raise ValueError("binned X must have shape (n_features, n_samples)") + n_features, n_samples = binned.shape + + grad = np.asarray(grad, dtype=np.float64) + hess = np.asarray(hess, dtype=np.float64) + if grad.ndim != 2 or hess.shape != grad.shape: + raise ValueError("grad and hess must have matching (n_samples, n_outputs) shapes") + if grad.shape[0] != n_samples: + raise ValueError(f"grad has {grad.shape[0]} samples, expected {n_samples}") + if not np.all(np.isfinite(grad)) or not np.all(np.isfinite(hess)): + raise ValueError("grad and hess must contain only finite values") + if np.any(hess < 0.0): + raise ValueError("hess must be non-negative for vector Newton trees") + if max_depth < 0: + raise ValueError("max_depth must be non-negative") + if reg_lambda <= 0.0: + raise ValueError("reg_lambda must be strictly positive") + if leaf_reg_lambda is not None and leaf_reg_lambda <= 0.0: + raise ValueError("leaf_reg_lambda must be strictly positive or None") + resolved_leaf_reg_lambda = reg_lambda if leaf_reg_lambda is None else leaf_reg_lambda + + n_outputs = grad.shape[1] + max_nodes = 2 ** (max_depth + 1) - 1 + features = np.full(max_nodes, -1, dtype=np.int32) + thresholds = np.zeros(max_nodes, dtype=np.uint8) + left_children = np.full(max_nodes, -1, dtype=np.int32) + right_children = np.full(max_nodes, -1, dtype=np.int32) + missing_go_left = np.ones(max_nodes, dtype=np.bool_) + values = np.zeros((max_nodes, n_outputs), dtype=np.float32) + + node_samples: dict[int, NDArray] = {0: np.arange(n_samples, dtype=np.int32)} + leaves: dict[int, NDArray] = {} + active = [0] + deepest = 0 + + for depth in range(max_depth): + next_active: list[int] = [] + for node_id in active: + sample_idx = node_samples[node_id] + split = _find_best_vector_split( + binned[:, sample_idx], + grad[sample_idx], + hess[sample_idx], + min_child_weight=min_child_weight, + reg_lambda=reg_lambda, + reg_alpha=reg_alpha, + min_gain=min_gain, + ) + if split.feature < 0: + leaves[node_id] = sample_idx + continue + + bins = binned[split.feature, sample_idx] + goes_left = bins <= split.threshold + if split.missing_go_left: + goes_left |= bins == MISSING_BIN + else: + goes_left &= bins != MISSING_BIN + + left_idx = sample_idx[goes_left] + right_idx = sample_idx[~goes_left] + if left_idx.size == 0 or right_idx.size == 0: + leaves[node_id] = sample_idx + continue + + left_id = 2 * node_id + 1 + right_id = left_id + 1 + features[node_id] = split.feature + thresholds[node_id] = split.threshold + left_children[node_id] = left_id + right_children[node_id] = right_id + missing_go_left[node_id] = split.missing_go_left + node_samples[left_id] = left_idx + node_samples[right_id] = right_idx + next_active.extend((left_id, right_id)) + deepest = max(deepest, depth + 1) + active = next_active + if not active: + break + + for node_id in active: + leaves[node_id] = node_samples[node_id] + + for node_id, sample_idx in leaves.items(): + sum_grad = np.sum(grad[sample_idx], axis=0, dtype=np.float64) + sum_hess = np.sum(hess[sample_idx], axis=0, dtype=np.float64) + values[node_id] = _leaf_value( + sum_grad, + sum_hess, + resolved_leaf_reg_lambda, + reg_alpha, + ).astype(np.float32) + + n_nodes = max([0, *node_samples]) + 1 + return TreeStructure( + features=features[:n_nodes], + thresholds=thresholds[:n_nodes], + left_children=left_children[:n_nodes], + right_children=right_children[:n_nodes], + values=VectorLeaves(values[:n_nodes], n_outputs=n_outputs), + n_nodes=n_nodes, + depth=deepest, + n_features=n_features, + missing_go_left=missing_go_left[:n_nodes], + ) diff --git a/src/openboost/_distributions.py b/src/openboost/_distributions.py index 7b519f2..74d7656 100644 --- a/src/openboost/_distributions.py +++ b/src/openboost/_distributions.py @@ -431,6 +431,74 @@ def nll_gradient( 'loc': (grad_loc.astype(np.float32), hess_loc.astype(np.float32)), 'scale': (grad_scale.astype(np.float32), hess_scale.astype(np.float32)), } + + def crps( + self, + y: NDArray, + params: dict[str, NDArray], + ) -> NDArray: + """Per-sample Gaussian continuous ranked probability score. + + Lower is better. The implementation uses the closed-form Normal + score in float64 so the same objective can be used both for training + diagnostics and finite-difference correctness tests. + """ + from scipy.special import erf + + y = np.asarray(y, dtype=np.float64) + loc = np.asarray(params['loc'], dtype=np.float64) + scale = np.asarray(params['scale'], dtype=np.float64) + z = (y - loc) / scale + cdf_contrast = erf(z / np.sqrt(2.0)) # 2 * Phi(z) - 1 + pdf = np.exp(-0.5 * z ** 2) / np.sqrt(2.0 * np.pi) + return scale * ( + z * cdf_contrast + 2.0 * pdf - 1.0 / np.sqrt(np.pi) + ) + + def crps_gradient( + self, + y: NDArray, + params: dict[str, NDArray], + ) -> dict[str, GradHess]: + """Gaussian CRPS gradient with positive expected curvature. + + Raw parameters are ``loc`` and ``log(scale)``. The exact CRPS + Hessian is indefinite for sufficiently large standardized residuals, + so it is not suitable for OpenBoost's positive-Hessian tree solver. + Instead this method returns the expected CRPS Hessian under the + current Normal prediction: + + ``diag(1 / (sqrt(pi) * scale), scale / (2 * sqrt(pi)))``. + + This is a score-specific, strictly positive curvature surrogate. It + is deliberately distinct from the Fisher information used by the NLL + natural-gradient objective. + """ + from scipy.special import erf + + y = np.asarray(y, dtype=np.float64) + loc = np.asarray(params['loc'], dtype=np.float64) + scale = np.asarray(params['scale'], dtype=np.float64) + z = (y - loc) / scale + cdf_contrast = erf(z / np.sqrt(2.0)) # 2 * Phi(z) - 1 + pdf = np.exp(-0.5 * z ** 2) / np.sqrt(2.0 * np.pi) + sqrt_pi = np.sqrt(np.pi) + + grad_loc = -cdf_contrast + grad_scale = scale * (2.0 * pdf - 1.0 / sqrt_pi) + curvature_loc = 1.0 / (sqrt_pi * scale) + curvature_scale = scale / (2.0 * sqrt_pi) + + return { + 'loc': ( + grad_loc.astype(np.float32), + curvature_loc.astype(np.float32), + ), + 'scale': ( + grad_scale.astype(np.float32), + curvature_scale.astype(np.float32), + ), + } def fisher_information( self, diff --git a/src/openboost/_models/__init__.py b/src/openboost/_models/__init__.py index 810829a..e2fffbb 100644 --- a/src/openboost/_models/__init__.py +++ b/src/openboost/_models/__init__.py @@ -34,6 +34,7 @@ NGBoostTweedie, ) from ._gam import OpenBoostGAM +from ._histogram_boost import HistogramBoost, HistogramDistributionOutput # Phase 15: Linear Leaf GBDT from ._linear_leaf import LinearLeafGBDT, LinearLeafTree @@ -52,6 +53,8 @@ "MultiClassGradientBoosting", "DART", "OpenBoostGAM", + "HistogramBoost", + "HistogramDistributionOutput", # Phase 13: sklearn-compatible wrappers "OpenBoostRegressor", "OpenBoostClassifier", diff --git a/src/openboost/_models/_boosting.py b/src/openboost/_models/_boosting.py index 12ab4cc..4fe69dc 100644 --- a/src/openboost/_models/_boosting.py +++ b/src/openboost/_models/_boosting.py @@ -3,13 +3,12 @@ Provides a scikit-learn-like API for training gradient boosting models with both built-in and custom loss functions. -This module implements batched training that keeps computation on the GPU -without returning to Python between trees, achieving performance competitive -with XGBoost. +This module implements GPU-aware training paths and reusable tree-building +primitives. Performance claims require workload-specific benchmark artifacts. Phase 13: Added callback support for early stopping, logging, etc. -Phase 17: Added GOSS sampling and mini-batch training for large-scale datasets. -Phase 18: Added multi-GPU support via Ray for data-parallel training. +Phase 17: Added GOSS sampling and low-level mini-batch primitives. +Phase 18: Added an experimental multi-GPU path via Ray. """ from __future__ import annotations @@ -70,6 +69,16 @@ def _is_levelwise_growth(growth) -> bool: ) +def _validate_batch_size(batch_size: int | None) -> None: + """Reject the reserved high-level mini-batch option until it is implemented.""" + if batch_size is not None: + raise NotImplementedError( + "batch_size is reserved but high-level mini-batch training is not " + "implemented. Leave batch_size=None; the low-level mini-batch " + "histogram helpers are not an end-to-end model.fit path." + ) + + def _compute_loss_value(loss, pred, y, **kwargs) -> float: """Compute scalar loss using the true loss formula for known objectives. @@ -141,7 +150,8 @@ class GradientBoosting(PersistenceMixin): - 'goss': Gradient-based One-Side Sampling (LightGBM-style) goss_top_rate: Fraction of top-gradient samples to keep (for GOSS). goss_other_rate: Fraction of remaining samples to sample (for GOSS). - batch_size: Mini-batch size for large datasets. If None, process all at once. + batch_size: Reserved for a future high-level mini-batch training path. + Any non-None value currently raises NotImplementedError. growth: Tree growth strategy: - 'levelwise': XGBoost-style level-wise growth (default) - 'leafwise': LightGBM-style best-first growth (see max_leaves) @@ -178,11 +188,11 @@ class GradientBoosting(PersistenceMixin): ) ``` - Multi-GPU training: + Experimental multi-GPU training: ```python model = ob.GradientBoosting(n_trees=100, n_gpus=4) - model.fit(X, y) # Data parallel across 4 GPUs + model.fit(X, y) # Requires independent parity/scaling validation ``` """ @@ -250,6 +260,8 @@ def fit( ) ``` """ + _validate_batch_size(self.batch_size) + # Clear any previous fit self.trees_ = [] @@ -413,7 +425,8 @@ def _fit_multigpu( Each GPU holds a shard of the data and computes local histograms, which are aggregated on the driver to build global trees. - This approach provides near-linear scaling for large datasets. + This path is experimental until single-device parity and repeated + two-/four-GPU scaling results are checked into the repository. """ if MultiGPUContext is None: raise ImportError( @@ -1329,6 +1342,8 @@ def fit( """ from .._loss import softmax_gradient + _validate_batch_size(self.batch_size) + # Clear previous fit self.trees_ = [] diff --git a/src/openboost/_models/_distributional.py b/src/openboost/_models/_distributional.py index a4151e0..cda1d19 100644 --- a/src/openboost/_models/_distributional.py +++ b/src/openboost/_models/_distributional.py @@ -66,6 +66,9 @@ #: Metrics accepted by ``fit(eval_metric=...)``. EVAL_METRICS = ('nll', 'crps', 'pinball', 'interval_score') +#: Objectives accepted by ``training_objective``. +TRAINING_OBJECTIVES = ('nll', 'crps') + def _validate_exposure(exposure, n_samples: int, context: str = "fit") -> NDArray: """Validate an exposure vector: positive, finite, shape (n_samples,). @@ -138,6 +141,10 @@ class DistributionalGBDT(PersistenceMixin): subsample: Row sampling ratio (0.0-1.0) colsample_bytree: Column sampling ratio (0.0-1.0) n_bins: Number of bins for histogram building + training_objective: Objective used to fit distribution parameters. + ``'nll'`` preserves the likelihood/Fisher path. ``'crps'`` is + currently available for Normal predictions and uses Gaussian + CRPS with a positive score-specific expected curvature. Attributes: trees_: Dict mapping param_name -> list of trees @@ -174,6 +181,7 @@ class DistributionalGBDT(PersistenceMixin): subsample: float = 1.0 colsample_bytree: float = 1.0 n_bins: int = 254 + training_objective: Literal['nll', 'crps'] = 'nll' # Fitted attributes (not init) trees_: dict[str, list[TreeStructure]] = field(default_factory=dict, init=False, repr=False) @@ -243,6 +251,19 @@ def fit( # Get distribution instance self.distribution_ = get_distribution(self.distribution) + if self.training_objective not in TRAINING_OBJECTIVES: + raise ValueError( + f"Unknown training_objective '{self.training_objective}'. " + f"Available: {', '.join(TRAINING_OBJECTIVES)}." + ) + if self.training_objective == 'crps' and not isinstance( + self.distribution_, Normal + ): + raise ValueError( + "training_objective='crps' currently supports only the " + "Normal distribution." + ) + y = np.asarray(y, dtype=np.float32).ravel() n_samples = len(y) @@ -425,11 +446,12 @@ def fit( params_report = self._constrained_params( raw_preds, exposure_param, train_log_offset ) + if self.training_objective == 'crps': + train_values = self.distribution_.crps(y, params_report) + else: + train_values = self.distribution_.nll(y, params_report) state.train_loss = float( - np.average( - self.distribution_.nll(y, params_report), - weights=sample_weight, - ) + np.average(train_values, weights=sample_weight) ) if last_metric is not None: # Early stopping monitors the LAST eval set's metric @@ -539,6 +561,8 @@ def _compute_gradients( Subclasses can override for different gradient computation. """ + if self.training_objective == 'crps': + return self.distribution_.crps_gradient(y, params) return self.distribution_.nll_gradient(y, params) def _predict_raw(self, X: NDArray | BinnedArray) -> dict[str, NDArray]: @@ -749,8 +773,10 @@ class NaturalBoost(DistributionalGBDT): Uses natural gradient instead of ordinary gradient, leading to faster convergence by accounting for the geometry of the parameter space. - Natural gradient: F^{-1} @ ordinary_gradient - where F is the Fisher information matrix. + With the default NLL objective, the natural gradient is + ``F^{-1} @ ordinary_gradient`` where F is the Fisher information matrix. + Normal CRPS training instead uses its score-specific expected curvature; + it does not reuse the NLL Fisher information. Key advantages over standard GBDT: - Full probability distributions, not just point estimates @@ -803,6 +829,12 @@ def _compute_gradients( Natural gradient = F^{-1} @ ordinary_gradient where F is the Fisher information matrix. """ + if self.training_objective == 'crps': + # Gaussian CRPS uses its own positive expected curvature rather + # than the NLL Fisher information. Returning gradient/curvature + # directly lets the tree solver perform the corresponding + # second-order leaf updates. + return self.distribution_.crps_gradient(y, params) return self.distribution_.natural_gradient(y, params) diff --git a/src/openboost/_models/_histogram_boost.py b/src/openboost/_models/_histogram_boost.py new file mode 100644 index 0000000..608b995 --- /dev/null +++ b/src/openboost/_models/_histogram_boost.py @@ -0,0 +1,539 @@ +"""Shared-tree histogram distribution boosting with a CRPS objective.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from .._array import BinnedArray, array +from .._core._growth import TreeStructure +from .._core._vector_tree import fit_vector_tree +from .._persistence import PersistenceMixin +from .._validation import validate_sample_weight, validate_X, validate_y + + +def _softmax(logits: NDArray) -> NDArray: + shifted = np.asarray(logits, dtype=np.float64) + if not np.all(np.isfinite(shifted)): + raise FloatingPointError("histogram logits contain non-finite values") + shifted = shifted - np.max(shifted, axis=1, keepdims=True) + exp = np.exp(shifted) + return exp / np.sum(exp, axis=1, keepdims=True) + + +def _continuous_crps_terms( + y: NDArray, + bin_edges: NDArray, + *, + normalize: bool = True, +) -> tuple[NDArray, NDArray]: + """Return normalized terms for exact piecewise-uniform histogram CRPS. + + CRPS has the energy representation + ``E|X-y| - 0.5 E|X-X'|``. Each histogram bin represents a uniform + conditional density, not a point mass at its midpoint. The returned first + term has shape ``(n_samples, n_bins)`` and the pairwise-distance matrix has + shape ``(n_bins, n_bins)``. With ``normalize=True``, both are divided by + mean bin width so tree regularization remains invariant to target units. + """ + y = np.asarray(y, dtype=np.float64).reshape(-1) + bin_edges = np.asarray(bin_edges, dtype=np.float64).reshape(-1) + if bin_edges.size < 3: + raise ValueError("bin_edges must describe at least two bins") + if not np.all(np.isfinite(y)) or not np.all(np.isfinite(bin_edges)): + raise ValueError("y and bin_edges must contain only finite values") + widths = np.diff(bin_edges) + if np.any(widths <= 0.0): + raise ValueError("bin_edges must be strictly increasing") + midpoints = 0.5 * (bin_edges[:-1] + bin_edges[1:]) + + y_column = y[:, None] + lower = bin_edges[:-1][None, :] + upper = bin_edges[1:][None, :] + distance_to_target = np.where( + y_column < lower, + midpoints[None, :] - y_column, + np.where( + y_column > upper, + y_column - midpoints[None, :], + ((y_column - lower) ** 2 + (upper - y_column) ** 2) / (2.0 * widths[None, :]), + ), + ) + + pairwise_distance = np.abs(midpoints[:, None] - midpoints[None, :]) + np.fill_diagonal(pairwise_distance, widths / 3.0) + normalization = float(np.mean(widths)) if normalize else 1.0 + return distance_to_target / normalization, pairwise_distance / normalization + + +def _continuous_crps_loss( + logits: NDArray, + y: NDArray, + bin_edges: NDArray, +) -> NDArray: + """Exact per-row CRPS for the represented piecewise-uniform histogram.""" + return _continuous_crps_from_probabilities( + _softmax(logits), + y, + bin_edges, + normalize=True, + ) + + +def _continuous_crps_from_probabilities( + probabilities: NDArray, + y: NDArray, + bin_edges: NDArray, + *, + normalize: bool = False, +) -> NDArray: + """Exact per-row CRPS for already-normalized histogram probabilities.""" + probabilities = np.asarray(probabilities, dtype=np.float64) + if probabilities.ndim != 2: + raise ValueError("probabilities must have shape (n_samples, n_bins)") + if not np.all(np.isfinite(probabilities)) or np.any(probabilities < 0.0): + raise ValueError("probabilities must be finite and non-negative") + totals = np.sum(probabilities, axis=1, keepdims=True) + if np.any(totals <= 0.0): + raise ValueError("each probability row must have positive mass") + probabilities = probabilities / totals + distance_to_target, pairwise_distance = _continuous_crps_terms( + y, + bin_edges, + normalize=normalize, + ) + if distance_to_target.shape != probabilities.shape: + raise ValueError("y and bin_edges must match the probability rows and bins") + first = np.sum(probabilities * distance_to_target, axis=1) + second = 0.5 * np.einsum( + "ni,ij,nj->n", + probabilities, + pairwise_distance, + probabilities, + ) + return first - second + + +def _crps_grad_gn( + logits: NDArray, + y: NDArray, + bin_edges: NDArray, + *, + curvature_scale: float = 1.0, + sample_weight: NDArray | None = None, + curvature_floor: float = 1e-6, +) -> tuple[NDArray, NDArray]: + """Gradient and PSD diagonal curvature for exact continuous CRPS. + + If ``a_j = E|U_j-y|`` and ``D_jk = E|U_j-U_k|`` for uniform histogram + bins, CRPS is ``a.T @ p - 0.5 * p.T @ D @ p``. The gradient is exact. + Curvature is the diagonal of ``J.T @ (-D) @ J`` for the softmax Jacobian + ``J``; ``-D`` is positive semidefinite on the probability-simplex tangent + space. The indefinite residual term from differentiating ``J`` is + deliberately excluded. + """ + logits = np.asarray(logits, dtype=np.float64) + if logits.ndim != 2: + raise ValueError("logits must have shape (n_samples, n_distribution_bins)") + n_samples, n_outputs = logits.shape + if curvature_scale <= 0.0: + raise ValueError("curvature_scale must be strictly positive") + + probabilities = _softmax(logits) + distance_to_target, pairwise_distance = _continuous_crps_terms(y, bin_edges) + if distance_to_target.shape != (n_samples, n_outputs): + raise ValueError("y and bin_edges must match the logit rows and outputs") + + probability_distance = probabilities @ pairwise_distance + grad_probability = distance_to_target - probability_distance + centered = grad_probability - np.sum(probabilities * grad_probability, axis=1, keepdims=True) + gradient = probabilities * centered + + probability_quadratic = np.sum( + probabilities * probability_distance, + axis=1, + keepdims=True, + ) + distance_diagonal = np.diag(pairwise_distance)[None, :] + curvature = ( + curvature_scale + * probabilities + * probabilities + * (2.0 * probability_distance - distance_diagonal - probability_quadratic) + ) + curvature = np.maximum(curvature, curvature_floor) + + if sample_weight is not None: + weight = np.asarray(sample_weight, dtype=np.float64).reshape(-1, 1) + if weight.shape[0] != n_samples: + raise ValueError("sample_weight must have one entry per logit row") + gradient *= weight + curvature *= weight + + return gradient.astype(np.float32), curvature.astype(np.float32) + + +@dataclass +class HistogramDistributionOutput: + """Piecewise-uniform predictive distributions on one shared bin grid.""" + + probas: NDArray + bin_edges: NDArray + + def __post_init__(self) -> None: + self.probas = np.asarray(self.probas, dtype=np.float64) + self.bin_edges = np.asarray(self.bin_edges, dtype=np.float64).reshape(-1) + if self.probas.ndim != 2: + raise ValueError("probas must have shape (n_samples, n_bins)") + if self.bin_edges.shape != (self.probas.shape[1] + 1,): + raise ValueError("bin_edges must have n_bins + 1 entries") + if not np.all(np.isfinite(self.bin_edges)): + raise ValueError("bin_edges must contain only finite values") + if np.any(np.diff(self.bin_edges) <= 0.0): + raise ValueError("bin_edges must be strictly increasing") + if not np.all(np.isfinite(self.probas)) or np.any(self.probas < 0.0): + raise ValueError("probas must be finite and non-negative") + totals = np.sum(self.probas, axis=1, keepdims=True) + if np.any(totals <= 0.0): + raise ValueError("each probability row must have positive mass") + self.probas = self.probas / totals + + @property + def bin_midpoints(self) -> NDArray: + return 0.5 * (self.bin_edges[:-1] + self.bin_edges[1:]) + + def mean(self) -> NDArray: + return self.probas @ self.bin_midpoints + + def variance(self) -> NDArray: + mean = self.mean() + widths = np.diff(self.bin_edges) + second_centered = (self.bin_midpoints[None, :] - mean[:, None]) ** 2 + widths[ + None, : + ] ** 2 / 12.0 + return np.sum(self.probas * second_centered, axis=1) + + def std(self) -> NDArray: + return np.sqrt(self.variance()) + + def crps(self, y: NDArray) -> NDArray: + """Return exact per-row continuous ranked probability scores.""" + return _continuous_crps_from_probabilities(self.probas, y, self.bin_edges) + + def tempered(self, temperature: float) -> HistogramDistributionOutput: + """Return the same histogram grid with temperature-scaled probabilities.""" + if not np.isfinite(temperature) or temperature <= 0.0: + raise ValueError("temperature must be finite and strictly positive") + if temperature == 1.0: + probabilities = self.probas.copy() + else: + log_probabilities = np.full_like(self.probas, -np.inf) + positive = self.probas > 0.0 + log_probabilities[positive] = np.log(self.probas[positive]) / temperature + log_probabilities -= np.max(log_probabilities, axis=1, keepdims=True) + probabilities = np.exp(log_probabilities) + return HistogramDistributionOutput(probabilities, self.bin_edges.copy()) + + def subdivide(self, factor: int) -> HistogramDistributionOutput: + """Refine each uniform bin without changing the represented density.""" + if isinstance(factor, bool) or not isinstance(factor, (int, np.integer)) or factor < 1: + raise ValueError("factor must be a positive integer") + factor = int(factor) + if factor == 1: + return HistogramDistributionOutput(self.probas.copy(), self.bin_edges.copy()) + n_bins = self.probas.shape[1] + refined_edges = np.empty(n_bins * factor + 1, dtype=np.float64) + fractions = np.arange(factor, dtype=np.float64) / factor + for index, (lower, upper) in enumerate( + zip(self.bin_edges[:-1], self.bin_edges[1:], strict=True) + ): + refined_edges[index * factor : (index + 1) * factor] = lower + fractions * ( + upper - lower + ) + refined_edges[-1] = self.bin_edges[-1] + refined_probabilities = np.repeat(self.probas / factor, factor, axis=1) + return HistogramDistributionOutput(refined_probabilities, refined_edges) + + def quantile(self, q: float) -> NDArray: + if not 0.0 <= q <= 1.0: + raise ValueError("q must lie in [0, 1]") + if q == 0.0: + return np.full(self.probas.shape[0], self.bin_edges[0]) + if q == 1.0: + return np.full(self.probas.shape[0], self.bin_edges[-1]) + cdf = np.cumsum(self.probas, axis=1) + indices = np.argmax(cdf >= q, axis=1) + rows = np.arange(self.probas.shape[0]) + previous = np.where(indices == 0, 0.0, cdf[rows, np.maximum(indices - 1, 0)]) + mass = self.probas[rows, indices] + fraction = np.divide( + q - previous, + mass, + out=np.zeros_like(mass), + where=mass > 0.0, + ) + widths = np.diff(self.bin_edges) + return self.bin_edges[indices] + np.clip(fraction, 0.0, 1.0) * widths[indices] + + def interval(self, alpha: float = 0.1) -> tuple[NDArray, NDArray]: + if not 0.0 < alpha < 1.0: + raise ValueError("alpha must lie in (0, 1)") + return self.quantile(alpha / 2.0), self.quantile(1.0 - alpha / 2.0) + + def sample(self, n_samples: int = 1, seed: int | None = None) -> NDArray: + if n_samples < 1: + raise ValueError("n_samples must be at least 1") + rng = np.random.default_rng(seed) + uniforms = rng.random((self.probas.shape[0], n_samples)) + cdf = np.cumsum(self.probas, axis=1) + samples = np.empty_like(uniforms) + widths = np.diff(self.bin_edges) + for row in range(self.probas.shape[0]): + indices = np.searchsorted(cdf[row], uniforms[row], side="left") + previous = np.where(indices == 0, 0.0, cdf[row, np.maximum(indices - 1, 0)]) + mass = self.probas[row, indices] + fraction = np.divide( + uniforms[row] - previous, + mass, + out=np.zeros(n_samples, dtype=np.float64), + where=mass > 0.0, + ) + samples[row] = self.bin_edges[indices] + np.clip(fraction, 0.0, 1.0) * widths[indices] + return samples + + +@dataclass +class HistogramBoost(PersistenceMixin): + """CPU shared-tree boosting for a flexible histogram distribution. + + Each boosting round fits one tree structure with a vector of logit updates + in every leaf. The objective is the exact continuous CRPS of the + represented piecewise-uniform histogram and therefore produces a monotone + CDF by construction without independent-quantile crossing. + + This first implementation supports numeric CPU input (including NaNs). + Categorical splits, CUDA, callbacks, and evaluation sets intentionally raise + or remain outside this API until their vector paths are implemented. + """ + + n_distribution_bins: int = 50 + n_trees: int = 100 + learning_rate: float = 0.05 + max_depth: int = 6 + min_child_weight: float = 1e-3 + reg_lambda: float = 1.0 + leaf_reg_lambda: float | None = None + reg_alpha: float = 0.0 + min_gain: float = 0.0 + n_feature_bins: int = 254 + curvature_scale: float = 1.0 + base_smoothing: float = 1.0 + + trees_: list[TreeStructure] = field(default_factory=list, init=False, repr=False) + X_binned_: BinnedArray | None = field(default=None, init=False, repr=False) + base_logits_: NDArray | None = field(default=None, init=False, repr=False) + target_bin_edges_: NDArray | None = field(default=None, init=False, repr=False) + target_bin_midpoints_: NDArray | None = field(default=None, init=False, repr=False) + n_features_in_: int | None = field(default=None, init=False) + + _PARAM_NAMES = ( + "n_distribution_bins", + "n_trees", + "learning_rate", + "max_depth", + "min_child_weight", + "reg_lambda", + "leaf_reg_lambda", + "reg_alpha", + "min_gain", + "n_feature_bins", + "curvature_scale", + "base_smoothing", + ) + + def get_params(self, deep: bool = True) -> dict[str, Any]: # noqa: ARG002 + """Return constructor parameters for sklearn cloning.""" + return {name: getattr(self, name) for name in self._PARAM_NAMES} + + def set_params(self, **params: Any) -> HistogramBoost: + unknown = sorted(set(params) - set(self._PARAM_NAMES)) + if unknown: + raise ValueError(f"Unknown parameter(s): {unknown}") + for name, value in params.items(): + setattr(self, name, value) + return self + + def _validate_params(self) -> None: + if self.n_distribution_bins < 2: + raise ValueError("n_distribution_bins must be at least 2") + if self.n_trees < 0: + raise ValueError("n_trees must be non-negative") + if self.learning_rate <= 0.0: + raise ValueError("learning_rate must be strictly positive") + if self.max_depth < 0: + raise ValueError("max_depth must be non-negative") + if self.min_child_weight < 0.0: + raise ValueError("min_child_weight must be non-negative") + if self.reg_lambda <= 0.0: + raise ValueError("reg_lambda must be strictly positive") + if self.leaf_reg_lambda is not None and self.leaf_reg_lambda <= 0.0: + raise ValueError("leaf_reg_lambda must be strictly positive or None") + if self.reg_alpha < 0.0 or self.min_gain < 0.0: + raise ValueError("reg_alpha and min_gain must be non-negative") + if not 2 <= self.n_feature_bins <= 254: + raise ValueError("n_feature_bins must lie in [2, 254]") + if self.curvature_scale <= 0.0: + raise ValueError("curvature_scale must be strictly positive") + if self.base_smoothing < 0.0: + raise ValueError("base_smoothing must be non-negative") + + def _make_target_grid(self, y: NDArray) -> tuple[NDArray, NDArray]: + lower = float(np.min(y)) + upper = float(np.max(y)) + if lower == upper: + half_span = max(abs(lower) * 1e-6, 1e-6) + edges = np.linspace( + lower - half_span, + upper + half_span, + self.n_distribution_bins + 1, + ) + else: + spacing = (upper - lower) / (self.n_distribution_bins - 1) + edges = np.linspace( + lower - 0.5 * spacing, + upper + 0.5 * spacing, + self.n_distribution_bins + 1, + ) + return edges, 0.5 * (edges[:-1] + edges[1:]) + + def fit( + self, + X: Any, + y: Any, + sample_weight: Any | None = None, + ) -> HistogramBoost: + """Fit on numeric CPU data; target grid state is learned from train y only.""" + self._validate_params() + if isinstance(X, BinnedArray): + raise TypeError( + "HistogramBoost.fit expects raw numeric X so it can own and persist " + "the training bin transform" + ) + X_valid = validate_X(X, allow_binned=False, allow_nan=True, context="fit") + y_valid = validate_y(y, n_samples=X_valid.shape[0], task="regression") + weights = validate_sample_weight(sample_weight, X_valid.shape[0]) + if weights is not None and float(np.sum(weights)) <= 0.0: + raise ValueError("sample_weight must contain positive total weight") + if weights is not None and np.any(weights == 0.0): + # A zero-weight observation must be equivalent to removing it. In + # particular, it must not influence feature bins or the target + # support learned below. + positive_weight = weights > 0.0 + X_valid = X_valid[positive_weight] + y_valid = y_valid[positive_weight] + weights = weights[positive_weight] + + self.X_binned_ = array(X_valid, n_bins=self.n_feature_bins, device="cpu") + if self.X_binned_.any_categorical: + raise NotImplementedError("HistogramBoost currently supports numeric features only") + self.n_features_in_ = X_valid.shape[1] + self.target_bin_edges_, self.target_bin_midpoints_ = self._make_target_grid(y_valid) + labels = np.searchsorted(self.target_bin_edges_[1:-1], y_valid, side="right").astype( + np.int64 + ) + labels = np.clip(labels, 0, self.n_distribution_bins - 1) + + count_weights = ( + np.ones_like(y_valid, dtype=np.float64) + if weights is None + else weights.astype(np.float64) + ) + counts = np.bincount( + labels, + weights=count_weights, + minlength=self.n_distribution_bins, + ).astype(np.float64) + # ``base_smoothing`` is a total Dirichlet concentration, distributed + # evenly so changing the number of bins does not change prior strength. + counts += self.base_smoothing / self.n_distribution_bins + probabilities = np.maximum(counts, np.finfo(np.float64).tiny) + probabilities /= np.sum(probabilities) + self.base_logits_ = np.log(probabilities) + self.base_logits_ -= np.mean(self.base_logits_) + + self.trees_ = [] + logits = np.broadcast_to( + self.base_logits_, (X_valid.shape[0], self.n_distribution_bins) + ).copy() + for _ in range(self.n_trees): + grad, hess = _crps_grad_gn( + logits, + y_valid, + self.target_bin_edges_, + curvature_scale=self.curvature_scale, + sample_weight=weights, + ) + tree = fit_vector_tree( + self.X_binned_, + grad, + hess, + max_depth=self.max_depth, + min_child_weight=self.min_child_weight, + reg_lambda=self.reg_lambda, + leaf_reg_lambda=self.leaf_reg_lambda, + reg_alpha=self.reg_alpha, + min_gain=self.min_gain, + ) + self.trees_.append(tree) + update = self.learning_rate * np.asarray(tree(self.X_binned_)) + if not np.all(np.isfinite(update)): + raise FloatingPointError("HistogramBoost produced a non-finite tree update") + logits += update + if not np.all(np.isfinite(logits)): + raise FloatingPointError("HistogramBoost produced non-finite training logits") + return self + + def _predict_logits(self, X: Any) -> NDArray: + if self.X_binned_ is None or self.base_logits_ is None or self.n_features_in_ is None: + raise ValueError("HistogramBoost is not fitted. Call fit before predict.") + if isinstance(X, BinnedArray): + raise TypeError("HistogramBoost.predict expects raw X") + X_valid = validate_X(X, allow_binned=False, allow_nan=True, context="predict") + if X_valid.shape[1] != self.n_features_in_: + raise ValueError(f"X has {X_valid.shape[1]} features, expected {self.n_features_in_}") + X_binned = self.X_binned_.transform(X_valid) + logits = np.broadcast_to( + self.base_logits_, (X_valid.shape[0], self.n_distribution_bins) + ).copy() + for tree in self.trees_: + update = self.learning_rate * np.asarray(tree(X_binned)) + if not np.all(np.isfinite(update)): + raise FloatingPointError("HistogramBoost produced a non-finite tree update") + logits += update + if not np.all(np.isfinite(logits)): + raise FloatingPointError("HistogramBoost produced non-finite prediction logits") + return logits + + def predict_distribution(self, X: Any) -> HistogramDistributionOutput: + if self.target_bin_edges_ is None: + raise ValueError("HistogramBoost is not fitted. Call fit before predict.") + return HistogramDistributionOutput( + probas=_softmax(self._predict_logits(X)), + bin_edges=self.target_bin_edges_, + ) + + def predict(self, X: Any) -> NDArray: + return self.predict_distribution(X).mean() + + def score(self, X: Any, y: Any) -> float: + y_true = np.asarray(y, dtype=np.float64).reshape(-1) + prediction = self.predict(X) + residual = np.sum((y_true - prediction) ** 2) + total = np.sum((y_true - np.mean(y_true)) ** 2) + return float(1.0 - residual / total) if total > 0.0 else 0.0 + + def __sklearn_is_fitted__(self) -> bool: + return self.base_logits_ is not None and self.X_binned_ is not None diff --git a/src/openboost/_models/_sklearn.py b/src/openboost/_models/_sklearn.py index 37a1f8d..bf39da2 100644 --- a/src/openboost/_models/_sklearn.py +++ b/src/openboost/_models/_sklearn.py @@ -114,7 +114,8 @@ class OpenBoostRegressor(BaseEstimator, RegressorMixin): goss_other_rate : float, default=0.1 Fraction of remaining samples to sample (for GOSS). batch_size : int, optional - Mini-batch size for large datasets. If None, process all at once. + Reserved for future high-level mini-batch training. Non-None values + currently raise NotImplementedError. early_stopping_rounds : int, optional Stop training if validation score doesn't improve for this many rounds. Requires eval_set to be passed to fit(). @@ -357,7 +358,8 @@ class OpenBoostClassifier(BaseEstimator, ClassifierMixin): goss_other_rate : float, default=0.1 Fraction of remaining samples to sample (for GOSS). batch_size : int, optional - Mini-batch size for large datasets. + Reserved for future high-level mini-batch training. Non-None values + currently raise NotImplementedError. early_stopping_rounds : int, optional Stop if validation doesn't improve. verbose : int, default=0 @@ -642,6 +644,11 @@ class OpenBoostDistributionalRegressor(BaseEstimator, RegressorMixin): use_natural_gradient : bool, default=True If True, use NGBoost (natural gradient). Recommended for faster convergence and better uncertainty calibration. + training_objective : {'nll', 'crps'}, default='nll' + Objective used to fit the distribution parameters. CRPS training is + currently supported only for the Normal distribution and uses a + positive score-specific expected curvature. This is independent of + ``eval_metric``. early_stopping_rounds : int, optional Stop training if the validation metric (``eval_metric``) doesn't improve for this many rounds. Requires eval_set to be passed to @@ -704,6 +711,7 @@ def __init__( reg_lambda: float = 1.0, n_bins: int = 254, use_natural_gradient: bool = True, + training_objective: Literal['nll', 'crps'] = 'nll', early_stopping_rounds: int | None = None, verbose: int = 0, eval_metric: Literal['nll', 'crps', 'pinball', 'interval_score'] = 'nll', @@ -718,6 +726,7 @@ def __init__( self.reg_lambda = reg_lambda self.n_bins = n_bins self.use_natural_gradient = use_natural_gradient + self.training_objective = training_objective self.early_stopping_rounds = early_stopping_rounds self.verbose = verbose self.eval_metric = eval_metric @@ -780,6 +789,7 @@ def fit( min_child_weight=self.min_child_weight, reg_lambda=self.reg_lambda, n_bins=self.n_bins, + training_objective=self.training_objective, ) all_callbacks = list(callbacks) if callbacks else [] diff --git a/src/openboost/_persistence.py b/src/openboost/_persistence.py index 0287cd9..2993cc0 100644 --- a/src/openboost/_persistence.py +++ b/src/openboost/_persistence.py @@ -17,6 +17,8 @@ T = TypeVar("T", bound="PersistenceMixin") +_SERIALIZATION_VERSION = 4 + def _to_numpy(arr: Any) -> np.ndarray | None: """Convert array to numpy, handling GPU arrays. @@ -84,14 +86,14 @@ def _tree_to_dict(tree: TreeStructure) -> dict[str, Any]: data["level_thresholds"] = _to_numpy(tree.level_thresholds) # Phase 14: Missing value handling - if hasattr(tree, "missing_go_left") and tree.missing_go_left is not None: + if tree.missing_go_left is not None: data["missing_go_left"] = _to_numpy(tree.missing_go_left) # Phase 14.3: Categorical support - if hasattr(tree, "is_categorical") and tree.is_categorical is not None: - data["is_categorical"] = _to_numpy(tree.is_categorical) - if hasattr(tree, "category_masks") and tree.category_masks is not None: - data["category_masks"] = _to_numpy(tree.category_masks) + if tree.is_categorical_split is not None: + data["is_categorical_split"] = _to_numpy(tree.is_categorical_split) + if tree.cat_bitsets is not None: + data["cat_bitsets"] = _to_numpy(tree.cat_bitsets) return data @@ -153,11 +155,19 @@ def _dict_to_tree(data: dict[str, Any]) -> TreeStructure: if values_type == "scalar": values = ScalarLeaves(values_arr) elif values_type == "vector": - values = VectorLeaves(values_arr) + values = VectorLeaves(values_arr, n_outputs=values_arr.shape[1]) else: values = values_arr - tree = TreeStructure( + # Accept the pre-Phase 14.3 draft key names for compatibility with any + # model states produced while those names were in use. + is_categorical_split = data.get( + "is_categorical_split", + data.get("is_categorical"), + ) + cat_bitsets = data.get("cat_bitsets", data.get("category_masks")) + + return TreeStructure( features=data["features"], thresholds=data["thresholds"], left_children=data["left_children"], @@ -169,20 +179,11 @@ def _dict_to_tree(data: dict[str, Any]) -> TreeStructure: is_symmetric=data.get("is_symmetric", False), level_features=data.get("level_features"), level_thresholds=data.get("level_thresholds"), + missing_go_left=data.get("missing_go_left"), + is_categorical_split=is_categorical_split, + cat_bitsets=cat_bitsets, ) - # Phase 14: Missing value handling - if "missing_go_left" in data: - tree.missing_go_left = data["missing_go_left"] - - # Phase 14.3: Categorical support - if "is_categorical" in data: - tree.is_categorical = data["is_categorical"] - if "category_masks" in data: - tree.category_masks = data["category_masks"] - - return tree - class PersistenceMixin: """Mixin class providing save/load functionality for models. @@ -224,7 +225,10 @@ def _to_state_dict(self) -> dict[str, Any]: Returns: Dictionary containing all model state """ - state = {"__class__": type(self).__name__, "_serialization_version": 1} + state = { + "__class__": type(self).__name__, + "_serialization_version": _SERIALIZATION_VERSION, + } for attr in self._get_persist_attrs(): value = getattr(self, attr, None) @@ -252,6 +256,9 @@ def _to_state_dict(self) -> dict[str, Any]: if value is not None: state["_bin_edges"] = value.bin_edges state["_n_features"] = value.n_features + state["_binning_version"] = vars(value).get( + "binning_version", 1 + ) if hasattr(value, "has_missing"): state["_has_missing"] = _to_numpy(value.has_missing) if hasattr(value, "is_categorical"): @@ -284,7 +291,6 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: """ import warnings - _CURRENT_SERIALIZATION_VERSION = 1 saved_version = state.get("_serialization_version") if saved_version is None: warnings.warn( @@ -293,14 +299,23 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: UserWarning, stacklevel=2, ) - elif saved_version > _CURRENT_SERIALIZATION_VERSION: + elif saved_version > _SERIALIZATION_VERSION: warnings.warn( f"Model was saved with serialization version {saved_version}, " - f"but current version is {_CURRENT_SERIALIZATION_VERSION}. " + f"but current version is {_SERIALIZATION_VERSION}. " "Some features may not load correctly.", UserWarning, stacklevel=2, ) + elif saved_version < 2 and np.any(state.get("_is_categorical", False)): + warnings.warn( + "This model uses categorical features and was saved with " + "serialization version 1, which did not reliably preserve " + "categorical tree routing. Retrain and resave the model before " + "using its predictions in production.", + UserWarning, + stacklevel=2, + ) trees_type = state.get("_trees_type", "list") @@ -341,8 +356,6 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: # Restore bin edges for transform if "_bin_edges" in state: - import numpy as np - from ._array import BinnedArray # Create a minimal BinnedArray with just bin edges for transform @@ -351,6 +364,10 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: is_categorical = state.get("_is_categorical", np.array([], dtype=np.bool_)) category_maps = state.get("_category_maps", []) n_categories = state.get("_n_categories", np.array([], dtype=np.int32)) + # States written before serialization version 4 used the legacy + # top-bin clipping rule. Preserve it so loading an old model does + # not silently change predictions. + binning_version = state.get("_binning_version", 1) # Create placeholder data (empty, just need structure for transform) placeholder_data = np.zeros((n_features, 0), dtype=np.uint8) @@ -365,6 +382,7 @@ def _from_state_dict(self, state: dict[str, Any]) -> None: is_categorical=is_categorical if isinstance(is_categorical, np.ndarray) else np.array(is_categorical, dtype=np.bool_), category_maps=category_maps, n_categories=n_categories if isinstance(n_categories, np.ndarray) else np.array(n_categories, dtype=np.int32), + binning_version=int(binning_version), ) # Reconstruct _loss_fn from stored loss name/config @@ -527,6 +545,7 @@ def load(path: str | Path) -> PersistenceMixin: NaturalBoostTweedie, ) from ._models._gam import OpenBoostGAM + from ._models._histogram_boost import HistogramBoost from ._models._linear_leaf import LinearLeafGBDT _CLASS_MAP: dict[str, type[PersistenceMixin]] = { @@ -536,6 +555,7 @@ def load(path: str | Path) -> PersistenceMixin: MultiClassGradientBoosting, DART, OpenBoostGAM, + HistogramBoost, DistributionalGBDT, NaturalBoost, NaturalBoostNormal, diff --git a/src/openboost/_validation.py b/src/openboost/_validation.py index 48c9c84..dcbb540 100644 --- a/src/openboost/_validation.py +++ b/src/openboost/_validation.py @@ -252,8 +252,10 @@ def validate_sample_weight( if sample_weight is None: return None - if not isinstance(sample_weight, np.ndarray): - sample_weight = np.asarray(sample_weight, dtype=np.float32) + try: + sample_weight = np.asarray(sample_weight, dtype=np.float64) + except (TypeError, ValueError) as exc: + raise ValueError("sample_weight must contain numeric values.") from exc if sample_weight.ndim != 1: raise ValueError( @@ -272,10 +274,17 @@ def validate_sample_weight( f"Min value: {np.min(sample_weight)}" ) - if np.any(np.isnan(sample_weight)): - raise ValueError("sample_weight contains NaN values.") + if not np.all(np.isfinite(sample_weight)): + raise ValueError("sample_weight must contain only finite values.") - return sample_weight.astype(np.float32) + with np.errstate(over="ignore", invalid="ignore"): + sample_weight_float32 = sample_weight.astype(np.float32) + if not np.all(np.isfinite(sample_weight_float32)): + raise ValueError( + "sample_weight must remain finite when converted to float32." + ) + + return sample_weight_float32 def validate_eval_set( diff --git a/tests/test_binning_correctness.py b/tests/test_binning_correctness.py index b116338..0753340 100644 --- a/tests/test_binning_correctness.py +++ b/tests/test_binning_correctness.py @@ -53,6 +53,27 @@ def test_transform_out_of_range_values(self): assert np.all(test_binned.data < 255), "Out-of-range values should not be missing bin" assert np.all(test_binned.data >= 0), "Bins should be non-negative" + def test_transform_preserves_corrected_top_bin(self): + """The largest and above-range values stay in the final numeric bin.""" + X_train = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + X_test = np.array([[0.0], [1.0], [2.0], [3.0]], dtype=np.float32) + + binned = ob.array(X_train, n_bins=2) + transformed = binned.transform(X_test) + + np.testing.assert_array_equal(transformed.data[0], [0, 0, 1, 1]) + + def test_transform_without_version_uses_legacy_routing(self): + """Directly unpickled pre-version metadata keeps its old top-bin rule.""" + X = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + binned = ob.array(X, n_bins=2) + del binned.binning_version + + transformed = binned.transform(X) + + assert transformed.binning_version == 1 + np.testing.assert_array_equal(transformed.data[0], 0) + class TestBinEdges: """Verify bin edge properties.""" @@ -142,11 +163,48 @@ def test_two_unique_values(self): """Two distinct values should produce two bins.""" X = np.array([[0.0], [0.0], [1.0], [1.0]], dtype=np.float32) - binned = ob.array(X) + binned = ob.array(X, n_bins=2) unique_bins = np.unique(binned.data[0, :]) assert len(unique_bins) == 2, f"Two values should produce 2 bins, got {len(unique_bins)}" + def test_low_cardinality_values_keep_distinct_top_bin(self): + """Quantile binning must not merge the highest ordinal level.""" + X = np.repeat( + np.array([[1.0], [2.0], [3.0], [4.0]], dtype=np.float32), + repeats=4, + axis=0, + ) + + binned = ob.array(X, n_bins=4) + + assert np.unique(binned.data[0]).tolist() == [0, 1, 2, 3] + + def test_top_bin_with_missing_values(self): + """The NaN path preserves both numeric bins plus the missing bin.""" + X = np.array([[1.0], [1.0], [2.0], [2.0], [np.nan]], dtype=np.float32) + + binned = ob.array(X, n_bins=2) + + np.testing.assert_array_equal(binned.data[0], [0, 0, 1, 1, 255]) + + def test_top_bin_enables_binary_tree_split(self): + """A depth-one tree can split the two values retained by binning.""" + X = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + y = np.array([0.0, 0.0, 1.0, 1.0], dtype=np.float32) + binned = ob.array(X, n_bins=2) + + tree = ob.fit_tree( + binned, + -y, + np.ones_like(y), + max_depth=1, + reg_lambda=0.0, + ) + + assert tree.features[0] == 0 + np.testing.assert_array_equal(tree(binned), y) + def test_very_large_values(self): """Large values should not cause overflow.""" X = np.array([[1e10, -1e10], [1e15, -1e15]], dtype=np.float32) diff --git a/tests/test_categorical.py b/tests/test_categorical.py index 868a4f9..71eaab2 100644 --- a/tests/test_categorical.py +++ b/tests/test_categorical.py @@ -178,6 +178,16 @@ def test_categorical_split_found(self): class TestGradientBoostingWithCategorical: """Tests for GradientBoosting with categorical features.""" + + def test_fit_rejects_unrepresentable_categorical_split(self): + """Tree training fails before silently truncating a category bitset.""" + categories = np.tile(np.arange(65, dtype=np.float32), 4) + X_binned = array(categories[:, None], categorical_features=[0]) + y = (categories % 2).astype(np.float32) + + model = GradientBoosting(n_trees=1, max_depth=1) + with pytest.raises(ValueError, match="maximum supported is 64"): + model.fit(X_binned, y) def test_fit_with_categorical(self): """GradientBoosting fits with categorical features.""" diff --git a/tests/test_distribution_gradients.py b/tests/test_distribution_gradients.py index 0e1eeb1..ece53c4 100644 --- a/tests/test_distribution_gradients.py +++ b/tests/test_distribution_gradients.py @@ -118,6 +118,49 @@ def test_normal(self): } _check_family_gradients(Normal(), y, raw) + def test_normal_crps_raw_gradients_and_expected_curvature(self): + """Gaussian CRPS derivatives match FD; training curvature stays positive.""" + dist = Normal() + z_values = np.array([-6.0, -2.0, -0.5, 0.0, 0.5, 2.0, 6.0]) + scales = np.array([0.05, 1.0, 20.0]) + z = np.tile(z_values, len(scales)) + scale = np.repeat(scales, len(z_values)) + loc = np.linspace(-1.0, 1.0, len(z)) + y = loc + z * scale + raw = {'loc': loc, 'scale': np.log(scale)} + params = _params_from_raw(dist, raw) + grads = dist.crps_gradient(y, params) + + for name in dist.param_names: + fd = _fd_grad_raw( + dist, + y, + raw, + name, + objective=dist.crps, + eps=1e-5, + ) + assert_allclose(grads[name][0], fd, rtol=5e-4, atol=5e-5) + assert np.all(np.isfinite(grads[name][1])) + assert np.all(grads[name][1] > 0) + + sqrt_pi = np.sqrt(np.pi) + assert_allclose(grads['loc'][1], 1 / (sqrt_pi * scale), rtol=1e-6) + assert_allclose(grads['scale'][1], scale / (2 * sqrt_pi), rtol=1e-6) + + def test_normal_crps_loss_matches_public_metric(self): + from openboost import crps_gaussian + + dist = Normal() + y = np.array([-2.0, 0.2, 3.0]) + params = { + 'loc': np.array([-1.5, 0.0, 2.0]), + 'scale': np.array([0.2, 1.0, 4.0]), + } + assert np.mean(dist.crps(y, params)) == pytest.approx( + crps_gaussian(y, params['loc'], params['scale']), rel=1e-12 + ) + def test_lognormal(self): y = np.array([0.05, 1.0, 4.0, 50.0]) raw = { @@ -400,6 +443,7 @@ def nll_fn(y, p): expected_hess = 2.0 * resid ** 2 / params['scale'] ** 2 assert_allclose(grads['scale'][1], expected_hess, rtol=1e-2) + @pytest.mark.jax def test_jax_gradient_matches_numerical_path(self): """JAX path must differentiate through the link, like the numerical path. @@ -474,6 +518,7 @@ def fail_autodiff(y, params): assert_allclose(actual[name][0], expected[name][0]) assert_allclose(actual[name][1], expected[name][1]) + @pytest.mark.jax def test_numpy_nll_falls_back_when_jax_is_installed(self): """Plain numpy NLLs remain correct when optional JAX is installed.""" pytest.importorskip('jax') diff --git a/tests/test_distributional.py b/tests/test_distributional.py index 7ae233b..29347b5 100644 --- a/tests/test_distributional.py +++ b/tests/test_distributional.py @@ -960,6 +960,68 @@ def test_crps_metric_decreases(self): assert vals[-1] < vals[0] assert np.mean(vals[-5:]) < np.mean(vals[:5]) + def test_crps_training_reduces_crps_and_reports_objective(self): + """Explicit CRPS training improves its objective and logs CRPS, not NLL.""" + from openboost import HistoryCallback, NaturalBoost, crps_gaussian + + X, y = self._make_data(seed=41, n=400, noise=0.5) + initial = crps_gaussian( + y, + np.full_like(y, np.mean(y)), + np.full_like(y, np.std(y) + 1e-6), + ) + history = HistoryCallback() + model = NaturalBoost( + distribution='normal', + training_objective='crps', + n_trees=50, + max_depth=3, + learning_rate=0.1, + ) + model.fit(X, y, callbacks=[history]) + + output = model.predict_distribution(X) + final = crps_gaussian(y, output.params['loc'], output.params['scale']) + final_nll = model.nll(X, y) + assert final < initial + assert history.history['train_loss'][-1] == pytest.approx(final, rel=1e-5) + assert history.history['train_loss'][-1] != pytest.approx(final_nll, rel=1e-3) + + def test_training_objective_is_explicit_and_validated(self): + """CRPS is opt-in, independent of eval_metric, and Normal-only.""" + from openboost import NaturalBoost + + X, y = self._make_data(seed=42, n=100) + default = NaturalBoost( + distribution='normal', n_trees=5, max_depth=2, learning_rate=0.05 + ) + explicit_nll = NaturalBoost( + distribution='normal', + training_objective='nll', + n_trees=5, + max_depth=2, + learning_rate=0.05, + ) + default.fit(X, y, eval_set=[(X, y)], eval_metric='crps') + explicit_nll.fit(X, y) + for name in ('loc', 'scale'): + assert_allclose( + default.predict_params(X)[name], + explicit_nll.predict_params(X)[name], + rtol=0, + atol=0, + ) + + with pytest.raises(ValueError, match="supports only the Normal"): + NaturalBoost( + distribution='poisson', training_objective='crps', n_trees=1 + ).fit(X, np.maximum(np.rint(y - y.min()), 0)) + + with pytest.raises(ValueError, match="Unknown training_objective"): + NaturalBoost( + distribution='normal', training_objective='mystery', n_trees=1 + ).fit(X, y) + def test_pinball_and_interval_metrics(self): """pinball (with quantiles) and interval_score (with level) run.""" from openboost import NaturalBoost diff --git a/tests/test_histogram_boost.py b/tests/test_histogram_boost.py new file mode 100644 index 0000000..fb3062d --- /dev/null +++ b/tests/test_histogram_boost.py @@ -0,0 +1,389 @@ +"""Correctness tests for shared-vector histogram distribution boosting.""" + +from __future__ import annotations + +import warnings + +import numpy as np +import pytest + +import openboost as ob +from openboost._core._vector_tree import _find_best_vector_split, fit_vector_tree +from openboost._models._histogram_boost import ( + _continuous_crps_from_probabilities, + _continuous_crps_loss, + _continuous_crps_terms, + _crps_grad_gn, +) + + +def test_continuous_crps_logit_gradient_matches_finite_difference(): + rng = np.random.default_rng(4) + logits = rng.normal(size=(3, 5)) + y = np.array([-0.2, 1.3, 4.8]) + bin_edges = np.array([-1.0, 0.0, 0.7, 2.0, 3.5, 5.0]) + grad, _ = _crps_grad_gn(logits, y, bin_edges) + + eps = 1e-6 + numerical = np.empty_like(logits) + for row in range(logits.shape[0]): + for output in range(logits.shape[1]): + plus = logits.copy() + minus = logits.copy() + plus[row, output] += eps + minus[row, output] -= eps + numerical[row, output] = ( + _continuous_crps_loss(plus, y, bin_edges)[row] + - _continuous_crps_loss(minus, y, bin_edges)[row] + ) / (2 * eps) + + np.testing.assert_allclose(grad, numerical, rtol=2e-5, atol=2e-6) + np.testing.assert_allclose(np.sum(grad, axis=1), 0.0, atol=1e-7) + + +def test_continuous_crps_includes_uniform_within_bin_distance(): + # A Uniform(0, 1) forecast observed at 0.5 has CRPS 1/12. The second + # far-away bin has negligible softmax mass at these logits. + loss = _continuous_crps_loss( + np.array([[50.0, -50.0]]), + np.array([0.5]), + np.array([0.0, 1.0, 2.0]), + ) + assert loss[0] == pytest.approx(1.0 / 12.0) + probability_loss = _continuous_crps_from_probabilities( + np.array([[1.0, 0.0]]), + np.array([0.5]), + np.array([0.0, 1.0, 2.0]), + ) + assert probability_loss[0] == pytest.approx(1.0 / 12.0) + + +def test_continuous_crps_psd_diagonal_matches_explicit_jacobian(): + logits = np.array([[0.4, -0.2, 0.1, 0.7]]) + y = np.array([1.2]) + bin_edges = np.array([-0.5, 0.0, 1.5, 2.25, 4.0]) + scale = 1.7 + _, hess = _crps_grad_gn( + logits, + y, + bin_edges, + curvature_scale=scale, + curvature_floor=0.0, + ) + + p = np.exp(logits[0] - np.max(logits[0])) + p /= p.sum() + _, pairwise_distance = _continuous_crps_terms(y, bin_edges) + jacobian = np.diag(p) - np.outer(p, p) + expected = scale * np.diag(jacobian.T @ (-pairwise_distance) @ jacobian) + + np.testing.assert_allclose(hess[0], expected, rtol=2e-6, atol=1e-8) + assert np.all(hess >= 0.0) + + +def test_crps_sample_weight_scales_gradient_and_curvature(): + logits = np.zeros((3, 4)) + y = np.array([-0.5, 1.2, 4.0]) + bin_edges = np.array([-1.0, 0.0, 1.0, 2.0, 3.0]) + weights = np.array([0.0, 2.0, 5.0]) + grad, hess = _crps_grad_gn(logits, y, bin_edges) + weighted_grad, weighted_hess = _crps_grad_gn( + logits, + y, + bin_edges, + sample_weight=weights, + ) + np.testing.assert_allclose(weighted_grad, grad * weights[:, None]) + np.testing.assert_allclose(weighted_hess, hess * weights[:, None]) + + +def _brute_split(binned, grad, hess, reg_lambda): + total_grad = grad.sum(axis=0) + total_hess = hess.sum(axis=0) + + def score(g, h): + return np.mean(g * g / (h + reg_lambda)) + + parent = score(total_grad, total_hess) + best = None + for feature in range(binned.shape[0]): + for threshold in range(254): + for missing_left in (True, False): + left = binned[feature] <= threshold + if missing_left: + left |= binned[feature] == ob.MISSING_BIN + else: + left &= binned[feature] != ob.MISSING_BIN + if not np.any(left) or np.all(left): + continue + gain = ( + score(grad[left].sum(axis=0), hess[left].sum(axis=0)) + + score(grad[~left].sum(axis=0), hess[~left].sum(axis=0)) + - parent + ) + candidate = (gain, feature, threshold, missing_left) + if best is None or candidate[0] > best[0]: + best = candidate + return best + + +def test_vector_split_and_leaf_values_match_brute_force(): + binned = np.array( + [ + [0, 0, 1, 1, ob.MISSING_BIN], + [0, 1, 0, 1, 0], + ], + dtype=np.uint8, + ) + grad = np.array([[-2.0, -1.0], [-1.5, -0.5], [1.0, 2.0], [1.5, 2.5], [-0.5, 0.5]]) + hess = np.ones_like(grad) + reg_lambda = 1.0 + expected = _brute_split(binned, grad, hess, reg_lambda) + split = _find_best_vector_split( + binned, + grad, + hess, + min_child_weight=0.0, + reg_lambda=reg_lambda, + reg_alpha=0.0, + min_gain=0.0, + ) + assert (split.feature, split.threshold, split.missing_go_left) == expected[1:] + assert split.gain == pytest.approx(expected[0]) + + tree = fit_vector_tree( + binned, + grad, + hess, + max_depth=1, + min_child_weight=0.0, + reg_lambda=reg_lambda, + ) + prediction = tree.predict(binned) + assert prediction.shape == grad.shape + left = binned[split.feature] <= split.threshold + if split.missing_go_left: + left |= binned[split.feature] == ob.MISSING_BIN + else: + left &= binned[split.feature] != ob.MISSING_BIN + np.testing.assert_allclose( + prediction[left][0], + -grad[left].sum(axis=0) / (hess[left].sum(axis=0) + reg_lambda), + ) + + leaf_reg_lambda = 0.1 + decoupled = fit_vector_tree( + binned, + grad, + hess, + max_depth=1, + min_child_weight=0.0, + reg_lambda=reg_lambda, + leaf_reg_lambda=leaf_reg_lambda, + ) + np.testing.assert_array_equal(decoupled.features, tree.features) + decoupled_prediction = decoupled.predict(binned) + np.testing.assert_allclose( + decoupled_prediction[left][0], + -grad[left].sum(axis=0) / (hess[left].sum(axis=0) + leaf_reg_lambda), + ) + + +def test_histogram_boost_defaults_are_frozen_and_sklearn_cloneable(): + model = ob.HistogramBoost() + assert model.n_distribution_bins == 50 + assert model.n_trees == 100 + assert model.learning_rate == 0.05 + assert model.max_depth == 6 + assert model.curvature_scale == 1.0 + assert model.reg_lambda == 1.0 + assert model.leaf_reg_lambda is None + + sklearn = pytest.importorskip("sklearn.base") + cloned = sklearn.clone(model) + assert cloned.get_params() == model.get_params() + assert not cloned.__sklearn_is_fitted__() + + +def test_histogram_boost_fit_predict_distribution_and_no_crossing(): + rng = np.random.default_rng(12) + X = rng.normal(size=(80, 3)).astype(np.float32) + X[3, 1] = np.nan + y = (X[:, 0] + rng.normal(scale=0.4, size=80)).astype(np.float32) + model = ob.HistogramBoost( + n_distribution_bins=12, + n_trees=4, + learning_rate=0.05, + max_depth=2, + n_feature_bins=16, + ).fit(X, y) + + edges_before = model.target_bin_edges_.copy() + dist = model.predict_distribution(np.array([[100.0, 0.0, 0.0], X[0]])) + assert dist.probas.shape == (2, 12) + np.testing.assert_allclose(dist.probas.sum(axis=1), 1.0) + assert np.all(np.diff(np.cumsum(dist.probas, axis=1), axis=1) >= -1e-12) + assert np.all(np.diff(np.column_stack([dist.quantile(q) for q in [0.1, 0.5, 0.9]])) >= 0) + np.testing.assert_array_equal(model.target_bin_edges_, edges_before) + assert model.predict(X[:5]).shape == (5,) + + +def test_histogram_distribution_output_moments_quantiles_and_sampling(): + dist = ob.HistogramDistributionOutput( + probas=np.array([[0.25, 0.75], [1.0, 0.0]]), + bin_edges=np.array([0.0, 1.0, 2.0]), + ) + np.testing.assert_allclose(dist.mean(), [1.25, 0.5]) + np.testing.assert_allclose(dist.quantile(0.5), [4.0 / 3.0, 0.5]) + lower, upper = dist.interval(0.2) + assert np.all(lower <= upper) + samples1 = dist.sample(20, seed=7) + samples2 = dist.sample(20, seed=7) + np.testing.assert_array_equal(samples1, samples2) + assert samples1.shape == (2, 20) + assert np.all((samples1 >= 0.0) & (samples1 <= 2.0)) + + sharper = dist.tempered(0.5) + np.testing.assert_allclose(sharper.probas.sum(axis=1), 1.0) + assert sharper.probas[0, 1] > dist.probas[0, 1] + np.testing.assert_array_equal(dist.tempered(1.0).probas, dist.probas) + np.testing.assert_allclose( + dist.crps(np.array([0.5, 0.5])), + [25.0 / 48.0, 1.0 / 12.0], + ) + refined = dist.subdivide(4) + assert refined.probas.shape == (2, 8) + np.testing.assert_allclose(refined.mean(), dist.mean()) + np.testing.assert_allclose(refined.variance(), dist.variance()) + np.testing.assert_allclose( + refined.crps(np.array([0.5, 0.5])), + dist.crps(np.array([0.5, 0.5])), + ) + with pytest.raises(ValueError, match="strictly positive"): + dist.tempered(0.0) + with pytest.raises(ValueError, match="positive integer"): + dist.subdivide(0) + + with pytest.raises(ValueError, match="bin_edges must contain only finite"): + ob.HistogramDistributionOutput( + probas=np.array([[0.5, 0.5]]), + bin_edges=np.array([0.0, np.nan, 2.0]), + ) + + +def test_histogram_boost_sample_weight_controls_base_distribution(): + X = np.arange(8, dtype=np.float32).reshape(-1, 1) + y = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]) + unweighted = ob.HistogramBoost( + n_distribution_bins=3, + n_trees=0, + base_smoothing=0.0, + ).fit(X, y) + weighted = ob.HistogramBoost( + n_distribution_bins=3, + n_trees=0, + base_smoothing=0.0, + ).fit(X, y, sample_weight=np.array([10, 10, 10, 10, 1, 1, 1, 1])) + + p_unweighted = unweighted.predict_distribution(X[:1]).probas[0] + p_weighted = weighted.predict_distribution(X[:1]).probas[0] + assert p_unweighted[0] == pytest.approx(p_unweighted[-1]) + assert p_weighted[0] / p_weighted[-1] == pytest.approx(10.0) + with pytest.raises(ValueError, match="positive total weight"): + weighted.fit(X, y, sample_weight=np.zeros(len(y))) + with pytest.raises(ValueError, match="only finite"): + weighted.fit(X, y, sample_weight=np.full(len(y), np.inf)) + with pytest.raises(ValueError, match="finite when converted to float32"): + weighted.fit(X, y, sample_weight=np.full(len(y), 1e300, dtype=np.float64)) + + +@pytest.mark.parametrize( + ("outlier_X", "outlier_y"), + [ + (np.array([1.5, -0.5], dtype=np.float32), 1_000.0), + (np.array([1e6, -1e6], dtype=np.float32), 1.5), + ], + ids=("target-outlier", "feature-outlier"), +) +def test_histogram_boost_zero_weight_outlier_matches_dropped_row(outlier_X, outlier_y): + X = np.array( + [ + [-2.0, 1.0], + [-1.0, 0.5], + [0.0, 0.0], + [1.0, -0.5], + [2.0, -1.0], + [3.0, -1.5], + ], + dtype=np.float32, + ) + y = np.array([-1.0, -0.5, 0.0, 1.0, 2.0, 2.5], dtype=np.float32) + X_with_outlier = np.vstack([X, outlier_X]) + y_with_outlier = np.append(y, np.float32(outlier_y)) + weights = np.append(np.ones(len(y), dtype=np.float32), np.float32(0.0)) + params = { + "n_distribution_bins": 6, + "n_trees": 3, + "max_depth": 2, + "n_feature_bins": 8, + } + + dropped = ob.HistogramBoost(**params).fit(X, y) + weighted = ob.HistogramBoost(**params).fit( + X_with_outlier, + y_with_outlier, + sample_weight=weights, + ) + + np.testing.assert_array_equal(weighted.target_bin_edges_, dropped.target_bin_edges_) + for weighted_edges, dropped_edges in zip( + weighted.X_binned_.bin_edges, + dropped.X_binned_.bin_edges, + strict=True, + ): + np.testing.assert_array_equal(weighted_edges, dropped_edges) + np.testing.assert_allclose( + weighted.predict_distribution(X).probas, + dropped.predict_distribution(X).probas, + rtol=0.0, + atol=0.0, + ) + + +def test_histogram_boost_base_smoothing_is_total_prior_weight(): + X = np.arange(4, dtype=np.float32).reshape(-1, 1) + y = np.zeros(4, dtype=np.float32) + model = ob.HistogramBoost( + n_distribution_bins=5, + n_trees=0, + base_smoothing=1.0, + ).fit(X, y) + + probabilities = model.predict_distribution(X[:1]).probas[0] + labels = np.searchsorted(model.target_bin_edges_[1:-1], y, side="right") + counts = np.bincount(labels, minlength=5) + expected = (counts + 1.0 / 5.0) / (len(y) + 1.0) + np.testing.assert_allclose(probabilities, expected) + + +def test_histogram_boost_persistence_preserves_vector_predictions(tmp_path): + rng = np.random.default_rng(2) + X = rng.normal(size=(50, 2)).astype(np.float32) + y = (X[:, 0] - 0.5 * X[:, 1]).astype(np.float32) + model = ob.HistogramBoost( + n_distribution_bins=8, + n_trees=3, + max_depth=2, + n_feature_bins=10, + ).fit(X, y) + expected = model.predict_distribution(X[:7]).probas + path = tmp_path / "histogram.joblib" + model.save(path) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + restored = ob.HistogramBoost.load(path) + auto_restored = ob.load(path) + np.testing.assert_allclose(restored.predict_distribution(X[:7]).probas, expected) + np.testing.assert_allclose(auto_restored.predict_distribution(X[:7]).probas, expected) + assert restored.trees_[0].leaf_values_array.ndim == 2 diff --git a/tests/test_large_scale.py b/tests/test_large_scale.py index d071b12..d9dc040 100644 --- a/tests/test_large_scale.py +++ b/tests/test_large_scale.py @@ -317,6 +317,30 @@ def test_goss_config_validation(self): # Integration Tests with GradientBoosting # ============================================================================= +class TestUnsupportedHighLevelBatching: + """High-level models must not silently ignore ``batch_size``.""" + + @pytest.mark.parametrize( + "model,y", + [ + (ob.GradientBoosting(n_trees=1, batch_size=16), np.arange(64)), + ( + ob.MultiClassGradientBoosting( + n_classes=2, + n_trees=1, + batch_size=16, + ), + np.arange(64) % 2, + ), + ], + ) + def test_batch_size_fails_fast(self, model, y): + X = np.arange(128, dtype=np.float32).reshape(64, 2) + + with pytest.raises(NotImplementedError, match="batch_size"): + model.fit(X, y) + + class TestGOSSIntegration: """Integration tests for GOSS with GradientBoosting.""" diff --git a/tests/test_performance_check.py b/tests/test_performance_check.py new file mode 100644 index 0000000..9e54bec --- /dev/null +++ b/tests/test_performance_check.py @@ -0,0 +1,58 @@ +"""Tests for the CI performance comparison harness.""" + +import json +from pathlib import Path + +from benchmarks.check_performance import ( + check_regression, + collect_provenance, + load_baselines, +) + + +def _result(**overrides): + result = { + "fit_time_median": 1.0, + "predict_time_median": 0.1, + "peak_memory_mb": 10.0, + "mse": 0.05, + "r2": 0.95, + "n_samples": 5000, + "n_features": 10, + "n_trees": 100, + "max_depth": 6, + } + result.update(overrides) + return result + + +def test_equal_results_have_no_regression(): + baseline = _result() + + assert check_regression(_result(), baseline) == [] + + +def test_runtime_and_quality_regressions_are_reported(): + baseline = _result() + current = _result(fit_time_median=1.21, mse=0.061) + + regressions = check_regression(current, baseline) + + assert any("fit_time_median" in item for item in regressions) + assert any("mse" in item for item in regressions) + + +def test_load_baselines_uses_explicit_path(tmp_path): + baseline_path = tmp_path / "parent.json" + baseline_path.write_text(json.dumps(_result())) + + assert load_baselines(baseline_path) == _result() + + +def test_provenance_records_source_commit_and_environment(): + provenance = collect_provenance(Path.cwd()) + + assert len(provenance["git_commit"]) == 40 + assert provenance["python_version"] + assert provenance["numpy_version"] + assert provenance["openboost_version"] diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 900e50f..8f437c5 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -73,6 +73,53 @@ def test_save_load_basic(self, regression_data, tmp_path): # Predictions should match np.testing.assert_allclose(pred_before, pred_after, rtol=1e-5) + def test_save_load_preserves_categorical_tree_state(self, tmp_path): + """Categorical split routing survives a model round trip.""" + import openboost as ob + + categories = np.tile( + np.array([0.0, 1.0, 2.0, np.nan], dtype=np.float32), + 60, + ) + X = categories[:, None] + y = np.select( + [categories == 0.0, categories == 1.0, categories == 2.0], + [4.0, -3.0, 2.0], + default=7.0, + ).astype(np.float32) + + X_binned = ob.array(X, categorical_features=[0]) + model = ob.GradientBoosting(n_trees=8, max_depth=2, learning_rate=0.2) + model.fit(X_binned, y) + + assert any( + tree.is_categorical_split is not None + and np.any(tree.is_categorical_split[: tree.n_nodes]) + for tree in model.trees_ + ) + + state = model._to_state_dict() + assert state["_serialization_version"] == 4 + assert state["_binning_version"] == 2 + assert all("is_categorical_split" in tree for tree in state["trees_"]) + assert all("cat_bitsets" in tree for tree in state["trees_"]) + + pred_before = model.predict(X) + save_path = tmp_path / "categorical_model.joblib" + model.save(save_path) + loaded = ob.GradientBoosting.load(save_path) + pred_after = loaded.predict(X) + + for expected, actual in zip(model.trees_, loaded.trees_, strict=True): + np.testing.assert_array_equal( + expected.is_categorical_split, + actual.is_categorical_split, + ) + np.testing.assert_array_equal(expected.cat_bitsets, actual.cat_bitsets) + np.testing.assert_array_equal(expected.missing_go_left, actual.missing_go_left) + + np.testing.assert_allclose(pred_before, pred_after, rtol=0, atol=0) + def test_save_load_with_different_losses(self, regression_data, tmp_path): """Test save/load with various loss functions.""" import openboost as ob @@ -280,6 +327,28 @@ def test_save_load_normal(self, regression_data, tmp_path): np.testing.assert_allclose(interval_before[0], interval_after[0], rtol=1e-5) np.testing.assert_allclose(interval_before[1], interval_after[1], rtol=1e-5) + def test_save_load_crps_objective(self, regression_data, tmp_path): + """CRPS training semantics survive a persistence round trip.""" + import openboost as ob + + X, y = regression_data + model = ob.NaturalBoostNormal( + training_objective='crps', n_trees=10, max_depth=3 + ) + model.fit(X[:400], y[:400]) + pred_before = model.predict_distribution(X[400:]) + + save_path = tmp_path / "crps-model.joblib" + model.save(save_path) + loaded = ob.NaturalBoost.load(save_path) + pred_after = loaded.predict_distribution(X[400:]) + + assert loaded.training_objective == 'crps' + for name in ('loc', 'scale'): + np.testing.assert_allclose( + pred_before.params[name], pred_after.params[name], rtol=1e-6 + ) + def test_save_load_poisson(self, tmp_path): """Test save/load for NaturalBoost with Poisson distribution.""" import openboost as ob @@ -401,6 +470,31 @@ def test_classifier_pickle(self, binary_data, tmp_path): class TestPersistenceEdgeCases: """Test edge cases for persistence.""" + def test_legacy_numeric_binning_routing_survives_load(self): + """Loading a pre-fix model preserves its legacy top-bin routing.""" + import openboost as ob + + X = np.array([[1.0], [1.0], [2.0], [2.0]], dtype=np.float32) + y = np.array([0.0, 0.0, 1.0, 1.0], dtype=np.float32) + metadata = ob.array(X, n_bins=2) + metadata.binning_version = 1 + legacy_binned = metadata.transform(X) + assert np.unique(legacy_binned.data).tolist() == [0] + + model = ob.GradientBoosting(n_trees=1, max_depth=1) + model.fit(legacy_binned, y) + pred_before = model.predict(X) + state = model._to_state_dict() + state["_serialization_version"] = 3 + state.pop("_binning_version") + + loaded = ob.GradientBoosting() + loaded._from_state_dict(state) + + assert loaded.X_binned_.binning_version == 1 + np.testing.assert_array_equal(loaded.X_binned_.transform(X).data, 0) + np.testing.assert_allclose(loaded.predict(X), pred_before, rtol=0, atol=0) + def test_load_wrong_class_raises(self, regression_data, tmp_path): """Test that loading with wrong class raises error.""" import openboost as ob diff --git a/tests/test_scoringbench_candidate_evaluation.py b/tests/test_scoringbench_candidate_evaluation.py new file mode 100644 index 0000000..b90eb1f --- /dev/null +++ b/tests/test_scoringbench_candidate_evaluation.py @@ -0,0 +1,80 @@ +"""Tests for the preregistered HistogramBoost acceptance evaluator.""" + +import pytest +from benchmarks.scoringbench.evaluate_crps_candidate import evaluate_records + + +def _records( + candidate_crps=0.99, + xgb_crps=1.0, + cat_crps=1.02, + candidate_coverage=0.9, + baseline_coverage=0.8, +): + result = [] + for model, crps, coverage, interval, rmse in ( + ("openboost_histogram_cpu", candidate_crps, candidate_coverage, 2.0, 1.0), + ("xgboost_quantile", xgb_crps, baseline_coverage, 2.0, 1.0), + ("catboost_quantile", cat_crps, baseline_coverage, 2.1, 1.02), + ): + for fold in range(5): + result.append( + { + "dataset": "example", + "model": model, + "fold": fold, + "crps": crps, + "coverage_90": coverage, + "interval_score_90": interval, + "rmse": rmse, + "error": None, + } + ) + return result + + +def test_development_candidate_passes_every_frozen_guardrail(): + result = evaluate_records( + _records(), + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + ) + + assert result["selected_strong_baseline"] == "xgboost_quantile" + assert result["comparisons"]["candidate_fold_wins"] == 5 + assert result["development_pass"] is True + assert result["accepted"] is True + + +def test_confirmation_requires_strict_crps_win_and_four_folds(): + result = evaluate_records( + _records(candidate_crps=1.0), + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + phase="confirmation", + ) + + assert result["development_pass"] is True + assert result["confirmation_dataset_win"] is False + assert result["accepted"] is False + + +def test_candidate_with_good_crps_but_bad_coverage_fails(): + result = evaluate_records( + _records(candidate_coverage=0.7), + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + ) + + assert result["guardrails"]["coverage_error_at_most_0_05"] is False + assert result["guardrails"]["coverage_error_improves_by_0_02"] is False + assert result["accepted"] is False + + +def test_incomplete_rows_fail_closed(): + with pytest.raises(ValueError, match="incomplete rows"): + evaluate_records( + _records()[:-1], + candidate="openboost_histogram_cpu", + baselines=("xgboost_quantile", "catboost_quantile"), + ) diff --git a/tests/test_scoringbench_provenance.py b/tests/test_scoringbench_provenance.py new file mode 100644 index 0000000..84d4148 --- /dev/null +++ b/tests/test_scoringbench_provenance.py @@ -0,0 +1,339 @@ +"""Tests for ScoringBench artifact provenance that need no external checkout.""" + +from pathlib import Path + +import pytest +from benchmarks.scoringbench.run import ( + _audit_records, + _build_parser, + _ci_state, + _enforce_dataset_role_lock, + _load_dataset_registry, + _model_parameters, + _resolve_dataset_registry_path, + _select_datasets, + _validate_selected_datasets, + _verify_dataset_files, + _working_directory, +) + + +def test_ci_state_is_none_outside_github_actions(monkeypatch): + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + assert _ci_state() is None + + +def test_ci_state_distinguishes_tested_merge_from_source_head(monkeypatch): + values = { + "GITHUB_ACTIONS": "true", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REPOSITORY": "jxucoder/openboost", + "GITHUB_REF": "refs/pull/19/merge", + "GITHUB_SHA": "merge-sha", + "OPENBOOST_SOURCE_SHA": "head-sha", + "GITHUB_HEAD_REF": "codex/scoringbench-release-hardening", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "2", + } + for name, value in values.items(): + monkeypatch.setenv(name, value) + + assert _ci_state() == { + "provider": "github_actions", + "event_name": "pull_request", + "repository": "jxucoder/openboost", + "ref": "refs/pull/19/merge", + "tested_sha": "merge-sha", + "source_sha": "head-sha", + "head_ref": "codex/scoringbench-release-hardening", + "run_id": "123", + "run_attempt": "2", + } + + +def test_working_directory_contains_upstream_output_and_restores_on_error(tmp_path): + original = Path.cwd() + artifact_dir = tmp_path / "artifact" + + with pytest.raises(RuntimeError, match="stop"), _working_directory(artifact_dir): + assert Path.cwd() == artifact_dir + Path("datasets.json").write_text("[]\n") + raise RuntimeError("stop") + + assert Path.cwd() == original + assert (artifact_dir / "datasets.json").read_text() == "[]\n" + + +def test_named_quality_shard_validates_only_selected_dataset(): + class Args: + dataset_index = None + dataset_name = ["Abalone"] + + registry = [ + {"name": "Abalone", "source": "openml", "id": 183}, + {"name": "large_unused", "source": "openml", "id": 999}, + ] + validated = [] + + def validate(datasets): + validated.extend(datasets) + return datasets + + result = _validate_selected_datasets(registry, Args(), validate) + + assert result == [registry[0]] + assert validated == [registry[0]] + + +def _complete_record(dataset="example", model="openboost_cpu", fold=0): + return { + "dataset": dataset, + "model": model, + "fold": fold, + "crps": 0.2, + "log_score": 0.4, + "rmse": 0.5, + "coverage_90": 0.9, + "interval_score_90": 1.2, + "train_time": 2.0, + } + + +def test_outcome_audit_accepts_exact_complete_distributional_rows(): + outcome = _audit_records( + [_complete_record(fold=0), _complete_record(fold=1)], + [{"name": "example"}], + ["openboost_cpu"], + n_folds=2, + n_repeats=1, + ) + + assert outcome["status"] == "complete" + assert outcome["expected_rows"] == 2 + assert outcome["valid_rows"] == 2 + + +def test_outcome_audit_publishes_missing_error_and_invalid_metric_rows(): + invalid = _complete_record(model="ngboost", fold=0) + invalid["log_score"] = float("nan") + error = _complete_record(fold=1) + error.update(error="model exploded", error_type="RuntimeError") + + outcome = _audit_records( + [_complete_record(fold=0), error, invalid], + [{"name": "example"}], + ["openboost_cpu", "ngboost"], + n_folds=2, + n_repeats=1, + ) + + assert outcome["status"] == "incomplete" + assert outcome["expected_rows"] == 4 + assert outcome["valid_rows"] == 1 + assert outcome["missing_rows"] == [{"dataset": "example", "model": "ngboost", "fold": 1}] + assert outcome["error_rows"][0]["error"] == "model exploded" + assert outcome["invalid_metric_rows"][0]["metrics"] == ["log_score"] + + +def test_load_dataset_registry_accepts_frozen_scoringbench_list(tmp_path): + path = tmp_path / "datasets.json" + path.write_text('[{"name": "alpha", "source": "pmlb", "url": "https://example"}]') + + assert _load_dataset_registry(path) == [ + {"name": "alpha", "source": "pmlb", "url": "https://example"} + ] + + +def test_registry_path_is_resolved_before_artifact_working_directory(tmp_path, monkeypatch): + registry = tmp_path / "datasets.json" + registry.write_text('[{"name": "alpha"}]') + monkeypatch.chdir(tmp_path) + + resolved = _resolve_dataset_registry_path("datasets.json") + with _working_directory(tmp_path / "artifact"): + assert _load_dataset_registry(resolved) == [{"name": "alpha"}] + + +def test_verify_dataset_files_accepts_matching_pinned_bytes(tmp_path): + raw = tmp_path / "pinned.tsv.gz" + raw.write_bytes(b"frozen dataset bytes") + expected = __import__("hashlib").sha256(raw.read_bytes()).hexdigest() + calls = [] + + def ensure_cached(name, url, filename): + calls.append((name, url, filename)) + return raw + + verified = _verify_dataset_files( + [ + { + "name": "example", + "source": "pmlb", + "url": "https://example.test/example.tsv.gz", + "raw_sha256": expected, + } + ], + ensure_cached, + ) + + assert calls == [("example", "https://example.test/example.tsv.gz", "example.tsv.gz")] + assert verified == [ + { + "name": "example", + "url": "https://example.test/example.tsv.gz", + "sha256": expected, + "size_bytes": len(b"frozen dataset bytes"), + } + ] + + +def test_verify_dataset_files_fails_closed_on_hash_mismatch(tmp_path): + raw = tmp_path / "pinned.tsv.gz" + raw.write_bytes(b"different bytes") + + with pytest.raises(ValueError, match="raw dataset hash mismatch"): + _verify_dataset_files( + [ + { + "name": "example", + "source": "pmlb", + "url": "https://example.test/example.tsv.gz", + "raw_sha256": "0" * 64, + } + ], + lambda *_args: raw, + ) + + +def test_confirmation_dataset_requires_explicit_unlock(): + datasets = [ + { + "name": "held_out", + "openboost_role": "untouched_confirmation", + } + ] + + with pytest.raises(ValueError, match="confirmation dataset is still locked"): + _enforce_dataset_role_lock(datasets, allow_confirmation=False) + + _enforce_dataset_role_lock(datasets, allow_confirmation=True) + + +def test_stable_strided_shards_cover_registry_exactly_once(): + registry = [{"name": f"dataset_{index}"} for index in range(7)] + + class Args: + dataset_index = None + dataset_name = None + shard_count = 3 + shard_index = 0 + + selected = [] + for shard_index in range(Args.shard_count): + Args.shard_index = shard_index + selected.extend(_select_datasets(registry, Args())) + + assert sorted(dataset["name"] for dataset in selected) == sorted( + dataset["name"] for dataset in registry + ) + assert len(selected) == len({dataset["name"] for dataset in selected}) + + +def test_strong_baseline_defaults_match_scoringbench_registered_budgets(): + args = _build_parser().parse_args( + ["--models", "openboost_cpu,ngboost,xgboost_quantile,xgblss,catboost_quantile"] + ) + + assert args.models == [ + "openboost_cpu", + "ngboost", + "xgboost_quantile", + "xgblss", + "catboost_quantile", + ] + assert args.n_trees == 500 + assert args.histogram_rounds == 100 + assert args.histogram_bins == 50 + assert args.histogram_learning_rate == 0.05 + assert args.histogram_max_depth == 6 + assert args.histogram_curvature_scale == 1.0 + assert args.histogram_v2_temperature_grid == (0.5, 0.7, 0.85, 1.0, 1.2) + assert args.histogram_v2_calibration_fraction == 0.2 + assert args.histogram_v2_calibration_seed == 42 + assert args.histogram_v2_evaluation_subdivisions == 2 + assert args.xgboost_rounds == 100 + assert args.xgboost_quantiles == 50 + assert args.xgblss_rounds == 100 + assert args.catboost_rounds == 1000 + assert args.reg_lambda == 1.0 + assert args.min_child_weight == 1.0 + assert args.training_objective == "nll" + assert args.development_run is False + + parameters = _model_parameters(args) + assert parameters["openboost_cpu"]["model_params"] == { + "reg_lambda": 1.0, + "min_child_weight": 1.0, + "training_objective": "nll", + } + assert parameters["openboost_histogram_cpu"] == { + "n_distribution_bins": 50, + "n_trees": 100, + "learning_rate": 0.05, + "max_depth": 6, + "n_feature_bins": 254, + "curvature_scale": 1.0, + } + assert parameters["openboost_histogram_cpu_v2"] == { + "n_distribution_bins": 50, + "n_trees": 100, + "learning_rate": 0.05, + "max_depth": 6, + "n_feature_bins": 254, + "curvature_scale": 1.0, + "temperature_grid": (0.5, 0.7, 0.85, 1.0, 1.2), + "calibration_fraction": 0.2, + "calibration_seed": 42, + "evaluation_subdivisions": 2, + } + assert parameters["xgboost_quantile"]["num_boost_round"] == 100 + assert parameters["xgblss"]["num_boost_round"] == 100 + assert parameters["catboost_quantile"]["iterations"] == 1000 + assert parameters["catboost_quantile"]["catboost_params"]["allow_writing_files"] is False + + +def test_development_parameters_are_explicit_in_manifest_constructor_contract(): + args = _build_parser().parse_args( + [ + "--models", + "openboost_cpu", + "--development-run", + "--n-trees", + "250", + "--learning-rate", + "0.04", + "--training-objective", + "crps", + "--max-depth", + "2", + "--reg-lambda", + "3.0", + "--min-child-weight", + "5.0", + ] + ) + + assert args.development_run is True + assert _model_parameters(args)["openboost_cpu"] == { + "backend": "cpu", + "n_trees": 250, + "learning_rate": 0.04, + "max_depth": 2, + "n_quantiles": 99, + "model_params": { + "reg_lambda": 3.0, + "min_child_weight": 5.0, + "training_objective": "crps", + }, + } diff --git a/tests/test_sklearn.py b/tests/test_sklearn.py index 1525098..a9a746e 100644 --- a/tests/test_sklearn.py +++ b/tests/test_sklearn.py @@ -579,22 +579,28 @@ def test_clone_get_params_with_new_params(self): reg = OpenBoostDistributionalRegressor( distribution='gamma', n_estimators=15, + training_objective='nll', eval_metric='interval_score', quantiles=[0.1, 0.9], interval_alpha=0.2, ) params = reg.get_params() assert params['eval_metric'] == 'interval_score' + assert params['training_objective'] == 'nll' assert params['quantiles'] == [0.1, 0.9] assert params['interval_alpha'] == 0.2 reg_clone = clone(reg) assert reg_clone is not reg assert reg_clone.eval_metric == 'interval_score' + assert reg_clone.training_objective == 'nll' assert reg_clone.quantiles == [0.1, 0.9] assert reg_clone.interval_alpha == 0.2 - reg.set_params(eval_metric='nll', quantiles=None) + reg.set_params( + training_objective='crps', eval_metric='nll', quantiles=None + ) + assert reg.training_objective == 'crps' assert reg.eval_metric == 'nll' assert reg.quantiles is None