From 472017e0dc74c13b9aeb9d5d225bf95c3ba2e8da Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Jul 2026 04:14:30 +0200 Subject: [PATCH] Add a workflow to run the versions benchmark on demand A manual (workflow_dispatch) workflow that launches the versions benchmark for a given ClickHouse version, a version prefix, or master. The runner only launches: it assumes the same federated IAM role as the other benchmark workflows and calls versions/run-benchmark.sh once per (version, machine type). Each machine downloads the prepared Native files, runs run-version.sh, sends its result to the sink and self-terminates. master needs no script changes: list-versions.sh already maps it to the "local" provider, which installs the development build on the machine. The datasets / tries / timeout inputs fall through to the script's defaults when empty, and the repository and branch default to the ones the workflow was dispatched from, so an unmerged branch can be benchmarked as is. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 4 +- .github/workflows/versions-benchmark.yml | 74 ++++++++++++++++++++++++ versions/README.md | 8 +++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/versions-benchmark.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c33857367a..d14947cb66 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -16,6 +16,7 @@ apart from the runs of main and are excluded by `collect-results.sh`. | `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | | `benchmark-manual.yml` | manual | any systems, machines, repository and branch | | `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label; `machine:all`, `machine:all-amd` and `machine:all-arm` expand to the daily-run machine sets); adding such a label relaunches the benchmark | +| `versions-benchmark.yml` | manual | the [versions benchmark](../../versions/README.md) (`versions/run-benchmark.sh`): one machine per (ClickHouse version, machine type), for an exact version, a version prefix, or `master` (the development build, installed on the machine with `curl https://clickhouse.com/ \| sh`). Sends its result to the same sink, tagged `kind: versions-benchmark` | | `collect-results.yml` | every 30 minutes | nothing - it collects the runs of the last day from the sink database (`collect-new-results.py`): commits result files and posts pastila.nl log links to the corresponding PR, or maintains one automated results PR per system for the runs of main | ## Setup @@ -41,7 +42,8 @@ apart from the runs of main and are excluded by `collect-results.sh`. `ec2:DescribeImages`, and `ec2:DescribeInstanceTypes`. The role's ARN and the region (us-east-1) are set in - `.github/actions/launch-benchmark/action.yml`. + `.github/actions/launch-benchmark/action.yml` and, for the versions + benchmark, in `.github/workflows/versions-benchmark.yml`. 2. An environment named `benchmark-approval` with required reviewers. It gates the PR workflow: nothing is launched for a pull request until a diff --git a/.github/workflows/versions-benchmark.yml b/.github/workflows/versions-benchmark.yml new file mode 100644 index 0000000000..12c1900943 --- /dev/null +++ b/.github/workflows/versions-benchmark.yml @@ -0,0 +1,74 @@ +name: "Run the versions benchmark" + +run-name: "Versions benchmark: ${{ inputs.versions }} on ${{ inputs.machines || 'c7a.4xlarge' }}" + +on: + workflow_dispatch: + inputs: + versions: + description: "ClickHouse versions to benchmark (space-separated): an exact version (26.6.1.1193, 1.1.54378), a prefix (26.6, 24), or master for the development build" + required: true + default: "master" + machines: + description: "EC2 instance types (space-separated)" + default: "c7a.4xlarge" + datasets: + description: "Datasets to load (space-separated; empty: all of hits ssb mgbench tpch tpcds coffeeshop ontime uk job taxi)" + default: "" + tries: + description: "Runs of every query: 1 cold + the rest hot (empty: 6)" + default: "" + timeout: + description: "Time limit for run-version.sh on the machine, seconds (empty: 18000)" + default: "" + repo: + description: "Repository the machine will clone (empty: the repository of this workflow run)" + default: "" + branch: + description: "Branch to clone (empty: the branch this workflow was dispatched from)" + default: "" + +permissions: + id-token: write # To assume the federated IAM role. + contents: read + +jobs: + launch: + runs-on: ubuntu-latest + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 + + # The same role as the main benchmark workflows; its trust policy only + # admits workflow runs of this repository. + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::264113785604:role/ClickBenchLauncher + aws-region: us-east-1 + role-session-name: clickbench-versions-${{ github.run_id }} + + # One self-terminating machine per (version, machine type): it downloads + # the prepared Native files, runs versions/run-version.sh and sends the + # result to the sink. The runner only launches. + - shell: bash + env: + versions: ${{ inputs.versions }} + machines: ${{ inputs.machines }} + datasets: ${{ inputs.datasets }} + tries: ${{ inputs.tries }} + timeout: ${{ inputs.timeout }} + repo: ${{ inputs.repo || github.repository }} + branch: ${{ inputs.branch || github.ref_name }} + run: | + [ -n "$versions" ] || { echo "No versions given"; exit 1; } + : "${machines:=c7a.4xlarge}" + failed="" + for m in $machines; do + for v in $versions; do + machine="$m" ./versions/run-benchmark.sh "$v" || failed="$failed $v/$m" + done + done + if [ -n "$failed" ]; then + echo "Failed to launch:$failed" + exit 1 + fi diff --git a/versions/README.md b/versions/README.md index 38ff6f6bd6..46c72a506c 100644 --- a/versions/README.md +++ b/versions/README.md @@ -126,6 +126,14 @@ result JSON (enriched with the machine type, `kind:"versions-benchmark"`) plus the log to `sink.data` on play.clickhouse.com. A server-side materialized view turns those into the published report, exactly as the main benchmark does. +The same thing on demand from GitHub: the **Run the versions benchmark** +workflow (`.github/workflows/versions-benchmark.yml`, `workflow_dispatch`) takes +the versions (exact, a prefix, or `master`), the machine types, and optionally +the datasets / tries / timeout / repository / branch, and launches one machine +per (version, machine type) with `run-benchmark.sh`. Its defaults for the +repository and branch are the ones the workflow was dispatched from, so a branch +can be tested without merging it first. + Notes: all datasets run by default (`datasets="hits ssb mgbench tpch tpcds coffeeshop taxi"`); the taxi table is narrowed to the five columns its queries use (~15 GB), so it no longer dominates. Pass a subset via `datasets=` to skip