diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c33857367a..d14947cb66 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -16,6 +16,7 @@ apart from the runs of main and are excluded by `collect-results.sh`. | `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | | `benchmark-manual.yml` | manual | any systems, machines, repository and branch | | `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label; `machine:all`, `machine:all-amd` and `machine:all-arm` expand to the daily-run machine sets); adding such a label relaunches the benchmark | +| `versions-benchmark.yml` | manual | the [versions benchmark](../../versions/README.md) (`versions/run-benchmark.sh`): one machine per (ClickHouse version, machine type), for an exact version, a version prefix, or `master` (the development build, installed on the machine with `curl https://clickhouse.com/ \| sh`). Sends its result to the same sink, tagged `kind: versions-benchmark` | | `collect-results.yml` | every 30 minutes | nothing - it collects the runs of the last day from the sink database (`collect-new-results.py`): commits result files and posts pastila.nl log links to the corresponding PR, or maintains one automated results PR per system for the runs of main | ## Setup @@ -41,7 +42,8 @@ apart from the runs of main and are excluded by `collect-results.sh`. `ec2:DescribeImages`, and `ec2:DescribeInstanceTypes`. The role's ARN and the region (us-east-1) are set in - `.github/actions/launch-benchmark/action.yml`. + `.github/actions/launch-benchmark/action.yml` and, for the versions + benchmark, in `.github/workflows/versions-benchmark.yml`. 2. An environment named `benchmark-approval` with required reviewers. It gates the PR workflow: nothing is launched for a pull request until a diff --git a/.github/workflows/versions-benchmark.yml b/.github/workflows/versions-benchmark.yml new file mode 100644 index 0000000000..12c1900943 --- /dev/null +++ b/.github/workflows/versions-benchmark.yml @@ -0,0 +1,74 @@ +name: "Run the versions benchmark" + +run-name: "Versions benchmark: ${{ inputs.versions }} on ${{ inputs.machines || 'c7a.4xlarge' }}" + +on: + workflow_dispatch: + inputs: + versions: + description: "ClickHouse versions to benchmark (space-separated): an exact version (26.6.1.1193, 1.1.54378), a prefix (26.6, 24), or master for the development build" + required: true + default: "master" + machines: + description: "EC2 instance types (space-separated)" + default: "c7a.4xlarge" + datasets: + description: "Datasets to load (space-separated; empty: all of hits ssb mgbench tpch tpcds coffeeshop ontime uk job taxi)" + default: "" + tries: + description: "Runs of every query: 1 cold + the rest hot (empty: 6)" + default: "" + timeout: + description: "Time limit for run-version.sh on the machine, seconds (empty: 18000)" + default: "" + repo: + description: "Repository the machine will clone (empty: the repository of this workflow run)" + default: "" + branch: + description: "Branch to clone (empty: the branch this workflow was dispatched from)" + default: "" + +permissions: + id-token: write # To assume the federated IAM role. + contents: read + +jobs: + launch: + runs-on: ubuntu-latest + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 + + # The same role as the main benchmark workflows; its trust policy only + # admits workflow runs of this repository. + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::264113785604:role/ClickBenchLauncher + aws-region: us-east-1 + role-session-name: clickbench-versions-${{ github.run_id }} + + # One self-terminating machine per (version, machine type): it downloads + # the prepared Native files, runs versions/run-version.sh and sends the + # result to the sink. The runner only launches. + - shell: bash + env: + versions: ${{ inputs.versions }} + machines: ${{ inputs.machines }} + datasets: ${{ inputs.datasets }} + tries: ${{ inputs.tries }} + timeout: ${{ inputs.timeout }} + repo: ${{ inputs.repo || github.repository }} + branch: ${{ inputs.branch || github.ref_name }} + run: | + [ -n "$versions" ] || { echo "No versions given"; exit 1; } + : "${machines:=c7a.4xlarge}" + failed="" + for m in $machines; do + for v in $versions; do + machine="$m" ./versions/run-benchmark.sh "$v" || failed="$failed $v/$m" + done + done + if [ -n "$failed" ]; then + echo "Failed to launch:$failed" + exit 1 + fi diff --git a/versions/README.md b/versions/README.md index 38ff6f6bd6..46c72a506c 100644 --- a/versions/README.md +++ b/versions/README.md @@ -126,6 +126,14 @@ result JSON (enriched with the machine type, `kind:"versions-benchmark"`) plus the log to `sink.data` on play.clickhouse.com. A server-side materialized view turns those into the published report, exactly as the main benchmark does. +The same thing on demand from GitHub: the **Run the versions benchmark** +workflow (`.github/workflows/versions-benchmark.yml`, `workflow_dispatch`) takes +the versions (exact, a prefix, or `master`), the machine types, and optionally +the datasets / tries / timeout / repository / branch, and launches one machine +per (version, machine type) with `run-benchmark.sh`. Its defaults for the +repository and branch are the ones the workflow was dispatched from, so a branch +can be tested without merging it first. + Notes: all datasets run by default (`datasets="hits ssb mgbench tpch tpcds coffeeshop taxi"`); the taxi table is narrowed to the five columns its queries use (~15 GB), so it no longer dominates. Pass a subset via `datasets=` to skip