From 153a3af716d275e2e62afdb772fa0a3ff44c0f8d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Jul 2026 04:31:04 +0200 Subject: [PATCH] Collect the versions benchmark results hourly The versions benchmark has no materialized view: a machine POSTs its whole result JSON to sink.data with kind = "versions-benchmark", and fetch-results.sh reads the results back out of those rows. This adds the automation around it, mirroring collect-results.yml for the main benchmark. versions/collect-new-results.sh fetches the results that arrived in the last day, rebuilds data.generated.js when they changed, and commits both as one automated pull request on auto-results/versions, which it merges (these are ClickHouse's own results, trusted like the clickhouse* systems are by the main collector). Runs that sent a log but no complete result -- still running, crashed, or an incomplete load -- are reported in the job summary, since nothing else would show them. fetch-results.sh grows SINCE_HOURS: fetch only the versions that ran within that window and leave the other files alone, instead of rebuilding results/ from every version in the sink. That is what makes the job cheap enough to run hourly; the full refresh is still available (FULL=1 / the workflow's full input), guarded against publishing a mass deletion if the query goes wrong. The collector also restores result files that came back byte-different but identical in content: jq reformats numbers (0.020 -> 0.02) whenever it has to reconstruct a document, which would otherwise commit a few hundred formatting-only diffs every hour. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 1 + .../workflows/versions-collect-results.yml | 50 ++++++ versions/README.md | 31 +++- versions/collect-new-results.sh | 166 ++++++++++++++++++ versions/fetch-results.sh | 17 +- 5 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/versions-collect-results.yml create mode 100755 versions/collect-new-results.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d14947cb66..3fead957e4 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -17,6 +17,7 @@ apart from the runs of main and are excluded by `collect-results.sh`. | `benchmark-manual.yml` | manual | any systems, machines, repository and branch | | `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label; `machine:all`, `machine:all-amd` and `machine:all-arm` expand to the daily-run machine sets); adding such a label relaunches the benchmark | | `versions-benchmark.yml` | manual | the [versions benchmark](../../versions/README.md) (`versions/run-benchmark.sh`): one machine per (ClickHouse version, machine type), for an exact version, a version prefix, or `master` (the development build, installed on the machine with `curl https://clickhouse.com/ \| sh`). Sends its result to the same sink, tagged `kind: versions-benchmark` | +| `versions-collect-results.yml` | hourly | nothing - it collects the versions benchmark results of the last day (`versions/collect-new-results.sh`). That benchmark has no materialized view: every machine POSTs its whole result JSON to `sink.data` with `kind: versions-benchmark`, so the results are read out of those rows by `versions/fetch-results.sh`, `versions/data.generated.js` is rebuilt from them, and the changes are committed as one automated, auto-merged pull request (branch `auto-results/versions`). Runs that sent a log but no complete result are reported in the job summary | | `collect-results.yml` | every 30 minutes | nothing - it collects the runs of the last day from the sink database (`collect-new-results.py`): commits result files and posts pastila.nl log links to the corresponding PR, or maintains one automated results PR per system for the runs of main | ## Setup diff --git a/.github/workflows/versions-collect-results.yml b/.github/workflows/versions-collect-results.yml new file mode 100644 index 0000000000..9a46587b63 --- /dev/null +++ b/.github/workflows/versions-collect-results.yml @@ -0,0 +1,50 @@ +name: "Collect versions benchmark results" + +on: + workflow_dispatch: + inputs: + since_hours: + description: "Look at the results that arrived in the last N hours" + default: "24" + full: + description: "Refetch every version and rebuild versions/results from scratch" + type: boolean + default: false + schedule: + - cron: '23 * * * *' + +permissions: + contents: write # To push the results branch. + pull-requests: write # To open and merge the automated pull request. + +# Two overlapping runs would fetch and publish the same results twice. +concurrency: + group: versions-collect-results + +jobs: + collect: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + # The versions benchmark has no materialized view: results are read out of the + # raw sink.data rows (kind: versions-benchmark) with clickhouse-client. + - name: Install the ClickHouse client + working-directory: ${{ runner.temp }} # not the checkout: it must stay clean + run: | + curl -fsSL https://clickhouse.com/ | sh + sudo ln -sf "$PWD/clickhouse" /usr/local/bin/clickhouse-client + + - env: + GH_TOKEN: ${{ github.token }} + CLICKBENCH_DB_PASSWORD: ${{ secrets.CLICKBENCH_DB_PASSWORD }} + SINCE_HOURS: ${{ inputs.since_hours || '24' }} + FULL: ${{ inputs.full && '1' || '' }} + run: | + if [ -z "$CLICKBENCH_DB_PASSWORD" ]; then + echo "CLICKBENCH_DB_PASSWORD is not set. Skipping." + exit 0 + fi + export CONNECTION_PARAMS="--user clickbench --password $CLICKBENCH_DB_PASSWORD --host play.clickhouse.com --secure" + ./versions/collect-new-results.sh diff --git a/versions/README.md b/versions/README.md index 46c72a506c..715bc71172 100644 --- a/versions/README.md +++ b/versions/README.md @@ -123,8 +123,9 @@ retry). The VM installs Docker, downloads the prepared Native files from image from source if the version has none (`clickhouse-built:*`, using the tag + GCC from `build-from-source/versions.txt`), runs `run-version.sh`, and POSTs the result JSON (enriched with the machine type, `kind:"versions-benchmark"`) plus -the log to `sink.data` on play.clickhouse.com. A server-side materialized view -turns those into the published report, exactly as the main benchmark does. +the log to `sink.data` on play.clickhouse.com. Unlike the main benchmark, there +is no materialized view here: the whole result is one row of `sink.data`, and +`fetch-results.sh` reads it back out of those rows (see below). The same thing on demand from GitHub: the **Run the versions benchmark** workflow (`.github/workflows/versions-benchmark.yml`, `workflow_dispatch`) takes @@ -140,6 +141,32 @@ use (~15 GB), so it no longer dominates. Pass a subset via `datasets=` to skip some. While this branch is unmerged, pass `branch=versions-benchmark-rework`. Missing dataset files in the bucket are skipped (their queries report null). +## Collecting the results + +`fetch-results.sh` reads the results back from the sink — for each version the +latest complete run (`argMax(content, time)` over the `kind: versions-benchmark` +rows) — adds the release date, drops incomplete loads, and writes +`results/.json`; `apply-minvers.py` then normalises every file to the +current `queries/*.minver`. The committed result files are produced solely by +this script; don't edit them by hand. + +```bash +CONNECTION_PARAMS='--user clickbench --password *** --host play.clickhouse.com --secure' \ + ./fetch-results.sh # every version; rebuilds results/ from scratch +SINCE_HOURS=24 ... ./fetch-results.sh # only what arrived in the last 24 hours +./generate-results.sh # rebuild data.generated.js for the page +``` + +This runs by itself: the **Collect versions benchmark results** workflow +(`.github/workflows/versions-collect-results.yml`) calls +`collect-new-results.sh` every hour, which fetches the last day's results, +rebuilds `data.generated.js` when they changed, and commits both as an +automated, auto-merged pull request (branch `auto-results/versions`). Runs that +sent a log but no complete result — still running, crashed, or an incomplete +load — are listed in the job summary. Dispatch it manually with `full: true` to +refetch every version (it refuses to publish if that loses more than a tenth of +the result files, which would mean a broken query rather than lost results). + ## Query set 344 queries in a fixed order: mgbench (15) + Star Schema Benchmark (13) + diff --git a/versions/collect-new-results.sh b/versions/collect-new-results.sh new file mode 100755 index 0000000000..9b7dc5581c --- /dev/null +++ b/versions/collect-new-results.sh @@ -0,0 +1,166 @@ +#!/bin/bash -e + +# Collect new Versions Benchmark results from the sink database and publish them. +# Run hourly by .github/workflows/versions-collect-results.yml. +# +# Unlike the main benchmark, this one has no materialized view: a machine POSTs its +# whole result JSON to sink.data (kind = "versions-benchmark", see cloud-init.sh.in), +# so results are read straight from those rows. fetch-results.sh does that reading; +# this script is the automation around it: +# +# 1. fetch the results that arrived in the last SINCE_HOURS hours (FULL=1: refetch +# every version and rebuild results/ from scratch), +# 2. regenerate data.generated.js, +# 3. if anything changed, commit results/ + data.generated.js to the branch +# auto-results/versions, open a pull request and merge it (these results are +# ClickHouse's own, so they are trusted, as the main collector trusts the +# clickhouse* systems), +# 4. report what arrived, including runs that sent a log but no usable result. +# +# CONNECTION_PARAMS='--user clickbench --password *** --host play.clickhouse.com --secure' \ +# ./collect-new-results.sh +# +# DRY_RUN=1 does everything except pushing, opening and merging. + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${HERE}" +CH() { clickhouse-client ${CONNECTION_PARAMS} "$@"; } + +SINCE_HOURS="${SINCE_HOURS:-24}" +FULL="${FULL:-}" +BRANCH="${BRANCH:-auto-results/versions}" +DRY_RUN="${DRY_RUN:-}" +BOT_NAME="github-actions[bot]" +BOT_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" + +note() { + echo "$@" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then echo "$@" >> "${GITHUB_STEP_SUMMARY}"; fi +} + +# --- fetch ------------------------------------------------------------------------- + +before=$(ls results/*.json 2>/dev/null | wc -l) +if [ -n "${FULL}" ]; then SINCE_HOURS=""; fi +SINCE_HOURS="${SINCE_HOURS}" ./fetch-results.sh +after=$(ls results/*.json 2>/dev/null | wc -l) + +# A full refresh that loses a large part of the results means the query or the sink +# went wrong, not that the results are gone: never commit such a mass deletion. +if [ -n "${FULL}" ] && [ "${after}" -lt $(( before * 9 / 10 )) ]; then + note "Refusing to publish: the full fetch produced ${after} result files, was ${before}." + exit 1 +fi + +# fetch-results.sh and apply-minvers.py re-serialise the files they touch, so a result can +# come back byte-different but identical in content: jq rewrites 0.020 as 0.02 whenever it +# has to reconstruct a document (and different jq versions differ in when they do). Left +# alone, that would make this job commit a few hundred formatting-only diffs every hour, so +# restore every file whose content did not actually change. +restored=0 +prefix=$(git rev-parse --show-prefix) +while read -r f; do + [ -z "${f}" ] && continue + if git show "HEAD:${prefix}${f}" 2>/dev/null | python3 -c ' +import json, sys +old = json.load(sys.stdin) +with open(sys.argv[1]) as fh: + new = json.load(fh) +sys.exit(0 if old == new else 1)' "${f}"; then + git checkout -q -- "${f}" + restored=$((restored + 1)) + fi +done < <(git diff --name-only --relative -- results) +if [ "${restored}" != 0 ]; then + echo "kept ${restored} result file(s) unchanged (re-serialisation only)" >&2 +fi + +# --- what arrived ------------------------------------------------------------------ + +# Every run writes '=== benchmarking (image ...)' into its log, which is sent +# to the sink whether or not the benchmark produced a result. Versions with a log but no +# result file of their own are runs that are still going, crashed, or loaded incompletely +# (fetch-results.sh skips those) -- worth reporting, since nothing else would show them. +window="${SINCE_HOURS:-24}" +started=$(CH --query " + SELECT DISTINCT extract(content, '=== benchmarking ([^ ]+) ') AS v + FROM sink.data + WHERE time >= now() - INTERVAL ${window} HOUR AND v != '' + ORDER BY v + FORMAT TSV") +# The runs that did finish, by the same criterion fetch-results.sh uses (a complete +# result array). Compared with the logs above, not with the result files: a version +# benchmarked before the window still has its file, so the files say nothing about +# whether this run of it finished. +complete=$(CH --query " + SELECT DISTINCT JSONExtractString(content, 'version') AS v + FROM sink.data + WHERE JSONExtractString(content, 'kind') = 'versions-benchmark' AND v != '' + AND length(JSONExtractArrayRaw(content, 'result')) = 344 + AND time >= now() - INTERVAL ${window} HOUR + ORDER BY v + FORMAT TSV") + +versions=$(git status --porcelain -- results | awk '{print $NF}' \ + | sed 's|.*/||; s|\.json$||' | sort -V | tr '\n' ' ' | xargs || true) + +for v in ${started}; do + if ! grep -qxF "${v}" <<<"${complete}"; then + note "The run of \`${v}\` has not produced a result yet (still running, failed, or an incomplete load)." + fi +done + +if [ -z "${versions}" ]; then + note "No new versions benchmark results in the last ${window} hours." + exit 0 +fi + +# The page data is derived from the result files, so it is rebuilt only when they change +# (regenerating it unconditionally would rewrite it on every run: jq reformats the numbers +# the same way it does in the result files). +./generate-results.sh + +# --- publish ----------------------------------------------------------------------- + +count=$(wc -w <<<"${versions}") +if [ "${count}" -le 5 ]; then + title="versions: results for ${versions// /, }" +else + title="versions: results for ${count} versions" +fi +body="Collected from the sink by \`versions/collect-new-results.sh\`. + +Versions: ${versions// /, }. + +The result files are generated: they are fetched from \`sink.data\` (rows with +\`kind: versions-benchmark\`, sent by the machines the versions benchmark runs on), +and \`data.generated.js\` is rebuilt from them. Do not edit them by hand." + +note "New or updated results: ${versions// /, }." +if [ -n "${DRY_RUN}" ]; then + note "DRY_RUN: would commit and merge \"${title}\"" + git status --short -- results data.generated.js + exit 0 +fi + +git add -- results data.generated.js +git -c "user.name=${BOT_NAME}" -c "user.email=${BOT_EMAIL}" commit -q -m "${title}" -m "${body}" +git push -q --force origin "HEAD:refs/heads/${BRANCH}" + +url=$(gh pr list --head "${BRANCH}" --state open --json url --jq '.[0].url') +if [ -z "${url}" ]; then + url=$(gh pr create --head "${BRANCH}" --base main --title "${title}" --body "${body}") + note "Opened ${url}" +else + note "Updated ${url}" +fi + +# Retry: GitHub may still be computing mergeability right after the push. +for attempt in 1 2 3; do + if gh pr merge "${url}" --merge --delete-branch; then + note "Merged ${url}" + exit 0 + fi + sleep 10 +done +note "Could not merge ${url}; it is left open." diff --git a/versions/fetch-results.sh b/versions/fetch-results.sh index 55cd0c03e4..f72f3eb910 100755 --- a/versions/fetch-results.sh +++ b/versions/fetch-results.sh @@ -10,11 +10,21 @@ # ./fetch-results.sh # # Then regenerate the page data with ./generate-results.sh. +# +# By default every version in the sink is refetched and results/ is rebuilt from +# scratch. SINCE_HOURS= instead fetches only the versions that sent a result within +# the last n hours and leaves the other files alone -- the incremental mode the hourly +# collector uses (collect-new-results.sh). HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "${HERE}" CH() { clickhouse-client ${CONNECTION_PARAMS} "$@"; } +SINCE_HOURS="${SINCE_HOURS:-}" +[[ "${SINCE_HOURS}" =~ ^[0-9]*$ ]] || { echo "SINCE_HOURS must be a number of hours" >&2; exit 1; } +WINDOW="" +if [ -n "${SINCE_HOURS}" ]; then WINDOW="AND time >= now() - INTERVAL ${SINCE_HOURS} HOUR"; fi + mkdir -p results # Release-date lookup. list-versions.sh resolves it (column 3) for published and tagged @@ -52,11 +62,14 @@ mapfile -t VERSIONS < <(CH --query " FROM sink.data WHERE JSONExtractString(content, 'kind') = 'versions-benchmark' AND v != '' AND length(JSONExtractArrayRaw(content, 'result')) = 344 + ${WINDOW} ORDER BY v FORMAT TSV") -echo "fetching ${#VERSIONS[@]} versions from the sink" >&2 -rm -f results/*.json +echo "fetching ${#VERSIONS[@]} versions from the sink${SINCE_HOURS:+ (results of the last ${SINCE_HOURS}h)}" >&2 +# A full fetch rebuilds the directory; an incremental one only overwrites the versions +# it fetched, keeping the files of everything that did not run in the window. +if [ -z "${SINCE_HOURS}" ]; then rm -f results/*.json; fi for v in "${VERSIONS[@]}"; do [ -z "${v}" ] && continue # argMax over time keeps the most recent run for this version.