diff --git a/ctrlb/.gitignore b/ctrlb/.gitignore new file mode 100644 index 0000000000..1769190623 --- /dev/null +++ b/ctrlb/.gitignore @@ -0,0 +1,4 @@ +# Benchmark byproducts — never commit these. +run.log +# Any local engine data dir. +data/ diff --git a/ctrlb/benchmark.sh b/ctrlb/benchmark.sh new file mode 100755 index 0000000000..5ae07af4f2 --- /dev/null +++ b/ctrlb/benchmark.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# ClickBench entry point for CtrlB (Parquet, single). Sets the harness env vars, +# runs the shared ClickBench driver, then turns its output into +# results//.json — so one ./benchmark.sh yields a result file +# you can read exactly like the other engines' (see make-result.py). + +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +export BENCH_RESTARTABLE=yes +export BENCH_DURABLE=yes +export BENCH_TRIES="${BENCH_TRIES:-3}" + +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" + +# Run the shared ClickBench driver, capturing stdout so we can BOTH show it live +# and turn it into the committed results JSON. `tee` preserves the standard text +# output (Load time / [t1,t2,t3] / Data size); PIPESTATUS keeps the driver's real +# exit code (tee's is always 0). +HERE="$(cd "$(dirname "$0")" && pwd)" +LOG="${CTRLB_RESULT_LOG:-$HERE/run.log}" + +../lib/benchmark-common.sh | tee "$LOG" +status=${PIPESTATUS[0]} + +# Assemble results//c6a.4xlarge.json from the captured log + template.json, +# so the result is readable exactly like the other engines'. Non-fatal: if this +# can't run, the raw log ($LOG) is still there to convert by hand with +# make-result.py. +if [ "$status" -ne 0 ]; then + echo "WARN: benchmark driver exited $status — not writing results JSON." >&2 +elif ! command -v python3 > /dev/null 2>&1; then + echo "WARN: python3 not found — run: python3 make-result.py \"$LOG\"" >&2 +else + python3 "$HERE/make-result.py" "$LOG" --machine "${CTRLB_MACHINE:-c6a.4xlarge}" \ + || echo "WARN: results JSON not written; run: python3 make-result.py \"$LOG\"" >&2 +fi + +exit "$status" diff --git a/ctrlb/check b/ctrlb/check new file mode 100755 index 0000000000..c9be78c2a9 --- /dev/null +++ b/ctrlb/check @@ -0,0 +1,11 @@ +#!/bin/bash +# Readiness probe. Exit 0 once the engine answers a health check. This is a +# SELECT 1-level signal that does NOT require the `hits` view, so the initial +# pre-download (viewless) start still passes. bench_check_loop calls this +# repeatedly until it succeeds or times out. +set -e +# shellcheck disable=SC1091 +source "$(dirname "$0")/config.sh" + +curl -sf -o /dev/null "$BASE_URL/healthz" 2>/dev/null \ + || curl -sf -o /dev/null "$BASE_URL/api/health" diff --git a/ctrlb/config.sh b/ctrlb/config.sh new file mode 100644 index 0000000000..ceed438cf7 --- /dev/null +++ b/ctrlb/config.sh @@ -0,0 +1,132 @@ +# Shared config + helpers for the CtrlB ClickBench system dir. +# Sourced (not executed) by every hook script: `source "$(dirname "$0")/config.sh"`. + +# Absolute path to THIS system directory — where the harness downloads +# hits.parquet and runs the hook scripts from. +CTRLB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ---- lake (docker) ---------------------------------------------------------- +# CtrlB runs as a Docker container (image pulled in ./install). The image serves +# the HTTP API on 5080. We publish that to the host and mount THIS system +# directory into the container at the SAME path, so the hits.parquet the harness +# downloads to $PARQUET_PATH is visible inside the container at the identical +# path and the CREATE EXTERNAL TABLE LOCATION resolves unchanged. +CTRLB_IMAGE="${CTRLB_IMAGE:-ghcr.io/ctrlb-hq/ctrlb-public:c8eead4}" +CONTAINER_NAME="${CONTAINER_NAME:-ctrlb}" +LAKE_STARTUP="${LAKE_STARTUP:-30}" + +# ---- query endpoint -------------------------------------------------------- +BASE_URL="${BASE_URL:-http://localhost:5080}" +ORG_ID="${ORG_ID:-default}" +SEARCH_URL="${SEARCH_URL:-${BASE_URL}/api/${ORG_ID}/_search_parquet?type=logs}" +START_TIME="${START_TIME:-1}" +END_TIME="${END_TIME:-9999999999999999}" + +# ---- data ------------------------------------------------------------------ +# The harness downloads into the system dir (its cwd during bench_download). +PARQUET_PATH="${PARQUET_PATH:-$CTRLB_DIR/hits.parquet}" + +# post_sql -> runs a statement and prints the endpoint's server-side +# execution_time on stdout, non-zero (diagnostics to stderr) on failure. Used +# ONLY for the untimed session setup in ./start (CREATE EXTERNAL TABLE / VIEW); +# the presence of execution_time in the response is just a success signal here. +# Benchmark queries are measured by time_sql (below), not this. +post_sql() { + local sql="$1" response et + response="$(curl -s -X POST "$SEARCH_URL" \ + -H 'Content-Type: application/json' \ + -d "$(printf '{"sql": %s, "start_time": %s, "end_time": %s}' \ + "$(printf '%s' "$sql" | jq -Rs .)" "$START_TIME" "$END_TIME")")" + et="$(printf '%s' "$response" | jq -r '.execution_time // empty' 2>/dev/null)" + if [ -z "$et" ]; then + echo "post_sql failed: $sql" >&2 + echo "resp: $(printf '%s' "$response" | head -c 300)" >&2 + return 1 + fi + printf '%s' "$et" +} + +# time_sql -> prints the CLIENT-SIDE WALL time (seconds) of +# the full HTTP round-trip on stdout, and writes the FULL response body to +# . The wall time (curl %{time_total}) covers sending the query, +# the server processing it, AND receiving the entire result body. +# +# The body is downloaded in full (so the result really is sent back over the wire +# — no server-side output suppression) and then VALIDATED as a genuine success +# payload: a non-200 status, OR a body that does not parse / is missing the +# expected fields (.success == true, .execution_time present) fails the try. We +# do NOT use .execution_time as the timing — it is only a structural sanity check +# so an error that somehow arrives with a 200 is not counted as a success. +# LC_ALL=C forces a '.' decimal so the harness's number parser accepts the time. +time_sql() { + local sql="$1" outfile="$2" out http_code time_total + out="$(LC_ALL=C curl -s -o "$outfile" -w '%{http_code} %{time_total}' \ + -X POST "$SEARCH_URL" \ + -H 'Content-Type: application/json' \ + -d "$(printf '{"sql": %s, "start_time": %s, "end_time": %s}' \ + "$(printf '%s' "$sql" | jq -Rs .)" "$START_TIME" "$END_TIME")")" + http_code="${out%% *}" + time_total="${out##* }" + if [ "$http_code" != "200" ]; then + echo "time_sql failed (HTTP ${http_code:-none}): $sql" >&2 + return 1 + fi + if ! jq -e '.success == true and (.execution_time != null)' "$outfile" > /dev/null 2>&1; then + echo "time_sql: HTTP 200 but response is not a valid success payload for: $sql" >&2 + echo "resp: $(head -c 300 "$outfile" 2>/dev/null)" >&2 + return 1 + fi + printf '%s' "$time_total" +} + +# Poll until the lake answers a health check, or LAKE_STARTUP seconds elapse. +wait_for_health() { + local waited=0 + while [ "$waited" -lt "$LAKE_STARTUP" ]; do + if curl -s -o /dev/null "$BASE_URL/healthz" 2>/dev/null \ + || curl -s -o /dev/null "$BASE_URL/api/health" 2>/dev/null; then + return 0 + fi + sleep 1 + waited=$((waited + 1)) + done + return 1 +} + +# Run docker the right way: directly if this user can reach the daemon, else via +# sudo (we already require passwordless sudo). This makes every docker call work +# whether or not the invoking user is in the 'docker' group — no group setup and +# no re-login needed, which matters on a fresh VM (a `usermod -aG docker` would +# not even take effect in the current session). The direct-vs-sudo decision is +# made once and cached per process in _DKR. +dkr() { + if [ -z "${_DKR:-}" ]; then + if docker info > /dev/null 2>&1; then + _DKR=direct + else + _DKR=sudo + fi + fi + if [ "$_DKR" = sudo ]; then + sudo docker "$@" + else + docker "$@" + fi +} + +# Stop and remove the CtrlB container (idempotent; -f also kills a running one). +stop_container() { + dkr rm -f "$CONTAINER_NAME" > /dev/null 2>&1 || true +} + +# Run the CtrlB container detached: use host networking so the lake's HTTP API on +# 5080 is reachable no matter which interface it binds inside the container, and +# mount this system dir read-only so the downloaded hits.parquet is readable +# inside at the same $PARQUET_PATH. The image has the benchmark env vars baked in. +# --rm so each restart begins from clean state. +start_container() { + dkr run -d --rm --name "$CONTAINER_NAME" \ + --network host \ + -v "$CTRLB_DIR":"$CTRLB_DIR":ro \ + "$CTRLB_IMAGE" > /dev/null +} diff --git a/ctrlb/create.sql b/ctrlb/create.sql new file mode 100644 index 0000000000..c42463ea8b --- /dev/null +++ b/ctrlb/create.sql @@ -0,0 +1,2 @@ +CREATE EXTERNAL TABLE hits_raw STORED AS PARQUET LOCATION 'hits.parquet' OPTIONS ('binary_as_string' 'true'); +CREATE VIEW hits AS SELECT * EXCEPT("EventDate"), CAST(CAST("EventDate" AS INT) AS DATE) AS "EventDate" FROM hits_raw; diff --git a/ctrlb/data-size b/ctrlb/data-size new file mode 100755 index 0000000000..09de1f39eb --- /dev/null +++ b/ctrlb/data-size @@ -0,0 +1,7 @@ +#!/bin/bash +# Storage size in bytes: CtrlB queries the parquet in place, so its "size" is +# the parquet file itself (same basis as the other in-place Parquet engines). +set -e +# shellcheck disable=SC1091 +source "$(dirname "$0")/config.sh" +wc -c < "$PARQUET_PATH" diff --git a/ctrlb/flush-caches b/ctrlb/flush-caches new file mode 100755 index 0000000000..b5a92129a7 --- /dev/null +++ b/ctrlb/flush-caches @@ -0,0 +1,15 @@ +#!/bin/bash +# Per-system cache hook, invoked by the harness's bench_flush_caches AFTER it has +# already run `sync; sudo tee /proc/sys/vm/drop_caches` itself. So the OS page +# cache is dropped either way (the harness requires passwordless sudo regardless). +# +# We just re-assert the page-cache drop for good measure. CtrlB runs as a +# `docker run --rm` container that is torn down and recreated on every per-query +# restart, so its entire on-disk state (metadata cache, sqlite, wal) lives in the +# container's ephemeral filesystem and is wiped automatically each restart — there +# is no persistent host directory for this hook to clear. +set -u + +sync +echo 3 | sudo -n tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true +exit 0 diff --git a/ctrlb/install b/ctrlb/install new file mode 100755 index 0000000000..3b8b1b3d55 --- /dev/null +++ b/ctrlb/install @@ -0,0 +1,46 @@ +#!/bin/bash +# ClickBench install hook — run once, before the benchmark. +# +# REQUIRES passwordless sudo — to install Docker if absent and to drop the OS +# page cache (`echo 3 > /proc/sys/vm/drop_caches`), which needs root. The +# ClickBench driver already assumes passwordless sudo (its bench_flush_caches +# runs `sudo tee /proc/sys/vm/drop_caches`), so this is consistent with every +# other system. If sudo is missing we abort before doing anything. +set -eu +# shellcheck disable=SC1091 +source "$(dirname "$0")/config.sh" # CTRLB_IMAGE is defined here (single source) + +# --- passwordless sudo (needed to install Docker if missing + drop the cache) - +if ! sudo -n true 2>/dev/null; then + echo "ERROR: passwordless sudo is required (to install Docker if missing and to" >&2 + echo " drop the OS page cache). Configure sudo, or run as root." >&2 + exit 1 +fi + +# --- Docker: install if missing on this fresh VM, then verify it is usable ---- +if ! command -v docker > /dev/null 2>&1; then + echo "Docker not found — installing via https://get.docker.com ..." >&2 + curl -fsSL https://get.docker.com | sudo sh +fi +# Make sure the daemon is up (systemd or sysv), then confirm we can talk to it. +# `dkr` (config.sh) uses the daemon directly if this user can reach it, else via +# sudo — so being outside the 'docker' group is fine as long as sudo works. +sudo systemctl enable --now docker > /dev/null 2>&1 \ + || sudo service docker start > /dev/null 2>&1 || true +if ! dkr info > /dev/null 2>&1; then + echo "ERROR: Docker is installed but the daemon is not reachable, even via sudo." >&2 + echo " Ensure the Docker service is running. Refusing to run." >&2 + exit 1 +fi + +# 1) Pull the CtrlB image. +dkr pull "$CTRLB_IMAGE" + +# 2) Right after the pull and before the container is spun up (in ./start): +# flush dirty pages to disk (sync), then drop the page cache. The pull dirties +# a large amount of memory; flushing here keeps ./load's own sync cheap so the +# load time stays honest instead of blowing up as writeback of the pull. +sync +echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null + +exit 0 diff --git a/ctrlb/load b/ctrlb/load new file mode 100755 index 0000000000..35b36f8712 --- /dev/null +++ b/ctrlb/load @@ -0,0 +1,8 @@ +#!/bin/bash +# CtrlB reads hits.parquet in place — there is no ingestion step. The `hits` +# view is (re)created in ./start after each restart, not here. The harness wraps +# this script in its load-time timer and appends its own `sync` afterward; that +# sync (flushing the freshly-downloaded parquet to disk) is the reported +# "Load time", exactly as for the other read-in-place parquet engines. +set -e +exit 0 diff --git a/ctrlb/make-result.py b/ctrlb/make-result.py new file mode 100644 index 0000000000..46f95be768 --- /dev/null +++ b/ctrlb/make-result.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Assemble a ClickBench results//.json from a benchmark log. + +This is a HELPER, not a ClickBench hook — run it after `./benchmark.sh | tee LOG` +to turn the raw run output into the JSON that gets committed in the PR. + +It reads the standard ClickBench log format emitted by ./benchmark.sh: + - one or more `Load time: ` lines (summed, then rounded to an int like + every peer result file) + - a `Data size: ` line (falls back to the parquet-single + constant if absent) + - 43 lines of `[cold, hot1, hot2],` (the per-query runtimes) +plus template.json (system / proprietary / hardware / tuned / tags), and writes a +VALID results JSON to results//.json with the same field order +and shape as the other systems' files. + +Usage: + ./make-result.py LOG + ./make-result.py LOG --machine c6a.4xlarge --date 2026-07-23 + ./make-result.py LOG --template template.json --outdir results + +Exits non-zero (with a clear message) if the log is malformed, so a bad file is +never written into results/. +""" +import argparse +import datetime +import json +import os +import re +import sys + +# hits.parquet (single) is a fixed-size file; every peer reports this exact size. +DEFAULT_DATA_SIZE = 14779976446 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("log", help="benchmark output captured via `| tee LOG`") + ap.add_argument("--machine", default="c6a.4xlarge", + help="hardware label + result filename (default: c6a.4xlarge)") + ap.add_argument("--date", default=None, + help="run date YYYY-MM-DD (default: today in UTC)") + ap.add_argument("--template", default=None, + help="path to template.json (default: alongside this script)") + ap.add_argument("--outdir", default=None, + help="results directory (default: