diff --git a/runner-maintenance/job-started-hook.sh b/runner-maintenance/job-started-hook.sh new file mode 100755 index 0000000..a69e2d3 --- /dev/null +++ b/runner-maintenance/job-started-hook.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Point ACTIONS_RUNNER_HOOK_JOB_STARTED at this file (via the runner's .env) to +# have every job check disk headroom before it starts. +# +# Before the job rather than after: a job that fails on ENOSPC has already burnt +# its full build time, and the failure surfaces as something unrelated-looking +# deep in a linker (`ar: ... No space left on device`). +# +# Never fails the job -- see the note at the end of runner-disk-clean.sh. +set -uo pipefail +exec "$(dirname "$0")/runner-disk-clean.sh" --if-low diff --git a/runner-maintenance/runner-disk-clean.service b/runner-maintenance/runner-disk-clean.service new file mode 100644 index 0000000..ddb5381 --- /dev/null +++ b/runner-maintenance/runner-disk-clean.service @@ -0,0 +1,16 @@ +# Nightly disk sweep for a self-hosted GitHub Actions runner. +# Installed alongside runner-disk-clean.timer; see runner-disk-clean.sh for why +# the sweep is age-based rather than a wipe. +[Unit] +Description=Reclaim disk on the GitHub Actions runner + +[Service] +Type=oneshot +# Runs as the runner's own user so it can only ever delete state that user +# created -- not as root, which would happily remove anything the paths resolve to. +User=actions-runner +Environment=RUNNER_HOME=/opt/actions-runner +ExecStart=/opt/actions-runner/runner-maintenance/runner-disk-clean.sh --scheduled +# Cleanup is never urgent enough to slow a build that is running right now. +Nice=19 +IOSchedulingClass=idle diff --git a/runner-maintenance/runner-disk-clean.sh b/runner-maintenance/runner-disk-clean.sh new file mode 100755 index 0000000..f105de2 --- /dev/null +++ b/runner-maintenance/runner-disk-clean.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Reclaim disk on a self-hosted GitHub Actions runner. +# +# The fleet is non-ephemeral: every job leaves its build state behind, and +# nothing ever removes it. On a Rust repo that accumulates fast -- a single +# Protect checkout is ~17 GB of target/, over half of it incremental state that +# is worthless the moment the job ends -- so a 60 GB runner fills after a +# handful of branches and then fails jobs late and confusingly with ENOSPC. +# +# Age-based throughout, never "delete everything": jobs run back to back on +# these hosts and share ~/.cargo, so anything a *running* job might be holding +# has to survive. MAX_AGE_DAYS is the guard -- state that has not been touched +# in that long cannot belong to an in-flight job. +# +# Two entry points, same script: +# --scheduled nightly timer; prunes anything older than MAX_AGE_DAYS +# --if-low no-op unless free space is under MIN_FREE_GB, then prunes +# with a shorter age cut. Called from the runner's +# ACTIONS_RUNNER_HOOK_JOB_STARTED hook, i.e. before a job runs +# rather than after it has already half-filled the disk. +set -uo pipefail + +RUNNER_HOME="${RUNNER_HOME:-/opt/actions-runner}" +WORK_DIR="${RUNNER_WORK_DIR:-$RUNNER_HOME/_work}" +CARGO_HOME="${CARGO_HOME:-$RUNNER_HOME/.cargo}" +MIN_FREE_GB="${MIN_FREE_GB:-15}" +MAX_AGE_DAYS="${MAX_AGE_DAYS:-7}" +# Under pressure the same sweep runs with a tighter cut. Still far longer than +# any single job, so it cannot race one. +LOW_DISK_AGE_DAYS="${LOW_DISK_AGE_DAYS:-1}" + +free_gb() { df -BG --output=avail "$WORK_DIR" | tail -n1 | tr -dc '0-9'; } +log() { echo "[runner-disk-clean] $*"; } + +sweep() { + local age="$1" + log "sweeping state older than ${age}d (free: $(free_gb) GB)" + + # Incremental compilation state: per-working-copy, never reused across jobs. + # Biggest single win and the safest thing here. + find "$WORK_DIR" -type d -name incremental -path '*/target/*' -mtime "+${age}" \ + -prune -exec rm -rf {} + 2>/dev/null + + # Whole target/ dirs from branches nobody has built since. A later job on that + # branch just recompiles; a stale tree costs GB indefinitely. + find "$WORK_DIR" -mindepth 2 -maxdepth 4 -type d -name target -mtime "+${age}" \ + -prune -exec rm -rf {} + 2>/dev/null + + # Workspace checkouts for repos the fleet no longer builds. actions/checkout + # re-clones; keeping them buys nothing once they are cold. + find "$WORK_DIR" -mindepth 1 -maxdepth 1 -type d -mtime "+$((age * 4))" \ + -exec rm -rf {} + 2>/dev/null + + # Extracted crate sources are regenerated from the .crate archives beside + # them, so this is cache, not state. registry/cache and git/db are left alone: + # deleting those means re-downloading on the next job. + find "$CARGO_HOME/registry/src" -mindepth 2 -maxdepth 2 -type d -mtime "+${age}" \ + -exec rm -rf {} + 2>/dev/null + + # node_modules trees under cold checkouts, npm's own cache, and dangling + # docker layers. `docker system prune` without --all deliberately: images a + # running job pulled stay, only unreferenced layers and stopped containers go. + find "$WORK_DIR" -type d -name node_modules -mtime "+${age}" \ + -prune -exec rm -rf {} + 2>/dev/null + npm cache verify >/dev/null 2>&1 + command -v docker >/dev/null && docker system prune --force --filter "until=${age}d" >/dev/null 2>&1 + + log "done (free: $(free_gb) GB)" +} + +case "${1:---scheduled}" in + --scheduled) + sweep "$MAX_AGE_DAYS" + ;; + --if-low) + avail="$(free_gb)" + if [ "${avail:-0}" -ge "$MIN_FREE_GB" ]; then + log "free ${avail} GB >= ${MIN_FREE_GB} GB threshold, nothing to do" + exit 0 + fi + log "free ${avail} GB below ${MIN_FREE_GB} GB threshold" + sweep "$LOW_DISK_AGE_DAYS" + # Deliberately does not fail the job when still short: a job that might have + # squeaked through should not be pre-emptively killed. The log line is the + # signal that the fleet needs a bigger disk or fewer repos. + [ "$(free_gb)" -lt "$MIN_FREE_GB" ] && log "WARNING: still under threshold after sweep" + ;; + *) + echo "usage: $0 [--scheduled|--if-low]" >&2 + exit 2 + ;; +esac +exit 0 diff --git a/runner-maintenance/runner-disk-clean.timer b/runner-maintenance/runner-disk-clean.timer new file mode 100644 index 0000000..f89c3e8 --- /dev/null +++ b/runner-maintenance/runner-disk-clean.timer @@ -0,0 +1,16 @@ +# Nightly, not hourly: the sweep only removes state older than a week, so +# running it more often finds nothing. The low-disk hook (see +# runner-disk-clean.sh --if-low) is what handles a runner filling up mid-day. +[Unit] +Description=Nightly disk reclamation on the GitHub Actions runner + +[Timer] +OnCalendar=*-*-* 03:30:00 +# The fleet's runners would otherwise all sweep on the same minute; stagger so +# they do not go IO-bound together if any share a host or storage backend. +RandomizedDelaySec=30m +# A runner that was powered off overnight still gets swept on next boot. +Persistent=true + +[Install] +WantedBy=timers.target