ops(runners): periodic and low-disk-triggered cleaning for self-hosted runners

runner-maintenance/job-started-hook.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Point ACTIONS_RUNNER_HOOK_JOB_STARTED at this file (via the runner's .env) to
+# have every job check disk headroom before it starts.
+#
+# Before the job rather than after: a job that fails on ENOSPC has already burnt
+# its full build time, and the failure surfaces as something unrelated-looking
+# deep in a linker (`ar: ... No space left on device`).
+#
+# Never fails the job -- see the note at the end of runner-disk-clean.sh.
+set -uo pipefail
+exec "$(dirname "$0")/runner-disk-clean.sh" --if-low

runner-maintenance/runner-disk-clean.service

@@ -0,0 +1,16 @@
+# Nightly disk sweep for a self-hosted GitHub Actions runner.
+# Installed alongside runner-disk-clean.timer; see runner-disk-clean.sh for why
+# the sweep is age-based rather than a wipe.
+[Unit]
+Description=Reclaim disk on the GitHub Actions runner
+
+[Service]
+Type=oneshot
+# Runs as the runner's own user so it can only ever delete state that user
+# created -- not as root, which would happily remove anything the paths resolve to.
+User=actions-runner
+Environment=RUNNER_HOME=/opt/actions-runner
+ExecStart=/opt/actions-runner/runner-maintenance/runner-disk-clean.sh --scheduled
+# Cleanup is never urgent enough to slow a build that is running right now.
+Nice=19
+IOSchedulingClass=idle

runner-maintenance/runner-disk-clean.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# Reclaim disk on a self-hosted GitHub Actions runner.
+#
+# The fleet is non-ephemeral: every job leaves its build state behind, and
+# nothing ever removes it. On a Rust repo that accumulates fast -- a single
+# Protect checkout is ~17 GB of target/, over half of it incremental state that
+# is worthless the moment the job ends -- so a 60 GB runner fills after a
+# handful of branches and then fails jobs late and confusingly with ENOSPC.
+#
+# Age-based throughout, never "delete everything": jobs run back to back on
+# these hosts and share ~/.cargo, so anything a *running* job might be holding
+# has to survive. MAX_AGE_DAYS is the guard -- state that has not been touched
+# in that long cannot belong to an in-flight job.
+#
+# Two entry points, same script:
+#   --scheduled   nightly timer; prunes anything older than MAX_AGE_DAYS
+#   --if-low      no-op unless free space is under MIN_FREE_GB, then prunes
+#                 with a shorter age cut. Called from the runner's
+#                 ACTIONS_RUNNER_HOOK_JOB_STARTED hook, i.e. before a job runs
+#                 rather than after it has already half-filled the disk.
+set -uo pipefail
+
+RUNNER_HOME="${RUNNER_HOME:-/opt/actions-runner}"
+WORK_DIR="${RUNNER_WORK_DIR:-$RUNNER_HOME/_work}"
+CARGO_HOME="${CARGO_HOME:-$RUNNER_HOME/.cargo}"
+MIN_FREE_GB="${MIN_FREE_GB:-15}"
+MAX_AGE_DAYS="${MAX_AGE_DAYS:-7}"
+# Under pressure the same sweep runs with a tighter cut. Still far longer than
+# any single job, so it cannot race one.
+LOW_DISK_AGE_DAYS="${LOW_DISK_AGE_DAYS:-1}"
+
+free_gb() { df -BG --output=avail "$WORK_DIR" | tail -n1 | tr -dc '0-9'; }
+log() { echo "[runner-disk-clean] $*"; }
+
+sweep() {
+  local age="$1"
+  log "sweeping state older than ${age}d (free: $(free_gb) GB)"
+
+  # Incremental compilation state: per-working-copy, never reused across jobs.
+  # Biggest single win and the safest thing here.
+  find "$WORK_DIR" -type d -name incremental -path '*/target/*' -mtime "+${age}" \
+    -prune -exec rm -rf {} + 2>/dev/null
+
+  # Whole target/ dirs from branches nobody has built since. A later job on that
+  # branch just recompiles; a stale tree costs GB indefinitely.
+  find "$WORK_DIR" -mindepth 2 -maxdepth 4 -type d -name target -mtime "+${age}" \
+    -prune -exec rm -rf {} + 2>/dev/null
+
+  # Workspace checkouts for repos the fleet no longer builds. actions/checkout
+  # re-clones; keeping them buys nothing once they are cold.
+  find "$WORK_DIR" -mindepth 1 -maxdepth 1 -type d -mtime "+$((age * 4))" \
+    -exec rm -rf {} + 2>/dev/null
+
+  # Extracted crate sources are regenerated from the .crate archives beside
+  # them, so this is cache, not state. registry/cache and git/db are left alone:
+  # deleting those means re-downloading on the next job.
+  find "$CARGO_HOME/registry/src" -mindepth 2 -maxdepth 2 -type d -mtime "+${age}" \
+    -exec rm -rf {} + 2>/dev/null
+
+  # node_modules trees under cold checkouts, npm's own cache, and dangling
+  # docker layers. `docker system prune` without --all deliberately: images a
+  # running job pulled stay, only unreferenced layers and stopped containers go.
+  find "$WORK_DIR" -type d -name node_modules -mtime "+${age}" \
+    -prune -exec rm -rf {} + 2>/dev/null
+  npm cache verify >/dev/null 2>&1
+  command -v docker >/dev/null && docker system prune --force --filter "until=${age}d" >/dev/null 2>&1
+
+  log "done (free: $(free_gb) GB)"
+}
+
+case "${1:---scheduled}" in
+  --scheduled)
+    sweep "$MAX_AGE_DAYS"
+    ;;
+  --if-low)
+    avail="$(free_gb)"
+    if [ "${avail:-0}" -ge "$MIN_FREE_GB" ]; then
+      log "free ${avail} GB >= ${MIN_FREE_GB} GB threshold, nothing to do"
+      exit 0
+    fi
+    log "free ${avail} GB below ${MIN_FREE_GB} GB threshold"
+    sweep "$LOW_DISK_AGE_DAYS"
+    # Deliberately does not fail the job when still short: a job that might have
+    # squeaked through should not be pre-emptively killed. The log line is the
+    # signal that the fleet needs a bigger disk or fewer repos.
+    [ "$(free_gb)" -lt "$MIN_FREE_GB" ] && log "WARNING: still under threshold after sweep"
+    ;;
+  *)
+    echo "usage: $0 [--scheduled|--if-low]" >&2
+    exit 2
+    ;;
+esac
+exit 0

runner-maintenance/runner-disk-clean.timer

@@ -0,0 +1,16 @@
+# Nightly, not hourly: the sweep only removes state older than a week, so
+# running it more often finds nothing. The low-disk hook (see
+# runner-disk-clean.sh --if-low) is what handles a runner filling up mid-day.
+[Unit]
+Description=Nightly disk reclamation on the GitHub Actions runner
+
+[Timer]
+OnCalendar=*-*-* 03:30:00
+# The fleet's runners would otherwise all sweep on the same minute; stagger so
+# they do not go IO-bound together if any share a host or storage backend.
+RandomizedDelaySec=30m
+# A runner that was powered off overnight still gets swept on next boot.
+Persistent=true
+
+[Install]
+WantedBy=timers.target