From 4028d593e3e76820eb90c6b775958f224d0fc73a Mon Sep 17 00:00:00 2001 From: aaron Date: Thu, 20 Aug 2026 06:11:08 -0400 Subject: [PATCH] =?UTF-8?q?chore(ops):=20version=20the=20fleet=20ops=20scr?= =?UTF-8?q?ipts=20=E2=80=94=20fleet-status,=20fleet-alert,=20papers-digest?= =?UTF-8?q?=20(#427)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three production launchd scripts lived only on the mini's disk, hand-maintained, outside any repo: a mini disk failure loses them, and scripts that page humans had no review trail. Now versioned in ops/ alongside scripts/: - fleet-status.sh (5 min) — fleet health JSON for the dashboard + server guard - fleet-alert.sh (15 min) — device transitions -> Slack #fleet (noise-gated) - papers-digest/daily.sh (daily ~09:00) — frozen-bundle digest -> Slack #papers - launchd/ — reference copies of the three plists - install.sh — deploy scripts to ~/.amico/ops/ (idempotent, state-safe) - README.md — what runs where, on what cadence, posting where; the frozen-bundle upgrade pattern (copy dist + refresh sha sidecar) ~/.amico/ops/ becomes a deploy target, not the source of truth. Closes #427 --- ops/README.md | 57 +++++++ ops/fleet-alert.sh | 62 ++++++++ ops/fleet-status.sh | 145 ++++++++++++++++++ ops/install.sh | 21 +++ .../co.harmoniqs.amicode-papers-digest.plist | 14 ++ ops/launchd/co.harmoniqs.fleet-alert.plist | 9 ++ ops/launchd/co.harmoniqs.fleet-status.plist | 21 +++ ops/papers-digest/daily.sh | 10 ++ 8 files changed, 339 insertions(+) create mode 100644 ops/README.md create mode 100755 ops/fleet-alert.sh create mode 100755 ops/fleet-status.sh create mode 100755 ops/install.sh create mode 100644 ops/launchd/co.harmoniqs.amicode-papers-digest.plist create mode 100644 ops/launchd/co.harmoniqs.fleet-alert.plist create mode 100644 ops/launchd/co.harmoniqs.fleet-status.plist create mode 100755 ops/papers-digest/daily.sh diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 00000000..001a91ef --- /dev/null +++ b/ops/README.md @@ -0,0 +1,57 @@ +# ops/ — the fleet's production scripts + +These scripts run ON THE MINI (the canonical server) under launchd. This repo is the +source of truth; `~/.amico/ops/` on the mini is a **deploy target** — deploy with +`ops/install.sh`, never edit the deployed copies by hand. + +## What runs where + +| script | cadence (launchd) | reads | writes / posts | +|---|---|---|---| +| `fleet-status.sh` | every 5 min (`co.harmoniqs.fleet-status`) | ssh probes (mini/macbook/erlich), canonical chat DB, server lsof, `~/.amico/sync.log`, local repo scan | `~/.amico/ops/fleet-status.json` (the dashboard widget's input); macOS notification on server-guard state change | +| `fleet-alert.sh` | every 15 min (`co.harmoniqs.fleet-alert`) | `fleet-status.json`, state file | **Slack `#fleet`** — device transitions only (noise-gated; always-on hosts `mini erlich` notify, laptops never do); down->24h re-reminds once daily | +| `papers-digest/daily.sh` | daily ~09:00 (`co.harmoniqs.amicode-papers-digest`) | the frozen bundle | **Slack `#papers`** — top-5 quant-ph digest; appends to `papers-digest/log.txt` | + +The launchd plists themselves are versioned alongside (`ops/launchd/`) — reference +copies; installing them is a one-time `launchctl load` on the mini (paths inside are +absolute to `/Users/aaron`). + +## Runtime state (NOT in this repo, never overwritten by deploy) + +`fleet-status.json`, `fleet-status.guard-state`, `fleet-alert.state`, +`fleet-alert.launchd.{out,err}`, `papers-digest/{log.txt,launchd.*}` — all live under +`~/.amico/ops/` on the mini and belong to the running system. `install.sh` touches +none of them. + +## The frozen-bundle pattern (papers-digest) + +`papers-digest/daily.sh` runs a **frozen bundle**, never a repo checkout — branches +move; production must not. The bundle lives on the mini at +`~/.amico/ops/papers-digest/bin/`: + +- `amico.js` — the compiled `amico` CLI dist (built from this repo) +- `amico.js.sha256` — its sidecar + +**Upgrade procedure** (the server pattern): + +```sh +# from a build of this repo (pnpm build in packages/cli or the dist pipeline): +cp /amico.js ~/.amico/ops/papers-digest/bin/amico.js +cd ~/.amico/ops/papers-digest/bin && shasum -a 256 amico.js > amico.js.sha256 +``` + +The sha sidecar is what an operator compares against to know what's deployed; the +digest job never needs a restart (it execs the bundle each run). + +## Deploy + +From a checkout of this repo on the mini: + +```sh +ops/install.sh # copies the three scripts to ~/.amico/ops/ (idempotent) +``` + +The install script copies scripts ONLY — no plists (one-time, by hand), no state +files, no bundle. After editing anything here: merge, then deploy, then +`launchctl kickstart` the affected agent if the change should take effect before its +next interval. diff --git a/ops/fleet-alert.sh b/ops/fleet-alert.sh new file mode 100755 index 00000000..924a4e17 --- /dev/null +++ b/ops/fleet-alert.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# fleet-alert.sh — the MISSING half of fleet-status: transitions → Slack. +# fleet-status.sh (launchd, 5 min) writes the JSON; this reads it, diffs +# against a state file, and posts DEVICE TRANSITIONS only to #fleet. +# Noise-gated: one post per transition; steady states never post; a device +# down >24h re-reminds once daily. The 2026-08-18 inciting incident: erlich +# dark 16h, zero alerts — the widget was green-passive. +# +# 2026-08-19 re-point (the #amicode thread): alerts move to #fleet, and only +# ALERT_DEVICES notify — always-on hosts (mini, erlich). Laptops (macbook) +# stay in fleet-status.json for the dashboard but never post: "I shouldn't +# get a notification even in #fleet everytime aaron closes his laptop" (jj). +set -uo pipefail +export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" +STATUS="$HOME/.amico/ops/fleet-status.json" +STATE="$HOME/.amico/ops/fleet-alert.state" +CH="fleet" +ALERT_DEVICES="mini erlich" # space-separated; devices here and ONLY here notify +[ -f "$STATUS" ] || exit 0 +python3 - "$STATUS" "$STATE" "$ALERT_DEVICES" << 'PY' > /tmp/fleet-alert-msg 2>/dev/null +import json, sys, time, subprocess, os +status, statef, alert_dev = sys.argv[1], sys.argv[2], set(sys.argv[3].split()) +now = time.time() +try: + cur = {d["name"]: d["reachable"] for d in json.load(open(status)).get("devices", [])} +except Exception: + sys.exit(0) +prev, meta = {}, {} +try: + j = json.load(open(statef)); prev = j.get("devices", {}); meta = j.get("meta", {}) +except Exception: + pass +msgs, changed = [], False +for name, up in cur.items(): + was = prev.get(name) + if was is None and not up: + continue # first run ever: don't announce pre-existing darkness silently recorded + if was != up: + changed = True + if not up: meta[f"down_since:{name}"] = now + else: meta.pop(f"down_since:{name}", None); meta.pop(f"reminded:{name}", None) + if name not in alert_dev: + continue # monitored for the dashboard, never posted (laptops sleep — not news) + msgs.append(f"{'🟢' if up else '🔴'} `{name}` {'reachable' if up else 'UNREACHABLE'}" + (f" (was {'reachable' if was else 'unreachable'})" if was is not None else "")) +for name in list(cur): + if name not in alert_dev: + continue + ds = meta.get(f"down_since:{name}") + if ds and not cur[name] and now - ds > 86400 and meta.get(f"reminded:{name}", 0) < now - 86400: + msgs.append(f"⏰ still down >24h: `{name}`"); meta[f"reminded:{name}"] = now +json.dump({"devices": cur, "meta": meta}, open(statef, "w")) +if msgs: + text = "*fleet:* " + " · ".join(msgs) + token = open(os.path.expanduser("~/.amico/slack/token")).read().strip() + channels = json.load(open(os.path.expanduser("~/.amico/slack/channels.json"))) + cid = channels.get("fleet") + if cid: + subprocess.run(["curl", "-sS", "-X", "POST", "https://slack.com/api/chat.postMessage", + "-H", f"Authorization: Bearer {token}", "-H", "Content-type: application/json", + "--data-binary", json.dumps({"channel": cid, "text": text})], capture_output=True) +PY +exit 0 diff --git a/ops/fleet-status.sh b/ops/fleet-status.sh new file mode 100755 index 00000000..3648055c --- /dev/null +++ b/ops/fleet-status.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# fleet-status.sh — collect fleet health into ~/.amico/ops/fleet-status.json +# for the Amicode dashboard widget. Run on the canonical server (the mini); +# safe to run any time, read-only everywhere. Wired to launchd every 5 min. +set -uo pipefail + +OUT="$HOME/.amico/ops/fleet-status.json" +TMP="$OUT.tmp" +DB="$HOME/.local/share/opencode/opencode-dev.db" +SYNCLOG="$HOME/.amico/sync.log" + +# --- devices: alias|host pairs (edit here as the fleet grows) --------------- +DEVICES=("mini:127.0.0.1" "macbook:macbook" "erlich:erlich") + +# Bounded ssh probe — no GNU timeout on macOS, and ConnectTimeout alone does +# NOT bound an in-band stall (observed 2026-08-08: Tailscale SSH to erlich +# parks on an interactive re-auth banner forever, hanging the whole script so +# fleet-status.json silently goes stale). Background + kill after 8 s. +ssh_probe() { + local host="$1" pid killer rc + ssh -o BatchMode=yes -o ConnectTimeout=5 "$host" true 2>/dev/null & + pid=$! + ( sleep 8; kill "$pid" 2>/dev/null ) & killer=$! + wait "$pid" 2>/dev/null; rc=$? + kill "$killer" 2>/dev/null; wait "$killer" 2>/dev/null + return $rc +} + +dev_rows="" +for pair in "${DEVICES[@]}"; do + name="${pair%%:*}"; host="${pair#*:}" + if [ "$host" = "127.0.0.1" ]; then + reachable=true; detail="this machine" + else + if ssh_probe "$host"; then + reachable=true; detail="ssh ok" + else + reachable=false; detail="ssh failed" + fi + fi + dev_rows="$dev_rows{\"name\":\"$name\",\"reachable\":$reachable,\"detail\":\"$detail\"}," +done +dev_rows="[${dev_rows%,}]" + +# --- canonical chat DB -------------------------------------------------------- +sessions="null"; last_session="null" +if [ -f "$DB" ]; then + read -r sessions last_session <<<"$(sqlite3 "$DB" "SELECT COUNT(*), COALESCE(datetime(MAX(time_updated)/1000,'unixepoch'),'') FROM session;" 2>/dev/null | awk -F'|' '{print $1, $2}')" + sessions="${sessions:-null}"; last_session="${last_session:-null}" +fi +server_code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 4 http://127.0.0.1:4096/session 2>/dev/null || echo 000)" + +# --- server guard: is the running server holding the CANONICAL db? ------------ +# (2026-08-08 incident: a vendor binary refresh flipped the build channel and +# the server silently served a fresh opencode-local.db for hours — panels saw +# an empty history while opencode-dev.db sat untouched on disk. Now caught +# here within one 5-min cycle, with a notification on state transition.) +guard_ok=true; guard_notes="" +srv_pid="$(launchctl list 2>/dev/null | awk '/co\.harmoniqs\.amicode-server/ {print $1}')" +srv_db="none"; served="null" +if [ -n "$srv_pid" ] && [ "$srv_pid" != "-" ]; then + srv_db="$(lsof -p "$srv_pid" 2>/dev/null | grep -oE 'opencode-[a-z]+\.db' | sort -u | head -1)" + srv_db="${srv_db:-none}" + if [ "$srv_db" != "opencode-dev.db" ]; then + guard_ok=false; guard_notes="${guard_notes}server holds ${srv_db} not opencode-dev.db; " + fi +else + guard_ok=false; guard_notes="${guard_notes}server not running; " +fi +if [ "$server_code" = "200" ]; then + served="$(curl -s --max-time 6 'http://127.0.0.1:4096/session?limit=1000' 2>/dev/null \ + | python3 -c 'import json,sys +d=json.load(sys.stdin) +print(len(d if isinstance(d,list) else d.get("sessions",[])))' 2>/dev/null || echo null)" + served="${served:-null}" + # the /session endpoint filters by the server's project, so it never matches + # the on-disk count exactly — flag only a dramatic shortfall (< 50 %). + if [ "$served" != "null" ] && [ "$sessions" != "null" ] && [ "$sessions" -ge 100 ] 2>/dev/null; then + if [ "$served" -lt $(( sessions / 2 )) ] 2>/dev/null; then + guard_ok=false; guard_notes="${guard_notes}serving only ${served} of ${sessions} sessions; " + fi + fi +fi + +# notify once per distinct bad state (launchd re-runs every 5 min) +GSTATE="$HOME/.amico/ops/fleet-status.guard-state" +prev_guard="$(cat "$GSTATE" 2>/dev/null || echo ok)" +if [ "$guard_ok" = false ]; then + sig="bad: ${guard_notes}" + if [ "$prev_guard" != "$sig" ]; then + osascript -e "display notification \"${guard_notes}— see ~/.amico/ops/fleet-status.json\" with title \"Amicode fleet: chat server\"" 2>/dev/null || true + fi + echo "$sig" > "$GSTATE" +else + echo ok > "$GSTATE" +fi + +# --- vault sync freshness ----------------------------------------------------- +sync_age_min="null"; sync_clean="null" +if [ -f "$SYNCLOG" ]; then + last_done="$(grep "done$" "$SYNCLOG" | tail -1 | sed -E 's/^\[([^]]+)\].*/\1/')" + if [ -n "$last_done" ]; then + last_epoch="$(date -j -f "%Y-%m-%dT%H:%M:%S%z" "$last_done" +%s 2>/dev/null || echo 0)" + now_epoch="$(date +%s)" + [ "$last_epoch" -gt 0 ] && sync_age_min=$(( (now_epoch - last_epoch) / 60 )) + fi + if tail -20 "$SYNCLOG" | grep -qE "fatal:|CONFLICT|no tracking information"; then + sync_clean=false + else + sync_clean=true + fi +fi + +# --- code repos (no daemon by design — only commits cross machines; the ritual +# is wip-sync.sh). Read-only scan of LOCAL ~/harmoniqs repos: dirty count + +# ahead/behind as of last fetch (no network in a 5-min launchd job). ------- +repo_rows="" +shopt -s nullglob +for gd in "$HOME"/harmoniqs/*/.git "$HOME"/harmoniqs/packages/*/.git "$HOME"/harmoniqs/demos/*/.git; do + r="${gd%/.git}" + name="$(basename "$r")" + branch="$(git -C "$r" branch --show-current 2>/dev/null)"; branch="${branch:-DETACHED}" + dirty="$(git -C "$r" status --porcelain 2>/dev/null | wc -l | tr -d ' ')" + ahead=0; behind=0 + ab="$(git -C "$r" rev-list --left-right --count '@{upstream}...HEAD' 2>/dev/null)" + if [ -n "$ab" ]; then behind="${ab%%[[:space:]]*}"; ahead="${ab##*[[:space:]]}"; fi + wips="$( { git -C "$r" branch --list 'wip/*' --format='%(refname:short)' 2>/dev/null; \ + git -C "$r" branch -r --list 'origin/wip/*' --format='%(refname:short)' 2>/dev/null | sed 's|^origin/||'; } \ + | sort -u | tr '\n' ',' )" + repo_rows="$repo_rows{\"name\":\"$name\",\"branch\":\"$branch\",\"dirty\":$dirty,\"ahead\":$ahead,\"behind\":$behind,\"wip_branches\":\"${wips%,}\"}," +done +shopt -u nullglob +repo_rows="[${repo_rows%,}]" + +cat > "$TMP" <" diff --git a/ops/launchd/co.harmoniqs.amicode-papers-digest.plist b/ops/launchd/co.harmoniqs.amicode-papers-digest.plist new file mode 100644 index 00000000..7401267c --- /dev/null +++ b/ops/launchd/co.harmoniqs.amicode-papers-digest.plist @@ -0,0 +1,14 @@ + + + + Labelco.harmoniqs.amicode-papers-digest + ProgramArguments + /bin/bash/Users/aaron/.amico/ops/papers-digest/daily.sh + + StartCalendarInterval + Hour9 + Minute0 + + StandardOutPath/Users/aaron/.amico/ops/papers-digest/launchd.out + StandardErrorPath/Users/aaron/.amico/ops/papers-digest/launchd.err + diff --git a/ops/launchd/co.harmoniqs.fleet-alert.plist b/ops/launchd/co.harmoniqs.fleet-alert.plist new file mode 100644 index 00000000..aefdb43b --- /dev/null +++ b/ops/launchd/co.harmoniqs.fleet-alert.plist @@ -0,0 +1,9 @@ + + + + Labelco.harmoniqs.fleet-alert + ProgramArguments/bin/bash/Users/aaron/.amico/ops/fleet-alert.sh + StartInterval900 + StandardOutPath/Users/aaron/.amico/ops/fleet-alert.launchd.out + StandardErrorPath/Users/aaron/.amico/ops/fleet-alert.launchd.err + diff --git a/ops/launchd/co.harmoniqs.fleet-status.plist b/ops/launchd/co.harmoniqs.fleet-status.plist new file mode 100644 index 00000000..8fc15af1 --- /dev/null +++ b/ops/launchd/co.harmoniqs.fleet-status.plist @@ -0,0 +1,21 @@ + + + + + Label + co.harmoniqs.fleet-status + ProgramArguments + + /bin/bash + /Users/aaron/.amico/ops/fleet-status.sh + + StartInterval + 300 + RunAtLoad + + StandardOutPath + /tmp/fleet-status.log + StandardErrorPath + /tmp/fleet-status.log + + diff --git a/ops/papers-digest/daily.sh b/ops/papers-digest/daily.sh new file mode 100755 index 00000000..3395f08b --- /dev/null +++ b/ops/papers-digest/daily.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# papers-digest daily — the intelligent arXiv digest to #papers (#412). +# Runs the FROZEN bundle (never the repo checkout — branches move); upgrade = +# copy a new dist/amico.js over bin/ + refresh the sidecar (the server pattern). +set -euo pipefail +export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" +BIN="$HOME/.amico/ops/papers-digest/bin/amico.js" +LOG="$HOME/.amico/ops/papers-digest/log.txt" +echo "[$(date -u +%FT%TZ)] digest run" >> "$LOG" +node "$BIN" papers digest --feed quant-ph --top 5 --post papers >> "$LOG" 2>&1 || echo "[$(date -u +%FT%TZ)] digest FAILED (see above)" >> "$LOG"