chore(ops): version the fleet ops scripts — fleet-status, fleet-alert, papers-digest

ops/README.md

@@ -0,0 +1,57 @@
+# ops/ — the fleet's production scripts
+
+These scripts run ON THE MINI (the canonical server) under launchd. This repo is the
+source of truth; `~/.amico/ops/` on the mini is a **deploy target** — deploy with
+`ops/install.sh`, never edit the deployed copies by hand.
+
+## What runs where
+
+| script | cadence (launchd) | reads | writes / posts |
+|---|---|---|---|
+| `fleet-status.sh` | every 5 min (`co.harmoniqs.fleet-status`) | ssh probes (mini/macbook/erlich), canonical chat DB, server lsof, `~/.amico/sync.log`, local repo scan | `~/.amico/ops/fleet-status.json` (the dashboard widget's input); macOS notification on server-guard state change |
+| `fleet-alert.sh` | every 15 min (`co.harmoniqs.fleet-alert`) | `fleet-status.json`, state file | **Slack `#fleet`** — device transitions only (noise-gated; always-on hosts `mini erlich` notify, laptops never do); down->24h re-reminds once daily |
+| `papers-digest/daily.sh` | daily ~09:00 (`co.harmoniqs.amicode-papers-digest`) | the frozen bundle | **Slack `#papers`** — top-5 quant-ph digest; appends to `papers-digest/log.txt` |
+
+The launchd plists themselves are versioned alongside (`ops/launchd/`) — reference
+copies; installing them is a one-time `launchctl load` on the mini (paths inside are
+absolute to `/Users/aaron`).
+
+## Runtime state (NOT in this repo, never overwritten by deploy)
+
+`fleet-status.json`, `fleet-status.guard-state`, `fleet-alert.state`,
+`fleet-alert.launchd.{out,err}`, `papers-digest/{log.txt,launchd.*}` — all live under
+`~/.amico/ops/` on the mini and belong to the running system. `install.sh` touches
+none of them.
+
+## The frozen-bundle pattern (papers-digest)
+
+`papers-digest/daily.sh` runs a **frozen bundle**, never a repo checkout — branches
+move; production must not. The bundle lives on the mini at
+`~/.amico/ops/papers-digest/bin/`:
+
+- `amico.js` — the compiled `amico` CLI dist (built from this repo)
+- `amico.js.sha256` — its sidecar
+
+**Upgrade procedure** (the server pattern):
+
+```sh
+# from a build of this repo (pnpm build in packages/cli or the dist pipeline):
+cp <dist>/amico.js ~/.amico/ops/papers-digest/bin/amico.js
+cd ~/.amico/ops/papers-digest/bin && shasum -a 256 amico.js > amico.js.sha256
+```
+
+The sha sidecar is what an operator compares against to know what's deployed; the
+digest job never needs a restart (it execs the bundle each run).
+
+## Deploy
+
+From a checkout of this repo on the mini:
+
+```sh
+ops/install.sh            # copies the three scripts to ~/.amico/ops/ (idempotent)
+```
+
+The install script copies scripts ONLY — no plists (one-time, by hand), no state
+files, no bundle. After editing anything here: merge, then deploy, then
+`launchctl kickstart` the affected agent if the change should take effect before its
+next interval.

ops/fleet-alert.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# fleet-alert.sh — the MISSING half of fleet-status: transitions → Slack.
+# fleet-status.sh (launchd, 5 min) writes the JSON; this reads it, diffs
+# against a state file, and posts DEVICE TRANSITIONS only to #fleet.
+# Noise-gated: one post per transition; steady states never post; a device
+# down >24h re-reminds once daily. The 2026-08-18 inciting incident: erlich
+# dark 16h, zero alerts — the widget was green-passive.
+#
+# 2026-08-19 re-point (the #amicode thread): alerts move to #fleet, and only
+# ALERT_DEVICES notify — always-on hosts (mini, erlich). Laptops (macbook)
+# stay in fleet-status.json for the dashboard but never post: "I shouldn't
+# get a notification even in #fleet everytime aaron closes his laptop" (jj).
+set -uo pipefail
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"
+STATUS="$HOME/.amico/ops/fleet-status.json"
+STATE="$HOME/.amico/ops/fleet-alert.state"
+CH="fleet"
+ALERT_DEVICES="mini erlich"   # space-separated; devices here and ONLY here notify
+[ -f "$STATUS" ] || exit 0
+python3 - "$STATUS" "$STATE" "$ALERT_DEVICES" << 'PY' > /tmp/fleet-alert-msg 2>/dev/null
+import json, sys, time, subprocess, os
+status, statef, alert_dev = sys.argv[1], sys.argv[2], set(sys.argv[3].split())
+now = time.time()
+try:
+    cur = {d["name"]: d["reachable"] for d in json.load(open(status)).get("devices", [])}
+except Exception:
+    sys.exit(0)
+prev, meta = {}, {}
+try:
+    j = json.load(open(statef)); prev = j.get("devices", {}); meta = j.get("meta", {})
+except Exception:
+    pass
+msgs, changed = [], False
+for name, up in cur.items():
+    was = prev.get(name)
+    if was is None and not up:
+        continue  # first run ever: don't announce pre-existing darkness silently recorded
+    if was != up:
+        changed = True
+        if not up: meta[f"down_since:{name}"] = now
+        else: meta.pop(f"down_since:{name}", None); meta.pop(f"reminded:{name}", None)
+        if name not in alert_dev:
+            continue  # monitored for the dashboard, never posted (laptops sleep — not news)
+        msgs.append(f"{'🟢' if up else '🔴'} `{name}` {'reachable' if up else 'UNREACHABLE'}" + (f" (was {'reachable' if was else 'unreachable'})" if was is not None else ""))
+for name in list(cur):
+    if name not in alert_dev:
+        continue
+    ds = meta.get(f"down_since:{name}")
+    if ds and not cur[name] and now - ds > 86400 and meta.get(f"reminded:{name}", 0) < now - 86400:
+        msgs.append(f"⏰ still down >24h: `{name}`"); meta[f"reminded:{name}"] = now
+json.dump({"devices": cur, "meta": meta}, open(statef, "w"))
+if msgs:
+    text = "*fleet:* " + " · ".join(msgs)
+    token = open(os.path.expanduser("~/.amico/slack/token")).read().strip()
+    channels = json.load(open(os.path.expanduser("~/.amico/slack/channels.json")))
+    cid = channels.get("fleet")
+    if cid:
+        subprocess.run(["curl", "-sS", "-X", "POST", "https://slack.com/api/chat.postMessage",
+            "-H", f"Authorization: Bearer {token}", "-H", "Content-type: application/json",
+            "--data-binary", json.dumps({"channel": cid, "text": text})], capture_output=True)
+PY
+exit 0

ops/fleet-status.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+# fleet-status.sh — collect fleet health into ~/.amico/ops/fleet-status.json
+# for the Amicode dashboard widget. Run on the canonical server (the mini);
+# safe to run any time, read-only everywhere. Wired to launchd every 5 min.
+set -uo pipefail
+
+OUT="$HOME/.amico/ops/fleet-status.json"
+TMP="$OUT.tmp"
+DB="$HOME/.local/share/opencode/opencode-dev.db"
+SYNCLOG="$HOME/.amico/sync.log"
+
+# --- devices: alias|host pairs (edit here as the fleet grows) ---------------
+DEVICES=("mini:127.0.0.1" "macbook:macbook" "erlich:erlich")
+
+# Bounded ssh probe — no GNU timeout on macOS, and ConnectTimeout alone does
+# NOT bound an in-band stall (observed 2026-08-08: Tailscale SSH to erlich
+# parks on an interactive re-auth banner forever, hanging the whole script so
+# fleet-status.json silently goes stale). Background + kill after 8 s.
+ssh_probe() {
+  local host="$1" pid killer rc
+  ssh -o BatchMode=yes -o ConnectTimeout=5 "$host" true 2>/dev/null &
+  pid=$!
+  ( sleep 8; kill "$pid" 2>/dev/null ) & killer=$!
+  wait "$pid" 2>/dev/null; rc=$?
+  kill "$killer" 2>/dev/null; wait "$killer" 2>/dev/null
+  return $rc
+}
+
+dev_rows=""
+for pair in "${DEVICES[@]}"; do
+  name="${pair%%:*}"; host="${pair#*:}"
+  if [ "$host" = "127.0.0.1" ]; then
+    reachable=true; detail="this machine"
+  else
+    if ssh_probe "$host"; then
+      reachable=true; detail="ssh ok"
+    else
+      reachable=false; detail="ssh failed"
+    fi
+  fi
+  dev_rows="$dev_rows{\"name\":\"$name\",\"reachable\":$reachable,\"detail\":\"$detail\"},"
+done
+dev_rows="[${dev_rows%,}]"
+
+# --- canonical chat DB --------------------------------------------------------
+sessions="null"; last_session="null"
+if [ -f "$DB" ]; then
+  read -r sessions last_session <<<"$(sqlite3 "$DB" "SELECT COUNT(*), COALESCE(datetime(MAX(time_updated)/1000,'unixepoch'),'') FROM session;" 2>/dev/null | awk -F'|' '{print $1, $2}')"
+  sessions="${sessions:-null}"; last_session="${last_session:-null}"
+fi
+server_code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 4 http://127.0.0.1:4096/session 2>/dev/null || echo 000)"
+
+# --- server guard: is the running server holding the CANONICAL db? ------------
+# (2026-08-08 incident: a vendor binary refresh flipped the build channel and
+#  the server silently served a fresh opencode-local.db for hours — panels saw
+#  an empty history while opencode-dev.db sat untouched on disk. Now caught
+#  here within one 5-min cycle, with a notification on state transition.)
+guard_ok=true; guard_notes=""
+srv_pid="$(launchctl list 2>/dev/null | awk '/co\.harmoniqs\.amicode-server/ {print $1}')"
+srv_db="none"; served="null"
+if [ -n "$srv_pid" ] && [ "$srv_pid" != "-" ]; then
+  srv_db="$(lsof -p "$srv_pid" 2>/dev/null | grep -oE 'opencode-[a-z]+\.db' | sort -u | head -1)"
+  srv_db="${srv_db:-none}"
+  if [ "$srv_db" != "opencode-dev.db" ]; then
+    guard_ok=false; guard_notes="${guard_notes}server holds ${srv_db} not opencode-dev.db; "
+  fi
+else
+  guard_ok=false; guard_notes="${guard_notes}server not running; "
+fi
+if [ "$server_code" = "200" ]; then
+  served="$(curl -s --max-time 6 'http://127.0.0.1:4096/session?limit=1000' 2>/dev/null \
+    | python3 -c 'import json,sys
+d=json.load(sys.stdin)
+print(len(d if isinstance(d,list) else d.get("sessions",[])))' 2>/dev/null || echo null)"
+  served="${served:-null}"
+  # the /session endpoint filters by the server's project, so it never matches
+  # the on-disk count exactly — flag only a dramatic shortfall (< 50 %).
+  if [ "$served" != "null" ] && [ "$sessions" != "null" ] && [ "$sessions" -ge 100 ] 2>/dev/null; then
+    if [ "$served" -lt $(( sessions / 2 )) ] 2>/dev/null; then
+      guard_ok=false; guard_notes="${guard_notes}serving only ${served} of ${sessions} sessions; "
+    fi
+  fi
+fi
+
+# notify once per distinct bad state (launchd re-runs every 5 min)
+GSTATE="$HOME/.amico/ops/fleet-status.guard-state"
+prev_guard="$(cat "$GSTATE" 2>/dev/null || echo ok)"
+if [ "$guard_ok" = false ]; then
+  sig="bad: ${guard_notes}"
+  if [ "$prev_guard" != "$sig" ]; then
+    osascript -e "display notification \"${guard_notes}— see ~/.amico/ops/fleet-status.json\" with title \"Amicode fleet: chat server\"" 2>/dev/null || true
+  fi
+  echo "$sig" > "$GSTATE"
+else
+  echo ok > "$GSTATE"
+fi
+
+# --- vault sync freshness -----------------------------------------------------
+sync_age_min="null"; sync_clean="null"
+if [ -f "$SYNCLOG" ]; then
+  last_done="$(grep "done$" "$SYNCLOG" | tail -1 | sed -E 's/^\[([^]]+)\].*/\1/')"
+  if [ -n "$last_done" ]; then
+    last_epoch="$(date -j -f "%Y-%m-%dT%H:%M:%S%z" "$last_done" +%s 2>/dev/null || echo 0)"
+    now_epoch="$(date +%s)"
+    [ "$last_epoch" -gt 0 ] && sync_age_min=$(( (now_epoch - last_epoch) / 60 ))
+  fi
+  if tail -20 "$SYNCLOG" | grep -qE "fatal:|CONFLICT|no tracking information"; then
+    sync_clean=false
+  else
+    sync_clean=true
+  fi
+fi
+
+# --- code repos (no daemon by design — only commits cross machines; the ritual
+#     is wip-sync.sh). Read-only scan of LOCAL ~/harmoniqs repos: dirty count +
+#     ahead/behind as of last fetch (no network in a 5-min launchd job). -------
+repo_rows=""
+shopt -s nullglob
+for gd in "$HOME"/harmoniqs/*/.git "$HOME"/harmoniqs/packages/*/.git "$HOME"/harmoniqs/demos/*/.git; do
+  r="${gd%/.git}"
+  name="$(basename "$r")"
+  branch="$(git -C "$r" branch --show-current 2>/dev/null)"; branch="${branch:-DETACHED}"
+  dirty="$(git -C "$r" status --porcelain 2>/dev/null | wc -l | tr -d ' ')"
+  ahead=0; behind=0
+  ab="$(git -C "$r" rev-list --left-right --count '@{upstream}...HEAD' 2>/dev/null)"
+  if [ -n "$ab" ]; then behind="${ab%%[[:space:]]*}"; ahead="${ab##*[[:space:]]}"; fi
+  wips="$( { git -C "$r" branch --list 'wip/*' --format='%(refname:short)' 2>/dev/null; \
+             git -C "$r" branch -r --list 'origin/wip/*' --format='%(refname:short)' 2>/dev/null | sed 's|^origin/||'; } \
+           | sort -u | tr '\n' ',' )"
+  repo_rows="$repo_rows{\"name\":\"$name\",\"branch\":\"$branch\",\"dirty\":$dirty,\"ahead\":$ahead,\"behind\":$behind,\"wip_branches\":\"${wips%,}\"},"
+done
+shopt -u nullglob
+repo_rows="[${repo_rows%,}]"
+
+cat > "$TMP" <<EOF
+{
+  "collected_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+  "devices": $dev_rows,
+  "chat_db": { "sessions": $sessions, "last_session": "$last_session", "server_http": "$server_code" },
+  "server_guard": { "ok": $guard_ok, "pid": "$srv_pid", "db_file": "$srv_db", "served_sessions": $served, "notes": "${guard_notes% }" },
+  "vault_sync": { "age_minutes": $sync_age_min, "clean": $sync_clean },
+  "repos": $repo_rows
+}
+EOF
+mv "$TMP" "$OUT"

ops/install.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# install.sh — deploy the versioned ops scripts to ~/.amico/ops/ on the mini.
+# Idempotent; copies scripts ONLY (never launchd plists, never runtime state,
+# never the papers-digest frozen bundle — see ops/README.md for those).
+set -euo pipefail
+
+DEST="$HOME/.amico/ops"
+SRC="$(cd "$(dirname "$0")" && pwd)"
+
+mkdir -p "$DEST/papers-digest"
+
+install -m 0755 "$SRC/fleet-status.sh"          "$DEST/fleet-status.sh"
+install -m 0755 "$SRC/fleet-alert.sh"           "$DEST/fleet-alert.sh"
+install -m 0755 "$SRC/papers-digest/daily.sh"   "$DEST/papers-digest/daily.sh"
+
+echo "deployed to $DEST:"
+echo "  fleet-status.sh   (launchd co.harmoniqs.fleet-status, every 5 min)"
+echo "  fleet-alert.sh    (launchd co.harmoniqs.fleet-alert, every 15 min)"
+echo "  papers-digest/daily.sh (launchd co.harmoniqs.amicode-papers-digest, daily ~09:00)"
+echo "state files, plists, and the frozen bundle were left untouched."
+echo "to activate before the next interval: launchctl kickstart -k gui/$(id -u)/<agent-label>"

ops/launchd/co.harmoniqs.amicode-papers-digest.plist

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0"><dict>
+  <key>Label</key><string>co.harmoniqs.amicode-papers-digest</string>
+  <key>ProgramArguments</key><array>
+    <string>/bin/bash</string><string>/Users/aaron/.amico/ops/papers-digest/daily.sh</string>
+  </array>
+  <key>StartCalendarInterval</key><dict>
+    <key>Hour</key><integer>9</integer>
+    <key>Minute</key><integer>0</integer>
+  </dict>
+  <key>StandardOutPath</key><string>/Users/aaron/.amico/ops/papers-digest/launchd.out</string>
+  <key>StandardErrorPath</key><string>/Users/aaron/.amico/ops/papers-digest/launchd.err</string>
+</dict></plist>

ops/launchd/co.harmoniqs.fleet-alert.plist

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0"><dict>
+  <key>Label</key><string>co.harmoniqs.fleet-alert</string>
+  <key>ProgramArguments</key><array><string>/bin/bash</string><string>/Users/aaron/.amico/ops/fleet-alert.sh</string></array>
+  <key>StartInterval</key><integer>900</integer>
+  <key>StandardOutPath</key><string>/Users/aaron/.amico/ops/fleet-alert.launchd.out</string>
+  <key>StandardErrorPath</key><string>/Users/aaron/.amico/ops/fleet-alert.launchd.err</string>
+</dict></plist>

ops/launchd/co.harmoniqs.fleet-status.plist

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>co.harmoniqs.fleet-status</string>
+    <key>ProgramArguments</key>
+    <array>
+        <string>/bin/bash</string>
+        <string>/Users/aaron/.amico/ops/fleet-status.sh</string>
+    </array>
+    <key>StartInterval</key>
+    <integer>300</integer>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>StandardOutPath</key>
+    <string>/tmp/fleet-status.log</string>
+    <key>StandardErrorPath</key>
+    <string>/tmp/fleet-status.log</string>
+</dict>
+</plist>

ops/papers-digest/daily.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# papers-digest daily — the intelligent arXiv digest to #papers (#412).
+# Runs the FROZEN bundle (never the repo checkout — branches move); upgrade =
+# copy a new dist/amico.js over bin/ + refresh the sidecar (the server pattern).
+set -euo pipefail
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"
+BIN="$HOME/.amico/ops/papers-digest/bin/amico.js"
+LOG="$HOME/.amico/ops/papers-digest/log.txt"
+echo "[$(date -u +%FT%TZ)] digest run" >> "$LOG"
+node "$BIN" papers digest --feed quant-ph --top 5 --post papers >> "$LOG" 2>&1 || echo "[$(date -u +%FT%TZ)] digest FAILED (see above)" >> "$LOG"