feat(mail): M365 Graph sync + bot ETL loop (OO+Gmail)

.gitignore

@@ -14,3 +14,9 @@ go.work.local
 models/
 # Purged from git history. Do not re-add.
 docs/crm-associations-proof.md
+
+# mount scaffold for the 8TB volume, never part of the repo
+mnt/
+
+# go build ./bin/mail/sync.go drops a binary named `sync` in cwd
+/sync

AGENTS.md

@@ -66,12 +66,20 @@ var/          kb.lbug, var/mail/*, caches (gitignored)
 ```bash
 bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail   # raw message.json + attachments
 bin/mail/sync.go --source gmail --query 'from:example.com' --out var/mail  # Gmail search (default in:inbox)
+bin/mail/sync.go --source m365 --env ~/.config/brain/mail.env --out var/mail  # Microsoft Graph (delta)
 bin/mail/import.go --from-raw var/mail                                  # message.json → message.md (convert only)
 bin/brain/index.go --rebuild --with-facts --with-chats
+bin/stack/start-mail-sync                                               # compose ETL: sync→import every 300s
 ```
 
 - `sync` (Go) downloads messages + attachments; Gmail uses paginated list +
-  `body.attachmentId` (not partId) for attachments.
+  `body.attachmentId` (not partId) for attachments. Sources: `onlyoffice`,
+  `gmail`, `m365` (client-credentials + delta link; commit after success).
+- Compose `mail-sync` / `bin/stack/start-mail-sync`: ETL loop (default
+  `onlyoffice,gmail`, 300s). On `new>0` runs import; full `--rebuild` only if
+  `MAIL_SYNC_INDEX=1`. Secrets: `~/.config/brain/mail.env` + `~/.gmail-mcp`.
+  Case wrappers (e.g. family `gmail-sync-la-quinta.sh`) and ai-bot
+  `gmail-reauth.sh` reuse this sync/OAuth — do not fork corpus download.
 - `import` converts body + attachments to markdown. PDFs use poppler
   `pdftotext -layout` fast path (~15ms); textless/scanned PDFs use
   `pdftoppm` + tesseract `eng+deu` (`bin/mail/ocr.go`). Optional

Dockerfile

@@ -6,6 +6,15 @@
 # API: Go + ladybug via Zig CGO (no CPython).
 # Index: Python write path (profile `index` until brain/add is v2).
 
+# --- mail-sync: standalone M365/OnlyOffice/Gmail puller (pure Go, no CGO) ---
+FROM golang:1.26-bookworm AS mail-build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY bin/mail ./bin/mail
+COPY internal ./internal
+RUN CGO_ENABLED=0 go build -o /mail-sync ./bin/mail/sync.go
+
 # --- Python sidecar (Ladybug write / rebuild) ---
 FROM python:3.12-slim AS index
 
@@ -28,6 +37,7 @@ RUN python -m pip install --no-cache-dir -r /tmp/requirements.lock.txt \
 COPY . .
 RUN chmod +x /app/bin/docker-entrypoint \
     && chown -R 2dph:2dph /app
+COPY --from=mail-build /mail-sync /app/bin/mail-sync
 USER 2dph
 
 ENV PATH="/app/bin:${PATH}" \

PLAN.md

@@ -86,7 +86,7 @@ detective method: **a fact needs ≥2 independent sources or it is
     mail/ocr.go             tesseract eng+deu (pdftoppm scans)
     md/import               (deprecated; bin/markdown/import.go)
     brain/extract  brain/audit       brain/deduce    (thinking wrapper)
-    stack/start start-assistant stop status
+    stack/start start-assistant start-mail-sync stop status
     web/search               (deprecated shim → web/search.go)
     db/psql-yq               (vendored)
     ssh-tunnel               onlyoffice pg tunnel 5433
@@ -141,8 +141,9 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
 
 ## Mail pipeline (done)
 
-1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail/OnlyOffice download.
-   Gmail attachments key off `body.attachmentId`, not MIME `partId`.
+1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail / OnlyOffice / M365
+   Graph download. Gmail attachments key off `body.attachmentId`, not MIME
+   `partId`. M365 uses client-credentials + delta link (commit after success).
 2. `bin/mail/import.go --from-raw` — message.json → message.md; PDFs via
    `pdftotext -layout` (~15ms); textless/scanned PDFs `pdftoppm` + tesseract
    `eng+deu`. ICS sidecars
@@ -151,7 +152,11 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
    corrupts its WAL on bulk-insert into an already-indexed DB. Conversion and
    indexing stay separate for crash safety. `bin/mail/index_mail` is a
    deprecation shim.
-4. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable
+4. Compose `mail-sync` / `bin/stack/start-mail-sync` — ETL loop (default
+   `onlyoffice,gmail`, 300s): sync → import on `new>0`; full rebuild only if
+   `MAIL_SYNC_INDEX=1`. Bot digests (ai-bot) and case wrappers reuse sync/OAuth;
+   they do not replace the corpus path.
+5. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable
    via `bin/brain/search.go`.
 
 ## CI/CD pipeline (D15)

README.md

@@ -119,6 +119,8 @@ Mail is a first-class corpus (retrievable through the same search):
 
 ```bash
 bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail  # raw sync (Go)
+bin/mail/sync.go --source m365 --env ~/.config/brain/mail.env          # Microsoft 365 Graph
+bin/stack/start-mail-sync                                              # compose ETL (300s; no auto-rebuild)
 bin/mail/import.go --from-raw var/mail                                  # JSON → markdown
 bin/brain/add.go --text T --root facts --source "a.md x b.md"
 bin/brain/index.go --rebuild --with-facts --with-chats                  # facts extract + chats md

bin/docker-entrypoint

@@ -5,6 +5,7 @@
 #   serve | search | watch
 # Index image (Python write path, compose profile `index`):
 #   index | extract | audit | search (deprecated python wrapper)
+#   mail-sync [N]   ETL loop: sync -> import; optional index (default 300s)
 #
 # Usage comment starts at line 2 (self-describing convention).
 set -euo pipefail
@@ -34,5 +35,27 @@ case "$CMD" in
 	serve) exec /app/bin/serve "$@" ;;
 	extract) exec "$KB_PY" /app/bin/facts/extract "$@" ;;
 	audit) exec "$KB_PY" /app/bin/facts/audit "$@" ;;
+	mail-sync)
+		# ETL: pull mail, convert to md when new>0. Full --rebuild is opt-in
+		# (MAIL_SYNC_INDEX=1) — ~29k leaf rebuild is minutes, not a 10s loop.
+		interval="${1:-300}"
+		[ "$interval" -gt 0 ] 2>/dev/null || interval=300
+		: "${MAIL_SYNC_ENV:=/secret/mail.env}"
+		: "${MAIL_SYNC_SRC:=onlyoffice,gmail}"
+		: "${MAIL_SYNC_OUT:=/app/var/mail}"
+		: "${MAIL_SYNC_INDEX:=0}"
+		while true; do
+			out="$("/app/bin/mail-sync" --source "$MAIL_SYNC_SRC" --env "$MAIL_SYNC_ENV" --out "$MAIL_SYNC_OUT" 2>&1)" || true
+			echo "$out"
+			new="$(printf '%s\n' "$out" | sed -n 's/.*new=\([0-9]*\).*/\1/p' | tail -1)"
+			if [ -n "$new" ] && [ "$new" -gt 0 ] 2>/dev/null; then
+				"$KB_PY" /app/bin/mail/import --from-raw "$MAIL_SYNC_OUT" 2>&1 | tail -1
+				if [ "$MAIL_SYNC_INDEX" = "1" ]; then
+					"$KB_PY" /app/bin/kb/index --rebuild --with-mail 2>&1 | tail -1
+				fi
+			fi
+			sleep "$interval"
+		done
+		;;
 	*) echo "unknown command: $CMD" >&2; exit 2 ;;
 esac

bin/mail/sync.go

@@ -1,8 +1,9 @@
 //usr/bin/env go run "$0" "$@"; exit
-// bin/mail/sync.go - async download of OnlyOffice and Gmail mail to var/mail/.
+// bin/mail/sync.go - async download of OnlyOffice, Gmail and M365 mail to var/mail/.
 //
-//	./bin/mail/sync.go --source onlyoffice,gmail --limit 50 --workers 8
+//	./bin/mail/sync.go --source onlyoffice,gmail,m365 --limit 50 --workers 8
 //	./bin/mail/sync.go --source gmail --force
+//	./bin/mail/sync.go --source m365 --env .secrets/m365.env
 //	./bin/mail/sync.go --dry-run
 //
 // Writes raw message.json + attachments under var/mail/<folder>/<id>/; run

bin/mail/sync/cli.go

@@ -55,7 +55,7 @@ func bind(v *flagVals) *flaggy.Parser {
 	p.Bool(&v.force, "", "force", "overwrite existing message.json")
 	p.Bool(&v.dryRun, "", "dry-run", "list counts without writing")
 	p.String(&v.query, "", "query", "Gmail search query")
-	p.String(&v.srcs, "", "source", "comma list: onlyoffice,gmail")
+	p.String(&v.srcs, "", "source", "comma list: onlyoffice,gmail,m365")
 	return p
 }
 
@@ -110,6 +110,24 @@ func ParseCLI(args []string) (CLIConfig, int, error) {
 				CredentialsPath: filepath.Join(home, ".gmail-mcp", "credentials.json"),
 				KeysPath:        filepath.Join(home, ".gmail-mcp", "gcp-oauth.keys.json"),
 			}
+		case "m365":
+			tenant := pick(envVars["M365_TENANT"], envVars["MS_TENANT"])
+			cid := pick(envVars["M365_CLIENT_ID"], envVars["MS_CLIENT_ID"])
+			sec := pick(envVars["M365_CLIENT_SECRET"], envVars["MS_CLIENT_SECRET"])
+			users := pick(envVars["M365_USERS"], envVars["MS_USERS"])
+			if tenant == "" || cid == "" || sec == "" || users == "" {
+				return CLIConfig{}, 2, fmt.Errorf("m365 source needs M365_TENANT/CLIENT_ID/CLIENT_SECRET/USERS in %s", v.env)
+			}
+			var userList []string
+			for _, u := range strings.Split(users, ",") {
+				if u = strings.TrimSpace(u); u != "" {
+					userList = append(userList, u)
+				}
+			}
+			if len(userList) == 0 {
+				return CLIConfig{}, 2, fmt.Errorf("m365 source: M365_USERS empty")
+			}
+			cfg.M365 = &M365Credentials{Tenant: tenant, ClientID: cid, ClientSecret: sec, Users: userList}
 		default:
 			return CLIConfig{}, 2, fmt.Errorf("unknown source %q", s)
 		}
@@ -126,7 +144,7 @@ func Main(args []string) int {
 		return code
 	}
 	if cfg.Help {
-		fmt.Fprintln(os.Stderr, "usage: bin/mail/sync.go [--source onlyoffice,gmail] [--query GMAIL_Q] [--limit N] [--offset N] [--workers N] [--force] [--dry-run]")
+		fmt.Fprintln(os.Stderr, "usage: bin/mail/sync.go [--source onlyoffice,gmail,m365] [--query GMAIL_Q] [--limit N] [--offset N] [--workers N] [--force] [--dry-run]")
 		return 0
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
@@ -169,7 +187,8 @@ func readEnv(path string) map[string]string {
 		if !ok {
 			continue
 		}
-		if strings.HasPrefix(k, "ONLYOFFICE_") || strings.HasPrefix(k, "OO_") {
+		if strings.HasPrefix(k, "ONLYOFFICE_") || strings.HasPrefix(k, "OO_") ||
+			strings.HasPrefix(k, "M365_") || strings.HasPrefix(k, "MS_") {
 			out[k] = v
 		}
 	}