Fix search ranking and --hop, make the evidence rule and CI gates actually check something

.dockerignore

@@ -10,5 +10,4 @@ __pycache__
 .cache
 .secrets
 .skills-tmp
-serve/serve
 docs/.build

.github/workflows/ci.yml

@@ -31,26 +31,31 @@ jobs:
         run: |
           bash -n bin/db/psql-yq
           bash -n bin/db/ssh-tunnel
-          bash -n bin/kb-watch
           bash -n bin/docker-entrypoint
 
       - name: Python unit tests (offline, vendored tools)
         run: |
-          uv run python -m unittest discover -s tools -t .
+          uv run python -m unittest discover -s bin/tools -t .
 
-      - name: Go serve tests (async, goroutine-bounded)
+      - name: Go tests (server + watch packages)
         run: |
           go vet ./...
           go test ./... -count=1
-        working-directory: serve
 
-      - name: facts/audit self (lexicon consistency, no network)
-        run: |
-          ./bin/facts/audit self 2>/dev/null || echo "audit: not yet implemented; gate skipped"
+      - name: facts/audit self (repo invariants, no network)
+        run: uv run bin/facts/audit self
 
-      - name: kb/eval recall gate
-        run: |
-          ./bin/kb/eval 2>/dev/null || echo "eval: not yet implemented; gate skipped"
+      - name: Cache the embedding model
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/huggingface
+          key: hf-potion-multilingual-128M
+
+      - name: Build the brain index (repo corpus)
+        run: uv run bin/kb/index --rebuild
+
+      - name: kb/eval recall gate (recall@5 >= 0.95)
+        run: uv run bin/kb/eval
 
   release:
     name: Release (semver)

.gitignore

@@ -8,4 +8,5 @@ __pycache__/
 .DS_Store
 *.env
 .env
-.secrets/
+.secrets/
+lib-ladybug/

AGENTS.md

@@ -10,6 +10,13 @@ Read first: [PLAN](PLAN.md) → [docs](docs/).
 > ≥2 independent sources of evidence, or the finding is `(not confirmed)`.
 > Link the lexicon yaml path that backs each claim.
 
+Independent means **different kind and different origin**, not two strings:
+`runtime | declared | netconfig | doc | vcs | external` × the system of record
+it came from. Two compose files are one kind and prove nothing together. Every
+observation needs a locator (`README.md:89`) so the claim can be re-checked.
+Enforced in `bin/tools/factsrules.py`, re-checked by `bin/facts/audit db` over
+`(Evidence)-[:SUPPORTS]->(Leaf)`.
+
 - `facts` root = assertions backed by ≥2 independent sources (docker ps ×
   compose × ssh-config × docs).
 - `info` root = descriptive/narrative leafs, searchable, never asserted as fact.
@@ -36,23 +43,45 @@ PLAN.md       decisions + execution + open questions
 docs/         published docs
 skills/       in-project agent skills (vendored, no external links)
 bin/          self-describing tools bin/{subject}/{method} (shebang)
-bin/kb-watch  corpus watcher (mtimes, no inotify deps)
+bin/serve.go  async Go HTTP server entry (self-executing go run shebang)
+bin/watch/    corpus watcher Go package (mtimes, no inotify deps)
+bin/server/   async Go HTTP server (goroutines, bounded worker pool)
+bin/mail/     mail pipeline: sync (Go), import (md), index_mail (rebuild)
+bin/tools/    vendored python libs behind bin/* (kblib, yamlout, websearch)
 bin/docker-entrypoint  container entrypoint (brain index|search|serve|watch)
-serve/        async Go HTTP server (goroutines, bounded worker pool)
-tools/        vendored python libs behind bin/* (yamlout, websearch)
 compose.yaml  docker composition (root level, not docker/)
-Dockerfile    multi-stage: python deps + static Go serve
-var/          kb.lbug, caches (gitignored)
+Dockerfile    multi-stage: python deps + static Go binaries
+var/          kb.lbug, var/mail/*, caches (gitignored)
 .venv/        ladybug + model2vec + mistune
 ```
 
+## Mail pipeline
+
+```bash
+bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail   # raw message.json + attachments
+bin/mail/import --from-raw var/mail                                     # message.json → message.md (convert only)
+bin/mail/index_mail                                                     # rebuild brain incl. all mail (fresh DB)
+```
+
+- `sync` (Go) downloads messages + attachments; Gmail uses paginated list +
+  `body.attachmentId` (not partId) for attachments.
+- `import` converts body + attachments to markdown. PDFs use poppler
+  `pdftotext -layout` fast path (~15ms); textless/scanned PDFs fall back to
+  docling (isolated subprocess — its native onnx can segfault the parent).
+  Conversion never touches the brain DB (crash safety).
+- `index_mail` always rebuilds from scratch (repo corpus + mail). Ladybug
+  corrupts its WAL when brand-new leafs are bulk-inserted while FTS/vector
+  indexes exist; a fresh DB with indexes created last is the only safe path.
+  Keep conversion + indexing separate so a conversion crash can't leave the
+  DB mid-transaction.
+
 ## Tools
 
 ```bash
-bin/facts/audit ["self"|"facts"|"info"|"stale"]   # 2-source + staleness gate
+bin/facts/audit ["self"|"db"]                     # repo invariants | evidence over kb.lbug
+bin/facts/crm [--dry-run]                         # proof person↔company/company↔project (ooCRM × corpus SoT)
 bin/kb/search "query" [--hop N] [--repo X]        # deduction search → YAML
-bin/md/tables                                     # what the graph holds → YAML
-bin/brain/deduce "question"                       # thinking wrapper
+bin/kb/stats                                      # what the graph holds → YAML
 ```
 
 Never start a shell command with `cd` — use the tool working-directory

Dockerfile

@@ -14,23 +14,27 @@ COPY requirements.lock.txt /tmp/requirements.lock.txt
 RUN python -m pip install --no-cache-dir -r /tmp/requirements.lock.txt \
     && rm /tmp/requirements.lock.txt
 
-# Go serve: static binary, no interpreter at runtime
-FROM golang:1.25 AS serve-build
-WORKDIR /src/serve
-COPY serve/go.mod serve/go.sum* ./
-COPY serve .
-RUN CGO_ENABLED=0 go build -o /serve -ldflags="-s -w" .
+# Go services: static binaries, no interpreter at runtime
+FROM golang:1.25 AS go-build
+WORKDIR /src
+COPY go.mod ./
+COPY bin/server ./bin/server
+COPY bin/watch ./bin/watch
+RUN CGO_ENABLED=0 go build -o /serve ./bin/server \
+    && CGO_ENABLED=0 go build -o /watch ./bin/watch
 
-# runtime: python toolchain + Go server
+# runtime: python toolchain + Go services
 FROM base
 COPY . .
-COPY --from=serve-build /serve /app/serve/serve
-RUN chmod +x /app/bin/kb-watch /app/bin/docker-entrypoint \
+COPY --from=go-build /serve /app/bin/serve
+COPY --from=go-build /watch /app/bin/watch
+RUN chmod +x /app/bin/docker-entrypoint \
     && chown -R 2dph:2dph /app
 USER 2dph
 
 ENV PATH="/app/bin:${PATH}" \
-    KB_PY=python3
+    KB_PY=python3 \
+    KB_ROOT=/app
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD python -c "import model2vec, ladybug, mistune; print('ok')" || exit 1
 

PLAN.md

@@ -25,13 +25,13 @@ detective method: **a fact needs ≥2 independent sources or it is
 | # | Question | Answer |
 |---|----------|--------|
 | D1 | RAG corpus | ops stack (chat, onlyoffice, gitea/NPM, searchxng, observability, ai-bot, mcp-servers, `~/.ssh/config`) + portfolio. Exclude `office.dev` + jobs/applications. |
-| D2 | skill merging | integrate skills **in this project** `skills/`; skip gitea / brain-detective-depe ndent skills. |
+| D2 | skill merging | integrate skills **in this project** `skills/`; skip gitea / brain-dependent skills. |
 | D3 | web search | import `web-search`, retire local `searxng-ops`. Vendored here, no remote link. |
 | D4 | embeddings | **model2vec** `minishlab/potion-multilingual-128M` instead of embeddinggemma. |
 | D5 | parser | **mistune** for MD → leaf extraction (duckdb-md documented as future optional SQL/export layer, not v1). |
 | D6 | graph engine | **LadybugDB** (Kuzu successor, MIT, embedded, native FTS+vector+Cypher). Python binding for `bin/*`; Go shebang for golang tools. |
 | D7 | db access | `db-yaml`/`psql-yq`-style, read-only, YAML out. OnlyOffice Postgres via SSH tunnel (`127.0.0.1:5433`). |
-| D8 | evidence | detective method: ≥2 independent sources or `(not confirmed)`. Auto-pair docker ps × compose × ssh-config × docs. |
+| D8 | evidence | detective method: ≥2 independent sources or `(not confirmed)`. Auto-pair docker ps × compose × ssh-config × docs. Independence = different `kind` **and** different `origin`, each with a locator; stored as `(Evidence)-[:SUPPORTS]->(Leaf)`, never as a substring of `Leaf.source`. |
 | D9 | facts/goal model | Who / What / How / Where / When + evidence + confidence on every edge. |
 | D10 | versioning | everything is a leaf with `sha256 + observed_at + source_rev`; `File-[:HAS_VERSION]->Commit-[:AUTHORED]->Person`. Stale = `source_rev` < git HEAD. |
 | D11 | strong/weak | `root` column: `facts` (strong) vs `info` (weak). Answer is `confirmed` only from facts root. |
@@ -50,12 +50,12 @@ detective method: **a fact needs ≥2 independent sources or it is
   skills/                   in-project skills (web-search, db-yaml, kb-search, agent-cost, diataxis-docs, …)
   bin/
     facts/extract           auto-pair 2 sources → lexicon yaml + graph
-    facts/audit             ["self"|"facts"|"info"|"stale"] 2-source + staleness gate
+    facts/audit             ["self"|"db"] repo invariants | evidence over kb.lbug
     kb/index                build FTS + HNSW from corpus
     kb/search               deduction: facts → info → web-search; --hop N
     kb/get  kb/stats  kb/eval
-    md/import  md/select  md/tables  md/gaps     (mistune)
-    brain/extract  brain/audit   brain/deduce    (thinking wrapper)
+    md/import                                   (mistune; select/tables/gaps planned)
+                                                (brain/* thinking wrappers planned)
     web/search               (vendored)
     db/psql-yq               (vendored)
     ssh-tunnel               onlyoffice pg tunnel 5433
@@ -93,18 +93,50 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
 
 - OQ1: mutually-contradicting evidence — how to resolve (authority weighting,
   temporal freshness, audit adjudication).
-- OQ2: OCR pipeline for pdfs/images/docs (late phase).
+- OQ2: OCR pipeline for pdfs/images/docs — mostly solved: poppler pdftotext
+  fast-path for born-digital PDFs, docling fallback for the ~5% textless ones.
 - OQ3: optional duckdb-md layer for `SELECT … FORMAT MARKDOWN` export/write-back.
+- OQ4: YAML-first storage for leafs — deferred: JSON is ~10x faster to
+  serialize and unambiguous; YAML only where humans edit files.
+- OQ6: re-verification of evidence. `Evidence.value_hash` exists in the schema
+  but is not filled: nothing re-runs a locator to check the observation still
+  says what it said. Until it does, `confirmed` means "was true when observed",
+  and staleness cannot be detected — which is what the planned `audit stale`
+  mode needs. Wants `audit db --recheck`: re-read each locator, compare the
+  hash, demote facts whose evidence moved to `hypothesis`.
+- OQ5: gate the Go search path in CI. `bin/kbsearch` is a nested module that
+  needs the native ladybug library (`lib-ladybug/`, gitignored), so neither
+  `go test ./...` nor `bin/kb/eval` covers it; `kb/eval` measures the python
+  BM25 path in kblib. Ranking logic is unit-tested in
+  bin/kbsearch/rank_test.go, but that file only compiles where the library
+  is present. Needs a CI step that fetches/builds ladybug.
+
+## Mail pipeline (done)
+
+1. `bin/mail/sync.go` (Go, 8 workers) — paginated Gmail/OnlyOffice download.
+   Gmail attachments key off `body.attachmentId`, not MIME `partId`.
+2. `bin/mail/import --from-raw` — message.json → message.md; PDFs via
+   `pdftotext -layout` (~15ms) with docling subprocess fallback; ICS sidecars
+   Latin-1→UTF-8 normalized.
+3. `bin/mail/index_mail` — fresh rebuild (repo corpus + mail) because ladybug
+   corrupts its WAL on bulk-insert into an already-indexed DB. Conversion and
+   indexing stay separate for crash safety.
+4. Result: 17,835 messages → 28,918 info leafs, FTS + HNSW healthy, searchable
+   via `bin/kb/search`.
 
 ## CI/CD pipeline (D15)
 
 `.github/workflows/ci.yml`:
 
-1. go vet + go test ./... (Go tools)
+1. go vet + go test ./... (Go tools; excludes the nested kbsearch module, OQ5)
 2. python -m unittest discover + pytest (Py tools)
-3. bin/facts/audit self  (lexicon internal consistency)
-4. bin/kb/eval            (recall@5 ≥ 0.95, gates index regressions)
-5. md-docs build/lint if docs tooling arrives.
+3. bin/facts/audit self  (evidence rule vs fixture, tool convention, doc/mode match)
+4. bin/kb/index --rebuild (repo corpus; HF model from actions/cache)
+5. bin/kb/eval            (recall@5 ≥ 0.95, gates index regressions)
+6. md-docs build/lint if docs tooling arrives.
+
+Every gate is fail-closed: no step swallows its exit code. A gate that cannot
+be run is removed or tracked as an open question, never faked with `|| true`.
 
 Feedback loop: every commit → PR → CI → green/gate → merge. Same discipline as
 `db/tech-poc`: contract first where there is an OpenAPI/message shape.

README.md

@@ -29,7 +29,7 @@ graph TB
 
     subgraph dph["2dph tools"]
         EX["bin/facts/extract<br/>2-source pairing"]
-        AU["bin/facts/audit<br/>confidence + staleness"]
+        AU["bin/facts/audit<br/>2-source + convention gate"]
         IDX["bin/kb/index<br/>chunk + embed"]
         MD["bin/md/import<br/>mistune leaves"]
         SR["bin/kb/search<br/>deduction + --hop"]
@@ -77,9 +77,14 @@ Every assertion is `Who / What / How / Where / When + evidence + confidence`,
 mirroring the detective detective skill: **≥2 independent sources confirm a
 fact; conflicting sources or a single source → `hypothesis` → `(not confirmed)`.**
 
+Independence is checked, not assumed: two observations count only if they differ
+in **kind** (runtime / declared / netconfig / doc / vcs / external) *and* in
+**origin** (which system of record produced them), and each must carry a locator
+you can go back and look at. Two compose files are not two sources.
+
 | root | meaning | used for answers |
 |------|---------|------------------|
-| `facts` | assertions backed by ≥2 sources (`confirmed`) | yes, with evidence links |
+| `facts` | assertions backed by ≥2 independent observations (`confirmed`) | yes, with evidence links |
 | `info` | descriptive/narrative leafs (how-tos, notes) | context only, marked `(not confirmed)` |
 
 ## Deduction search
@@ -93,6 +98,15 @@ bin/kb/stats                                      # index health
 bin/kb/eval                                       # recall@5 gate
 ```
 
+Mail is a first-class corpus (retrievable through the same search):
+
+```bash
+bin/mail/sync.go --source onlyoffice,gmail --workers 8 --out var/mail  # raw sync (Go)
+bin/mail/import --from-raw var/mail                                     # JSON → markdown
+bin/mail/index_mail                                                     # rebuild brain incl. mail
+bin/kb/search "Mietwagen Nürnberg invoice"                              # now answers from mail
+```
+
 ## Storage
 
 - **LadybugDB** — single `var/kb.lbug`, Cypher property graph, HNSW + BM25
@@ -115,8 +129,8 @@ touches network/db is read-only, throttled, cached. Tests gate every commit.
 ```bash
 uv venv .venv                                  # Python 3.12, uv-managed
 uv pip install -r requirements.lock.txt        # pinned toolchain
-bin/facts/audit self                           # lexicon consistency gate
-go test ./... && python -m unittest discover -s tools -t .
+bin/facts/audit self                           # evidence rule + tool convention gate
+go test ./... && python -m unittest discover -s bin/tools -t .
 ```
 
 Docker (optional, cached model + var volumes):
@@ -134,6 +148,6 @@ docker compose up brain-watch                  # auto re-index on change
   Neo4j + Qdrant + Matrix RAG brain
 - [agent-skills](https://github.com/eSlider/agent-skills) — upstream
   skills (`web-search`, `db-yaml`, …) that 2dph integrates
-- [detective](https://github.com/detective) — the two-source method
+- detective method — the two-source method
 
 See [PLAN.md](PLAN.md) for decisions, execution status, and v2 open questions.

bin/ci/semver

@@ -17,7 +17,7 @@ import subprocess
 import sys
 from pathlib import Path
 
-sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools"))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
 
 from semver import bump_type, bump_version  # noqa: E402
 

bin/db/ssh-tunnel

@@ -34,11 +34,13 @@ case "${1:-}" in
     ;;
   "")
     [ -f "$HOME/.ssh/config" ] || { echo "db/ssh-tunnel: ~/.ssh/config missing" >&2; exit 1; }
-    if db/ssh-tunnel --check; then
+    if "$0" --check; then
       echo "tunnel already up on ${SRC}"
       exit 0
     fi
     ssh -f -N -M -S "$HOME/.ssh/2dph-tunnel.sock" \
+      -o StrictHostKeyChecking=accept-new \
+      -o BatchMode=yes \
       -L "${SRC}:${DST}" -p "$SSH_PORT" "${SSH_USER}@${SSH_HOST}" \
       && echo "tunnel up on ${SRC} (-> vm:${DST})"
     exit 0

bin/docker-entrypoint

@@ -4,8 +4,8 @@
 #   brain                 shell (default)
 #   brain search <q>      bin/kb/search
 #   brain index           bin/kb/index
-#   brain watch <dir>     watchdog re-indexer
-#   brain serve           async Go HTTP server (serve/)
+#   brain watch <dir>     watchdog re-indexer (bin/kb/watch)
+#   brain serve           async Go HTTP server (bin/serve)
 #
 # Usage comment starts at line 2 (self-describing convention).
 set -euo pipefail
@@ -17,7 +17,7 @@ case "$CMD" in
   shell) exec bash ;;
   search) exec "$KB_PY" /app/bin/kb/search "$@" ;;
   index) exec "$KB_PY" /app/bin/kb/index "$@" ;;
-  watch) exec bash /app/bin/kb-watch "$@" ;;
-  serve) exec /app/serve/serve "$@" ;;
+  watch) exec /app/bin/watch "$@" ;;
+  serve) exec /app/bin/serve "$@" ;;
   *) echo "unknown command: $CMD" >&2; exit 2 ;;
 esac