feat: DuckDB quantiles in-process (D22 / #30)

.github/workflows/ci.yml

@@ -45,10 +45,10 @@ jobs:
         run: |
           uv run python -m unittest discover -s bin/tools -t .
 
-      - name: Go tests (root module, no ladybug cgo)
+      - name: Go tests (root module; duckdb-go CGO via gcc, no ladybug)
         run: |
-          go vet ./...
-          go test ./... -count=1
+          CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= go vet ./...
+          CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= go test ./... -count=1
 
       - name: brain ranking tests (no cgo / no ladybug)
         run: go test ./internal/brain/rank -count=1

.gitignore

@@ -12,3 +12,5 @@ __pycache__/
 lib-ladybug/
 go.work.local
 models/
+# Purged from git history. Do not re-add.
+docs/crm-associations-proof.md

AGENTS.md

@@ -48,7 +48,8 @@ bin/postgres/ query.go (read-only YAML)
 bin/git/      import.go (go-git history; Python shim execs it)
 bin/web/      search.go (SearXNG; Python shim execs it)
 bin/reasoner/ bakeoff.go (D18 CPU OpenAI tool-call bake-off)
-internal/     shared Go (brain/rank is cgo-free; chats parsers; gitlog; websearch; reasoner)
+internal/     shared Go (brain/rank is cgo-free; chats parsers; gitlog; websearch; reasoner; duckstats)
+bin/qa/       stats.go (DuckDB quantiles / JSONL count; gcc CGO, not Zig)
 bin/watch/    corpus watcher (used by bin/brain/watch.go)
 bin/tools/    vendored python libs behind bin/* (kblib, yamlout, websearch)
 bin/cgo/      zig zcc zc++ (CGO via zig cc, not gcc)
@@ -101,13 +102,16 @@ bin/git/import.go [REPO] [--json] [--limit N]     # go-git history → commit le
 bin/web/search.go "query" [--json]                # SearXNG; throttled ≠ absence
 bin/reasoner/bakeoff.go [--model ID] [--json]     # D18 CPU tool-call bake-off
 bin/postgres/query.go --profile onlyoffice -c 'SELECT 1'
+bin/qa/stats.go                                  # D22 DuckDB quantiles / JSONL (gcc CGO)
 bin/mail/ocr.go <image|pdf>                      # tesseract eng+deu (scans)
 bin/md/tables                                     # what the graph holds → YAML
 bin/brain/deduce "question"                       # thinking wrapper
 ```
 
 Never start a shell command with `cd` — use the tool working-directory
-parameter. Search before reading whole files.
+parameter. Search before reading whole files. For YAML/JSON/XML/CSV/TOML/HCL
+prefer mikefarah/yq (`skills/yq/SKILL.md`). For bulk rows and quantiles use
+duckdb-go (`internal/duckstats`, `skills/duckdb/SKILL.md`), not Ladybug.
 
 ## GitHub safety rules (ABSOLUTE — never violate)
 

PLAN.md

@@ -48,6 +48,7 @@ detective method: **a fact needs ≥2 independent sources or it is
 | D19 | git history | [go-git](https://github.com/go-git/go-git) via `bin/git/import.go`. No subprocess of the git binary. Conversion prints commit leafs; brain write is `bin/brain/index.go`. |
 | D20 | agent API | OpenAPI + MCP are generated from the same `internal/httpapi.Ops` table as `bin/brain/serve.go` handlers. `GET /openapi.json`, `POST /mcp` (JSON-RPC tools/list + tools/call). Tool names match OpenAPI paths (`search`/`get`/`stats`/`audit`/`ingest`). |
 | D21 | CGO | Ladybug/tokenizers CGO is compiled with **Zig** (`bin/cgo/zcc` → `zig cc -target …-linux-gnu`), not gcc. `bin/cgo/zig` pins Zig 0.14.1 + liblbug 0.19.1 + libtokenizers 1.27.0. Compose `target: api` has no CPython; write/rebuild is profile `index`. |
+| D22 | analytics | **duckdb-go** in-process (`internal/duckstats`, `bin/qa/stats.go`) for quantiles/JSONL. Links with **gcc/g++**, not Zig. Ladybug stays the graph; web-search cache stays modernc sqlite. Slice small structured docs with **mikefarah/yq**, not kislyuk/jq. [#30](https://git.produktor.io/eSlider/2dph/issues/30). |
 
 ## Architecture
 
@@ -110,7 +111,7 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
 
 - `bin/{subject}/{method}` — line 2 is a usage comment (mirrors `psql-yq`).
 - bash + python primary; golang via Go shebang when a compiled helper is right.
-- YAML default output, `--json` for machines. Slice with `yq`.
+- YAML default output, `--json` for machines. Slice with mikefarah/yq.
 - Everything that touches the network / DB is read-only, throttled, cached.
 - Tests (TDD) gate every commit; `gh` + CI/CD on every push.
 
@@ -122,8 +123,8 @@ Common props on every node/edge: `root`, `confidence`, `evidence[]`, `how`,
   `eng+deu` (`bin/mail/ocr.go`, `internal/ocr`). No gocv, no gosseract CGO
   (D21 Zig owns Ladybug CGO). Optional `OCR_ENGINE=paddle` / compose profile
   `ocr-paddle`. Docling left the default path. [#6](https://git.produktor.io/eSlider/2dph/issues/6).
-- OQ3: optional duckdb-md layer for `SELECT … FORMAT MARKDOWN` export/write-back.
-  [#30](https://git.produktor.io/eSlider/2dph/issues/30).
+- OQ3: **in** — duckdb-go (`internal/duckstats`, `bin/qa/stats.go`) for
+  quantiles / JSONL count. Not a second graph. [#30](https://git.produktor.io/eSlider/2dph/issues/30).
 - OQ4: YAML-first storage for leafs — deferred: JSON is ~10x faster to
   serialize and unambiguous; YAML only where humans edit files.
 

bin/qa/stats.go

@@ -0,0 +1,73 @@
+//usr/bin/env go run -tags=qa_stats "$0" "$@"; exit
+//go:build qa_stats
+//
+// bin/qa/stats.go - DuckDB quantiles over a JSON number array or JSONL count.
+//
+//	./bin/qa/stats.go <<< '[1,2,3,4,5]'
+//	./bin/qa/stats.go --jsonl rows.jsonl
+//
+// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
+// DuckDB CGO needs gcc/g++ (not Zig). After eval "$(bin/cgo/zig env)":
+//   CC=gcc CXX=g++ CGO_CFLAGS= CGO_LDFLAGS= ./bin/qa/stats.go
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/eSlider/2dph/internal/duckstats"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:]))
+}
+
+func run(args []string) int {
+	jsonl := ""
+	for i := 0; i < len(args); i++ {
+		a := args[i]
+		switch {
+		case a == "--jsonl" && i+1 < len(args):
+			i++
+			jsonl = args[i]
+		case strings.HasPrefix(a, "--jsonl="):
+			jsonl = strings.TrimPrefix(a, "--jsonl=")
+		case a == "-h" || a == "--help":
+			fmt.Fprintln(os.Stderr, "bin/qa/stats.go [--jsonl FILE]  # stdin = JSON [float,…]")
+			return 0
+		default:
+			fmt.Fprintln(os.Stderr, "unknown arg:", a)
+			return 2
+		}
+	}
+	if jsonl != "" {
+		n, err := duckstats.CountJSONL(jsonl)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return 1
+		}
+		fmt.Printf("n: %d\n", n)
+		return 0
+	}
+	raw, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return 1
+	}
+	var samples []float64
+	if err := json.Unmarshal(raw, &samples); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return 1
+	}
+	s, err := duckstats.Quantiles(samples)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return 1
+	}
+	fmt.Printf("n: %d\nmin: %g\np50: %g\np95: %g\nmax: %g\navg: %g\n",
+		s.N, s.Min, s.P50, s.P95, s.Max, s.Avg)
+	return 0
+}

bin/reasoner/bakeoff.go

@@ -7,7 +7,7 @@
 //	./bin/reasoner/bakeoff.go --model MichelRosselli/bonsai-27b:Q1_0 --json
 //
 // Measures OpenAI tool_calls (search/get/audit) and RSS from Ollama /api/ps, not VRAM.
-// PicoClaw is not in this repo; the tool names match internal/httpapi MCP ops.
+// PicoClaw is compose profile picoclaw; tool names match internal/httpapi MCP ops.
 // NOTE: never run `gofmt -w` on this file — it breaks the shebang.
 package main
 
@@ -17,6 +17,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/eSlider/2dph/internal/duckstats"
 	"github.com/eSlider/2dph/internal/reasoner"
 )
 
@@ -61,6 +62,14 @@ func run(args []string) int {
 	}
 	c := reasoner.Client{BaseURL: base, Model: model, Device: device}
 	rep := reasoner.Run(c)
+	lat := make([]float64, 0, len(rep.Prompts))
+	for _, p := range rep.Prompts {
+		lat = append(lat, float64(p.LatencyMS))
+	}
+	if st, err := duckstats.Quantiles(lat); err == nil {
+		rep.LatencyP50MS = st.P50
+		rep.LatencyP95MS = st.P95
+	}
 	raw, err := json.MarshalIndent(rep, "", "  ")
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -76,6 +85,8 @@ func run(args []string) int {
 		fmt.Printf("xml_leak: %d\n", rep.XMLLeak)
 		fmt.Printf("rss_mb: %d\n", rep.RSSMB)
 		fmt.Printf("vram_mb: %d\n", rep.VRAMMB)
+		fmt.Printf("latency_p50_ms: %g\n", rep.LatencyP50MS)
+		fmt.Printf("latency_p95_ms: %g\n", rep.LatencyP95MS)
 		for _, p := range rep.Prompts {
 			status := "fail"
 			if p.OK {

bin/tools/test_bin_layout.py

@@ -217,6 +217,32 @@ def test_gogit_is_direct_go_mod_require(self) -> None:
             if "go-git/go-git" in line:
                 self.assertNotIn("indirect", line)
 
+    def test_duckdb_go_is_direct_require(self) -> None:
+        text = (ROOT / "go.mod").read_text()
+        first = text.split("require (")[1].split(")")[0]
+        self.assertRegex(first, r"github.com/duckdb/duckdb-go/v2\s+v")
+        for line in first.splitlines():
+            if "duckdb/duckdb-go" in line:
+                self.assertNotIn("indirect", line)
+        skill = (ROOT / "skills" / "duckdb" / "SKILL.md").read_text()
+        self.assertIn("github.com/duckdb/duckdb-go", skill)
+        self.assertIn("Ladybug", skill)
+        self.assertIn("sqlite", skill.lower())
+        self.assertIn("gcc", skill.lower())
+        self.assertIn("Zig", skill)
+        plan = (ROOT / "PLAN.md").read_text()
+        self.assertIn("D22", plan)
+        self.assertIn("duckdb-go", plan)
+        self._assert_shebang("bin/qa/stats.go")
+        reasoner = (ROOT / "internal" / "reasoner" / "client.go").read_text()
+        self.assertNotIn("duckdb", reasoner)
+        self.assertNotIn("duckstats", reasoner)
+        bakeoff = (ROOT / "bin" / "reasoner" / "bakeoff.go").read_text()
+        self.assertIn("internal/duckstats", bakeoff)
+        webcache = (ROOT / "internal" / "websearch" / "cache.go").read_text()
+        self.assertNotIn("duckdb", webcache)
+        self.assertIn("modernc.org/sqlite", webcache)
+
     def test_cgo_uses_zig_not_gcc(self) -> None:
         for rel in ("bin/cgo/zig", "bin/cgo/zcc", "bin/cgo/zc++"):
             p = ROOT / rel

bin/tools/test_published_docs.py

@@ -121,7 +121,7 @@ def test_reasoner_docs_name_real_hf_ids_cpu_sidecar(self) -> None:
         self.assertIn("D18", plan)
         self.assertIn("Qwen/Qwen3.5-9B", plan)
         compose = (ROOT / "compose.yaml").read_text()
-        self.assertIn('profiles: ["reasoner"]', compose)
+        self.assertIn('"reasoner"', compose)
         self.assertIn("OLLAMA_NUM_GPU", compose)
         self.assertIn("127.0.0.1:11435", compose)
         dockerfile = (ROOT / "Dockerfile").read_text()

bin/tools/test_skills.py

@@ -47,3 +47,17 @@ def test_picoclaw_lists_tool_order(self) -> None:
         self.assertIn("throttled", skill.lower())
         self.assertIn("not a negative finding", agents)
         self.assertIn("Fact-check every", agents)
+
+    def test_yq_is_mikefarah_for_structured_data(self) -> None:
+        skill = (ROOT / "skills" / "yq" / "SKILL.md").read_text()
+        self.assertIn("https://github.com/mikefarah/yq", skill)
+        for fmt in ("YAML", "JSON", "XML", "CSV", "TOML", "HCL"):
+            self.assertIn(fmt, skill)
+        self.assertIn("not kislyuk", skill.lower())
+        plan = (ROOT / "PLAN.md").read_text()
+        self.assertIn("mikefarah/yq", plan)
+        agents = (ROOT / "AGENTS.md").read_text()
+        self.assertIn("mikefarah/yq", agents)
+        web = (ROOT / "skills" / "web-search" / "SKILL.md").read_text()
+        self.assertIn("| yq ", web)
+        self.assertNotIn("| jq ", web)

bin/tools/test_system_perf.py

@@ -0,0 +1,36 @@
+"""qa/system_perf.py is an offline-gated system test (no live brain in CI)."""
+from __future__ import annotations
+
+import ast
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class SystemPerfScriptTest(unittest.TestCase):
+    def test_script_compiles_and_is_read_only(self) -> None:
+        path = ROOT / "qa" / "system_perf.py"
+        src = path.read_text()
+        compile(src, str(path), "exec")
+        self.assertIn("--json", src)
+        self.assertIn("qwen3.5:9b", src)
+        self.assertIn("--picoclaw", src)
+        self.assertIn("BRAIN_URL", src)
+        self.assertIn("tools/list", src)
+        self.assertIn("tools/call", src)
+        self.assertIn("GATE_HEALTH_MS", src)
+        self.assertIn("GATE_GET_P50_MS", src)
+        self.assertNotIn("kb.lbug", src)
+        self.assertNotIn("password", src.lower())
+        self.assertNotIn("token", src.lower())
+
+    def test_script_does_not_write_ladybug(self) -> None:
+        tree = ast.parse((ROOT / "qa" / "system_perf.py").read_text())
+        writes = [
+            n.func.attr
+            for n in ast.walk(tree)
+            if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
+            and n.func.attr in {"write_text", "write_bytes", "dump"}
+        ]
+        self.assertEqual(writes, [], f"system_perf must not write files: {writes}")

compose.yaml

@@ -2,7 +2,7 @@
 #
 #   docker compose up -d brain                  # API (Zig CGO serve)
 #   docker compose --profile index run --rm index   # Python rebuild
-#   docker compose --profile picoclaw up brain-mcp
+#   docker compose --profile picoclaw up -d     # brain-mcp + CPU reasoner + PicoClaw gateway
 #   docker compose --profile reasoner up -d reasoner   # CPU Ollama :11435
 #   docker compose --profile searxng up -d
 #   OCR_ENGINE=paddle docker compose --profile ocr-paddle run --rm ocr-paddle
@@ -11,6 +11,14 @@
 
 name: 2dph
 
+networks:
+  default:
+    name: 2dph_sys
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.23.42.0/24
+
 services:
   brain:
     image: ghcr.io/eslider/2dph:api
@@ -98,8 +106,8 @@ services:
       - ./deploy/searxng/limiter.toml:/etc/searxng/limiter.toml:ro
     restart: unless-stopped
 
-  # MCP endpoint for an external agent (PicoClaw is not shipped here).
-  #   docker compose --profile picoclaw up brain-mcp
+  # MCP endpoint for PicoClaw (and any MCP client).
+  #   docker compose --profile picoclaw up -d
   brain-mcp:
     profiles: ["picoclaw"]
     image: ghcr.io/eslider/2dph:api
@@ -121,7 +129,7 @@ services:
   #   docker compose --profile reasoner up -d reasoner
   #   docker compose --profile reasoner exec reasoner ollama pull qwen3.5:9b
   reasoner:
-    profiles: ["reasoner"]
+    profiles: ["reasoner", "picoclaw"]
     image: docker.io/ollama/ollama:latest
     environment:
       OLLAMA_NUM_GPU: "0"
@@ -132,6 +140,26 @@ services:
       - reasoner-ollama:/root/.ollama
     restart: unless-stopped
 
+  # Official PicoClaw gateway. Config has no secrets (Ollama + HTTP MCP).
+  # Host network: brain/reasoner bind 127.0.0.1 only, so host.docker.internal
+  # (docker0) cannot reach them. Gateway 127.0.0.1:18790 (not the 18800 launcher).
+  # If :8630/:11435 are already bound, do not start brain-mcp/reasoner:
+  #   docker compose --profile picoclaw up -d --no-deps picoclaw
+  picoclaw:
+    profiles: ["picoclaw"]
+    image: docker.io/sipeed/picoclaw:v0.3.1
+    network_mode: host
+    depends_on:
+      - brain-mcp
+      - reasoner
+    environment:
+      PICOCLAW_GATEWAY_HOST: "127.0.0.1"
+    entrypoint: ["picoclaw", "gateway"]
+    volumes:
+      - picoclaw-home:/root/.picoclaw
+      - ./deploy/picoclaw/config.json:/root/.picoclaw/config.json:ro
+    restart: unless-stopped
+
   # Optional PP-OCRv5 (not default). Default OCR is tesseract eng+deu.
   #   OCR_ENGINE=paddle docker compose --profile ocr-paddle run --rm ocr-paddle
   ocr-paddle:
@@ -145,3 +173,4 @@ volumes:
   kb-model:
   kb-var:
   reasoner-ollama:
+  picoclaw-home:

deploy/picoclaw/config.json

@@ -0,0 +1,33 @@
+{
+  "agents": {
+    "defaults": {
+      "model_name": "qwen3.5-9b",
+      "max_tool_iterations": 8,
+      "max_tokens": 512,
+      "context_window": 8192
+    }
+  },
+  "model_list": [
+    {
+      "model_name": "qwen3.5-9b",
+      "model": "ollama/qwen3.5:9b",
+      "api_base": "http://127.0.0.1:11435/v1",
+      "request_timeout": 600
+    }
+  ],
+  "tools": {
+    "web": {
+      "enabled": false
+    },
+    "mcp": {
+      "enabled": true,
+      "servers": {
+        "2dph": {
+          "enabled": true,
+          "type": "http",
+          "url": "http://127.0.0.1:8630/mcp"
+        }
+      }
+    }
+  }
+}