SPIKE: SPDD leg 3 entity projection (structured markdown → __Entity__)

.env.example

@@ -20,9 +20,6 @@ GUIDE_PROFILE=user
 # NEO4J_PASSWORD=brahmsian
 # NEO4J_URI=bolt://localhost:7687
 
-# Discord bot token (optional — only needed for Discord integration)
-# DISCORD_TOKEN=your-discord-token
-
 # Encryption key for BYOK API keys cached client-side.
 # Generate with: openssl rand -base64 32
 EMBABEL_KEY_SECRET=

.github/workflows/export-seed.yml

@@ -57,7 +57,7 @@ jobs:
           echo "model.onnx: $(du -h "$MODEL_DIR/model.onnx" | cut -f1)"
           echo "tokenizer.json: $(du -h "$MODEL_DIR/tokenizer.json" | cut -f1)"
 
-      - name: Start guide with auto-seed
+      - name: Start guide
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ANTHROPIC_API_KEY: dummy-key
@@ -66,30 +66,51 @@ jobs:
             -DNEO4J_URI=bolt://localhost:7687 \
             -DNEO4J_USERNAME=neo4j \
             -DNEO4J_PASSWORD=testpassword \
-            -Dguide.reload-content-on-startup=true \
             -jar target/*.jar > /tmp/guide.log 2>&1 &
           echo "GUIDE_PID=$!" >> $GITHUB_ENV
 
-      - name: Wait for guide to be ready (ingestion completes in constructor)
+      - name: Wait for guide to be ready
         run: |
-          echo "Waiting for guide to start and finish ingestion..."
-          for i in $(seq 1 120); do
+          echo "Waiting for guide to start..."
+          for i in $(seq 1 60); do
             HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:1337/api/v1/data/stats 2>/dev/null || echo "000")
             if [ "$HTTP_CODE" = "200" ]; then
-              echo "Guide is ready after ~${i}0 seconds (HTTP $HTTP_CODE)"
+              echo "Guide is ready after ~${i}0 seconds"
               exit 0
             fi
-            # Show recent log output for progress
             echo "--- Attempt $i (${i}0s elapsed) HTTP=$HTTP_CODE ---"
             tail -5 /tmp/guide.log 2>/dev/null || true
             sleep 10
           done
-          echo "ERROR: Guide did not become ready within 20 minutes"
-          echo "=== Full guide log ==="
+          echo "ERROR: Guide did not start within 10 minutes"
           cat /tmp/guide.log
           exit 1
+        timeout-minutes: 12
+
+      - name: Trigger ingestion and wait for completion
+        run: |
+          echo "Triggering ingestion via load-references endpoint..."
+          HTTP_CODE=$(curl -s -o /tmp/ingestion-response.txt -w "%{http_code}" \
+            -X POST http://localhost:1337/api/v1/data/load-references)
+          echo "Ingestion completed with HTTP $HTTP_CODE"
+          cat /tmp/ingestion-response.txt
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "ERROR: Ingestion failed"
+            tail -100 /tmp/guide.log
+            exit 1
+          fi
         timeout-minutes: 25
 
+      - name: Remove Persona nodes before export
+        run: |
+          echo "Removing Persona nodes from export (managed by the app, not the seed)..."
+          RESULT=$(curl -sf \
+            -u "neo4j:testpassword" \
+            -H "Content-Type: application/json" \
+            "http://localhost:7474/db/neo4j/tx/commit" \
+            -d '{"statements": [{"statement": "MATCH (p:Persona) DETACH DELETE p RETURN count(p) AS removed"}]}')
+          echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'Removed {d[\"results\"][0][\"data\"][0][\"row\"][0]} Persona nodes')"
+
       - name: Tag all nodes with managedBy
         run: |
           echo "Tagging all nodes with managedBy=embabel..."

.gitignore

@@ -38,6 +38,7 @@ embabel-agent-api/src/main/resources/mcp/**
 # Ignore all personal profile files except the checked-in example
 scripts/user-config/application-*.yml
 !scripts/user-config/application-*.yml.example
+scripts/user-config/ingestion-git-revisions.json
 
 # Temporary files
 *.tmp

Dockerfile

@@ -19,4 +19,9 @@ COPY guide-app.jar app.jar
 
 EXPOSE 1337
 
-ENTRYPOINT ["java", "-jar", "app.jar"]
+ENTRYPOINT ["java", \
+  "-XX:+UseContainerSupport", \
+  "-XX:MaxRAMPercentage=75.0", \
+  "-XX:+UseG1GC", \
+  "-XX:+ParallelRefProcEnabled", \
+  "-jar", "app.jar"]

README.md

@@ -1,22 +1,22 @@
-![Build](https://github.com/embabel/embabel-agent/actions/workflows/maven.yml/badge.svg)
+<a href="https://hub.embabel.com"><img align="left" src="logo.png" width="240"></a>
 
-![Java](https://img.shields.io/badge/java-%23ED8B00.svg?style=for-the-badge&logo=openjdk&logoColor=white)
-![Spring](https://img.shields.io/badge/spring-%236DB33F.svg?style=for-the-badge&logo=spring&logoColor=white)
-![Apache Tomcat](https://img.shields.io/badge/apache%20tomcat-%23F8DC75.svg?style=for-the-badge&logo=apache-tomcat&logoColor=black)
-![Apache Maven](https://img.shields.io/badge/Apache%20Maven-C71A36?style=for-the-badge&logo=Apache%20Maven&logoColor=white)
-![ChatGPT](https://img.shields.io/badge/chatGPT-74aa9c?style=for-the-badge&logo=openai&logoColor=white)
-![JSON](https://img.shields.io/badge/JSON-000?logo=json&logoColor=fff)
-![GitHub Actions](https://img.shields.io/badge/github%20actions-%232671E5.svg?style=for-the-badge&logo=githubactions&logoColor=white)
-![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white)
-![IntelliJ IDEA](https://img.shields.io/badge/IntelliJIDEA-000000.svg?style=for-the-badge&logo=intellij-idea&logoColor=white)
+### **Embabel Guide : Chat and MCP Server**
 
-# Embabel Guide : Chat and MCP Server
+![Build](https://github.com/embabel/embabel-agent/actions/workflows/maven.yml/badge.svg) [![Embabel](https://img.shields.io/badge/Embabel-Agent_Framework-6C3483?style=for-the-badge)](https://github.com/embabel/embabel-agent) ![Java](https://img.shields.io/badge/java-%23ED8B00.svg?style=for-the-badge&logo=openjdk&logoColor=white) ![Spring](https://img.shields.io/badge/spring-%236DB33F.svg?style=for-the-badge&logo=spring&logoColor=white) ![Apache Maven](https://img.shields.io/badge/Apache%20Maven-C71A36?style=for-the-badge&logo=Apache%20Maven&logoColor=white) ![ChatGPT](https://img.shields.io/badge/chatGPT-74aa9c?style=for-the-badge&logo=openai&logoColor=white) ![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white)
 
-<img src="https://github.com/embabel/embabel-agent/blob/main/embabel-agent-api/images/315px-Meister_der_Weltenchronik_001.jpg?raw=true" width="180">
+<br clear="left"/>
 
 Guide exposes resources relating to the Embabel Agent Framework, such
 as documentation, relevant blogs and other content, and up-to-the-minute API information.
 
+This repository is the backend for [Embabel Hub](https://hub.embabel.com) — the Guide powers the
+natural-language "Talk to the Docs" experience there.
+
+## Links
+
+- [Embabel Hub](https://hub.embabel.com) — talk to the docs, powered by this Guide
+- [Embabel Agent Framework](https://github.com/embabel/embabel-agent)
+
 <p align="center">
   <a href="https://www.youtube.com/watch?v=hY6ZFMIJdd4" target="_blank">
     <img src="./guide-demo.png" alt="The Voice, The Word, and The Wheel" width="700">
@@ -45,6 +45,45 @@ To see stats on data, make a GET request or browse to http://localhost:1337/api/
 
 RAG content storage uses the `ChunkingContentElementRepository` interface from the `embabel-agent-rag-core` library. The default backend is Neo4j via `DrivineStore`. You can plug in other backends by providing a different `ChunkingContentElementRepository` bean.
 
+## Switching the Graph Database
+
+`DrivineStore` (from `embabel-agent-rag-graph`) supports three Cypher-speaking backends: **Neo4j** (default), **FalkorDB**, and **Memgraph**. The active backend is selected by Spring profile in `src/main/resources/application.yml` — each profile sets `database.dataSources.neo.type`, and `RagConfiguration` picks the matching `RagDialect` at startup.
+
+| Backend  | Profile    | Default port | Compose profile |
+|----------|------------|--------------|-----------------|
+| Neo4j    | `neo4j`    | `7687` (bolt)| `neo4j`         |
+| FalkorDB | `falkordb` | `6379`       | `falkordb`      |
+| Memgraph | `memgraph` | `7688` (bolt)| `memgraph`      |
+
+### Switching at startup
+
+Change `spring.profiles.active` in `application.yml`, or override at launch:
+
+```bash
+# Neo4j (default)
+./mvnw spring-boot:run -Dspring-boot.run.profiles=neo4j
+
+# FalkorDB
+./mvnw spring-boot:run -Dspring-boot.run.profiles=falkordb
+
+# Memgraph
+./mvnw spring-boot:run -Dspring-boot.run.profiles=memgraph
+```
+
+### Starting the matching container
+
+```bash
+docker compose --profile neo4j   up -d   # or falkordb / memgraph
+```
+
+Memgraph maps host port `7688` → container `7687` to avoid clashing with Neo4j, so you can run them side-by-side. FalkorDB uses Redis protocol on `6379`.
+
+### Browsing data
+
+- **Neo4j**: http://localhost:7474/browser/ (user `neo4j`, password `brahmsian`)
+- **FalkorDB**: http://localhost:3001 (FalkorDB Browser, started with the `falkordb` compose profile)
+- **Memgraph**: connect via [Memgraph Lab](https://memgraph.com/lab) to `bolt://localhost:7688`
+
 ## Viewing and Deleting Data
 
 Go to the Neo Browser at http://localhost:7474/browser/

codegen-gradle/build.gradle.kts

@@ -18,7 +18,7 @@ plugins {
 group = "com.embabel.guide"
 version = "0.1.0-SNAPSHOT"
 
-val drivineVersion = "0.0.28"
+val drivineVersion = "0.0.45"
 
 repositories {
     mavenCentral()
@@ -40,7 +40,6 @@ dependencies {
 
     // Dependencies needed for domain classes to compile
     implementation("com.embabel.agent:embabel-agent-api:0.3.2-SNAPSHOT")
-    implementation("com.embabel.agent:embabel-agent-discord:0.3.2-SNAPSHOT")
     implementation("com.fasterxml.jackson.core:jackson-annotations:2.18.2")
 }
 

compose.yaml

@@ -1,5 +1,7 @@
 services:
   neo4j:
+    profiles:
+      - neo4j
     image: neo4j:${NEO4J_VERSION:-2025.10.1-community-bullseye}
     container_name: embabel-neo4j
     ports:
@@ -55,7 +57,6 @@ services:
       - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j}
       - NEO4J_PASSWORD=${NEO4J_PASSWORD:-brahmsian}
       - OPENAI_API_KEY=${OPENAI_API_KEY}
-      - DISCORD_TOKEN=${DISCORD_TOKEN:-}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
@@ -83,6 +84,46 @@ services:
       - embabel-network
     restart: unless-stopped
 
+  memgraph:
+    profiles:
+      - memgraph
+    image: memgraph/memgraph-mage:latest
+    container_name: embabel-memgraph
+    ports:
+      - "${MEMGRAPH_BOLT_PORT:-7688}:7687"
+    command: ["--also-log-to-stderr", "--log-level=WARNING"]
+    volumes:
+      - memgraph_data:/var/lib/memgraph
+      - memgraph_log:/var/log/memgraph
+    networks:
+      - embabel-network
+
+  falkordb:
+    profiles:
+      - falkordb
+    image: falkordb/falkordb:latest
+    container_name: embabel-falkordb
+    ports:
+      - "${FALKORDB_PORT:-6379}:6379"
+    volumes:
+      - falkordb_data:/data
+    networks:
+      - embabel-network
+
+  falkordb-browser:
+    profiles:
+      - falkordb
+    image: falkordb/falkordb-browser:latest
+    container_name: embabel-falkordb-browser
+    ports:
+      - "${FALKORDB_BROWSER_PORT:-3001}:3000"
+    environment:
+      - FALKORDB_URL=redis://falkordb:6379
+    depends_on:
+      - falkordb
+    networks:
+      - embabel-network
+
 volumes:
   neo4j_data:
     driver: local
@@ -92,6 +133,12 @@ volumes:
     driver: local
   neo4j_plugins:
     driver: local
+  falkordb_data:
+    driver: local
+  memgraph_data:
+    driver: local
+  memgraph_log:
+    driver: local
 
 networks:
   embabel-network:

docs/spdd-branch-changes.md

@@ -0,0 +1,99 @@
+# Branch change summary for Guide developers
+
+Branch: `cursor/spike-spdd-dice-projection-17f4` (tracks upstream `main`).
+Audience: developers who work on Guide and want to understand what this branch adds,
+why, and what the blast radius is.
+
+**One paragraph:** the branch turns Guide into an optional *hybrid context backend* for
+an SDLC workflow: alongside the existing RAG chunk store, an opt-in projection writes
+**typed domain entities** (`__Entity__` nodes: `WorkId`, `Canvas`, `Area`, `Decision`,
+`Pitfall`, `Pattern`) into the same Neo4j and exposes typed-edge retrieval over HTTP and
+MCP. Everything is behind `guide.spdd-projection.enabled` (default **false**); with the
+flag off, runtime behavior matches upstream except for the additions listed under
+"Cross-cutting" below.
+
+## 1. New package `com.embabel.guide.spdd` (all opt-in)
+
+| Class | Role |
+|-------|------|
+| `SpddEntityDictionary` | `DataDictionary` built from `DynamicType`s — the entity schema, also used to validate label parameters on reads |
+| `SpddMarkdownProjectionService` | Parses structured markdown (`spdd/canvas/*.md`, `agent-context/memory/context-index.md`) and persists via `NamedEntityDataRepository.save` + `mergeRelationship` (merge-by-id, idempotent). Read side: `subgraphForWorkId`, `lessonsForArea`, `listByLabel` |
+| `SpddProjectionController` | Operator HTTP under `/api/v1/data/spdd-projection` (`load`, `stats`, `work/{workId}`, `area?name=`) with explicit status mapping (400 validation / 404 not found / 409 disabled) |
+| `SpddDomainTools` | `@LlmTool` methods exported to MCP as `spdd_*` via `McpToolExport.fromToolObject(ToolObject(...).withPrefix("spdd_"))`; failures return `{"error": …}` JSON |
+| `SpddProjectionConfiguration` | Beans: `DrivineNamedEntityDataRepository` wired with the SPDD dictionary + the MCP export. `@ConditionalOnProperty` on the enable flag |
+
+Graph model: `WorkId —canvas→ Canvas`, `WorkId —area→ Area`,
+`WorkId —decision/pitfall/pattern→ lesson`, `lesson —about→ Area`. The `about` edge is
+what makes lessons retrievable **across** work items by code area.
+
+Hardening baked in: per-request `rootPath` overrides must resolve under
+`default-root-path` or configured `allowed-roots` (the load endpoint is on the
+permit-all list, so arbitrary filesystem roots are rejected); a malformed source file is
+skipped and counted (`skippedFiles`) instead of failing the load; list reads accept only
+schema labels and are capped (50 default / 200 max).
+
+Uses only public library APIs (`NamedEntityData`, `NamedEntityDataRepository`,
+`RelationshipDirection`, `DataDictionary`). It does **not** touch the DICE proposition
+extraction pipeline, and no consumer project names are hardcoded — the coupling is to
+the SPDD directory conventions only.
+
+## 2. RAG ingest: git-incremental directories
+
+- `GitIncrementalDirectorySupport` + `GitIngestionRevisionStore` — when
+  `guide.git-ingestion.enabled=true`, directory ingest diffs against the last recorded
+  git revision and reprocesses only changed files. Subdirectory entries (e.g.
+  `repo/spdd/canvas`) are supported: Guide walks up to the `.git` root and scopes the
+  diff.
+- `DataManager` — hooks the incremental path into `loadReferences()`.
+- `RagContentMaintenanceService`, `RagMaintenanceController`,
+  `RagMaintenanceExceptionHandler` — operator endpoints for content-element purge
+  preview/purge and git revision reset.
+
+## 3. Cross-cutting changes (active regardless of the flag)
+
+- **`GuideProperties`** — new `spddProjection` block (`enabled`, `defaultRootPath`,
+  `allowedRoots`) and git-ingestion settings; `application.yml` documents both, defaults
+  off.
+- **`SecurityConfig`** — permit-all additions: POST `…/spdd-projection/load`,
+  `…/git-ingestion/revision/reset`, `…/content-elements/purge{,-preview}`; GET
+  `…/spdd-projection/stats`, `…/work/*`, `…/area`. Same local-operator posture as the
+  existing `…/data/load-references`.
+- **`PersonaSeedingService`** — startup resilience: fails fast with an actionable
+  message if the Drivine KSP query DSL is missing from the classpath, and persona
+  seeding failures no longer abort startup (RAG/MCP stay available).
+- **Build (`pom.xml`)** — `embabel-agent-rag-neo-drivine` pinned to timestamp
+  `0.1.2-20260224.010659-19` (newer snapshots require agent 0.4.0; Guide is on
+  0.3.5-SNAPSHOT); maven-enforcer rule fails the build early when the KSP-generated
+  DSL files are missing instead of dying at runtime.
+- **Scripts** — `append-ingest.sh` gains env-driven Neo4j/profile handling;
+  `run-mcp-guide-against-hub.sh` added; `application-*-spdd-projection.yml.example`
+  profile example under `scripts/user-config/`.
+
+## 4. Tests
+
+- `SpddMarkdownProjectionServiceTest` (16) — projection, idempotent reload, lesson/about
+  edges, root allowlist enforcement, blank/unknown input validation, list caps.
+- `SpddProjectionControllerTest` (8) — standalone MockMvc against the real service +
+  in-memory repository; verifies the 200/400/404 mapping.
+- `SpddDomainToolsTest` (8) — MCP JSON contract, `{"error": …}` on bad input.
+- `GitIncrementalDirectorySupportTest`, `RagMaintenanceControllerWebMvcTest`,
+  `DataManagerControllerWebMvcTest` — incremental ingest + maintenance endpoints.
+- Fixture: `src/test/resources/spdd-fixture/` (minimal canvas + context index).
+
+## 5. How to review / try it
+
+1. `git diff upstream/main...HEAD` — ~30 files; the spdd package and rag ingest classes
+   are the substance.
+2. Operator walkthrough: `docs/spdd-projection-ingest.md` (enable, persist, retrieve,
+   MCP tools).
+3. Quick sanity: enable the flag against a project following the SPDD layout, then
+   `POST …/spdd-projection/load` and `GET …/work/{workId}`.
+
+## 6. Known limitations
+
+- Schema uses `DynamicType` + `SimpleNamedEntityData` (spike speed); a permanent home
+  would promote to typed Kotlin `NamedEntity` classes.
+- Entity→chunk join (`findChunksForEntity`) exists at the store level but is not exposed
+  on the projection API yet.
+- `Operation`, session, and domain-keyword entities are declared in the schema roadmap
+  but not projected yet.