From dccbe6d50fa44934ad68a853054fe9d288f9c40c Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Wed, 10 Jun 2026 21:23:43 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Implement graphify Docker graph-report smoke","authority":"XY-889"} --- Makefile.toml | 9 + .../memory_projects_manifest.json | 70 +- .../tests/real_world_job_benchmark.rs | 57 +- .../research/comparison_external_projects.md | 4 +- .../research/research_projects_inventory.md | 4 +- scripts/graphify-docker-graph-report-smoke.py | 1317 +++++++++++++++++ scripts/real-world-live-adapters.sh | 29 + 7 files changed, 1449 insertions(+), 41 deletions(-) create mode 100755 scripts/graphify-docker-graph-report-smoke.py diff --git a/Makefile.toml b/Makefile.toml index e4ffcdc9..8348b19f 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -825,6 +825,7 @@ args = [ # | lightrag-docker-context-smoke | command | | # | graphrag-docker-smoke | command | | # | graphiti-zep-docker-temporal-smoke | command | | +# | graphify-docker-graph-report-smoke | command | | [tasks.ragflow-docker-smoke] workspace = false @@ -857,6 +858,14 @@ args = [ "set -euo pipefail; start=\"$(printenv ELF_GRAPHITI_ZEP_SMOKE_START || true)\"; status=0; if [ \"$start\" = \"1\" ]; then docker compose -f docker-compose.baseline.yml --profile graphiti-zep up -d graphiti-falkordb; fi; docker compose -f docker-compose.baseline.yml run --build --rm -e ELF_GRAPHITI_ZEP_SMOKE_RUN -e ELF_GRAPHITI_ZEP_SMOKE_REPORT_DIR -e ELF_GRAPHITI_ZEP_SMOKE_WORK_DIR -e ELF_GRAPHITI_ZEP_SMOKE_INSTALL -e ELF_GRAPHITI_ZEP_VERSION -e ELF_GRAPHITI_ZEP_PACKAGE -e ELF_GRAPHITI_ZEP_REF -e ELF_GRAPHITI_ZEP_API_BASE -e ELF_GRAPHITI_ZEP_API_KEY -e ELF_GRAPHITI_ZEP_LLM_MODEL -e ELF_GRAPHITI_ZEP_EMBEDDING_MODEL -e ELF_GRAPHITI_ZEP_FALKORDB_HOST -e ELF_GRAPHITI_ZEP_FALKORDB_PORT -e ELF_GRAPHITI_ZEP_FALKORDB_DATABASE -e ELF_GRAPHITI_ZEP_TIMEOUT_SECONDS -e ELF_GRAPHITI_ZEP_STARTUP_ATTEMPTS -e ELF_GRAPHITI_ZEP_STARTUP_INTERVAL_SECONDS baseline-runner python3 scripts/graphiti-zep-docker-temporal-smoke.py || status=$?; if [ \"$start\" = \"1\" ]; then docker compose -f docker-compose.baseline.yml --profile graphiti-zep stop graphiti-falkordb >/dev/null 2>&1 || true; fi; exit \"$status\"", ] +[tasks.graphify-docker-graph-report-smoke] +workspace = false +command = "bash" +args = [ + "-lc", + "set -euo pipefail; docker compose -f docker-compose.baseline.yml run --build --rm -e ELF_GRAPHIFY_SMOKE_RUN -e ELF_GRAPHIFY_SMOKE_REPORT_DIR -e ELF_GRAPHIFY_SMOKE_WORK_DIR -e ELF_GRAPHIFY_SMOKE_INSTALL -e ELF_GRAPHIFY_PACKAGE -e ELF_GRAPHIFY_REF -e ELF_GRAPHIFY_TIMEOUT_SECONDS -e ELF_GRAPHIFY_QUERY_BUDGET baseline-runner python3 scripts/graphify-docker-graph-report-smoke.py", +] + [tasks.real-world-memory-knowledge] workspace = false dependencies = [ diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index af688749..6dbe0c0b 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -1943,46 +1943,61 @@ "evidence_class": "research_gate", "docker_default": true, "host_global_installs_required": false, - "overall_status": "not_encoded", + "overall_status": "blocked", "setup": { - "status": "not_encoded", - "evidence": "XY-882 marks graphify as an adapter_candidate for a Docker-only CLI/materializer path, but no adapter is implemented." + "status": "blocked", + "evidence": "XY-889 adds a Docker-only graph/report smoke command. The checked-in manifest remains a research gate until a generated artifact reaches graphify graph/report output.", + "command": "cargo make graphify-docker-graph-report-smoke", + "artifact": "tmp/real-world-memory/graphify-smoke/graphify-smoke.json" }, "run": { - "status": "not_encoded", - "evidence": "No graphify graph/report build is encoded." + "status": "blocked", + "evidence": "The smoke installs graphify in a container-local venv, runs over a generated public corpus, and records typed setup/runtime failure if graph/report build or query output is unavailable.", + "command": "cargo make graphify-docker-graph-report-smoke", + "artifact": "tmp/real-world-memory/graphify-smoke/summary.json" }, "result": { - "status": "not_encoded", - "evidence": "No graph-navigation or knowledge-compilation result is claimed." + "status": "blocked", + "evidence": "No graph-navigation or knowledge-compilation quality result is claimed from the checked-in research gate. Generated smoke artifacts may become live_real_world only after graph.json, GRAPH_REPORT.md, and graphify query output map to generated evidence ids.", + "artifact": "tmp/real-world-memory/graphify-smoke/graphify-smoke.json" }, "capabilities": [ + { + "capability": "docker_cli_boundary", + "status": "blocked", + "evidence": "The smoke uses docker-compose.baseline.yml baseline-runner, a container-local Python venv, and isolated assistant config paths; it does not install host-global assistant hooks." + }, { "capability": "graph_report_generation", - "status": "not_encoded", - "evidence": "Graph reports and query output have a candidate scoring path, but they are not executed by the runner." + "status": "blocked", + "evidence": "The smoke captures graphify-out/graph.json, GRAPH_REPORT.md, cache metadata, command logs, build time, graph size, and report size when build succeeds." + }, + { + "capability": "real_world_job_adapter", + "status": "blocked", + "evidence": "The smoke maps node labels, edge types, confidence tags, source files, source locations, report text, and query output to generated real_world_job evidence ids when graphify reaches output." }, { "capability": "multimodal_code_graph", "status": "not_encoded", - "evidence": "Multimodal graph extraction is a reference capability but not scored." + "evidence": "Multimodal extraction for videos, images, PDFs, or broad codebase understanding is a reference capability but not scored by this smoke." }, { - "capability": "real_world_job_adapter", + "capability": "quality_or_scale_claim", "status": "not_encoded", - "evidence": "No graphify materializer exists." + "evidence": "The smoke does not claim broad graph quality, private corpus behavior, scale, or authoritative memory-store behavior." } ], "suites": [ { "suite_id": "knowledge_compilation", - "status": "not_encoded", - "evidence": "Graph report citation and lint behavior are not scored." + "status": "blocked", + "evidence": "The generated smoke can exercise graph/report evidence mapping for one generated knowledge-compilation fixture, but the checked-in record stays blocked until a live artifact reaches graph/report output." }, { "suite_id": "retrieval", - "status": "not_encoded", - "evidence": "Graph-guided query output is not mapped to required evidence." + "status": "blocked", + "evidence": "Graph-guided query output is mapped only for the generated smoke when available; broad retrieval quality scoring remains unclaimed." }, { "suite_id": "work_resume", @@ -1995,6 +2010,16 @@ "kind": "source", "ref": "https://github.com/safishamsi/graphify", "status": "real" + }, + { + "kind": "command", + "ref": "cargo make graphify-docker-graph-report-smoke", + "status": "blocked" + }, + { + "kind": "artifact", + "ref": "tmp/real-world-memory/graphify-smoke/graphify-smoke.json", + "status": "blocked" } ], "execution_metadata": { @@ -2010,14 +2035,15 @@ "evidence": "Official CLI, output artifact, query, and source-location contract." } ], - "setup_path": "Install graphify inside Docker, build a graph/report from a generated corpus, and export query evidence without installing host-global assistant hooks.", - "runtime_boundary": "Docker-only CLI/materializer run over mounted benchmark corpus.", - "resource_expectation": "Graph build cost scales with corpus and model choices; record build time, graph size, and generated report size.", + "setup_path": "Run cargo make graphify-docker-graph-report-smoke to install graphify in Docker, build graph/report artifacts from a generated public corpus, and export query evidence without installing host-global assistant hooks.", + "runtime_boundary": "docker-compose.baseline.yml baseline-runner, container-local Python venv, isolated HOME/config paths, generated public corpus, and artifacts under tmp/real-world-memory/graphify-smoke.", + "resource_expectation": "Graph build cost scales with corpus and model choices; generated artifacts record package reference, provider/model boundary, build time, graph size, report size, cache size, timeout, and retry behavior.", "retry_guidance": [ - "Start with a generated public code/document corpus.", - "Score graph-guided answers only when report nodes cite source evidence IDs." + "Run cargo make graphify-docker-graph-report-smoke first; setup/runtime failures must remain typed artifacts, not pass claims.", + "Do not use graphify host assistant hook installs or operator-owned assistant configuration as proof.", + "Score graph-guided answers only when graph.json, GRAPH_REPORT.md, and graphify query output map to generated evidence ids." ], - "research_depth": "D1 feasibility verdict: adapter_candidate (XY-882); research_gate only, adapter not encoded" + "research_depth": "D1 feasibility verdict plus XY-889 Docker graph/report smoke implementation; checked-in record remains research_gate unless a generated artifact reaches graphify output" }, "follow_up": { "title": "[ELF benchmark adapter] Implement graphify Docker graph-report adapter", diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 5b68232f..966a4b68 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -257,13 +257,13 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/overall_status_counts/blocked") .and_then(Value::as_u64), - Some(5) + Some(6) ); assert_eq!( report .pointer("/external_adapters/summary/overall_status_counts/not_encoded") .and_then(Value::as_u64), - Some(8) + Some(7) ); assert_eq!( report @@ -281,7 +281,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/suite_status_counts/blocked") .and_then(Value::as_u64), - Some(9) + Some(11) ); } @@ -297,6 +297,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { let lightrag = find_by_field(adapters, "/adapter_id", "lightrag_research_gate")?; let graphrag = find_by_field(adapters, "/adapter_id", "graphrag_research_gate")?; let graphiti_zep = find_by_field(adapters, "/adapter_id", "graphiti_zep_research_gate")?; + let graphify = find_by_field(adapters, "/adapter_id", "graphify_research_gate")?; let qmd_deep = find_by_field(adapters, "/adapter_id", "qmd_deep_profile_gate")?; assert_eq!(elf.pointer("/evidence_class").and_then(Value::as_str), Some("fixture_backed")); @@ -364,38 +365,64 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { Some("cargo make graphrag-docker-smoke") ); assert_eq!(graphrag.pointer("/suites/1/status").and_then(Value::as_str), Some("not_encoded")); + + assert_graphiti_zep_adapter(graphiti_zep); + assert_graphify_adapter(graphify); + assert_eq!( - graphiti_zep.pointer("/evidence_class").and_then(Value::as_str), - Some("research_gate") + qmd_deep.pointer("/capabilities/2/status").and_then(Value::as_str), + Some("unsupported") ); - assert_eq!(graphiti_zep.pointer("/overall_status").and_then(Value::as_str), Some("blocked")); + + Ok(()) +} + +fn assert_graphiti_zep_adapter(adapter: &Value) { + assert_eq!(adapter.pointer("/evidence_class").and_then(Value::as_str), Some("research_gate")); + assert_eq!(adapter.pointer("/overall_status").and_then(Value::as_str), Some("blocked")); assert_eq!( - graphiti_zep.pointer("/setup/command").and_then(Value::as_str), + adapter.pointer("/setup/command").and_then(Value::as_str), Some("cargo make graphiti-zep-docker-temporal-smoke") ); assert_eq!( - graphiti_zep.pointer("/run/command").and_then(Value::as_str), + adapter.pointer("/run/command").and_then(Value::as_str), Some( "ELF_GRAPHITI_ZEP_SMOKE_START=1 ELF_GRAPHITI_ZEP_SMOKE_RUN=1 cargo make graphiti-zep-docker-temporal-smoke" ) ); assert_eq!( - graphiti_zep.pointer("/suites/0/suite_id").and_then(Value::as_str), + adapter.pointer("/suites/0/suite_id").and_then(Value::as_str), Some("memory_evolution") ); - assert_eq!(graphiti_zep.pointer("/suites/0/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(adapter.pointer("/suites/0/status").and_then(Value::as_str), Some("blocked")); assert_eq!( - graphiti_zep.pointer("/execution_metadata/research_depth").and_then(Value::as_str), + adapter.pointer("/execution_metadata/research_depth").and_then(Value::as_str), Some( "D2 feasibility plus XY-888 Docker temporal smoke implementation; checked-in record remains research_gate unless a generated artifact reaches Graphiti search output" ) ); +} + +fn assert_graphify_adapter(adapter: &Value) { + assert_eq!(adapter.pointer("/evidence_class").and_then(Value::as_str), Some("research_gate")); + assert_eq!(adapter.pointer("/overall_status").and_then(Value::as_str), Some("blocked")); assert_eq!( - qmd_deep.pointer("/capabilities/2/status").and_then(Value::as_str), - Some("unsupported") + adapter.pointer("/setup/command").and_then(Value::as_str), + Some("cargo make graphify-docker-graph-report-smoke") + ); + assert_eq!( + adapter.pointer("/suites/0/suite_id").and_then(Value::as_str), + Some("knowledge_compilation") + ); + assert_eq!(adapter.pointer("/suites/0/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(adapter.pointer("/suites/1/suite_id").and_then(Value::as_str), Some("retrieval")); + assert_eq!(adapter.pointer("/suites/1/status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + adapter.pointer("/execution_metadata/research_depth").and_then(Value::as_str), + Some( + "D1 feasibility verdict plus XY-889 Docker graph/report smoke implementation; checked-in record remains research_gate unless a generated artifact reaches graphify output" + ) ); - - Ok(()) } fn assert_live_sweep_record(adapter: &Value) -> Result<()> { diff --git a/docs/guide/research/comparison_external_projects.md b/docs/guide/research/comparison_external_projects.md index f969544c..f9540823 100644 --- a/docs/guide/research/comparison_external_projects.md +++ b/docs/guide/research/comparison_external_projects.md @@ -106,7 +106,7 @@ Project-to-suite map: | llm-wiki | `rw.knowledge-synthesis`, `rw.resume-evidence` | Query/save/lint flows and topic-scoped wiki pages are a useful reference for turning retrieved memory into maintained project knowledge. | Run a corpus-to-wiki job, ask resume/decision questions, require page citations back to source memory, then mutate a stale source and prove lint/repair catches it. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium for derived-knowledge fit. | ELF is not yet stronger on derived knowledge pages; llm-wiki should inform rebuildable, evidence-cited dossiers rather than core storage. | | gbrain | `rw.knowledge-synthesis`, `rw.operator-continuity` | `compiled_truth`, timeline sections, backlinks, primary-home routing, and enrichment workflows model a living operational brain for project work. | Build or update pages from the real-world corpus, require current-truth plus timeline answers, and prove enrichment/backlink maintenance does not hide unsupported claims. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium for operator knowledge UX. | ELF should keep source notes authoritative; gbrain is a reference for presentation, enrichment, and maintenance loops. | | Always-On Memory Agent | `rw.consolidation-review`, `rw.operator-continuity` | The file/API/dashboard ingest loop and timer-based consolidation show how background memory formation becomes a user-visible product surface. | Run scheduled consolidation on a fixed corpus, record source rows and output insights, then score whether consolidation is reviewable, repeatable, and bounded against unsupported claims. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium for consolidation workflow reference. | ELF should borrow scheduling and operator controls while keeping deterministic writes and reviewable derived outputs. | -| graphify | `rw.graph-navigation`, `rw.knowledge-synthesis`, `rw.resume-evidence` | Deterministic code extraction, LLM-assisted graph building, honesty tags, graph reports, and assistant hooks are strong references for graph-compressed navigation over large corpora. | Generate graph/report artifacts from the benchmark corpus, require answers to use graph structure plus source evidence, and prove rebuild behavior after corpus edits. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium for graph-navigation reference. | ELF is stronger as a memory service; graphify is the reference for rebuildable graph reports and pre-search guidance. | +| graphify | `rw.graph-navigation`, `rw.knowledge-synthesis`, `rw.resume-evidence` | Deterministic code extraction, LLM-assisted graph building, honesty tags, graph reports, and assistant hooks are strong references for graph-compressed navigation over large corpora. | Generate graph/report artifacts from the benchmark corpus, require answers to use graph structure plus source evidence, and prove rebuild behavior after corpus edits. | Implementation-backed research gate: `cargo make graphify-docker-graph-report-smoke` records a Docker-only generated-corpus graph/report artifact; checked-in manifest remains blocked/research_gate and does not claim broad graph quality or rebuild strength. Confidence: medium for adapter feasibility, low for production-quality graph navigation. | ELF is stronger as a memory service; graphify is now a runnable reference for derived graph reports and pre-search guidance, but not yet a stronger end-to-end memory system. | | Letta | `rw.core-archival`, `rw.operator-continuity` | Core memory blocks, archival memory, and shared/read-only memory blocks map directly to always-loaded operating context versus retrievable memory. | Build a multi-agent job where core blocks must be attached/detached/shared read-only, while archival memory is retrieved separately and audited. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium for memory-semantics reference. | ELF has scoped notes but not first-class core/archival block ergonomics; Letta is the reference dimension. | | LangGraph | `rw.replay-regression`, `rw.resume-evidence` | Thread checkpoints, durable execution, replay, fork, and time travel define a strong model for debugging agent-state and memory-regression behavior. | Run an agent job with memory reads across checkpoints, replay/fork the thread after a stale-memory failure, and verify side-effect boundaries. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium for replay workflow reference. | ELF traces are useful but do not replace full agent checkpoint replay; LangGraph is the reference for replay-regression jobs. | | Graphiti / Zep | `rw.graph-temporal`, `rw.resume-evidence` | Temporal entities, relations, fact triples, validity windows, and graph search directly target stale/contradictory factual memory. | Add fact triples with validity changes, query current and historical answers, and score invalidation/append behavior under contradiction traps. | Docs-grounded D1; no benchmark adapter evidence. Confidence: medium-high for temporal-graph dimension. | ELF graph-lite covers evidence-linked validity windows and current/historical relation context; Graphiti/Zep remains the reference for broader temporal graph workflows. | @@ -120,7 +120,7 @@ XY-882 feasibility verdicts for RAG and graph-memory gates: | LightRAG | `adapter_candidate` | Docker Compose server with explicit LLM, embedding, rerank, storage, workspace, and data-volume configuration. | Context-only query modes can return the context prepared for the LLM; core APIs can insert documents with ids and source file paths. | [XY-886](https://linear.app/hack-ink/issue/XY-886/elf-benchmark-adapter-implement-lightrag-docker-context-export-adapter); no live pass claim. | | GraphRAG | `adapter_candidate` | Cost-bounded Docker Python CLI/API run over a generated tiny corpus with container-local parquet artifacts. | Output tables contain generated UUIDs, human-readable ids, source documents, text units, community reports, and text-unit links for graph summaries and relationships. | [XY-887](https://linear.app/hack-ink/issue/XY-887/elf-benchmark-adapter-implement-graphrag-cost-bounded-docker-adapter); no live pass claim. | | Graphiti / Zep | `adapter_candidate` | Docker-local FalkorDB or Neo4j plus Python SDK runner with provider config captured under benchmark artifacts. | Search results and fact triples expose UUIDs, fact text, and validity windows (`valid_at` / `invalid_at`) that map to memory-evolution scoring. | [XY-888](https://linear.app/hack-ink/issue/XY-888/elf-benchmark-adapter-implement-graphitizep-temporal-graph-adapter); no live pass claim. | -| graphify | `adapter_candidate` | Docker-only CLI/materializer using `pip install graphifyy` over a mounted corpus; host-global assistant hooks are out of scope. | `graph.json`, `GRAPH_REPORT.md`, and graph query output include edge types, confidence tags, source files, and source locations. | [XY-889](https://linear.app/hack-ink/issue/XY-889/elf-benchmark-adapter-implement-graphify-docker-graph-report-adapter); no live pass claim. | +| graphify | `adapter_candidate` | Docker-only CLI/materializer using `pip install graphifyy` over a mounted corpus; host-global assistant hooks are out of scope. | `graph.json`, `GRAPH_REPORT.md`, and graph query output include edge types, confidence tags, source files, and source locations. | [XY-889](https://linear.app/hack-ink/issue/XY-889/elf-benchmark-adapter-implement-graphify-docker-graph-report-adapter) adds `cargo make graphify-docker-graph-report-smoke`; generated artifacts may carry live status, while the checked-in research-gate record avoids broad quality claims. | | Letta | `research_only` | Docker server exists, but current docs require explicit embedding configuration and steer Letta Code evaluation toward non-Docker local/frontier-model exploration. | Core/archival memory and shared blocks remain useful semantics, but no contained evidence export is selected for this adapter batch. | No implementation issue. | | LangGraph | `research_only` | A Docker harness is possible, but the project is an agent-state/checkpoint framework rather than a standalone memory adapter. | Store search and checkpoints are references for replay-regression jobs, not a direct external memory output contract here. | No implementation issue. | | nanograph | `research_only` | Official positioning is one CLI / one folder / no server / no Docker. | Typed schema, query, CDC, and search ergonomics remain graph-lite DX inspiration. | No implementation issue. | diff --git a/docs/guide/research/research_projects_inventory.md b/docs/guide/research/research_projects_inventory.md index 960fcfec..a76a0d4f 100644 --- a/docs/guide/research/research_projects_inventory.md +++ b/docs/guide/research/research_projects_inventory.md @@ -30,7 +30,7 @@ Last updated: June 10, 2026. | [llm-wiki](https://github.com/nvk/llm-wiki) | D1 | Reviewed; XY-882 verdict `research_only` | `rw.knowledge-synthesis`, `rw.resume-evidence` | LLM-maintained wiki pattern, topic-scoped knowledge bases, query-save and lint workflows | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json` | | [gbrain](https://github.com/garrytan/gbrain) | D1 | Reviewed; XY-882 verdict `blocked` | `rw.knowledge-synthesis`, `rw.operator-continuity` | Operational knowledge brain, `compiled_truth` + timeline pages, enrichment and maintenance loops; blocked on Docker-local brain repo and database proof | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json` | | [Always-On Memory Agent](https://github.com/GoogleCloudPlatform/generative-ai/tree/main/gemini/agents/always-on-memory-agent) | D1 | Reviewed | `rw.consolidation-review`, `rw.operator-continuity` | Always-on multimodal ingest + scheduled consolidation loop with simple local ops surface | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json` | -| [graphify](https://github.com/safishamsi/graphify) | D1 | Reviewed; XY-882 verdict `adapter_candidate` | `rw.graph-navigation`, `rw.knowledge-synthesis`, `rw.resume-evidence` | Multimodal graph compression, deterministic code extraction, and graph/report outputs with source-file/source-location references | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json` | +| [graphify](https://github.com/safishamsi/graphify) | D1 | Reviewed; XY-882 verdict `adapter_candidate`; XY-889 adds Docker graph/report smoke | `rw.graph-navigation`, `rw.knowledge-synthesis`, `rw.resume-evidence` | Multimodal graph compression, deterministic code extraction, and graph/report outputs with source-file/source-location references; current ELF evidence is a generated-corpus Docker smoke, not broad graph-quality proof | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json`; `apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json` | | [Letta](https://github.com/letta-ai/letta) | D1 | Reviewed; XY-882 verdict `research_only` | `rw.core-archival`, `rw.operator-continuity` | Core vs archival memory split, shared blocks; not an implementation candidate until a supported contained server path can export evidence | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json` | | [LangGraph](https://docs.langchain.com/oss/python/langgraph/persistence) | D1 | Reviewed; XY-882 verdict `research_only` | `rw.replay-regression`, `rw.resume-evidence` | Checkpoint/replay mindset for quality regression workflows; not a standalone memory backend adapter | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json` | | [Graphiti / Zep](https://help.getzep.com/graphiti/core-concepts/temporal-awareness) | D1 | Reviewed; XY-882 verdict `adapter_candidate` | `rw.graph-temporal`, `rw.resume-evidence` | Temporal fact validity model with Docker-local graph-store options and UUID/fact/validity-window output | `docs/guide/research/comparison_external_projects.md`; `docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json`; `docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json` | @@ -51,7 +51,7 @@ evidence; they only decide whether an implementation follow-up is justified. | LightRAG | `adapter_candidate` | Follow-up issue: [XY-886](https://linear.app/hack-ink/issue/XY-886/elf-benchmark-adapter-implement-lightrag-docker-context-export-adapter), a Docker context-export adapter using explicit LLM/embedding config and source file-path citations. | | GraphRAG | `adapter_candidate` | Follow-up issue: [XY-887](https://linear.app/hack-ink/issue/XY-887/elf-benchmark-adapter-implement-graphrag-cost-bounded-docker-adapter), a cost-bounded Docker CLI/API adapter over a tiny corpus and parquet output tables. | | Graphiti / Zep | `adapter_candidate` | Follow-up issue: [XY-888](https://linear.app/hack-ink/issue/XY-888/elf-benchmark-adapter-implement-graphitizep-temporal-graph-adapter), a Docker-local temporal graph adapter that scores current/historical fact validity. | -| graphify | `adapter_candidate` | Follow-up issue: [XY-889](https://linear.app/hack-ink/issue/XY-889/elf-benchmark-adapter-implement-graphify-docker-graph-report-adapter), a Docker-only CLI/materializer adapter over `graph.json` and `GRAPH_REPORT.md`; host-global assistant hooks remain out of scope. | +| graphify | `adapter_candidate` | Follow-up issue: [XY-889](https://linear.app/hack-ink/issue/XY-889/elf-benchmark-adapter-implement-graphify-docker-graph-report-adapter), a Docker-only CLI/materializer adapter over `graph.json` and `GRAPH_REPORT.md`; host-global assistant hooks remain out of scope. The checked-in manifest remains a research gate, while generated smoke artifacts may carry live status. | | Letta | `research_only` | Keep as a core/archival memory reference until a supported contained path can export archival-memory evidence for scoring. | | LangGraph | `research_only` | Keep as a checkpoint/replay regression reference, not a standalone external memory adapter. | | nanograph | `research_only` | Keep as typed graph DX inspiration; official shape is no server/no Docker. | diff --git a/scripts/graphify-docker-graph-report-smoke.py b/scripts/graphify-docker-graph-report-smoke.py new file mode 100755 index 00000000..da1555a3 --- /dev/null +++ b/scripts/graphify-docker-graph-report-smoke.py @@ -0,0 +1,1317 @@ +#!/usr/bin/env python3 +"""Docker-contained graphify graph/report smoke for real-world adapters.""" + +from __future__ import annotations + +import csv +import json +import os +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +SCRIPT_DIR = Path(__file__).resolve().parent +ROOT_DIR = SCRIPT_DIR.parent +REPORT_DIR = Path( + os.environ.get( + "ELF_GRAPHIFY_SMOKE_REPORT_DIR", + ROOT_DIR / "tmp" / "real-world-memory" / "graphify-smoke", + ) +) +WORK_DIR = Path(os.environ.get("ELF_GRAPHIFY_SMOKE_WORK_DIR", REPORT_DIR / "work")) +OUT = Path(os.environ.get("ELF_GRAPHIFY_SMOKE_OUT", REPORT_DIR / "graphify-smoke.json")) +MANIFEST_OUT = Path( + os.environ.get( + "ELF_GRAPHIFY_SMOKE_MANIFEST_OUT", + REPORT_DIR / "memory_projects_manifest.graphify-smoke.json", + ) +) +SUMMARY_OUT = Path(os.environ.get("ELF_GRAPHIFY_SMOKE_SUMMARY_OUT", REPORT_DIR / "summary.json")) +FIXTURE_DIR = REPORT_DIR / "graphify-fixtures" +CORPUS_DIR = WORK_DIR / "generated-public-corpus" +OUTPUT_CAPTURE_DIR = REPORT_DIR / "graphify-out" +LOG_DIR = REPORT_DIR / "logs" + +RUN_ID = os.environ.get( + "ELF_GRAPHIFY_SMOKE_RUN_ID", + f"graphify-docker-smoke-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}", +) +RUN_GRAPHIFY = os.environ.get("ELF_GRAPHIFY_SMOKE_RUN", "1") == "1" +ALLOW_HOST = os.environ.get("ELF_GRAPHIFY_SMOKE_ALLOW_HOST", "0") == "1" +INSTALL_GRAPHIFY = os.environ.get("ELF_GRAPHIFY_SMOKE_INSTALL", "1") == "1" +GRAPHIFY_PACKAGE = os.environ.get("ELF_GRAPHIFY_PACKAGE", "graphifyy") +GRAPHIFY_REF = os.environ.get("ELF_GRAPHIFY_REF", f"pypi:{GRAPHIFY_PACKAGE}") +TIMEOUT_SECONDS = int(os.environ.get("ELF_GRAPHIFY_TIMEOUT_SECONDS", "600")) +QUERY_BUDGET = int(os.environ.get("ELF_GRAPHIFY_QUERY_BUDGET", "1200")) + + +@dataclass +class CorpusItem: + """Generated public corpus item with source mapping metadata.""" + + evidence_id: str + claim_id: str + title: str + file_name: str + text: str + expected: bool + kind: str = "document" + line: int = 1 + + +@dataclass +class StatusState: + """Typed status for generated graphify smoke artifacts.""" + + setup: str = "blocked" + run: str = "not_encoded" + result: str = "blocked" + overall: str = "blocked" + evidence_class: str = "research_gate" + failure_class: str = "graphify_live_run_disabled" + failure_reason: str = ( + "graphify graph/report execution is disabled; set ELF_GRAPHIFY_SMOKE_RUN=1 " + "to install and run graphify inside Docker." + ) + + +@dataclass +class CommandRecord: + """Captured command result without secret-bearing environment values.""" + + label: str + command: list[str] + status: str + elapsed_ms: float + stdout_artifact: str | None + stderr_artifact: str | None + returncode: int | None + reason: str + + +def utc_now() -> str: + """Return an RFC3339 UTC timestamp.""" + + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def rel(path: Path) -> str: + """Return a repository-relative path when possible.""" + + try: + return str(path.resolve().relative_to(ROOT_DIR)) + except ValueError: + return str(path) + + +def mkdirs() -> None: + """Create and reset output directories owned by this smoke.""" + + for path in (FIXTURE_DIR, OUTPUT_CAPTURE_DIR, LOG_DIR): + if path.exists(): + shutil.rmtree(path) + + for path in (REPORT_DIR, WORK_DIR, FIXTURE_DIR, OUTPUT_CAPTURE_DIR, LOG_DIR): + path.mkdir(parents=True, exist_ok=True) + + for path in (OUT, MANIFEST_OUT, SUMMARY_OUT, REPORT_DIR / "generated-corpus.csv"): + if path.exists(): + path.unlink() + + +def write_json(path: Path, payload: Any) -> None: + """Write stable, pretty JSON.""" + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def dir_size(path: Path) -> int: + """Return total file size for a directory or file.""" + + if not path.exists(): + return 0 + if path.is_file(): + return path.stat().st_size + + return sum(item.stat().st_size for item in path.rglob("*") if item.is_file()) + + +def file_count(path: Path) -> int: + """Return file count for a directory.""" + + if not path.exists(): + return 0 + + return sum(1 for item in path.rglob("*") if item.is_file()) + + +def command_available(command: str) -> bool: + """Return whether a command is on PATH.""" + + return shutil.which(command) is not None + + +def runtime_env() -> dict[str, str]: + """Return an isolated graphify runtime environment.""" + + home = WORK_DIR / "home" + return { + "HOME": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "XDG_CACHE_HOME": str(home / ".cache"), + "CODEX_HOME": str(home / ".codex"), + "CLAUDE_CONFIG_DIR": str(home / ".claude"), + "GEMINI_HOME": str(home / ".gemini"), + "PYTHONUNBUFFERED": "1", + "NO_COLOR": "1", + } + + +def run_command( + label: str, + command: list[str], + cwd: Path, + timeout: int = TIMEOUT_SECONDS, + extra_env: dict[str, str] | None = None, +) -> CommandRecord: + """Run a subprocess and capture stdout/stderr artifacts.""" + + cwd.mkdir(parents=True, exist_ok=True) + stdout_path = LOG_DIR / f"{label}.stdout.log" + stderr_path = LOG_DIR / f"{label}.stderr.log" + env = os.environ.copy() + + if extra_env: + env.update(extra_env) + + started = time.monotonic() + try: + proc = subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + elapsed_ms = (time.monotonic() - started) * 1000 + stdout_path.write_text(proc.stdout, encoding="utf-8") + stderr_path.write_text(proc.stderr, encoding="utf-8") + status = "pass" if proc.returncode == 0 else "incomplete" + reason = "Command completed." if proc.returncode == 0 else f"Command exited {proc.returncode}." + + return CommandRecord( + label=label, + command=command, + status=status, + elapsed_ms=elapsed_ms, + stdout_artifact=rel(stdout_path), + stderr_artifact=rel(stderr_path), + returncode=proc.returncode, + reason=reason, + ) + except subprocess.TimeoutExpired as err: + elapsed_ms = (time.monotonic() - started) * 1000 + stdout_path.write_text(err.stdout or "", encoding="utf-8") + stderr_path.write_text(err.stderr or "", encoding="utf-8") + + return CommandRecord( + label=label, + command=command, + status="incomplete", + elapsed_ms=elapsed_ms, + stdout_artifact=rel(stdout_path), + stderr_artifact=rel(stderr_path), + returncode=None, + reason=f"Command timed out after {timeout} seconds.", + ) + + +def command_to_json(record: CommandRecord) -> dict[str, Any]: + """Serialize a command record.""" + + return { + "label": record.label, + "status": record.status, + "command": record.command, + "elapsed_ms": round(record.elapsed_ms, 3), + "stdout_artifact": record.stdout_artifact, + "stderr_artifact": record.stderr_artifact, + "returncode": record.returncode, + "reason": record.reason, + } + + +def generated_corpus() -> list[CorpusItem]: + """Return the bounded generated-public graphify corpus.""" + + return [ + CorpusItem( + evidence_id="graphify-smoke-memory-service", + claim_id="memory_service_graph", + title="ELF Memory Service Graph Note", + file_name="elf_memory_service.py", + text=( + '"""Evidence ID graphify-smoke-memory-service.\n' + "ELF stores evidence-linked facts as notes and keeps Postgres as the " + "source of truth for graph/report validation.\n" + '"""\n\n' + "class ElfMemoryService:\n" + " \"\"\"Evidence ID graphify-smoke-memory-service maps memory notes " + "to source-backed graph nodes.\"\"\"\n\n" + " def attach_evidence(self, note_id: str, source_ref: str) -> tuple[str, str]:\n" + " \"\"\"Attach source_ref evidence to a note before retrieval.\"\"\"\n" + " return note_id, source_ref\n" + ), + expected=True, + ), + CorpusItem( + evidence_id="graphify-smoke-qdrant-rebuild", + claim_id="qdrant_rebuild_graph", + title="Qdrant Rebuild Graph Note", + file_name="qdrant_rebuild.py", + text=( + '"""Evidence ID graphify-smoke-qdrant-rebuild.\n' + "Qdrant is a derived, rebuildable index. The graphify smoke should " + "connect Qdrant rebuild evidence to the ELF memory service node and " + "preserve this source file as evidence for scoring.\n" + '"""\n\n' + "class QdrantRebuildIndex:\n" + " \"\"\"Evidence ID graphify-smoke-qdrant-rebuild maps rebuildable " + "index behavior to source evidence.\"\"\"\n\n" + " def rebuild_from_postgres_vectors(self, collection: str) -> str:\n" + " \"\"\"Rebuild the derived Qdrant collection from Postgres vectors.\"\"\"\n" + " return collection\n" + ), + expected=True, + ), + CorpusItem( + evidence_id="graphify-smoke-report-mapping", + claim_id="graph_report_mapping", + title="Graph Report Mapping Note", + file_name="graph_report_mapping.py", + text=( + '"""Evidence ID graphify-smoke-report-mapping.\n' + "GRAPH_REPORT.md and graph.json must be captured as derived adapter " + "artifacts, then mapped back to real_world_job evidence ids.\n" + '"""\n\n' + "def map_graph_report_to_evidence(graph_json: str, graph_report: str) -> str:\n" + " \"\"\"Return graphify-smoke-report-mapping when graph artifacts cite sources.\"\"\"\n" + " return f\"{graph_json}:{graph_report}\"\n" + ), + expected=True, + ), + CorpusItem( + evidence_id="graphify-smoke-stale-trap", + claim_id="stale_authority_trap", + title="Stale Graph Authority Trap", + file_name="stale_vector_authority.py", + text=( + '"""Evidence ID graphify-smoke-stale-trap.\n' + "Stale trap: graphify output is an authoritative ELF memory store. " + "This is intentionally false; graphify is only a derived graph/report adapter.\n" + '"""\n\n' + "def stale_authority_claim() -> str:\n" + " \"\"\"Return the stale claim that must not drive the answer.\"\"\"\n" + " return \"graphify is authoritative\"\n" + ), + expected=False, + ), + ] + + +def write_corpus(corpus: list[CorpusItem]) -> Path: + """Write graphify input files plus a CSV mapping copy.""" + + if CORPUS_DIR.exists(): + shutil.rmtree(CORPUS_DIR) + CORPUS_DIR.mkdir(parents=True, exist_ok=True) + csv_path = REPORT_DIR / "generated-corpus.csv" + + with csv_path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter( + handle, + fieldnames=("evidence_id", "claim_id", "title", "file_name", "line", "text"), + ) + writer.writeheader() + + for item in corpus: + line = evidence_line(item.text, item.evidence_id) + item.line = line + writer.writerow( + { + "evidence_id": item.evidence_id, + "claim_id": item.claim_id, + "title": item.title, + "file_name": item.file_name, + "line": line, + "text": item.text, + } + ) + (CORPUS_DIR / item.file_name).write_text(item.text, encoding="utf-8") + + (CORPUS_DIR / ".graphifyignore").write_text( + "graphify-out/\n__pycache__/\n*.pyc\n", + encoding="utf-8", + ) + + return csv_path + + +def evidence_line(text: str, evidence_id: str) -> int: + """Return the first line containing an evidence id.""" + + for index, line in enumerate(text.splitlines(), start=1): + if evidence_id in line: + return index + + return 1 + + +def install_graphify(command_records: list[CommandRecord]) -> Path | None: + """Create a venv and install graphify in the container-local work dir.""" + + venv_dir = WORK_DIR / ".venv" + python = venv_dir / "bin" / "python" + graphify = venv_dir / "bin" / "graphify" + + if INSTALL_GRAPHIFY: + venv_record = run_command("python-venv", [sys.executable, "-m", "venv", str(venv_dir)], WORK_DIR) + command_records.append(venv_record) + if venv_record.status != "pass": + return None + + install_record = run_command( + "graphify-install", + [str(python), "-m", "pip", "install", "--disable-pip-version-check", GRAPHIFY_PACKAGE], + WORK_DIR, + extra_env=runtime_env(), + ) + command_records.append(install_record) + if install_record.status != "pass": + return None + elif not graphify.exists(): + command_records.append( + CommandRecord( + label="graphify-install", + command=["graphify"], + status="incomplete", + elapsed_ms=0.0, + stdout_artifact=None, + stderr_artifact=None, + returncode=None, + reason="graphify install was disabled and no venv graphify executable exists.", + ) + ) + return None + + version_record = run_command("graphify-help", [str(graphify), "--help"], WORK_DIR, extra_env=runtime_env()) + command_records.append(version_record) + + return graphify if version_record.status == "pass" else None + + +def run_graphify(graphify: Path, command_records: list[CommandRecord]) -> Path | None: + """Run graphify build and query commands.""" + + build_record = run_command( + "graphify-build", + [str(graphify), str(CORPUS_DIR), "--no-viz"], + WORK_DIR, + extra_env=runtime_env(), + ) + command_records.append(build_record) + if build_record.status != "pass": + return None + + cluster_record = run_command( + "graphify-cluster-report", + [str(graphify), "cluster-only", str(CORPUS_DIR)], + WORK_DIR, + extra_env=runtime_env(), + ) + command_records.append(cluster_record) + + output_dir = find_graphify_output_dir() + + if output_dir is None: + command_records.append( + CommandRecord( + label="graphify-output-discovery", + command=["find", str(WORK_DIR), "-path", "*/graphify-out/graph.json"], + status="incomplete", + elapsed_ms=0.0, + stdout_artifact=None, + stderr_artifact=None, + returncode=None, + reason="graphify completed but graphify-out/graph.json was not found.", + ) + ) + return None + + copy_graphify_output(output_dir) + graph_json = OUTPUT_CAPTURE_DIR / "graph.json" + query_record = run_command( + "graphify-query", + [ + str(graphify), + "query", + "what connects the ELF memory service, Qdrant rebuild, and graph report evidence mapping?", + "--graph", + str(graph_json), + "--budget", + str(QUERY_BUDGET), + ], + WORK_DIR, + extra_env=runtime_env(), + ) + command_records.append(query_record) + + return OUTPUT_CAPTURE_DIR + + +def find_graphify_output_dir() -> Path | None: + """Find the graphify output directory generated by the CLI.""" + + candidates: list[Path] = [] + + for base in (WORK_DIR, CORPUS_DIR): + if not base.exists(): + continue + + for graph_path in base.rglob("graph.json"): + if ".venv" in graph_path.parts: + continue + if graph_path.parent.name == "graphify-out": + candidates.append(graph_path.parent) + + if not candidates: + return None + + candidates.sort(key=lambda path: path.stat().st_mtime if path.exists() else 0.0) + + return candidates[-1] + + +def copy_graphify_output(output_dir: Path) -> None: + """Copy graphify output artifacts into the report directory.""" + + if OUTPUT_CAPTURE_DIR.exists(): + shutil.rmtree(OUTPUT_CAPTURE_DIR) + shutil.copytree(output_dir, OUTPUT_CAPTURE_DIR) + + +def map_artifacts(corpus: list[CorpusItem], command_records: list[CommandRecord]) -> dict[str, Any]: + """Map graphify graph/report/query output to real_world_job evidence ids.""" + + graph_json = OUTPUT_CAPTURE_DIR / "graph.json" + graph_report = OUTPUT_CAPTURE_DIR / "GRAPH_REPORT.md" + graph_payload = read_json_or_none(graph_json) + nodes, edges = extract_graph_rows(graph_payload) + node_mappings = [map_graph_row("node", row, corpus) for row in nodes] + edge_mappings = [map_graph_row("edge", row, corpus) for row in edges] + report_mapping = map_text_artifact("graph_report", graph_report, corpus) + query_mapping = map_query_output(command_records, corpus) + mapped_ids: list[str] = [] + + for section in (node_mappings, edge_mappings): + for row in section: + for evidence_id in row["evidence_ids"]: + append_unique(mapped_ids, evidence_id) + + for row in (report_mapping, query_mapping): + for evidence_id in row["evidence_ids"]: + append_unique(mapped_ids, evidence_id) + + return { + "expected_evidence_ids": expected_ids(corpus), + "mapped_evidence_ids": mapped_ids, + "graph_json": { + "artifact": rel(graph_json) if graph_json.exists() else None, + "exists": graph_json.exists(), + "size_bytes": graph_json.stat().st_size if graph_json.exists() else 0, + }, + "graph_report": report_mapping, + "query_output": query_mapping, + "nodes": node_mappings, + "edges": edge_mappings, + } + + +def read_json_or_none(path: Path) -> Any | None: + """Read JSON and return None on missing or invalid payloads.""" + + if not path.exists(): + return None + + try: + return json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + + +def extract_graph_rows(payload: Any | None) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Extract node and edge rows from common graph JSON shapes.""" + + if not isinstance(payload, dict): + return [], [] + + nodes = payload.get("nodes") + edges = payload.get("edges") or payload.get("links") or payload.get("relationships") + + if nodes is None and isinstance(payload.get("elements"), dict): + elements = payload["elements"] + nodes = elements.get("nodes") + edges = elements.get("edges") + + return rows_from_value(nodes), rows_from_value(edges) + + +def rows_from_value(value: Any) -> list[dict[str, Any]]: + """Normalize a graph row container into dictionaries.""" + + if not isinstance(value, list): + return [] + + rows: list[dict[str, Any]] = [] + for item in value: + if isinstance(item, dict): + data = item.get("data") + rows.append(data if isinstance(data, dict) else item) + + return rows + + +def map_graph_row(kind: str, row: dict[str, Any], corpus: list[CorpusItem]) -> dict[str, Any]: + """Map one graph node or edge row to evidence ids.""" + + blob = json.dumps(row, sort_keys=True, default=str) + evidence_ids = evidence_from_text(blob, corpus) + return { + "kind": kind, + "row_id": str(row.get("id") or row.get("key") or row.get("source") or ""), + "label": first_text(row, ("label", "name", "title", "type", "kind")), + "edge_type": first_text(row, ("edge_type", "type", "relation", "relationship", "predicate")), + "confidence": first_text( + row, + ("confidence", "confidence_score", "confidence_tag", "extraction_status", "status"), + ), + "source_files": source_values(row), + "source_locations": source_location_values(row), + "evidence_ids": evidence_ids, + } + + +def first_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None: + """Return the first scalar text value for a set of keys.""" + + for key in keys: + value = row.get(key) + + if isinstance(value, (str, int, float)): + return str(value) + + return None + + +def source_values(value: Any) -> list[str]: + """Collect source file-ish values from a graph row.""" + + values: list[str] = [] + collect_source_values(value, values, ("source", "file", "path")) + + return values[:12] + + +def source_location_values(value: Any) -> list[str]: + """Collect source location-ish values from a graph row.""" + + values: list[str] = [] + collect_source_values(value, values, ("location", "line", "span", "range")) + + return values[:12] + + +def collect_source_values(value: Any, out: list[str], key_fragments: tuple[str, ...]) -> None: + """Recursively collect bounded source-related values.""" + + if isinstance(value, dict): + for key, item in value.items(): + key_lower = key.lower() + + if any(fragment in key_lower for fragment in key_fragments) and isinstance(item, (str, int, float)): + append_unique(out, str(item)) + else: + collect_source_values(item, out, key_fragments) + elif isinstance(value, list): + for item in value: + collect_source_values(item, out, key_fragments) + + +def map_text_artifact(kind: str, path: Path, corpus: list[CorpusItem]) -> dict[str, Any]: + """Map a text artifact to evidence ids.""" + + text = "" + if path.exists(): + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + text = "" + + return { + "kind": kind, + "artifact": rel(path) if path.exists() else None, + "exists": path.exists(), + "size_bytes": path.stat().st_size if path.exists() else 0, + "evidence_ids": evidence_from_text(text, corpus), + } + + +def map_query_output(command_records: list[CommandRecord], corpus: list[CorpusItem]) -> dict[str, Any]: + """Map graphify query stdout to evidence ids.""" + + query_record = next((record for record in command_records if record.label == "graphify-query"), None) + text = "" + artifact = query_record.stdout_artifact if query_record else None + + if artifact: + path = ROOT_DIR / artifact + if path.exists(): + text = path.read_text(encoding="utf-8") + + return { + "kind": "query_output", + "artifact": artifact, + "exists": bool(artifact and (ROOT_DIR / artifact).exists()), + "command_status": query_record.status if query_record else "not_encoded", + "evidence_ids": evidence_from_text(text, corpus), + } + + +def evidence_from_text(text: str, corpus: list[CorpusItem]) -> list[str]: + """Return evidence ids whose signatures appear in a text blob.""" + + evidence_ids: list[str] = [] + haystack = text.lower() + + for item in corpus: + signatures = ( + item.evidence_id, + slug(item.evidence_id), + item.file_name, + item.title, + f"{item.file_name}:{item.line}", + ) + + if any(signature.lower() in haystack for signature in signatures): + append_unique(evidence_ids, item.evidence_id) + + return evidence_ids + + +def append_unique(values: list[str], value: str) -> None: + """Append a value if absent.""" + + if value not in values: + values.append(value) + + +def expected_ids(corpus: list[CorpusItem]) -> list[str]: + """Return expected evidence ids for pass scoring.""" + + return [item.evidence_id for item in corpus if item.expected] + + +def mapping_outcome(mappings: dict[str, Any], command_records: list[CommandRecord]) -> tuple[str, str]: + """Return typed result status and explanation for evidence mapping.""" + + graph_build = next((record for record in command_records if record.label == "graphify-build"), None) + graph_query = next((record for record in command_records if record.label == "graphify-query"), None) + + if graph_build is None or graph_build.status != "pass": + return "incomplete", "graphify did not complete graph/report build for the generated corpus." + if not mappings["graph_json"]["exists"]: + return "incomplete", "graphify did not produce graph.json." + if not mappings["graph_report"]["exists"]: + return "incomplete", "graphify did not produce GRAPH_REPORT.md." + if graph_query is None or graph_query.status != "pass": + return "incomplete", "graphify query output was not available for scoring." + + missing = [ + evidence_id + for evidence_id in mappings["expected_evidence_ids"] + if evidence_id not in mappings["mapped_evidence_ids"] + ] + + if missing: + return "wrong_result", f"graphify output mappings missed expected evidence ids: {', '.join(missing)}." + + return "pass", "graphify graph/report/query output mapped to expected generated evidence ids." + + +def write_fixture(corpus: list[CorpusItem], status: StatusState, mapped_ids: list[str]) -> Path: + """Write a generated real_world_job fixture for the graphify smoke.""" + + fixture_path = FIXTURE_DIR / "knowledge" / "graphify_graph_report.json" + used_ids = [evidence_id for evidence_id in mapped_ids if evidence_id in expected_ids(corpus)] + response = { + "adapter_id": "graphify_docker_smoke", + "answer": { + "content": ( + "graphify connected the ELF memory service, Qdrant rebuild, and graph report mapping " + "through graph/report artifacts that cite generated source evidence." + if used_ids + else "" + ), + "claims": [ + { + "claim_id": "graphify_report_evidence_mapping", + "text": ( + "graphify graph/report artifacts map back to the generated ELF memory service, " + "Qdrant rebuild, and report mapping evidence ids." + ), + "evidence_ids": used_ids, + "confidence": "derived_from_graphify_graph_report_mapping", + } + ] + if used_ids + else [], + "evidence_ids": used_ids, + "pages": [ + { + "page_id": "graphify:graph-report", + "page_type": "concept", + "title": "graphify Graph Report", + "path": rel(OUTPUT_CAPTURE_DIR / "GRAPH_REPORT.md"), + "sections": [ + { + "section_id": "derived-graph-report", + "heading": "Derived Graph Report", + "role": "summary", + "content": "GRAPH_REPORT.md is a derived graphify artifact, not authoritative ELF memory.", + "evidence_ids": used_ids, + "timeline_event_ids": ["graphify-smoke-built-graph-report"], + "unsupported_reason": None if used_ids else "graphify output was not mapped.", + } + ], + "backlinks": used_ids, + "lint_findings": [], + } + ] + if (OUTPUT_CAPTURE_DIR / "GRAPH_REPORT.md").exists() + else [], + "latency_ms": 0.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0, + }, + }, + } + fixture: dict[str, Any] = { + "schema": "elf.real_world_job/v1", + "job_id": "graphify-graph-report-001", + "suite": "knowledge_compilation", + "title": "Map graphify graph/report output to generated evidence", + "corpus": { + "corpus_id": "graphify-generated-public-smoke", + "profile": "generated_public", + "items": [ + { + "evidence_id": item.evidence_id, + "kind": item.kind, + "text": item.text, + "source_ref": { + "schema": "source_ref/v1", + "resolver": "graphify_smoke/v1", + "ref": { + "run_id": RUN_ID, + "file": item.file_name, + "line": item.line, + "evidence_id": item.evidence_id, + }, + }, + "created_at": "2026-06-10T00:00:00Z", + } + for item in corpus + ], + "adapter_response": response, + }, + "timeline": [ + { + "event_id": "graphify-smoke-corpus-generated", + "ts": "2026-06-10T00:00:00Z", + "actor": "system", + "action": "generated_public_corpus", + "evidence_ids": expected_ids(corpus), + "summary": "The graphify smoke generated a tiny public corpus for source mapping.", + }, + { + "event_id": "graphify-smoke-built-graph-report", + "ts": "2026-06-10T00:01:00Z", + "actor": "system", + "action": "built_derived_graph_report", + "evidence_ids": used_ids, + "summary": "graphify built derived graph/report artifacts when the Docker smoke reached execution.", + }, + ], + "prompt": { + "role": "user", + "content": "What does graphify connect in the generated ELF graph/report smoke?", + "job_mode": "compile", + "constraints": ["cite_evidence", "avoid_stale_facts", "do_not_claim_authoritative_store"], + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "graphify_report_evidence_mapping", + "text": ( + "graphify connects the ELF memory service, Qdrant rebuild, and graph report " + "mapping through derived graph/report artifacts." + ), + } + ], + "must_not_include": ["graphify output is an authoritative ELF memory store."], + "evidence_links": {"graphify_report_evidence_mapping": expected_ids(corpus)}, + "answer_type": "compiled_knowledge", + "accepted_alternates": [], + "requires_caveat": True, + "requires_refusal": False, + }, + "required_evidence": [ + { + "evidence_id": item.evidence_id, + "claim_id": "graphify_report_evidence_mapping", + "requirement": "cite", + "quote": item.evidence_id, + } + for item in corpus + if item.expected + ], + "negative_traps": [ + { + "trap_id": "graphify-authoritative-store", + "type": "unsupported_claim", + "evidence_ids": ["graphify-smoke-stale-trap"], + "failure_if_used": True, + } + ], + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.25, + "max_points": 1.0, + "criteria": "States the graph/report connection without broad quality claims.", + }, + "evidence_grounding": { + "weight": 0.4, + "max_points": 1.0, + "criteria": "Maps graphify output back to generated evidence ids.", + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not treat graphify output as an authoritative ELF memory store.", + }, + "latency_resource": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Records build time, artifact sizes, provider boundary, and retry behavior.", + }, + }, + "pass_threshold": 0.75, + "hard_fail_rules": [], + }, + "allowed_uncertainty": { + "phrases": ["tiny generated corpus", "derived graph/report adapter"], + "fallback": "Report typed failure when graphify output cannot be mapped to evidence ids.", + }, + "operator_debug": None, + "encoding": {}, + "memory_evolution": None, + "tags": ["external_adapter", "generated_public", "graphify", "no_live_claim"], + } + + if status.result in {"blocked", "incomplete"}: + fixture["encoding"] = { + "status": status.result, + "reason": status.failure_reason, + } + + write_json(fixture_path, fixture) + + return fixture_path + + +def write_materialization( + status: StatusState, + corpus: list[CorpusItem], + fixture_path: Path, + corpus_csv: Path, + command_records: list[CommandRecord], + mappings: dict[str, Any], + started_at: float, +) -> dict[str, Any]: + """Write the primary smoke artifact.""" + + elapsed_ms = (time.monotonic() - started_at) * 1000 + graph_json = OUTPUT_CAPTURE_DIR / "graph.json" + graph_report = OUTPUT_CAPTURE_DIR / "GRAPH_REPORT.md" + cache_dir = OUTPUT_CAPTURE_DIR / "cache" + query_record = next((record for record in command_records if record.label == "graphify-query"), None) + payload = { + "schema": "elf.graphify_docker_graph_report_smoke/v1", + "generated_at": utc_now(), + "run_id": RUN_ID, + "adapter_id": "graphify_docker_smoke", + "evidence_class": status.evidence_class, + "status": { + "setup": status.setup, + "run": status.run, + "result": status.result, + "overall": status.overall, + "failure_class": status.failure_class, + "failure_reason": status.failure_reason, + }, + "artifacts": { + "generated_corpus_csv": rel(corpus_csv), + "generated_corpus_dir": rel(CORPUS_DIR), + "generated_fixture": rel(fixture_path), + "graph_output_dir": rel(OUTPUT_CAPTURE_DIR), + "graph_json": rel(graph_json) if graph_json.exists() else None, + "graph_report": rel(graph_report) if graph_report.exists() else None, + "query_output": query_record.stdout_artifact if query_record else None, + "manifest": rel(MANIFEST_OUT), + "summary": rel(SUMMARY_OUT), + }, + "docker_boundary": { + "compose_file": "docker-compose.baseline.yml", + "runner_service": "baseline-runner", + "runner": "scripts/graphify-docker-graph-report-smoke.py", + "host_global_installs_required": False, + "docker_only": True, + "assistant_hook_install_used": False, + "isolated_home": True, + }, + "model_provider_boundary": { + "package": GRAPHIFY_REF, + "package_spec": GRAPHIFY_PACKAGE, + "assistant_platform_hooks_used": False, + "host_global_assistant_config_used": False, + "operator_owned_provider_credentials_used": False, + "provider_or_model_name": "graphify CLI default; no model configured by this runner", + "live_run_enabled": RUN_GRAPHIFY, + }, + "resource_bounds": { + "generated_file_count": len(corpus), + "generated_input_chars": sum(len(item.text) for item in corpus), + "timeout_seconds": TIMEOUT_SECONDS, + "elapsed_ms": round(elapsed_ms, 3), + "graph_json_size_bytes": graph_json.stat().st_size if graph_json.exists() else 0, + "graph_report_size_bytes": graph_report.stat().st_size if graph_report.exists() else 0, + "graph_output_size_bytes": dir_size(OUTPUT_CAPTURE_DIR), + "cache_size_bytes": dir_size(cache_dir), + "cache_file_count": file_count(cache_dir), + }, + "retry_behavior": { + "max_attempts": 1, + "retries_performed": 0, + "retry_guidance": "Rerun the same Docker command after setup/runtime fixes; do not use host assistant hooks as proof.", + }, + "commands": [command_to_json(record) for record in command_records], + "evidence_mapping": mappings, + } + write_json(OUT, payload) + + return payload + + +def write_manifest(status: StatusState) -> dict[str, Any]: + """Write a generated external adapter manifest for this smoke.""" + + manifest = { + "schema": "elf.real_world_external_adapter_manifest/v1", + "manifest_id": f"graphify-docker-smoke-{RUN_ID}", + "docker_isolation": { + "default": True, + "compose_file": "docker-compose.baseline.yml", + "runner": "scripts/graphify-docker-graph-report-smoke.py", + "artifact_dir": "tmp/real-world-memory/graphify-smoke", + "host_global_installs_required": False, + "notes": [ + f"Generated by the graphify Docker graph/report smoke at {utc_now()}.", + "The smoke uses generated public source files and records typed setup/runtime failures.", + ], + }, + "adapters": [ + { + "adapter_id": "graphify_docker_smoke", + "project": "graphify", + "adapter_kind": "docker_cli_graph_report_smoke", + "evidence_class": status.evidence_class, + "docker_default": True, + "host_global_installs_required": False, + "overall_status": status.overall, + "setup": { + "status": status.setup, + "evidence": "The smoke installs graphify in a container-local Python venv and runs with isolated assistant config paths.", + "command": "cargo make graphify-docker-graph-report-smoke", + "artifact": rel(OUT), + }, + "run": { + "status": status.run, + "evidence": "The live path builds graphify graph/report artifacts from a generated public corpus and runs graphify query over graph.json.", + "command": "cargo make graphify-docker-graph-report-smoke", + "artifact": rel(OUT), + }, + "result": { + "status": status.result, + "evidence": status.failure_reason + if status.failure_reason + else "graphify graph.json, GRAPH_REPORT.md, and query output mapped to generated real_world_job evidence ids.", + "artifact": rel(OUT), + }, + "capabilities": [ + { + "capability": "docker_cli_boundary", + "status": status.setup, + "evidence": "The runner uses docker-compose.baseline.yml baseline-runner and does not install graphify or assistant hooks on the host.", + }, + { + "capability": "graph_report_generation", + "status": status.run, + "evidence": "The smoke captures graphify-out/graph.json, GRAPH_REPORT.md, cache metadata, and command logs when build succeeds.", + }, + { + "capability": "graph_query_evidence_mapping", + "status": status.result, + "evidence": "Node labels, edge types, confidence tags, source files, source locations, report text, and query output are scanned for generated evidence ids.", + }, + { + "capability": "quality_or_scale_claim", + "status": "not_encoded", + "evidence": "The smoke does not claim multimodal, private corpus, broad codebase-understanding, or large-corpus graph quality.", + }, + ], + "suites": [ + { + "suite_id": "knowledge_compilation", + "status": status.result, + "evidence": "Only the generated graph/report evidence-mapping job is represented.", + }, + { + "suite_id": "retrieval", + "status": status.result if status.result in {"pass", "wrong_result"} else status.run, + "evidence": "The smoke uses graphify query output only to support source mapping; broad retrieval quality is not scored.", + }, + { + "suite_id": "work_resume", + "status": "not_encoded", + "evidence": "Resume-answer behavior is not encoded by this graph/report smoke.", + }, + { + "suite_id": "production_ops", + "status": "not_encoded", + "evidence": "The smoke records resource bounds but does not encode backup, restore, provider credential, or private corpus operations.", + }, + ], + "evidence": [ + {"kind": "artifact", "ref": rel(OUT), "status": status.result}, + {"kind": "artifact", "ref": rel(OUTPUT_CAPTURE_DIR), "status": status.result}, + {"kind": "manifest", "ref": rel(MANIFEST_OUT), "status": status.overall}, + {"kind": "source", "ref": "https://github.com/safishamsi/graphify", "status": "real"}, + { + "kind": "source", + "ref": "https://github.com/safishamsi/graphify/blob/v3/README.md", + "status": "real", + }, + ], + "execution_metadata": { + "sources": [ + { + "label": "graphify repository", + "url": "https://github.com/safishamsi/graphify", + "evidence": "Official source for graphify graph extraction and query workflow.", + }, + { + "label": "graphify README", + "url": "https://github.com/safishamsi/graphify/blob/v3/README.md", + "evidence": "Official CLI, output artifact, query, confidence, and source-location contract.", + }, + { + "label": "graphify PyPI package", + "url": "https://pypi.org/project/graphifyy/", + "evidence": "Official package referenced by the graphify README.", + }, + ], + "setup_path": "Run cargo make graphify-docker-graph-report-smoke to install graphify in a container-local venv and build graph/report artifacts over generated public files.", + "runtime_boundary": "docker-compose.baseline.yml baseline-runner, isolated HOME/config paths, generated corpus, and artifacts under tmp/real-world-memory/graphify-smoke.", + "resource_expectation": f"graphify package {GRAPHIFY_REF}, generated_files=4, timeout_seconds={TIMEOUT_SECONDS}, query_budget={QUERY_BUDGET}.", + "retry_guidance": [ + "Rerun cargo make graphify-docker-graph-report-smoke after dependency or runtime fixes.", + "Do not use graphify install hooks, host-global Codex/Claude/Gemini config, or private corpora as proof.", + "Score only when graph.json, GRAPH_REPORT.md, and graphify query output map to generated evidence ids.", + ], + "research_depth": "D1 feasibility plus XY-889 Docker graph/report smoke implementation; generated artifact decides live evidence class.", + }, + "notes": [ + "The checked-in manifest record remains research_gate; generated smoke artifacts carry live status.", + "graphify output is treated as a derived graph/report adapter, not an authoritative ELF memory store.", + ], + } + ], + } + write_json(MANIFEST_OUT, manifest) + + return manifest + + +def write_summary(materialization: dict[str, Any], manifest: dict[str, Any]) -> None: + """Write a small summary artifact.""" + + write_json( + SUMMARY_OUT, + { + "schema": "elf.graphify_docker_smoke_summary/v1", + "generated_at": utc_now(), + "adapter_id": "graphify_docker_smoke", + "evidence_class": materialization["evidence_class"], + "materialization": materialization, + "manifest": { + "json": rel(MANIFEST_OUT), + "summary": manifest["adapters"][0]["overall_status"], + "suites": manifest["adapters"][0]["suites"], + }, + }, + ) + + +def slug(value: str) -> str: + """Return a small ASCII slug.""" + + out: list[str] = [] + last_dash = False + + for char in value.lower(): + if char.isascii() and char.isalnum(): + out.append(char) + last_dash = False + elif not last_dash and out: + out.append("-") + last_dash = True + + while out and out[-1] == "-": + out.pop() + + return "".join(out) or "item" + + +def main() -> int: + """Run the smoke and always emit typed artifacts when possible.""" + + started_at = time.monotonic() + mkdirs() + status = StatusState() + command_records: list[CommandRecord] = [] + corpus = generated_corpus() + corpus_csv = write_corpus(corpus) + mappings = { + "expected_evidence_ids": expected_ids(corpus), + "mapped_evidence_ids": [], + "graph_json": {"artifact": None, "exists": False, "size_bytes": 0}, + "graph_report": { + "kind": "graph_report", + "artifact": None, + "exists": False, + "size_bytes": 0, + "evidence_ids": [], + }, + "query_output": { + "kind": "query_output", + "artifact": None, + "exists": False, + "command_status": "not_encoded", + "evidence_ids": [], + }, + "nodes": [], + "edges": [], + } + + if not Path("/.dockerenv").exists() and not ALLOW_HOST: + status.setup = "incomplete" + status.result = "incomplete" + status.overall = "incomplete" + status.failure_class = "not_running_in_docker" + status.failure_reason = "graphify smoke must run inside Docker; use cargo make graphify-docker-graph-report-smoke." + elif not command_available("python3"): + status.setup = "incomplete" + status.result = "incomplete" + status.overall = "incomplete" + status.failure_class = "python_missing" + status.failure_reason = "python3 is required for the graphify smoke runner." + elif not RUN_GRAPHIFY: + pass + else: + graphify = install_graphify(command_records) + + if graphify is None: + status.setup = "incomplete" + status.result = "incomplete" + status.overall = "incomplete" + status.failure_class = "graphify_setup_failed" + status.failure_reason = "graphify installation or help command failed inside the Docker runner." + else: + status.setup = "pass" + output_dir = run_graphify(graphify, command_records) + + if output_dir is None: + status.run = "incomplete" + status.result = "incomplete" + status.overall = "incomplete" + status.failure_class = "graphify_build_failed" + status.failure_reason = "graphify did not build graph/report artifacts for the generated corpus." + else: + status.run = "pass" + status.evidence_class = "live_real_world" + mappings = map_artifacts(corpus, command_records) + result_status, reason = mapping_outcome(mappings, command_records) + status.result = result_status + status.overall = result_status + + if result_status == "pass": + status.failure_class = "" + status.failure_reason = "" + else: + status.failure_class = "graphify_evidence_mapping_failed" + status.failure_reason = reason + + fixture_path = write_fixture(corpus, status, mappings["mapped_evidence_ids"]) + materialization = write_materialization( + status, + corpus, + fixture_path, + corpus_csv, + command_records, + mappings, + started_at, + ) + manifest = write_manifest(status) + write_summary(materialization, manifest) + print(f"graphify smoke artifact: {OUT}") + print(f"graphify smoke manifest: {MANIFEST_OUT}") + print(f"graphify smoke summary: {SUMMARY_OUT}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/real-world-live-adapters.sh b/scripts/real-world-live-adapters.sh index 505086ec..3cd5ab31 100755 --- a/scripts/real-world-live-adapters.sh +++ b/scripts/real-world-live-adapters.sh @@ -31,6 +31,7 @@ rm -rf "${REPORT_DIR:?}/elf-fixtures" \ "${REPORT_DIR:?}/lightrag" \ "${REPORT_DIR:?}/graphrag" \ "${REPORT_DIR:?}/graphiti-zep" \ + "${REPORT_DIR:?}/graphify" \ "${REPORT_DIR:?}/summary.json" cd "${ROOT_DIR}" @@ -94,6 +95,11 @@ if [[ "${ELF_REAL_WORLD_LIVE_ENABLE_GRAPHITI_ZEP:-0}" == "1" ]]; then python3 scripts/graphiti-zep-docker-temporal-smoke.py fi +if [[ "${ELF_REAL_WORLD_LIVE_ENABLE_GRAPHIFY:-0}" == "1" ]]; then + ELF_GRAPHIFY_SMOKE_REPORT_DIR="${REPORT_DIR}/graphify" \ + python3 scripts/graphify-docker-graph-report-smoke.py +fi + jq -n \ --slurpfile elf_materialization "${REPORT_DIR}/elf-materialization.json" \ --slurpfile qmd_materialization "${REPORT_DIR}/qmd-materialization.json" \ @@ -182,6 +188,25 @@ if [[ -f "${REPORT_DIR}/graphiti-zep/summary.json" ]]; then mv "${REPORT_DIR}/summary.json.tmp" "${REPORT_DIR}/summary.json" fi +if [[ -f "${REPORT_DIR}/graphify/summary.json" ]]; then + jq \ + --slurpfile graphify_summary "${REPORT_DIR}/graphify/summary.json" \ + '.adapters += [ + { + adapter_id: $graphify_summary[0].adapter_id, + evidence_class: $graphify_summary[0].evidence_class, + materialization: $graphify_summary[0].materialization, + report: { + json: "tmp/real-world-memory/live-adapters/graphify/graphify-smoke.json", + markdown: null, + summary: $graphify_summary[0].materialization.status, + suites: $graphify_summary[0].manifest.suites + } + } + ]' "${REPORT_DIR}/summary.json" >"${REPORT_DIR}/summary.json.tmp" + mv "${REPORT_DIR}/summary.json.tmp" "${REPORT_DIR}/summary.json" +fi + echo "Live real-world adapter reports:" echo " ${REPORT_DIR}/elf-report.json" echo " ${REPORT_DIR}/elf-report.md" @@ -199,4 +224,8 @@ if [[ -f "${REPORT_DIR}/graphiti-zep/summary.json" ]]; then echo " ${REPORT_DIR}/graphiti-zep/graphiti-zep-smoke.json" echo " ${REPORT_DIR}/graphiti-zep/summary.json" fi +if [[ -f "${REPORT_DIR}/graphify/summary.json" ]]; then + echo " ${REPORT_DIR}/graphify/graphify-smoke.json" + echo " ${REPORT_DIR}/graphify/summary.json" +fi echo " ${REPORT_DIR}/summary.json"