diff --git a/Makefile.toml b/Makefile.toml index c1663f99..27f5c6c5 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -822,6 +822,7 @@ args = [ # | real-world-memory-knowledge-json | command | | # | real-world-memory-knowledge-report | command | | # | ragflow-docker-smoke | command | | +# | lightrag-docker-context-smoke | command | | [tasks.ragflow-docker-smoke] workspace = false @@ -830,6 +831,14 @@ args = [ "scripts/ragflow-docker-evidence-smoke.sh", ] +[tasks.lightrag-docker-context-smoke] +workspace = false +command = "bash" +args = [ + "-lc", + "set -euo pipefail; start=\"$(printenv ELF_LIGHTRAG_CONTEXT_START || true)\"; status=0; if [ \"$start\" = \"1\" ]; then docker compose -f docker-compose.baseline.yml --profile lightrag up -d lightrag; fi; docker compose -f docker-compose.baseline.yml run --build --rm baseline-runner bash scripts/lightrag-docker-context-smoke.sh || status=$?; if [ \"$start\" = \"1\" ]; then docker compose -f docker-compose.baseline.yml --profile lightrag stop lightrag lightrag-mock-provider >/dev/null 2>&1 || true; fi; exit \"$status\"", +] + [tasks.real-world-memory-knowledge] workspace = false dependencies = [ diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 5a9d25d4..07c16306 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -1164,48 +1164,58 @@ "overall_status": "blocked", "setup": { "status": "blocked", - "evidence": "XY-882 marks LightRAG as an adapter_candidate, but the runner still needs a Docker context-export adapter before any live result." + "evidence": "XY-886 adds a Docker-profile context-export smoke command. The checked-in manifest remains a research gate until a generated artifact reaches LightRAG context/source output.", + "command": "cargo make lightrag-docker-context-smoke", + "artifact": "tmp/real-world-memory/lightrag-context/lightrag-materialization.json" }, "run": { - "status": "not_encoded", - "evidence": "No LightRAG real_world_job adapter is encoded." + "status": "blocked", + "evidence": "The default smoke records a typed setup/runtime failure if the LightRAG API is unavailable; set ELF_LIGHTRAG_CONTEXT_START=1 to start the opt-in Docker service profile.", + "command": "ELF_LIGHTRAG_CONTEXT_START=1 cargo make lightrag-docker-context-smoke", + "artifact": "tmp/real-world-memory/lightrag-context/summary.json" }, "result": { "status": "blocked", - "evidence": "No graph-RAG quality claim is allowed until a Docker-safe adapter reaches query output." + "evidence": "No graph-RAG quality result is claimed from the checked-in research gate. Generated smoke artifacts may become live_real_world only after LightRAG returns context or references mapped to generated evidence ids.", + "artifact": "tmp/real-world-memory/lightrag-context/lightrag-report.json" }, "capabilities": [ { - "capability": "graph_augmented_rag_setup", - "status": "not_encoded", - "evidence": "XY-882 completed setup/output feasibility research; graph-augmented RAG execution is still not encoded." + "capability": "docker_service_setup", + "status": "blocked", + "evidence": "The opt-in compose profile records explicit LightRAG image, LLM, embedding, rerank, workspace, and Docker volume configuration without host-global installs." }, { "capability": "retrieved_context_export", "status": "blocked", - "evidence": "The adapter must prove it can extract evidence-bearing retrieved contexts for scoring." + "evidence": "The materializer calls /documents/texts, waits on /documents/track_status, and queries /query with only_need_context plus chunk references when the service is reachable." }, { "capability": "real_world_job_adapter", + "status": "blocked", + "evidence": "The LightRAG materializer rewrites generated retrieval fixtures with adapter_response evidence only when source paths or context map to required evidence ids." + }, + { + "capability": "quality_or_scale_claim", "status": "not_encoded", - "evidence": "No LightRAG fixture materializer or scorer mapping exists." + "evidence": "The smoke does not score broad graph-RAG quality, private corpora, scale, or comparative ranking claims." } ], "suites": [ { "suite_id": "retrieval", "status": "blocked", - "evidence": "Graph/vector retrieval output mapping needs research." + "evidence": "The generated smoke can exercise retrieval context/source mapping for retrieval fixtures, but the checked-in record stays blocked until a live artifact reaches query output." }, { "suite_id": "memory_evolution", - "status": "blocked", - "evidence": "Stale/corrected fact update behavior is not yet audited." + "status": "not_encoded", + "evidence": "LightRAG update/delete/current-versus-historical behavior is not encoded by the context-export smoke." }, { "suite_id": "operator_debugging_ux", "status": "not_encoded", - "evidence": "Trace or context-debug output is not mapped to benchmark scoring." + "evidence": "The smoke records context/source mappings, but full trace or viewer diagnostics are not mapped to benchmark scoring." } ], "evidence": [ @@ -1218,6 +1228,16 @@ "kind": "source", "ref": "https://github.com/HKUDS/LightRAG/blob/main/docs/DockerDeployment.md", "status": "real" + }, + { + "kind": "command", + "ref": "cargo make lightrag-docker-context-smoke", + "status": "blocked" + }, + { + "kind": "artifact", + "ref": "tmp/real-world-memory/lightrag-context/lightrag-materialization.json", + "status": "blocked" } ], "execution_metadata": { @@ -1243,14 +1263,15 @@ "evidence": "Official source-id and file-path citation reference." } ], - "setup_path": "Implement Docker Compose with explicit LLM, embedding, rerank, storage, workspace, and data-volume configuration, then export context-only query output.", - "runtime_boundary": "Docker-only service profile with generated corpus mounted as container-local input.", - "resource_expectation": "Graph extraction and local model choices may dominate runtime; record backend choices, cache sizes, and provider needs.", + "setup_path": "Run cargo make lightrag-docker-context-smoke for a typed preflight artifact; set ELF_LIGHTRAG_CONTEXT_START=1 to start the opt-in LightRAG Docker profile and attempt live context export.", + "runtime_boundary": "docker-compose.baseline.yml baseline-runner plus opt-in lightrag and lightrag-mock-provider services; generated source files and LightRAG data stay in Docker-mounted artifact paths and Docker volumes.", + "resource_expectation": "The default profile uses the official LightRAG image, a local OpenAI-compatible mock provider, 64-dimensional embeddings, rerank disabled for context queries, cargo/pip/Hugging Face caches, and Docker volumes for rag_storage, inputs, and prompts.", "retry_guidance": [ - "Run a tiny Docker ingest/query smoke with deterministic or local providers.", - "Verify returned contexts can be mapped to required evidence IDs." + "Run cargo make lightrag-docker-context-smoke first; a missing API must remain a typed incomplete artifact, not a pass claim.", + "Set ELF_LIGHTRAG_CONTEXT_START=1 only when Docker may pull/start the LightRAG service profile.", + "Score retrieval only when returned context, references.file_path, or references.content map to required evidence ids." ], - "research_depth": "D2 feasibility verdict: adapter_candidate (XY-882); research_gate only, adapter not encoded" + "research_depth": "D2 feasibility plus XY-886 context-export implementation; checked-in record remains research_gate unless a generated artifact reaches query output" }, "follow_up": { "title": "[ELF benchmark adapter] Implement LightRAG Docker context-export adapter", diff --git a/apps/elf-eval/src/bin/real_world_live_adapter.rs b/apps/elf-eval/src/bin/real_world_live_adapter.rs index 00a564b9..ac30d229 100644 --- a/apps/elf-eval/src/bin/real_world_live_adapter.rs +++ b/apps/elf-eval/src/bin/real_world_live_adapter.rs @@ -10,15 +10,16 @@ use std::{ path::{Path, PathBuf}, process::{Command, Stdio}, sync::Arc, - time::Instant, + time::{Duration, Instant}, }; use blake3::Hasher; use clap::{Parser, Subcommand, ValueEnum}; use color_eyre::{self, eyre}; +use reqwest::RequestBuilder; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use tokio::task::JoinSet; +use tokio::{task::JoinSet, time}; use uuid::Uuid; use elf_chunking::ChunkingConfig; @@ -89,6 +90,52 @@ struct QmdArgs { adapter_id: String, } +#[derive(Debug, Parser)] +struct LightragArgs { + /// Fixture file or directory containing real_world_job JSON fixtures. + #[arg(long, value_name = "PATH")] + fixtures: PathBuf, + /// Directory where generated real_world_job fixtures are written. + #[arg(long, value_name = "DIR")] + out_fixtures: PathBuf, + /// JSON evidence file for adapter setup/run/result details. + #[arg(long, value_name = "FILE")] + evidence_out: PathBuf, + /// Work directory for generated source files and command logs. + #[arg(long, value_name = "DIR")] + work_dir: PathBuf, + /// LightRAG API base URL reachable from the Docker runner. + #[arg(long, default_value = "http://lightrag:9621")] + api_base: String, + /// Optional LightRAG API bearer token. + #[arg(long)] + api_key: Option, + /// Adapter id embedded in generated adapter_response objects. + #[arg(long, default_value = "lightrag_live_real_world")] + adapter_id: String, + /// LightRAG query mode used for context export. + #[arg(long, default_value = "naive")] + query_mode: String, + /// Number of top results requested from LightRAG. + #[arg(long, default_value_t = 5)] + top_k: u32, + /// Number of chunk results requested from LightRAG. + #[arg(long, default_value_t = 5)] + chunk_top_k: u32, + /// Health-check attempts before returning a typed runtime failure. + #[arg(long, default_value_t = 30)] + startup_attempts: u32, + /// Delay between LightRAG health-check attempts. + #[arg(long, default_value_t = 2)] + startup_interval_seconds: u64, + /// Poll attempts for asynchronous document indexing. + #[arg(long, default_value_t = 60)] + index_attempts: u32, + /// Delay between document indexing status checks. + #[arg(long, default_value_t = 2)] + index_interval_seconds: u64, +} + #[derive(Debug)] struct LoadedJob { path: PathBuf, @@ -158,6 +205,8 @@ struct MaterializationEvidence { generated_fixtures: String, command_evidence: Vec, jobs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + metadata: Option, } #[derive(Debug, Serialize)] @@ -178,9 +227,13 @@ struct MaterializedJobEvidence { query: String, evidence_ids: Vec, returned_count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + indexing_latency_ms: Option, latency_ms: f64, trace_id: Option, failure: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + source_mappings: Vec, } #[derive(Debug, Serialize)] @@ -236,9 +289,11 @@ struct MaterializedJobInput { content: String, evidence_ids: Vec, latency_ms: f64, + indexing_latency_ms: Option, returned_count: usize, trace_id: Option, failure: Option, + source_mappings: Vec, } struct MaterializedOutput<'a> { @@ -250,6 +305,7 @@ struct MaterializedOutput<'a> { jobs: &'a [LoadedJob], materialized: &'a [MaterializedJob], command_evidence: Vec, + metadata: Option, } #[derive(Debug)] @@ -258,6 +314,21 @@ struct CorpusText { text: String, } +#[derive(Clone, Debug, Serialize)] +struct SourceMappingEvidence { + source: String, + evidence_ids: Vec, + mapping_status: String, + content_count: usize, +} + +#[derive(Debug)] +struct LightragSource { + evidence_id: String, + file_source: String, + artifact_path: PathBuf, +} + #[derive(Debug)] struct BaselineRuntime { config_path: PathBuf, @@ -380,6 +451,8 @@ enum CommandArgs { Elf(ElfArgs), /// Materialize adapter responses by running jobs through qmd's local CLI workflow. Qmd(QmdArgs), + /// Materialize adapter responses by exporting LightRAG query context and source mappings. + Lightrag(LightragArgs), } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, ValueEnum)] @@ -387,6 +460,7 @@ enum CommandArgs { enum AdapterKind { ElfServiceRuntime, QmdCliRuntime, + LightragApiContextExport, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] @@ -423,6 +497,7 @@ fn run_qmd(args: QmdArgs) -> color_eyre::Result<()> { reason: "qmd live adapter used collection add, update, embed, and query --json." .to_string(), }], + metadata: None, }) } @@ -575,13 +650,285 @@ fn materialize_qmd_job( content: selected.content, evidence_ids: selected.evidence_ids, latency_ms, + indexing_latency_ms: None, returned_count: entries.len(), trace_id: None, failure: None, + source_mappings: Vec::new(), }, )) } +fn lightrag_not_encoded_job(adapter_id: &str, loaded: &LoadedJob) -> Option { + match loaded.job.suite.as_str() { + "retrieval" => None, + _ => Some(materialized_declared_status_job( + adapter_id, + loaded, + MaterializationStatus::NotEncoded, + "LightRAG context-export smoke only maps retrieved context/source paths; this suite is not encoded for LightRAG scoring.".to_string(), + )), + } +} + +fn lightrag_failure_jobs( + adapter_id: &str, + jobs: &[LoadedJob], + stage: &str, + reason: String, +) -> Vec { + jobs.iter() + .map(|job| { + if let Some(declared) = declared_encoding_job(adapter_id, job) { + return declared; + } + if let Some(not_encoded) = lightrag_not_encoded_job(adapter_id, job) { + return not_encoded; + } + + materialized_job( + job, + adapter_id, + MaterializedJobInput { + content: String::new(), + evidence_ids: Vec::new(), + latency_ms: 0.0, + indexing_latency_ms: None, + returned_count: 0, + trace_id: None, + failure: Some(format!("{stage}: {reason}")), + source_mappings: Vec::new(), + }, + ) + }) + .collect() +} + +fn write_lightrag_corpus( + args: &LightragArgs, + loaded: &LoadedJob, + corpus: &[CorpusText], + run_slug: &str, +) -> color_eyre::Result> { + let job_slug = slug(&loaded.job.job_id); + let corpus_dir = args.work_dir.join("corpus").join(run_slug).join(&job_slug); + + fs::create_dir_all(&corpus_dir)?; + + corpus + .iter() + .map(|item| { + let file_name = format!("{}.md", slug(&item.evidence_id)); + let artifact_path = corpus_dir.join(&file_name); + let file_source = format!("elf-real-world/{run_slug}/{job_slug}/{file_name}"); + + fs::write(&artifact_path, format!("# {}\n\n{}\n", item.evidence_id, item.text))?; + + Ok(LightragSource { evidence_id: item.evidence_id.clone(), file_source, artifact_path }) + }) + .collect() +} + +fn lightrag_index_failed(status: &Value) -> bool { + status.get("documents").and_then(Value::as_array).into_iter().flatten().any(|doc| { + doc.get("status") + .and_then(Value::as_str) + .is_some_and(|status| status.to_ascii_lowercase().contains("fail")) + }) +} + +fn lightrag_index_processed(status: &Value, expected_docs: usize) -> bool { + let Some(documents) = status.get("documents").and_then(Value::as_array) else { + return false; + }; + + documents.len() >= expected_docs + && documents.iter().all(|doc| { + doc.get("status").and_then(Value::as_str).is_some_and(|status| { + let normalized = status.to_ascii_lowercase(); + + normalized.contains("processed") || normalized.contains("success") + }) + }) +} + +fn lightrag_keywords(query: &str) -> Vec { + terms(query).into_iter().take(12).collect() +} + +fn lightrag_source_mappings( + corpus: &[CorpusText], + sources: &[LightragSource], + response: &Value, +) -> Vec { + let mut mappings = Vec::new(); + + if let Some(references) = response.get("references").and_then(Value::as_array) { + for reference in references { + mappings.push(lightrag_reference_mapping(corpus, sources, reference)); + } + } + + if mappings.is_empty() + && let Some(context) = response.get("response").and_then(Value::as_str) + { + let evidence_ids = map_lightrag_evidence_ids(corpus, sources, context); + + if !evidence_ids.is_empty() { + mappings.push(SourceMappingEvidence { + source: "response_context".to_string(), + evidence_ids, + mapping_status: "matched_context".to_string(), + content_count: 1, + }); + } + } + + mappings +} + +fn lightrag_reference_mapping( + corpus: &[CorpusText], + sources: &[LightragSource], + reference: &Value, +) -> SourceMappingEvidence { + let source = reference + .get("file_path") + .and_then(Value::as_str) + .or_else(|| reference.get("reference_id").and_then(Value::as_str)) + .unwrap_or("unknown_source") + .to_string(); + let content = reference + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect::>(); + let joined_content = content.join("\n"); + let combined = format!("{source}\n{joined_content}"); + let evidence_ids = map_lightrag_evidence_ids(corpus, sources, combined.as_str()); + let mapping_status = if evidence_ids.is_empty() { + "unmatched" + } else if !joined_content.is_empty() { + "matched_reference_content" + } else { + "matched_reference_source" + }; + + SourceMappingEvidence { + source, + evidence_ids, + mapping_status: mapping_status.to_string(), + content_count: content.len(), + } +} + +fn map_lightrag_evidence_ids( + corpus: &[CorpusText], + sources: &[LightragSource], + haystack: &str, +) -> Vec { + let normalized_haystack = normalize_ascii_alnum_lowercase(haystack); + let mut evidence_ids = Vec::new(); + + for item in corpus { + let evidence_slug = slug(&item.evidence_id); + let signature = normalized_text_signature(item.text.as_str()); + let source_match = sources.iter().any(|source| { + source.evidence_id == item.evidence_id + && (haystack.contains(source.file_source.as_str()) + || haystack.contains(source.artifact_path.to_string_lossy().as_ref())) + }); + let id_match = haystack.contains(item.evidence_id.as_str()) + || haystack.contains(evidence_slug.as_str()) + || normalized_haystack.contains(evidence_slug.as_str()); + let content_match = + !signature.is_empty() && normalized_haystack.contains(signature.as_str()); + + if source_match || id_match || content_match { + push_unique(&mut evidence_ids, item.evidence_id.clone()); + } + } + + evidence_ids +} + +fn normalized_text_signature(text: &str) -> String { + normalize_ascii_alnum_lowercase(text).split_whitespace().take(8).collect::>().join(" ") +} + +fn lightrag_mapped_evidence_ids(mappings: &[SourceMappingEvidence]) -> Vec { + let mut evidence_ids = Vec::new(); + + for mapping in mappings { + for evidence_id in &mapping.evidence_ids { + push_unique(&mut evidence_ids, evidence_id.clone()); + } + } + + evidence_ids +} + +fn lightrag_api_base(args: &LightragArgs) -> String { + args.api_base.trim_end_matches('/').to_string() +} + +fn lightrag_metadata(args: &LightragArgs, run_slug: &str) -> Value { + serde_json::json!({ + "schema": "elf.lightrag_context_export_metadata/v1", + "run_slug": run_slug, + "api_base": lightrag_api_base(args), + "query": { + "mode": args.query_mode, + "only_need_context": true, + "include_references": true, + "include_chunk_content": true, + "enable_rerank": false, + "top_k": args.top_k, + "chunk_top_k": args.chunk_top_k + }, + "docker_boundary": { + "compose_file": "docker-compose.baseline.yml", + "service_profile": "lightrag", + "service": "lightrag", + "mock_provider_service": "lightrag-mock-provider", + "host_global_installs_required": false, + "workspace": "/app/data/rag_storage", + "input_dir": "/app/data/inputs", + "data_volumes": [ + "elf-live-baseline-lightrag-rag-storage", + "elf-live-baseline-lightrag-inputs", + "elf-live-baseline-lightrag-prompts" + ] + }, + "provider_boundaries": { + "llm_binding": "openai-compatible", + "embedding_binding": "openai-compatible", + "embedding_dim": 64, + "rerank_binding": "cohere-compatible", + "rerank_enabled_for_query": false, + "api_key_provided": args.api_key.as_deref().is_some_and(|key| !key.is_empty()), + "operator_owned_provider_credentials_used": false + }, + "cache_and_resource_envelope": { + "cargo_cache": "/usr/local/cargo", + "pip_cache": "/root/.cache/pip", + "huggingface_cache": "/root/.cache/huggingface", + "lightrag_storage": "/app/data/rag_storage", + "startup_attempts": args.startup_attempts, + "startup_interval_seconds": args.startup_interval_seconds, + "index_attempts": args.index_attempts, + "index_interval_seconds": args.index_interval_seconds + }, + "source_mapping": { + "corpus_file_source_template": "elf-real-world/{run_slug}/{job_slug}/{evidence_id}.md", + "mapping_inputs": ["references.file_path", "references.content", "response"], + "quality_claim": "none" + } + }) +} + fn materialized_job( loaded: &LoadedJob, adapter_id: &str, @@ -639,9 +986,11 @@ fn materialized_job( query: loaded.job.prompt.content.clone(), evidence_ids: input.evidence_ids, returned_count: input.returned_count, + indexing_latency_ms: input.indexing_latency_ms, latency_ms: input.latency_ms, trace_id: input.trace_id, failure: input.failure, + source_mappings: input.source_mappings, }, } } @@ -748,9 +1097,11 @@ fn materialized_declared_status_job( query: loaded.job.prompt.content.clone(), evidence_ids: Vec::new(), returned_count: 0, + indexing_latency_ms: None, latency_ms: 0.0, trace_id: None, failure, + source_mappings: Vec::new(), }, } } @@ -864,9 +1215,11 @@ fn failure_jobs( content: String::new(), evidence_ids: Vec::new(), latency_ms: 0.0, + indexing_latency_ms: None, returned_count: 0, trace_id: None, failure: Some(format!("{stage}: {reason}")), + source_mappings: Vec::new(), }, ) }) @@ -926,6 +1279,7 @@ fn write_materialized_output(output: MaterializedOutput<'_>) -> color_eyre::Resu generated_fixtures: output.out_fixtures.display().to_string(), command_evidence: output.command_evidence, jobs: output.materialized.iter().map(|job| clone_job_evidence(&job.evidence)).collect(), + metadata: output.metadata, }; if let Some(parent) = output.evidence_out.parent() { @@ -946,9 +1300,11 @@ fn clone_job_evidence(evidence: &MaterializedJobEvidence) -> MaterializedJobEvid query: evidence.query.clone(), evidence_ids: evidence.evidence_ids.clone(), returned_count: evidence.returned_count, + indexing_latency_ms: evidence.indexing_latency_ms, latency_ms: evidence.latency_ms, trace_id: evidence.trace_id, failure: evidence.failure.clone(), + source_mappings: evidence.source_mappings.clone(), } } @@ -1353,6 +1709,255 @@ fn split_long_token(token: &str) -> Vec { chunks } +async fn run_lightrag_async(args: LightragArgs) -> color_eyre::Result<()> { + let jobs = load_jobs(&args.fixtures)?; + let run_slug = short_hash(format!("{}:{}", args.adapter_id, Uuid::new_v4()).as_str()); + let result = materialize_lightrag_jobs(&args, &jobs, &run_slug).await; + let materialized = match result { + Ok(jobs) => jobs, + Err(err) => lightrag_failure_jobs( + &args.adapter_id, + &jobs, + "lightrag_api_context_export", + err.to_string(), + ), + }; + let status = aggregate_status(&materialized); + + write_materialized_output(MaterializedOutput { + adapter_id: &args.adapter_id, + adapter_kind: AdapterKind::LightragApiContextExport, + fixtures: &args.fixtures, + out_fixtures: &args.out_fixtures, + evidence_out: &args.evidence_out, + jobs: &jobs, + materialized: &materialized, + command_evidence: vec![CommandEvidence { + label: "lightrag_api_context_export".to_string(), + status, + command: "cargo run -p elf-eval --bin real_world_live_adapter -- lightrag" + .to_string(), + artifact: Some(args.evidence_out.display().to_string()), + reason: "LightRAG adapter used /documents/texts, /documents/track_status, and /query with only_need_context plus chunk references.".to_string(), + }], + metadata: Some(lightrag_metadata(&args, &run_slug)), + }) +} + +async fn materialize_lightrag_jobs( + args: &LightragArgs, + jobs: &[LoadedJob], + run_slug: &str, +) -> color_eyre::Result> { + fs::create_dir_all(&args.work_dir)?; + + let client = reqwest::Client::builder().timeout(Duration::from_secs(180)).build()?; + + wait_for_lightrag(args, &client).await?; + + let mut out = Vec::with_capacity(jobs.len()); + + for loaded in jobs { + out.push(materialize_lightrag_job(args, &client, loaded, run_slug).await?); + } + + Ok(out) +} + +async fn wait_for_lightrag( + args: &LightragArgs, + client: &reqwest::Client, +) -> color_eyre::Result<()> { + let mut last_error = String::new(); + + for _attempt in 1..=args.startup_attempts { + match lightrag_get_json(args, client, "/health").await { + Ok(_) => return Ok(()), + Err(err) => last_error = err.to_string(), + } + + time::sleep(Duration::from_secs(args.startup_interval_seconds)).await; + } + + Err(eyre::eyre!( + "LightRAG API did not become healthy at {} after {} attempts: {}", + lightrag_api_base(args), + args.startup_attempts, + last_error + )) +} + +async fn materialize_lightrag_job( + args: &LightragArgs, + client: &reqwest::Client, + loaded: &LoadedJob, + run_slug: &str, +) -> color_eyre::Result { + if let Some(job) = declared_encoding_job(&args.adapter_id, loaded) { + return Ok(job); + } + if let Some(job) = lightrag_not_encoded_job(&args.adapter_id, loaded) { + return Ok(job); + } + + let corpus = corpus_texts(loaded)?; + let sources = write_lightrag_corpus(args, loaded, &corpus, run_slug)?; + let indexed_at = Instant::now(); + let insert_response = insert_lightrag_texts(args, client, &corpus, &sources).await?; + + wait_for_lightrag_index(args, client, &insert_response, corpus.len()).await?; + + let indexing_latency_ms = indexed_at.elapsed().as_secs_f64() * 1_000.0; + let queried_at = Instant::now(); + let query_response = query_lightrag_context(args, client, loaded).await?; + let latency_ms = queried_at.elapsed().as_secs_f64() * 1_000.0; + let source_mappings = lightrag_source_mappings(&corpus, &sources, &query_response); + let evidence_ids = lightrag_mapped_evidence_ids(&source_mappings); + let selected = selected_required_corpus_texts(loaded, &corpus, &evidence_ids); + + Ok(materialized_job( + loaded, + &args.adapter_id, + MaterializedJobInput { + content: selected.content, + evidence_ids: selected.evidence_ids, + latency_ms, + indexing_latency_ms: Some(indexing_latency_ms), + returned_count: source_mappings.len(), + trace_id: None, + failure: None, + source_mappings, + }, + )) +} + +async fn insert_lightrag_texts( + args: &LightragArgs, + client: &reqwest::Client, + corpus: &[CorpusText], + sources: &[LightragSource], +) -> color_eyre::Result { + let request = serde_json::json!({ + "texts": corpus.iter().map(|item| item.text.as_str()).collect::>(), + "file_sources": sources.iter().map(|source| source.file_source.as_str()).collect::>(), + "chunking": { + "strategy": "fixed_token", + "params": { + "chunk_token_size": 320, + "chunk_overlap_token_size": 32 + } + } + }); + + lightrag_post_json(args, client, "/documents/texts", &request).await +} + +async fn wait_for_lightrag_index( + args: &LightragArgs, + client: &reqwest::Client, + insert_response: &Value, + expected_docs: usize, +) -> color_eyre::Result<()> { + let track_id = insert_response + .get("track_id") + .and_then(Value::as_str) + .ok_or_else(|| eyre::eyre!("LightRAG text insert response did not include track_id."))?; + let mut last_status = Value::Null; + + for _attempt in 1..=args.index_attempts { + let status = + lightrag_get_json(args, client, format!("/documents/track_status/{track_id}")).await?; + + if lightrag_index_failed(&status) { + return Err(eyre::eyre!( + "LightRAG document indexing failed for track_id {track_id}: {}", + serde_json::to_string(&status)? + )); + } + if lightrag_index_processed(&status, expected_docs) { + return Ok(()); + } + + last_status = status; + + time::sleep(Duration::from_secs(args.index_interval_seconds)).await; + } + + Err(eyre::eyre!( + "LightRAG document indexing did not finish for track_id {} after {} attempts: {}", + track_id, + args.index_attempts, + serde_json::to_string(&last_status)? + )) +} + +async fn query_lightrag_context( + args: &LightragArgs, + client: &reqwest::Client, + loaded: &LoadedJob, +) -> color_eyre::Result { + let keywords = lightrag_keywords(loaded.job.prompt.content.as_str()); + let request = serde_json::json!({ + "query": loaded.job.prompt.content, + "mode": args.query_mode, + "only_need_context": true, + "include_references": true, + "include_chunk_content": true, + "enable_rerank": false, + "top_k": args.top_k, + "chunk_top_k": args.chunk_top_k, + "hl_keywords": keywords, + "ll_keywords": keywords, + "stream": false + }); + + lightrag_post_json(args, client, "/query", &request).await +} + +async fn lightrag_get_json( + args: &LightragArgs, + client: &reqwest::Client, + path: impl AsRef, +) -> color_eyre::Result { + let url = format!("{}{}", lightrag_api_base(args), path.as_ref()); + let mut request = client.get(url); + + if let Some(api_key) = args.api_key.as_deref().filter(|key| !key.is_empty()) { + request = request.bearer_auth(api_key); + } + + lightrag_send_json(request).await +} + +async fn lightrag_post_json( + args: &LightragArgs, + client: &reqwest::Client, + path: &str, + body: &Value, +) -> color_eyre::Result { + let url = format!("{}{}", lightrag_api_base(args), path); + let mut request = client.post(url).json(body); + + if let Some(api_key) = args.api_key.as_deref().filter(|key| !key.is_empty()) { + request = request.bearer_auth(api_key); + } + + lightrag_send_json(request).await +} + +async fn lightrag_send_json(request: RequestBuilder) -> color_eyre::Result { + let response = request.send().await?; + let status = response.status(); + let body = response.text().await?; + + if !status.is_success() { + return Err(eyre::eyre!("LightRAG API returned HTTP {status}: {body}")); + } + + serde_json::from_str(&body) + .map_err(|err| eyre::eyre!("LightRAG API returned invalid JSON: {err}; body={body}")) +} + #[tokio::main] async fn main() -> color_eyre::Result<()> { color_eyre::install()?; @@ -1360,6 +1965,7 @@ async fn main() -> color_eyre::Result<()> { match Args::parse().command { CommandArgs::Elf(args) => run_elf(args).await, CommandArgs::Qmd(args) => run_qmd(args), + CommandArgs::Lightrag(args) => run_lightrag_async(args).await, } } @@ -1387,6 +1993,7 @@ async fn run_elf(args: ElfArgs) -> color_eyre::Result<()> { reason: "ELF live adapter used ElfService, worker indexing, and search_raw." .to_string(), }], + metadata: None, }) } @@ -1527,9 +2134,11 @@ async fn materialize_elf_job( content: selected.content, evidence_ids: selected.evidence_ids, latency_ms, + indexing_latency_ms: None, returned_count: response.items.len(), trace_id: Some(response.trace_id), failure: None, + source_mappings: Vec::new(), }, )) } diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index b3e0e99f..1ac9bfd2 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -281,7 +281,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/suite_status_counts/blocked") .and_then(Value::as_u64), - Some(11) + Some(10) ); } @@ -294,6 +294,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { let agentmemory = find_by_field(adapters, "/adapter_id", "agentmemory_live_baseline")?; let openviking = find_by_field(adapters, "/adapter_id", "openviking_live_baseline")?; let ragflow = find_by_field(adapters, "/adapter_id", "ragflow_research_gate")?; + let lightrag = find_by_field(adapters, "/adapter_id", "lightrag_research_gate")?; let qmd_deep = find_by_field(adapters, "/adapter_id", "qmd_deep_profile_gate")?; assert_eq!(elf.pointer("/evidence_class").and_then(Value::as_str), Some("fixture_backed")); @@ -341,6 +342,20 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { ragflow.pointer("/execution_metadata/sources/0/url").and_then(Value::as_str), Some("https://github.com/infiniflow/ragflow") ); + assert_eq!(lightrag.pointer("/evidence_class").and_then(Value::as_str), Some("research_gate")); + assert_eq!(lightrag.pointer("/overall_status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + lightrag.pointer("/setup/command").and_then(Value::as_str), + Some("cargo make lightrag-docker-context-smoke") + ); + assert_eq!( + lightrag.pointer("/run/command").and_then(Value::as_str), + Some("ELF_LIGHTRAG_CONTEXT_START=1 cargo make lightrag-docker-context-smoke") + ); + assert_eq!( + lightrag.pointer("/capabilities/3/status").and_then(Value::as_str), + Some("not_encoded") + ); assert_eq!( qmd_deep.pointer("/capabilities/2/status").and_then(Value::as_str), Some("unsupported") diff --git a/docker-compose.baseline.yml b/docker-compose.baseline.yml index 5793f66c..9d5c6972 100644 --- a/docker-compose.baseline.yml +++ b/docker-compose.baseline.yml @@ -22,6 +22,56 @@ services: volumes: - elf-live-baseline-qdrant-data:/qdrant/storage + lightrag-mock-provider: + profiles: + - lightrag + image: python:3.13-slim + environment: + ELF_LIGHTRAG_MOCK_EMBEDDING_DIM: ${ELF_LIGHTRAG_EMBEDDING_DIM:-64} + ELF_LIGHTRAG_MOCK_HOST: 0.0.0.0 + ELF_LIGHTRAG_MOCK_PORT: 8080 + command: + - python + - /app/scripts/lightrag-mock-openai-provider.py + volumes: + - ./scripts/lightrag-mock-openai-provider.py:/app/scripts/lightrag-mock-openai-provider.py:ro + + lightrag: + profiles: + - lightrag + image: ${ELF_LIGHTRAG_IMAGE:-ghcr.io/hkuds/lightrag:latest} + depends_on: + - lightrag-mock-provider + environment: + WORKING_DIR: /app/data/rag_storage + INPUT_DIR: /app/data/inputs + PROMPT_DIR: /app/data/prompts + HOST: 0.0.0.0 + PORT: 9621 + LLM_BINDING: ${ELF_LIGHTRAG_LLM_BINDING:-openai} + LLM_BINDING_HOST: ${ELF_LIGHTRAG_LLM_BINDING_HOST:-http://lightrag-mock-provider:8080/v1} + LLM_BINDING_API_KEY: ${ELF_LIGHTRAG_LLM_BINDING_API_KEY:-local-key} + LLM_MODEL: ${ELF_LIGHTRAG_LLM_MODEL:-elf-lightrag-mock} + EMBEDDING_BINDING: ${ELF_LIGHTRAG_EMBEDDING_BINDING:-openai} + EMBEDDING_BINDING_HOST: ${ELF_LIGHTRAG_EMBEDDING_BINDING_HOST:-http://lightrag-mock-provider:8080/v1} + EMBEDDING_BINDING_API_KEY: ${ELF_LIGHTRAG_EMBEDDING_BINDING_API_KEY:-local-key} + EMBEDDING_MODEL: ${ELF_LIGHTRAG_EMBEDDING_MODEL:-elf-lightrag-mock-embedding} + EMBEDDING_DIM: ${ELF_LIGHTRAG_EMBEDDING_DIM:-64} + RERANK_BY_DEFAULT: ${ELF_LIGHTRAG_RERANK_BY_DEFAULT:-False} + RERANK_BINDING: ${ELF_LIGHTRAG_RERANK_BINDING:-cohere} + RERANK_BINDING_HOST: ${ELF_LIGHTRAG_RERANK_BINDING_HOST:-http://lightrag-mock-provider:8080/rerank} + RERANK_BINDING_API_KEY: ${ELF_LIGHTRAG_RERANK_BINDING_API_KEY:-local-key} + RERANK_MODEL: ${ELF_LIGHTRAG_RERANK_MODEL:-elf-lightrag-mock-rerank} + MAX_ASYNC_LLM: ${ELF_LIGHTRAG_MAX_ASYNC_LLM:-1} + MAX_ASYNC_RERANK: ${ELF_LIGHTRAG_MAX_ASYNC_RERANK:-1} + MAX_PARALLEL_INSERT: ${ELF_LIGHTRAG_MAX_PARALLEL_INSERT:-1} + CHUNK_SIZE: ${ELF_LIGHTRAG_CHUNK_SIZE:-320} + CHUNK_OVERLAP_SIZE: ${ELF_LIGHTRAG_CHUNK_OVERLAP_SIZE:-32} + volumes: + - elf-live-baseline-lightrag-rag-storage:/app/data/rag_storage + - elf-live-baseline-lightrag-inputs:/app/data/inputs + - elf-live-baseline-lightrag-prompts:/app/data/prompts + baseline-runner: build: context: . @@ -100,6 +150,9 @@ volumes: elf-live-baseline-cargo-git: elf-live-baseline-cargo-registry: elf-live-baseline-huggingface-cache: + elf-live-baseline-lightrag-inputs: + elf-live-baseline-lightrag-prompts: + elf-live-baseline-lightrag-rag-storage: elf-live-baseline-npm-cache: elf-live-baseline-pip-cache: elf-live-baseline-postgres-data: diff --git a/scripts/lightrag-docker-context-smoke.sh b/scripts/lightrag-docker-context-smoke.sh new file mode 100644 index 00000000..feac9054 --- /dev/null +++ b/scripts/lightrag-docker-context-smoke.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPORT_DIR="${ELF_LIGHTRAG_CONTEXT_REPORT_DIR:-${ROOT_DIR}/tmp/real-world-memory/lightrag-context}" +FIXTURE_DIR="${ELF_LIGHTRAG_CONTEXT_FIXTURES:-${ROOT_DIR}/apps/elf-eval/fixtures/real_world_memory/retrieval}" +WORK_DIR="${ELF_LIGHTRAG_CONTEXT_WORK_DIR:-/bench/real-world-live-adapters/lightrag}" +API_BASE="${ELF_LIGHTRAG_API_BASE:-http://lightrag:9621}" +ADAPTER_ID="${ELF_LIGHTRAG_ADAPTER_ID:-lightrag_live_real_world}" +ADAPTER_NAME="${ELF_LIGHTRAG_ADAPTER_NAME:-LightRAG Docker context-export adapter}" +STARTUP_ATTEMPTS="${ELF_LIGHTRAG_STARTUP_ATTEMPTS:-6}" +STARTUP_INTERVAL_SECONDS="${ELF_LIGHTRAG_STARTUP_INTERVAL_SECONDS:-2}" +INDEX_ATTEMPTS="${ELF_LIGHTRAG_INDEX_ATTEMPTS:-60}" +INDEX_INTERVAL_SECONDS="${ELF_LIGHTRAG_INDEX_INTERVAL_SECONDS:-2}" + +if [[ ! -f "/.dockerenv" && "${ELF_LIGHTRAG_CONTEXT_ALLOW_HOST:-0}" != "1" ]]; then + echo "Refusing to run LightRAG context smoke outside Docker. Use cargo make lightrag-docker-context-smoke." >&2 + exit 1 +fi + +for cmd in cargo jq; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "Missing ${cmd} in LightRAG context smoke runner." >&2 + exit 1 + fi +done + +mkdir -p "${REPORT_DIR}" "${WORK_DIR}" +rm -rf "${REPORT_DIR:?}/lightrag-fixtures" \ + "${REPORT_DIR:?}/lightrag-materialization.json" \ + "${REPORT_DIR:?}/lightrag-report.json" \ + "${REPORT_DIR:?}/lightrag-report.md" \ + "${REPORT_DIR:?}/summary.json" + +cd "${ROOT_DIR}" + +cargo run -p elf-eval --bin real_world_live_adapter -- lightrag \ + --fixtures "${FIXTURE_DIR}" \ + --out-fixtures "${REPORT_DIR}/lightrag-fixtures" \ + --evidence-out "${REPORT_DIR}/lightrag-materialization.json" \ + --work-dir "${WORK_DIR}" \ + --api-base "${API_BASE}" \ + --adapter-id "${ADAPTER_ID}" \ + --startup-attempts "${STARTUP_ATTEMPTS}" \ + --startup-interval-seconds "${STARTUP_INTERVAL_SECONDS}" \ + --index-attempts "${INDEX_ATTEMPTS}" \ + --index-interval-seconds "${INDEX_INTERVAL_SECONDS}" + +MATERIALIZATION_STATUS="$(jq -r '.status' "${REPORT_DIR}/lightrag-materialization.json")" + +cargo run -p elf-eval --bin real_world_job_benchmark -- run \ + --fixtures "${REPORT_DIR}/lightrag-fixtures" \ + --out "${REPORT_DIR}/lightrag-report.json" \ + --run-id real-world-memory-live-lightrag \ + --adapter-id "${ADAPTER_ID}" \ + --adapter-name "${ADAPTER_NAME}" \ + --adapter-behavior docker_api_context_export \ + --adapter-storage-status "${MATERIALIZATION_STATUS}" \ + --adapter-runtime-status "${MATERIALIZATION_STATUS}" \ + --adapter-notes "Materialized by real_world_live_adapter through the LightRAG Docker API using generated source file paths, /documents/texts ingest, /query context export, and reference/content evidence mapping; non-executed suites remain typed non-pass records." + +cargo run -p elf-eval --bin real_world_job_benchmark -- publish \ + --report "${REPORT_DIR}/lightrag-report.json" \ + --out "${REPORT_DIR}/lightrag-report.md" + +jq -n \ + --slurpfile materialization "${REPORT_DIR}/lightrag-materialization.json" \ + --slurpfile report "${REPORT_DIR}/lightrag-report.json" \ + '{ + schema: "elf.lightrag_context_export_smoke/v1", + generated_at: (now | todateiso8601), + artifact_dir: (env.ELF_LIGHTRAG_CONTEXT_REPORT_DIR // "tmp/real-world-memory/lightrag-context"), + fixture_dir: (env.ELF_LIGHTRAG_CONTEXT_FIXTURES // "apps/elf-eval/fixtures/real_world_memory/retrieval"), + adapter_id: (env.ELF_LIGHTRAG_ADAPTER_ID // "lightrag_live_real_world"), + evidence_class: "live_real_world_when_materialization_passes", + materialization: $materialization[0], + report: { + json: "tmp/real-world-memory/lightrag-context/lightrag-report.json", + markdown: "tmp/real-world-memory/lightrag-context/lightrag-report.md", + summary: $report[0].summary, + suites: $report[0].suites + } + }' >"${REPORT_DIR}/summary.json" + +echo "LightRAG context-export smoke reports:" +echo " ${REPORT_DIR}/lightrag-materialization.json" +echo " ${REPORT_DIR}/lightrag-report.json" +echo " ${REPORT_DIR}/lightrag-report.md" +echo " ${REPORT_DIR}/summary.json" diff --git a/scripts/lightrag-mock-openai-provider.py b/scripts/lightrag-mock-openai-provider.py new file mode 100644 index 00000000..975261d2 --- /dev/null +++ b/scripts/lightrag-mock-openai-provider.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Small OpenAI-compatible mock provider for LightRAG Docker smokes.""" + +from __future__ import annotations + +import hashlib +import json +import os +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + + +EMBEDDING_DIM = int(os.environ.get("ELF_LIGHTRAG_MOCK_EMBEDDING_DIM", "64")) +HOST = os.environ.get("ELF_LIGHTRAG_MOCK_HOST", "0.0.0.0") +PORT = int(os.environ.get("ELF_LIGHTRAG_MOCK_PORT", "8080")) + + +def _read_json(handler: BaseHTTPRequestHandler) -> dict[str, Any]: + length = int(handler.headers.get("content-length", "0")) + if length == 0: + return {} + raw = handler.rfile.read(length) + return json.loads(raw.decode("utf-8")) + + +def _write_json(handler: BaseHTTPRequestHandler, status: int, payload: dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + handler.send_response(status) + handler.send_header("content-type", "application/json") + handler.send_header("content-length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + +def _embedding(text: str) -> list[float]: + vector = [0.0] * EMBEDDING_DIM + for term in "".join(ch.lower() if ch.isalnum() else " " for ch in text).split(): + if len(term) < 2: + continue + digest = hashlib.blake2b(term.encode("utf-8"), digest_size=8).digest() + index = int.from_bytes(digest[:4], "little") % EMBEDDING_DIM + vector[index] += 1.0 + norm = sum(value * value for value in vector) ** 0.5 + if norm > 0: + vector = [value / norm for value in vector] + return vector + + +def _chat_completion(request: dict[str, Any]) -> dict[str, Any]: + content = ( + '{"entities":[],"relationships":[],"summary":"No graph facts extracted by ' + 'the local LightRAG smoke provider."}' + ) + return { + "id": "elf-lightrag-mock-chat", + "object": "chat.completion", + "model": request.get("model", "elf-lightrag-mock"), + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": content}, + } + ], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +def _embeddings(request: dict[str, Any]) -> dict[str, Any]: + inputs = request.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + return { + "object": "list", + "model": request.get("model", "elf-lightrag-mock-embedding"), + "data": [ + {"object": "embedding", "index": index, "embedding": _embedding(str(text))} + for index, text in enumerate(inputs) + ], + "usage": {"prompt_tokens": 0, "total_tokens": 0}, + } + + +def _rerank(request: dict[str, Any]) -> dict[str, Any]: + documents = request.get("documents", []) + if not isinstance(documents, list): + documents = [] + return { + "id": "elf-lightrag-mock-rerank", + "results": [ + {"index": index, "relevance_score": 1.0 / (index + 1)} + for index, _document in enumerate(documents) + ], + } + + +class Handler(BaseHTTPRequestHandler): + """HTTP handler for the mock provider.""" + + def do_GET(self) -> None: + if self.path in {"/health", "/v1/health"}: + _write_json(self, 200, {"status": "ok"}) + return + _write_json(self, 404, {"error": "not_found"}) + + def do_POST(self) -> None: + try: + request = _read_json(self) + if self.path.endswith("/chat/completions"): + _write_json(self, 200, _chat_completion(request)) + elif self.path.endswith("/embeddings"): + _write_json(self, 200, _embeddings(request)) + elif self.path.endswith("/rerank") or self.path == "/rerank": + _write_json(self, 200, _rerank(request)) + else: + _write_json(self, 404, {"error": "not_found", "path": self.path}) + except Exception as exc: # noqa: BLE001 + _write_json(self, 500, {"error": "mock_provider_error", "detail": str(exc)}) + + def log_message(self, format: str, *args: Any) -> None: + return + + +if __name__ == "__main__": + server = ThreadingHTTPServer((HOST, PORT), Handler) + server.serve_forever() diff --git a/scripts/real-world-live-adapters.sh b/scripts/real-world-live-adapters.sh index 26609d25..094db251 100755 --- a/scripts/real-world-live-adapters.sh +++ b/scripts/real-world-live-adapters.sh @@ -28,6 +28,7 @@ rm -rf "${REPORT_DIR:?}/elf-fixtures" \ "${REPORT_DIR:?}/elf-report.md" \ "${REPORT_DIR:?}/qmd-report.json" \ "${REPORT_DIR:?}/qmd-report.md" \ + "${REPORT_DIR:?}/lightrag" \ "${REPORT_DIR:?}/summary.json" cd "${ROOT_DIR}" @@ -75,6 +76,12 @@ cargo run -p elf-eval --bin real_world_job_benchmark -- publish \ --report "${REPORT_DIR}/qmd-report.json" \ --out "${REPORT_DIR}/qmd-report.md" +if [[ "${ELF_REAL_WORLD_LIVE_ENABLE_LIGHTRAG:-0}" == "1" ]]; then + ELF_LIGHTRAG_CONTEXT_REPORT_DIR="${REPORT_DIR}/lightrag" \ + ELF_LIGHTRAG_CONTEXT_FIXTURES="${ELF_LIGHTRAG_CONTEXT_FIXTURES:-${FIXTURE_DIR}/retrieval}" \ + bash scripts/lightrag-docker-context-smoke.sh +fi + jq -n \ --slurpfile elf_materialization "${REPORT_DIR}/elf-materialization.json" \ --slurpfile qmd_materialization "${REPORT_DIR}/qmd-materialization.json" \ @@ -111,9 +118,27 @@ jq -n \ ] }' >"${REPORT_DIR}/summary.json" +if [[ -f "${REPORT_DIR}/lightrag/summary.json" ]]; then + jq \ + --slurpfile lightrag_summary "${REPORT_DIR}/lightrag/summary.json" \ + '.adapters += [ + { + adapter_id: $lightrag_summary[0].adapter_id, + evidence_class: $lightrag_summary[0].evidence_class, + materialization: $lightrag_summary[0].materialization, + report: $lightrag_summary[0].report + } + ]' "${REPORT_DIR}/summary.json" >"${REPORT_DIR}/summary.json.tmp" + mv "${REPORT_DIR}/summary.json.tmp" "${REPORT_DIR}/summary.json" +fi + echo "Live real-world adapter reports:" echo " ${REPORT_DIR}/elf-report.json" echo " ${REPORT_DIR}/elf-report.md" echo " ${REPORT_DIR}/qmd-report.json" echo " ${REPORT_DIR}/qmd-report.md" +if [[ -f "${REPORT_DIR}/lightrag/summary.json" ]]; then + echo " ${REPORT_DIR}/lightrag/lightrag-report.json" + echo " ${REPORT_DIR}/lightrag/lightrag-report.md" +fi echo " ${REPORT_DIR}/summary.json"