diff --git a/README.md b/README.md index 8e6f65f5..a27508a6 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,15 @@ provider-backed ELF evidence was required. Postgres graph-lite facts to show current, historical, future, sourced, inferred, ambiguous, stale, and superseded markers without introducing a separate graph database or replacing source evidence. +- Recall/debug panel after XY-1022: the June 20 follow-up adds + `elf.recall_debug_panel/v1` through service, HTTP, and MCP readback. The panel + groups Memory Note trace selected rows and retained dropped replay candidates, + Source Library document candidates, Knowledge Workspace page snippets, graph facts, + and Dreaming proposals with + authority layer, freshness state, source refs, stage reason, evidence class, and + replay command. Missing anchors remain explicit `not_requested` layers, so the + panel improves debug ergonomics without turning untested or blocked layers into + pass claims. - Operator-approved public-proxy addendum after XY-930: the June 19 follow-up runs `cargo make baseline-production-private-addendum` with a simulated/public-proxy production corpus manifest approved for this stage. The run records 12 documents, diff --git a/apps/elf-api/src/routes.rs b/apps/elf-api/src/routes.rs index 7465fcd0..8722da64 100644 --- a/apps/elf-api/src/routes.rs +++ b/apps/elf-api/src/routes.rs @@ -61,11 +61,11 @@ use elf_service::{ KnowledgePageSearchResponse, KnowledgePagesListRequest, KnowledgePagesListResponse, ListRequest, ListResponse, MemoryHistoryGetRequest, MemoryHistoryResponse, NoteFetchRequest, NoteFetchResponse, NoteProvenanceBundleResponse, NoteProvenanceGetRequest, PayloadLevel, - PublishNoteRequest, QueryPlan, RankingRequestOverride, RebuildReport, SearchDetailsRequest, - SearchDetailsResult, SearchExplainRequest, SearchExplainResponse, SearchIndexItem, - SearchRequest, SearchResponse, SearchSessionGetRequest, SearchTimelineGroup, - SearchTimelineRequest, SearchTrajectoryResponse, SearchTrajectorySummary, ShareScope, - SpaceGrantRevokeRequest, SpaceGrantRevokeResponse, SpaceGrantUpsertRequest, + PublishNoteRequest, QueryPlan, RankingRequestOverride, RebuildReport, RecallDebugPanelRequest, + RecallDebugPanelResponse, SearchDetailsRequest, SearchDetailsResult, SearchExplainRequest, + SearchExplainResponse, SearchIndexItem, SearchRequest, SearchResponse, SearchSessionGetRequest, + SearchTimelineGroup, SearchTimelineRequest, SearchTrajectoryResponse, SearchTrajectorySummary, + ShareScope, SpaceGrantRevokeRequest, SpaceGrantRevokeResponse, SpaceGrantUpsertRequest, SpaceGrantsListRequest, TextPositionSelector, TextQuoteSelector, TraceBundleGetRequest, TraceBundleResponse, TraceGetRequest, TraceGetResponse, TraceRecentListRequest, TraceRecentListResponse, TraceTrajectoryGetRequest, UnpublishNoteRequest, UpdateRequest, @@ -148,6 +148,7 @@ const VIEWER_HTML: &str = include_str!("../static/viewer.html"); consolidation_proposal_get, consolidation_proposal_review, dreaming_review_queue, + recall_debug_panel, knowledge_page_rebuild, knowledge_pages_list, knowledge_pages_search, @@ -181,6 +182,7 @@ const VIEWER_HTML: &str = include_str!("../static/viewer.html"); (name = "graph", description = "Graph query and predicate administration."), (name = "consolidation", description = "Reviewable derived consolidation proposals."), (name = "dreaming", description = "Dreaming review queue and derived memory organization."), + (name = "recall", description = "Cross-layer recall and debug readback."), (name = "knowledge", description = "Derived knowledge page rebuild and lint readback."), (name = "admin", description = "Local admin and operator inspection routes."), ) @@ -515,6 +517,18 @@ struct TraceBundleGetQuery { candidates_limit: Option, } +#[derive(Clone, Debug, Deserialize)] +struct RecallDebugPanelBody { + trace_id: Option, + query: Option, + docs_query: Option, + knowledge_query: Option, + graph_subject: Option, + graph_predicate: Option, + include_dreaming: Option, + limit: Option, +} + #[derive(Clone, Debug, Deserialize)] struct ShareScopeBody { space: String, @@ -753,6 +767,7 @@ pub fn admin_router(state: AppState) -> Router { routing::post(consolidation_proposal_review), ) .route("/v2/admin/dreaming/review-queue", routing::get(dreaming_review_queue)) + .route("/v2/admin/recall-debug/panel", routing::post(recall_debug_panel)) .route("/v2/admin/knowledge/pages", routing::get(knowledge_pages_list)) .route("/v2/admin/knowledge/pages/rebuild", routing::post(knowledge_page_rebuild)) .route("/v2/admin/knowledge/pages/search", routing::post(knowledge_pages_search)) @@ -3118,6 +3133,52 @@ async fn dreaming_review_queue( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/admin/recall-debug/panel", + tag = "recall", + request_body = Value, + responses( + (status = 200, description = "Cross-layer recall/debug panel.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] +async fn recall_debug_panel( + State(state): State, + headers: HeaderMap, + payload: Result, JsonRejection>, +) -> Result, ApiError> { + let ctx = RequestContext::from_headers(&headers)?; + let read_profile = required_read_profile(&headers)?; + let Json(payload) = payload.map_err(|err| { + tracing::warn!(error = %err, "Invalid request payload."); + + json_error(StatusCode::BAD_REQUEST, "INVALID_REQUEST", "Invalid request payload.", None) + })?; + let response = state + .service + .recall_debug_panel(RecallDebugPanelRequest { + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + agent_id: ctx.agent_id, + read_profile, + trace_id: payload.trace_id, + query: payload.query, + docs_query: payload.docs_query, + knowledge_query: payload.knowledge_query, + graph_subject: payload.graph_subject, + graph_predicate: payload.graph_predicate, + include_dreaming: payload.include_dreaming, + limit: payload.limit, + }) + .await?; + + Ok(Json(response)) +} + #[utoipa::path( post, path = "/v2/admin/knowledge/pages/rebuild", diff --git a/apps/elf-eval/fixtures/report_snapshots/2026-06-20-recall-debug-panel-report.json b/apps/elf-eval/fixtures/report_snapshots/2026-06-20-recall-debug-panel-report.json new file mode 100644 index 00000000..da62fd38 --- /dev/null +++ b/apps/elf-eval/fixtures/report_snapshots/2026-06-20-recall-debug-panel-report.json @@ -0,0 +1,115 @@ +{ + "schema": "elf.recall_debug_panel_report/v1", + "authority": "XY-1022", + "generated_at": "2026-06-20T00:00:00Z", + "service_contract": { + "response_schema": "elf.recall_debug_panel/v1", + "service_module": "packages/elf-service/src/recall_debug.rs", + "http_endpoint": "POST /v2/admin/recall-debug/panel", + "mcp_tool": "elf_recall_debug_panel", + "spec": "docs/spec/system_recall_debug_panel_v1.md", + "read_model_only": true, + "raw_sql_needed": false + }, + "layer_contract": { + "layer_count": 5, + "layers": [ + { + "layer": "memory_notes", + "anchor": "trace_id", + "selection_states": ["selected", "dropped"], + "authority_layer": "memory_note", + "source_ref_surface": "memory_notes.source_ref", + "replay_surface": "elf_admin_trace_bundle_get", + "evidence_class": "pass" + }, + { + "layer": "source_documents", + "anchor": "docs_query or query", + "selection_states": ["selected"], + "authority_layer": "source_library", + "source_ref_surface": "source_ref/v1 resolver elf_doc_ext/v1", + "replay_surface": "elf_docs_search_l0", + "effective_limit": 32, + "evidence_class": "pass" + }, + { + "layer": "knowledge_pages", + "anchor": "knowledge_query or query", + "selection_states": ["selected"], + "authority_layer": "derived_knowledge_page", + "source_ref_surface": "source_coverage plus section source refs", + "replay_surface": "elf_recall_debug_panel", + "evidence_class": "pass" + }, + { + "layer": "graph_facts", + "anchor": "graph_subject", + "selection_states": ["available"], + "authority_layer": "graph_fact", + "source_ref_surface": "evidence_note_ids and supersession ids", + "replay_surface": "elf_graph_report", + "evidence_class": "pass" + }, + { + "layer": "dreaming_proposals", + "anchor": "include_dreaming", + "selection_states": ["reviewable"], + "authority_layer": "reviewable_dreaming_proposal", + "source_ref_surface": "source_refs, source_snapshot, affected_refs", + "replay_surface": "elf_dreaming_review_queue", + "evidence_class": "pass" + } + ] + }, + "debug_invariants": { + "not_requested_layers_preserved": true, + "requested_layer_failures_preserved_as_blocked": true, + "selected_and_dropped_memory_candidates": true, + "evidence_class_counts_preserved": true, + "authority_layer_required": true, + "freshness_state_required": true, + "stage_reason_required": true, + "source_refs_required": true, + "replay_command_or_artifact_path_required_when_available": true, + "no_source_mutation": true, + "no_graph_mutation": true, + "no_proposal_review_mutation": true + }, + "command_evidence": [ + { + "command": "cargo test -p elf-service recall_debug -- --nocapture", + "status": "pass", + "purpose": "Unit-check summary counters and not_requested layer behavior." + }, + { + "command": "cargo test -p elf-mcp registers_all_tools -- --nocapture", + "status": "pass", + "purpose": "Guard MCP registration for elf_recall_debug_panel." + }, + { + "command": "cargo test -p elf-eval --test real_world_job_benchmark recall_debug_panel_report_wires_cross_layer_debug_contract -- --nocapture", + "status": "pass", + "purpose": "Guard service/API/MCP/docs/snapshot coverage for XY-1022." + } + ], + "claim_boundaries": { + "allowed": [ + "ELF exposes a typed cross-layer recall/debug read model.", + "Memory trace selected rows and retained dropped replay candidates are visible through trace bundles when candidate capture/retention preserved them.", + "Source documents, knowledge pages, graph facts, and Dreaming proposals can be inspected from one panel response when their anchors are supplied.", + "not_requested layers remain explicit instead of being hidden behind aggregate pass claims." + ], + "not_allowed": [ + "Do not claim the panel is a mutating UI.", + "Do not claim external competitor UI parity from this read model alone.", + "Do not claim graph facts, source documents, or high-impact memories can be changed through the panel.", + "Do not treat missing anchors as pass evidence." + ] + }, + "next_optimization_direction": [ + "Add a visual operator panel that groups rows by layer, authority, freshness, and stage reason.", + "Attach one-click replay to trace bundles, docs search, graph reports, and Dreaming queue filters.", + "Use XY-1023 to score full benchmark deltas and keep competitor debug advantages separate from ELF's typed cross-layer readback." + ] +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index dcad0d75..e17c6bc3 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -254,6 +254,10 @@ fn dreaming_review_queue_report_json_path() -> Result { report_snapshot_path("2026-06-20-dreaming-review-queue-report.json") } +fn recall_debug_panel_report_json_path() -> Result { + report_snapshot_path("2026-06-20-recall-debug-panel-report.json") +} + fn openmemory_ui_export_product_readback_report_json_path() -> Result { report_snapshot_path("2026-06-19-openmemory-ui-export-product-readback-report.json") } @@ -300,6 +304,14 @@ fn dreaming_review_queue_report_markdown_path() -> Result { .join("2026-06-20-dreaming-review-queue-report.md")) } +fn recall_debug_panel_report_markdown_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("evidence") + .join("benchmarking") + .join("2026-06-20-recall-debug-panel-report.md")) +} + fn openmemory_ui_export_product_readback_report_markdown_path() -> Result { Ok(workspace_root()? .join("docs") @@ -3676,6 +3688,127 @@ fn dreaming_review_queue_report_wires_reviewable_policy_contract() -> Result<()> Ok(()) } +#[test] +fn recall_debug_panel_report_wires_cross_layer_debug_contract() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + recall_debug_panel_report_json_path()?, + )?)?; + let markdown = fs::read_to_string(recall_debug_panel_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(benchmarking_index_path()?)?; + let readme = fs::read_to_string(readme_path()?)?; + let workspace = workspace_root()?; + let service = fs::read_to_string(workspace.join("packages/elf-service/src/recall_debug.rs"))?; + let service_lib = fs::read_to_string(workspace.join("packages/elf-service/src/lib.rs"))?; + let routes = fs::read_to_string(workspace.join("apps/elf-api/src/routes.rs"))?; + let mcp = fs::read_to_string(workspace.join("apps/elf-mcp/src/server.rs"))?; + let recall_spec = + fs::read_to_string(workspace.join("docs/spec/system_recall_debug_panel_v1.md"))?; + let service_spec = + fs::read_to_string(workspace.join("docs/spec/system_elf_memory_service_v2.md"))?; + let version_registry = + fs::read_to_string(workspace.join("docs/spec/system_version_registry.md"))?; + + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.recall_debug_panel_report/v1") + ); + assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-1022")); + assert_eq!( + report.pointer("/service_contract/response_schema").and_then(Value::as_str), + Some("elf.recall_debug_panel/v1") + ); + assert_eq!( + report.pointer("/service_contract/read_model_only").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + report.pointer("/service_contract/raw_sql_needed").and_then(Value::as_bool), + Some(false) + ); + assert_eq!(report.pointer("/layer_contract/layer_count").and_then(Value::as_u64), Some(5)); + + let layers = array_at(&report, "/layer_contract/layers")?; + + for (layer, authority, replay) in [ + ("memory_notes", "memory_note", "elf_admin_trace_bundle_get"), + ("source_documents", "source_library", "elf_docs_search_l0"), + ("knowledge_pages", "derived_knowledge_page", "elf_recall_debug_panel"), + ("graph_facts", "graph_fact", "elf_graph_report"), + ("dreaming_proposals", "reviewable_dreaming_proposal", "elf_dreaming_review_queue"), + ] { + let row = find_by_field(layers, "/layer", layer)?; + + assert_eq!(row.pointer("/authority_layer").and_then(Value::as_str), Some(authority)); + assert_eq!(row.pointer("/replay_surface").and_then(Value::as_str), Some(replay)); + assert_eq!(row.pointer("/evidence_class").and_then(Value::as_str), Some("pass")); + } + + let memory = find_by_field(layers, "/layer", "memory_notes")?; + let docs = find_by_field(layers, "/layer", "source_documents")?; + + assert!(array_contains_str(memory, "/selection_states", "selected")?); + assert!(array_contains_str(memory, "/selection_states", "dropped")?); + assert_eq!(docs.pointer("/effective_limit").and_then(Value::as_u64), Some(32)); + assert_eq!( + report.pointer("/debug_invariants/not_requested_layers_preserved").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + report + .pointer("/debug_invariants/selected_and_dropped_memory_candidates") + .and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + report + .pointer("/debug_invariants/requested_layer_failures_preserved_as_blocked") + .and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + report.pointer("/debug_invariants/no_source_mutation").and_then(Value::as_bool), + Some(true) + ); + assert!(service.contains("ELF_RECALL_DEBUG_PANEL_SCHEMA_V1")); + assert!(service.contains("pub async fn recall_debug_panel")); + assert!(service.contains("not_requested_layer")); + assert!(service.contains("blocked_layer")); + assert!(service.contains("public_error_class")); + assert!(service.contains("candidate_identity")); + assert!(service.contains("ORG_PROJECT_ID")); + assert!(service.contains("trace_bundle_get")); + assert!(service.contains("docs_search_l0")); + assert!(service.contains("knowledge_pages_search")); + assert!(service.contains("graph_report")); + assert!(service.contains("dreaming_review_queue")); + assert!(service_lib.contains("pub mod recall_debug")); + assert!(service_lib.contains("RecallDebugPanelResponse")); + assert!(routes.contains("/v2/admin/recall-debug/panel")); + assert!(routes.contains("async fn recall_debug_panel")); + assert!(routes.contains("RecallDebugPanelRequest")); + assert!(mcp.contains("elf_recall_debug_panel")); + assert!(mcp.contains("recall_debug_panel_schema")); + assert!(mcp.contains("/v2/admin/recall-debug/panel")); + assert!(recall_spec.contains("elf.recall_debug_panel/v1")); + assert!(recall_spec.contains("not_requested")); + assert!(recall_spec.contains("evidence_class = \"blocked\"")); + assert!(recall_spec.contains("effective `top_k` cap of 32")); + assert!(recall_spec.contains("selected`, `dropped`, `available`, or `reviewable`")); + assert!(service_spec.contains("POST /v2/admin/recall-debug/panel")); + assert!(service_spec.contains("system_recall_debug_panel_v1.md")); + assert!(version_registry.contains("elf.recall_debug_panel/v1")); + assert!(markdown.contains("Recall Debug Panel Report")); + assert!(markdown.contains("Missing anchors stay visible as `not_requested`")); + assert!(markdown.contains("retained dropped replay candidates")); + assert!(markdown.contains("effective cap of 32 rows")); + assert!(benchmarking_index.contains("2026-06-20-recall-debug-panel-report.md")); + assert!(readme.contains("Recall/debug panel after XY-1022")); + assert!(readme.contains("elf.recall_debug_panel/v1")); + assert!(readme.contains("retained dropped replay candidates")); + + Ok(()) +} + #[test] fn operator_approved_public_proxy_private_addendum_preserves_boundary() -> Result<()> { let report = serde_json::from_str::(&fs::read_to_string( diff --git a/apps/elf-mcp/src/server.rs b/apps/elf-mcp/src/server.rs index 16c55f9a..51693a84 100644 --- a/apps/elf-mcp/src/server.rs +++ b/apps/elf-mcp/src/server.rs @@ -373,6 +373,20 @@ impl ElfMcp { self.forward(HttpMethod::Get, "/v2/admin/dreaming/review-queue", params, None).await } + #[rmcp::tool( + name = "elf_recall_debug_panel", + description = "Build a cross-layer recall/debug panel over memory traces, source documents, knowledge pages, graph facts, and Dreaming proposals.", + input_schema = recall_debug_panel_schema() + )] + async fn elf_recall_debug_panel( + &self, + params: JsonObject, + ) -> Result { + reject_context_override_params(¶ms)?; + + self.forward(HttpMethod::Post, "/v2/admin/recall-debug/panel", params, None).await + } + #[rmcp::tool( name = "elf_searches_create", description = "Create a search session using quick-find or planned-search mode. Response includes optional trajectory_summary for staged retrieval progress.", @@ -868,6 +882,19 @@ fn take_optional_string(params: &mut JsonObject, key: &str) -> Result Result<(), ErrorData> { + for key in ["tenant_id", "project_id", "agent_id", "read_profile"] { + if params.contains_key(key) { + return Err(ErrorData::invalid_params( + format!("{key} is configured by the MCP server and must not be supplied."), + None, + )); + } + } + + Ok(()) +} + fn notes_structured_entity_schema() -> Value { serde_json::json!({ "type": "object", @@ -1268,6 +1295,73 @@ fn dreaming_review_queue_schema() -> Arc { })) } +fn recall_debug_panel_schema() -> Arc { + Arc::new(rmcp::object!({ + "type": "object", + "additionalProperties": false, + "properties": { + "trace_id": { "type": ["string", "null"], "format": "uuid" }, + "query": { "type": ["string", "null"] }, + "docs_query": { "type": ["string", "null"] }, + "knowledge_query": { "type": ["string", "null"] }, + "graph_subject": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["entity_id"], + "properties": { + "entity_id": { + "type": "string", + "format": "uuid" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["surface"], + "properties": { + "surface": { "type": "string" } + } + }, + { "type": "null" } + ] + }, + "graph_predicate": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["predicate_id"], + "properties": { + "predicate_id": { + "type": "string", + "format": "uuid" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["surface"], + "properties": { + "surface": { "type": "string" } + } + }, + { "type": "null" } + ] + }, + "include_dreaming": { "type": ["boolean", "null"] }, + "limit": { + "type": ["integer", "null"], + "minimum": 1, + "maximum": 100 + } + } + })) +} + fn searches_create_schema() -> Arc { let filter_schema = rmcp::object!({ "type": "object", @@ -1647,7 +1741,7 @@ mod tests { type RequestRecorder = Arc>>>; - const ALL_TOOL_DEFINITIONS: [ToolDefinition; 33] = [ + const ALL_TOOL_DEFINITIONS: [ToolDefinition; 34] = [ ToolDefinition::new( "elf_notes_ingest", HttpMethod::Post, @@ -1696,6 +1790,12 @@ mod tests { "/v2/admin/dreaming/review-queue", "List source-backed Dreaming review queue proposals with variants, affected refs, lint flags, policy gates, and review audit.", ), + ToolDefinition::new( + "elf_recall_debug_panel", + HttpMethod::Post, + "/v2/admin/recall-debug/panel", + "Build a cross-layer recall/debug panel over memory traces, source documents, knowledge pages, graph facts, and Dreaming proposals.", + ), ToolDefinition::new( "elf_searches_get", HttpMethod::Get, @@ -1903,6 +2003,7 @@ mod tests { "elf_space_grant_revoke", "elf_admin_traces_recent_list", "elf_dreaming_review_queue", + "elf_recall_debug_panel", "elf_admin_trace_get", "elf_admin_trajectory_get", "elf_admin_trace_item_get", @@ -2023,6 +2124,39 @@ mod tests { assert_eq!(mcp.api_base_for_path("/v2/searches"), "http://127.0.0.1:9000"); } + #[test] + fn recall_debug_panel_schema_rejects_context_override_fields() { + let schema = super::recall_debug_panel_schema(); + let properties = schema + .get("properties") + .and_then(Value::as_object) + .expect("recall debug panel schema is missing properties."); + + assert_eq!(schema.get("additionalProperties"), Some(&Value::Bool(false))); + + for key in ["tenant_id", "project_id", "agent_id", "read_profile"] { + assert!(!properties.contains_key(key), "{key} must not be a tool param."); + } + for key in ["graph_subject", "graph_predicate"] { + let one_of = properties + .get(key) + .and_then(Value::as_object) + .and_then(|schema| schema.get("oneOf")) + .and_then(Value::as_array) + .expect("selector schema is missing oneOf."); + + for branch in one_of.iter().filter_map(Value::as_object) { + if branch.get("type").and_then(Value::as_str) == Some("object") { + assert_eq!( + branch.get("additionalProperties"), + Some(&Value::Bool(false)), + "{key} selector object branches must be closed." + ); + } + } + } + } + #[test] fn off_mode_allows_requests_without_auth_header() { let headers = HeaderMap::new(); @@ -2216,6 +2350,30 @@ mod tests { assert!(description.contains("structured")); } + #[tokio::test] + async fn recall_debug_panel_rejects_context_override_params() { + let context = McpContext { + tenant_id: "tenant-a".to_string(), + project_id: "project-a".to_string(), + agent_id: "agent-a".to_string(), + read_profile: "private_plus_project".to_string(), + }; + let mcp = ElfMcp::new( + "http://127.0.0.1:1".to_string(), + "http://127.0.0.1:1".to_string(), + ElfContextHeaders::new(&context), + McpAuthState::Off, + ); + let params = serde_json::Map::from_iter([( + "tenant_id".to_string(), + Value::String("tenant-override".to_string()), + )]); + let result = mcp.elf_recall_debug_panel(params).await; + let err = result.expect_err("context override params must fail before forwarding."); + + assert!(format!("{err:?}").contains("tenant_id")); + } + #[tokio::test] async fn default_ingestion_profile_set_uses_put_admin_default_path() { let (admin_base, received) = spawn_recording_admin_server().await; diff --git a/docs/evidence/benchmarking/2026-06-20-recall-debug-panel-report.md b/docs/evidence/benchmarking/2026-06-20-recall-debug-panel-report.md new file mode 100644 index 00000000..2b6dfd5a --- /dev/null +++ b/docs/evidence/benchmarking/2026-06-20-recall-debug-panel-report.md @@ -0,0 +1,87 @@ +--- +type: Evidence +title: "Recall Debug Panel Report - June 20, 2026" +description: "Checked-in benchmark evidence record for the cross-layer recall/debug panel." +resource: docs/evidence/benchmarking/2026-06-20-recall-debug-panel-report.md +status: active +authority: current_state +owner: evidence +last_verified: 2026-06-20 +tags: + - docs + - evidence + - benchmarking + - recall +--- +# Recall Debug Panel Report - June 20, 2026 + +Goal: Close XY-1022 by exposing one typed recall/debug read model across Memory +Notes, Source Library documents, Knowledge Workspace pages, graph facts, and Dreaming +review proposals. + +Inputs: `packages/elf-service/src/recall_debug.rs`, `apps/elf-api/src/routes.rs`, +`apps/elf-mcp/src/server.rs`, `docs/spec/system_recall_debug_panel_v1.md`, and +`apps/elf-eval/fixtures/report_snapshots/2026-06-20-recall-debug-panel-report.json`. + +## Executive Judgment + +ELF now has `elf.recall_debug_panel/v1`, a read-only panel response that lets an +agent or operator inspect why recall candidates were selected, dropped, available, or +reviewable across the main Agent Knowledge OS layers. This is a product/debug surface +over existing authority layers, not a new mutating worker and not a replacement for +the underlying trace, docs, graph, knowledge, or proposal APIs. + +## Layer Coverage + +| Layer | Anchor | Selection states | Replay/readback | +| --- | --- | --- | --- | +| Memory Notes | `trace_id` | `selected`, `dropped` | `elf_admin_trace_bundle_get` | +| Source Library documents | `docs_query` or `query` | `selected` | `elf_docs_search_l0` | +| Knowledge Workspace pages | `knowledge_query` or `query` | `selected` | `elf_recall_debug_panel` with a page query | +| Graph facts | `graph_subject` | `available` | `elf_graph_report` | +| Dreaming proposals | `include_dreaming` | `reviewable` | `elf_dreaming_review_queue` | + +Each row exposes item refs, authority layer, freshness state, source refs or source +snapshots, score/rank when available, stage reason, evidence class, replay command, +and layer-specific debug artifacts. + +The panel-level `limit` is a per-layer request cap, but the Source Library layer +inherits the docs-search effective cap of 32 rows and reports requested/effective +limits in document row debug artifacts. + +## Command Evidence + +| Command | Status | Purpose | +| --- | --- | --- | +| `cargo test -p elf-service recall_debug -- --nocapture` | pass | Unit-check panel summary counters and `not_requested` layer behavior. | +| `cargo test -p elf-mcp registers_all_tools -- --nocapture` | pass | Guard MCP tool registration for `elf_recall_debug_panel`. | +| `cargo test -p elf-eval --test real_world_job_benchmark recall_debug_panel_report_wires_cross_layer_debug_contract -- --nocapture` | pass | Guard service, API, MCP, docs, README, and snapshot coverage for XY-1022. | + +## Claim Boundaries + +Allowed: + +- ELF exposes a typed cross-layer recall/debug read model. +- Memory trace selected rows and retained dropped replay candidates are visible + through trace bundles when candidate capture/retention preserved them. +- Source documents, knowledge pages, graph facts, and Dreaming proposals can be + inspected from one panel response when their anchors are supplied. +- Missing anchors stay visible as `not_requested` layers instead of hidden pass + claims. +- Requested layer readback failures stay visible as `blocked` layers instead of + failing or hiding the rest of the panel. + +Not allowed: + +- Do not claim the panel mutates notes, docs, pages, graph facts, or proposals. +- Do not claim external competitor UI parity from this read model alone. +- Do not treat missing anchors as pass evidence. +- Do not collapse blocked, incomplete, or wrong-result evidence into a broad win. + +## Next Optimization Direction + +The next useful layer is a visual operator panel that groups rows by layer, +authority, freshness, and stage reason, with one-click replay into trace bundles, +docs search, graph reports, and Dreaming queue filters. XY-1023 should then run the +full benchmark closeout and keep competitor debug advantages separate from ELF's +typed cross-layer readback. diff --git a/docs/evidence/benchmarking/index.md b/docs/evidence/benchmarking/index.md index 02e22aaf..6500a2e9 100644 --- a/docs/evidence/benchmarking/index.md +++ b/docs/evidence/benchmarking/index.md @@ -47,3 +47,4 @@ Routes to: Benchmarking evidence concepts under `docs/evidence/benchmarking/`. - `2026-06-20-graph-topic-map-report.md`: Graph Topic-Map Report - June 20, 2026; adds the ELF-native `elf.graph_report/v1` readback for Postgres graph-lite facts with sourced, inferred, ambiguous, stale, and superseded topic-map markers. - `2026-06-20-knowledge-workspace-version-diff-report.md`: Knowledge Workspace Version-Diff Report - June 20, 2026; proves ELF knowledge pages now expose previous-version diff metadata without perturbing page content hashes while preserving citation, lint, and source-of-truth boundaries. - `2026-06-20-live-knowledge-page-rebuild-lint-report.md`: Live Knowledge-Page Rebuild/Lint Report - June 20, 2026; adds a Docker-contained ELF service-native knowledge-page materialization command while preserving llm-wiki, gbrain, GraphRAG, RAGFlow, LightRAG, and graphify as separate comparison targets until they emit comparable scored page artifacts. +- `2026-06-20-recall-debug-panel-report.md`: Recall Debug Panel Report - June 20, 2026; adds `elf.recall_debug_panel/v1` as a typed cross-layer readback over memory traces, Source Library document candidates, Knowledge Workspace pages, graph facts, and Dreaming proposals while preserving not-requested and non-pass evidence classes. diff --git a/docs/spec/index.md b/docs/spec/index.md index 2dde84ef..37a0927b 100644 --- a/docs/spec/index.md +++ b/docs/spec/index.md @@ -45,6 +45,7 @@ Question this index answers: "what must remain true?" - `system_knowledge_pages_v1.md`: Derived Knowledge Pages v1 Specification. - `system_memory_summary_v1.md`: Reviewable Memory Summary v1 Specification. - `system_provenance_mapping_v1.md`: System: Note Provenance Mapping (v1). +- `system_recall_debug_panel_v1.md`: Recall Debug Panel v1 Specification. - `system_search_filter_expr_v1.md`: System: Search Filter Expression Contract v1. - `system_source_ref_doc_pointer_v1.md`: System: `source_ref` Doc Pointer Resolver (v1). - `system_version_registry.md`: System Version Registry. diff --git a/docs/spec/system_elf_memory_service_v2.md b/docs/spec/system_elf_memory_service_v2.md index f9d8dacb..5dc0a928 100644 --- a/docs/spec/system_elf_memory_service_v2.md +++ b/docs/spec/system_elf_memory_service_v2.md @@ -1123,6 +1123,23 @@ Behavior: - They must not mutate authoritative source notes, docs, events, traces, graph facts, or search traces. +Admin recall/debug panel: +- POST /v2/admin/recall-debug/panel + +Behavior: +- The endpoint returns `elf.recall_debug_panel/v1`, a read-only cross-layer panel + over Memory Note trace bundles, Source Library document search, Knowledge Workspace + page search, graph reports, and Dreaming review queue proposals. +- Each row must expose selection state, authority layer, freshness state, source refs + or source snapshots, score/rank where available, stage reason, evidence class, and + replay command or deterministic artifact path when available. +- Missing anchors must be represented as `not_requested` layers. The panel must not + collapse not-requested, incomplete, blocked, or wrong-result layers into a broad + pass claim. +- Requested layer failures must be represented as blocked layer evidence, so one + unavailable readback surface does not hide the other layer states. +- The detailed contract is defined in `system_recall_debug_panel_v1.md`. + Admin derived knowledge pages: - POST /v2/admin/knowledge/pages/rebuild - GET /v2/admin/knowledge/pages @@ -2368,7 +2385,7 @@ Original query: - X-ELF-Tenant-Id - X-ELF-Project-Id - X-ELF-Agent-Id - - X-ELF-Read-Profile (defaults to mcp.read_profile; may be overridden per tool call) + - X-ELF-Read-Profile (server-configured from mcp.read_profile; not client-controlled) - Tools map 1:1 to v2 endpoints: - elf_notes_ingest -> POST /v2/notes/ingest - elf_events_ingest -> POST /v2/events/ingest @@ -2403,6 +2420,7 @@ Original query: - elf_admin_trajectory_get -> GET /v2/admin/trajectories/{trace_id} - elf_admin_trace_item_get -> GET /v2/admin/trace-items/{item_id} - elf_admin_trace_bundle_get -> GET /v2/admin/traces/{trace_id}/bundle + - elf_recall_debug_panel -> POST /v2/admin/recall-debug/panel - elf_admin_note_provenance_get -> GET /v2/admin/notes/{note_id}/provenance - elf_admin_memory_history_get -> GET /v2/admin/notes/{note_id}/history - The MCP server must contain zero business logic or policy. diff --git a/docs/spec/system_recall_debug_panel_v1.md b/docs/spec/system_recall_debug_panel_v1.md new file mode 100644 index 00000000..f04e5e02 --- /dev/null +++ b/docs/spec/system_recall_debug_panel_v1.md @@ -0,0 +1,122 @@ +--- +type: Spec +title: "Recall Debug Panel v1 Specification" +description: "Define the cross-layer recall/debug panel readback contract for memory, source documents, knowledge pages, graph facts, and Dreaming proposals." +resource: docs/spec/system_recall_debug_panel_v1.md +status: active +authority: normative +owner: memory-service +last_verified: 2026-06-20 +tags: + - spec + - recall + - debug + - mcp +source_refs: [] +code_refs: + - packages/elf-service/src/recall_debug.rs + - apps/elf-api/src/routes.rs + - apps/elf-mcp/src/server.rs +related: + - docs/spec/system_elf_memory_service_v2.md + - docs/spec/system_doc_extension_v1_trajectory.md + - docs/spec/system_knowledge_pages_v1.md + - docs/spec/system_graph_memory_postgres_v1.md + - docs/spec/system_consolidation_proposals_v1.md +--- +# Recall Debug Panel v1 Specification + +Purpose: Define `elf.recall_debug_panel/v1`, the cross-layer readback surface for +replayable recall and operator debugging. +Status: normative +Read this when: You need to inspect why memory, source documents, knowledge pages, +graph facts, or Dreaming proposals were selected, dropped, or made available. +Not this document: Ranking math internals, document ingestion, graph mutation, page +rebuild, or proposal review state transitions. +Defines: Request anchors, layer rows, evidence classes, replay boundaries, and +authority/freshness fields for the recall/debug panel. + +## Contract + +The response schema is `elf.recall_debug_panel/v1`. + +The panel is a read model over existing authoritative surfaces: + +- Memory Notes: search traces, trace items, trajectory stages, replay candidates, and + note `source_ref` values. +- Source Library: `docs_search_l0` document chunk results and retrieval trajectory. +- Knowledge Workspace: admin project-level `knowledge_pages_search` derived page + sections, source refs, lint summary, trust state, and rebuild metadata. +- Graph facts: `elf.graph_report/v1` facts and topic-map status markers. +- Dreaming proposals: `elf.dreaming_review_queue/v1` reviewable proposal rows. + +The panel MUST NOT mutate notes, documents, pages, graph facts, or proposals. + +## Request Anchors + +`RecallDebugPanelRequest` requires tenant, project, agent, and read profile from the +authenticated request context. Client-provided anchors are optional: + +- `trace_id`: loads memory selected and dropped rows from a persisted search trace. +- `query`: shared query fallback for document and knowledge-page layers. +- `docs_query`: Source Library query override. +- `knowledge_query`: Knowledge Workspace query override. +- `graph_subject`: graph entity selector by `entity_id` or `surface`. +- `graph_predicate`: optional graph predicate selector by `predicate_id` or `surface`. +- `include_dreaming`: includes Dreaming review queue proposals when true. +- `limit`: per-layer row cap, clamped to the implementation maximum. + +If an anchor is absent, the corresponding layer MUST return `evidence_class = +"not_requested"` instead of pretending the layer was tested. +If an anchor is supplied but the backing typed readback fails, the corresponding +layer MUST return `evidence_class = "blocked"` instead of failing the whole panel. +The Source Library layer inherits the docs-search effective `top_k` cap of 32 +rows even when the panel-level `limit` is higher; returned document rows expose +the requested and effective limits in `debug_artifacts`. + +## Layer Rows + +Each returned row MUST include: + +- `layer`: one of `memory_notes`, `source_documents`, `knowledge_pages`, + `graph_facts`, or `dreaming_proposals`. +- `item_ref`: stable identifiers for replay or hydration. +- `selection_state`: `selected`, `dropped`, `available`, or `reviewable`. +- `authority_layer`: the system surface that owns the row. +- `freshness_state`: lifecycle, temporal, lint, or review state. +- `source_refs`: source refs, evidence note ids, source snapshots, or coverage + metadata that supports the row. +- `score` and `rank` when available. +- `rationale`: short reason for inclusion. +- `stage_reason`: stage name, diversity/drop reason, temporal marker, or review + policy reason. +- `replay_command`: MCP tool command or deterministic artifact path when available. +- `evidence_class`: row-level evidence class. +- `debug_artifacts`: layer-specific explain payloads. + +## Evidence Classes + +Allowed layer evidence classes are: + +- `pass`: the layer readback was executed through a typed ELF surface. +- `not_requested`: the client did not supply the anchor needed for that layer. +- `incomplete`: a requested layer lacks enough data to prove a recall/debug claim. +- `blocked`: an external dependency or authority boundary prevents execution. +- `wrong_result`: the layer executed but returned data that contradicts expected + evidence. + +The panel summary MUST preserve evidence class counts. Aggregate success MUST NOT +hide `not_requested`, `incomplete`, `blocked`, or `wrong_result` layers. + +## Replay Boundary + +The panel may return replay commands such as `elf_admin_trace_bundle_get`, +`elf_docs_search_l0`, `elf_graph_report`, and `elf_dreaming_review_queue`. These +commands are readback aids only. They MUST NOT bypass write policy, proposal review, +graph mutation rules, or source-library authority. + +## Raw SQL Boundary + +The panel MUST prefer typed service, HTTP, and MCP surfaces. `raw_sql_needed` is false +for normal rows. If a future layer requires raw database inspection to explain a +claim, it must mark that layer or row explicitly instead of hiding the gap. diff --git a/docs/spec/system_version_registry.md b/docs/spec/system_version_registry.md index 1add3326..4785b614 100644 --- a/docs/spec/system_version_registry.md +++ b/docs/spec/system_version_registry.md @@ -161,6 +161,18 @@ This document is normative. When a new versioned identifier is introduced, it mu - Consumers: `GET /v2/admin/traces/{trace_id}/bundle` API response, `apps/elf-api`, `apps/elf-mcp`. - Bump rule: Introduce a new identifier only if this response payload becomes incompatible. +### Recall debug panel schema + +- Identifier: `elf.recall_debug_panel/v1`. +- Type: Cross-layer recall/debug panel response payload identifier. +- Defined in: `packages/elf-service/src/recall_debug.rs` + (`ELF_RECALL_DEBUG_PANEL_SCHEMA_V1`) and + `docs/spec/system_recall_debug_panel_v1.md`. +- Consumers: `POST /v2/admin/recall-debug/panel` API response, `apps/elf-api`, + `apps/elf-mcp`, operator debugging workflows, and benchmark closeout reports. +- Bump rule: Introduce a new identifier only if layer names, selection states, + evidence-class semantics, replay fields, or required row keys become incompatible. + ### Search filter expression schema - Identifier: `search_filter_expr/v1`. diff --git a/packages/elf-service/src/lib.rs b/packages/elf-service/src/lib.rs index a776f838..c6910b50 100644 --- a/packages/elf-service/src/lib.rs +++ b/packages/elf-service/src/lib.rs @@ -20,6 +20,7 @@ pub mod list; pub mod notes; pub mod progressive_search; pub mod provenance; +pub mod recall_debug; pub mod search; pub mod sharing; pub mod structured_fields; @@ -113,6 +114,11 @@ pub use self::{ NoteProvenanceIngestDecision, NoteProvenanceNote, NoteProvenanceNoteVersion, NoteProvenanceRecentTrace, }, + recall_debug::{ + ELF_RECALL_DEBUG_PANEL_SCHEMA_V1, RecallDebugLayer, RecallDebugPanelRequest, + RecallDebugPanelRequestEcho, RecallDebugPanelResponse, RecallDebugPanelSummary, + RecallDebugRow, + }, search::{ BlendRankingOverride, BlendSegmentOverride, PayloadLevel, QueryPlan, QueryPlanBlendSegment, QueryPlanBudget, QueryPlanDynamicGate, QueryPlanFusionPolicy, QueryPlanIntent, diff --git a/packages/elf-service/src/recall_debug.rs b/packages/elf-service/src/recall_debug.rs new file mode 100644 index 00000000..9c10b593 --- /dev/null +++ b/packages/elf-service/src/recall_debug.rs @@ -0,0 +1,1040 @@ +//! Cross-layer recall/debug panel readback. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::FromRow; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::{ + DocsSearchL0Request, DreamingReviewQueueRequest, ElfService, Error, GraphQueryEntityRef, + GraphQueryPredicateRef, GraphReportRequest, KnowledgePageSearchItem, + KnowledgePageSearchRequest, Result, SearchExplainItem, SearchTrajectoryStage, + TraceBundleGetRequest, + access::ORG_PROJECT_ID, + search::{TraceBundleMode, TraceReplayCandidate}, +}; + +/// Schema identifier for recall/debug panel responses. +pub const ELF_RECALL_DEBUG_PANEL_SCHEMA_V1: &str = "elf.recall_debug_panel/v1"; + +const DEFAULT_RECALL_DEBUG_LIMIT: u32 = 25; +const MAX_RECALL_DEBUG_LIMIT: u32 = 100; +const MAX_RECALL_DEBUG_DOCS_LIMIT: u32 = 32; + +/// Request payload for the cross-layer recall/debug panel. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct RecallDebugPanelRequest { + /// Tenant that owns the readback. + pub tenant_id: String, + /// Project that owns the readback. + pub project_id: String, + /// Agent requesting the readback. + pub agent_id: String, + /// Read profile used for memory, document, and graph visibility. + pub read_profile: String, + /// Optional search trace anchor for memory selected/dropped rows. + pub trace_id: Option, + /// Shared query used when docs_query or knowledge_query are omitted. + pub query: Option, + /// Optional Source Library query. + pub docs_query: Option, + /// Optional Knowledge Workspace page query. + pub knowledge_query: Option, + /// Optional graph subject selector. + pub graph_subject: Option, + /// Optional graph predicate selector. + pub graph_predicate: Option, + /// Whether to include Dreaming review queue proposals. Omitted means not requested. + pub include_dreaming: Option, + /// Maximum rows per layer. + pub limit: Option, +} + +/// Cross-layer recall/debug panel response. +#[derive(Clone, Debug, Serialize)] +pub struct RecallDebugPanelResponse { + /// Response schema identifier. + pub schema: String, + #[serde(with = "crate::time_serde")] + /// Panel generation timestamp. + pub generated_at: OffsetDateTime, + /// Echo of the effective anchors used for this response. + pub request: RecallDebugPanelRequestEcho, + /// Aggregate panel summary. + pub summary: RecallDebugPanelSummary, + /// Cross-layer rows grouped by source layer. + pub layers: Vec, +} + +/// Stable request echo for panel responses. +#[derive(Clone, Debug, Serialize)] +pub struct RecallDebugPanelRequestEcho { + /// Search trace anchor used for memory rows. + pub trace_id: Option, + /// Effective Source Library query. + pub docs_query: Option, + /// Effective Knowledge Workspace query. + pub knowledge_query: Option, + /// Whether a graph subject was supplied. + pub graph_subject_supplied: bool, + /// Whether Dreaming proposals were included. + pub include_dreaming: bool, + /// Effective row cap per layer. + pub limit: u32, +} + +/// Aggregate panel counters. +#[derive(Clone, Debug, Default, Serialize)] +pub struct RecallDebugPanelSummary { + /// Number of returned layers. + pub layer_count: usize, + /// Total returned row count. + pub row_count: usize, + /// Rows selected by a retrieval or review stage. + pub selected_count: usize, + /// Rows dropped by a retrieval or review stage. + pub dropped_count: usize, + /// Rows available for inspection but not selected/dropped. + pub available_count: usize, + /// Layers skipped because no anchor was supplied. + pub not_requested_layer_count: usize, + /// Layers that require follow-up before they can prove a debug claim. + pub incomplete_layer_count: usize, + /// Rows or layers that require raw SQL to inspect. + pub raw_sql_needed_count: usize, + /// Rows with a replay command or deterministic artifact path. + pub replay_command_count: usize, + /// Evidence-class counts across layers. + pub evidence_class_counts: BTreeMap, +} + +/// One recall/debug source layer. +#[derive(Clone, Debug, Serialize)] +pub struct RecallDebugLayer { + /// Layer identifier. + pub layer: String, + /// Evidence class for this layer. + pub evidence_class: String, + /// Human-readable layer summary. + pub summary: String, + /// Query or object anchor used by the layer. + pub anchor: Option, + /// Number of returned rows. + pub row_count: usize, + /// Selected rows in this layer. + pub selected_count: usize, + /// Dropped rows in this layer. + pub dropped_count: usize, + /// Available review/inspection rows in this layer. + pub available_count: usize, + /// Whether raw SQL is needed to inspect this layer. + pub raw_sql_needed: bool, + /// Whether the layer includes replay commands or deterministic artifact paths. + pub replayable: bool, + /// Returned layer rows. + pub rows: Vec, +} + +/// One item in the recall/debug panel. +#[derive(Clone, Debug, Serialize)] +pub struct RecallDebugRow { + /// Layer identifier. + pub layer: String, + /// Stable item reference. + pub item_ref: Value, + /// Selection state such as selected, dropped, available, or reviewable. + pub selection_state: String, + /// Authority layer that owns the row. + pub authority_layer: String, + /// Freshness or temporal state. + pub freshness_state: String, + /// Source refs or source snapshots backing the row. + pub source_refs: Value, + /// Optional final score. + pub score: Option, + /// Optional rank within the layer. + pub rank: Option, + /// Short selection rationale. + pub rationale: Option, + /// Stage reason for selected/dropped status. + pub stage_reason: Option, + /// Replay command or deterministic artifact path when available. + pub replay_command: Option, + /// Row-level evidence class. + pub evidence_class: String, + /// Layer-specific debug artifacts. + pub debug_artifacts: Value, +} + +#[derive(Clone, Debug, FromRow)] +struct NoteDebugSourceRow { + note_id: Uuid, + status: String, + source_ref: Value, + updated_at: OffsetDateTime, +} + +impl ElfService { + /// Builds a cross-layer recall/debug panel from existing readback surfaces. + pub async fn recall_debug_panel( + &self, + req: RecallDebugPanelRequest, + ) -> Result { + let limit = + req.limit.unwrap_or(DEFAULT_RECALL_DEBUG_LIMIT).clamp(1, MAX_RECALL_DEBUG_LIMIT); + let docs_query = req + .docs_query + .clone() + .or_else(|| req.query.clone()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + let knowledge_query = req + .knowledge_query + .clone() + .or_else(|| req.query.clone()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + let include_dreaming = req.include_dreaming == Some(true); + let mut layers = Vec::new(); + + layers.push(self.recall_memory_layer(&req, limit).await.unwrap_or_else(|err| { + blocked_layer( + "memory_notes", + req.trace_id.map(|trace_id| trace_id.to_string()), + "Requested memory trace bundle could not be read.", + &err, + ) + })); + layers.push( + self.recall_docs_layer(&req, docs_query.as_deref(), limit).await.unwrap_or_else( + |err| { + blocked_layer( + "source_documents", + docs_query.clone(), + "Requested Source Library document search could not be read.", + &err, + ) + }, + ), + ); + layers.push( + self.recall_knowledge_layer(&req, knowledge_query.as_deref(), limit) + .await + .unwrap_or_else(|err| { + blocked_layer( + "knowledge_pages", + knowledge_query.clone(), + "Requested Knowledge Workspace page search could not be read.", + &err, + ) + }), + ); + layers.push(self.recall_graph_layer(&req, limit).await.unwrap_or_else(|err| { + blocked_layer( + "graph_facts", + req.graph_subject.as_ref().and_then(json_anchor), + "Requested graph report could not be read.", + &err, + ) + })); + layers.push( + self.recall_dreaming_layer(&req, include_dreaming, limit).await.unwrap_or_else(|err| { + blocked_layer( + "dreaming_proposals", + Some("include_dreaming=true".to_string()), + "Requested Dreaming review queue could not be read.", + &err, + ) + }), + ); + + let summary = summarize_layers(&layers); + + Ok(RecallDebugPanelResponse { + schema: ELF_RECALL_DEBUG_PANEL_SCHEMA_V1.to_string(), + generated_at: OffsetDateTime::now_utc(), + request: RecallDebugPanelRequestEcho { + trace_id: req.trace_id, + docs_query, + knowledge_query, + graph_subject_supplied: req.graph_subject.is_some(), + include_dreaming, + limit, + }, + summary, + layers, + }) + } + + async fn recall_memory_layer( + &self, + req: &RecallDebugPanelRequest, + limit: u32, + ) -> Result { + let Some(trace_id) = req.trace_id else { + return Ok(not_requested_layer( + "memory_notes", + "Supply trace_id to show selected and dropped Memory Note candidates.", + )); + }; + let bundle = self + .trace_bundle_get(TraceBundleGetRequest { + tenant_id: req.tenant_id.clone(), + project_id: req.project_id.clone(), + agent_id: req.agent_id.clone(), + trace_id, + mode: TraceBundleMode::Bounded, + stage_items_limit: Some(limit), + candidates_limit: Some(limit.saturating_mul(4).min(400)), + }) + .await?; + let selected_note_ids = + bundle.items.iter().map(|item| item.note_id).collect::>(); + let selected_candidate_keys = + bundle.items.iter().filter_map(search_item_candidate_key).collect::>(); + let candidate_note_ids = + bundle.candidates.as_ref().into_iter().flatten().map(|candidate| candidate.note_id); + let all_note_ids = + selected_note_ids.iter().copied().chain(candidate_note_ids).collect::>(); + let source_refs = self + .load_memory_note_debug_sources(req, all_note_ids.iter().copied().collect()) + .await?; + let replay_command = format!("elf_admin_trace_bundle_get trace_id={trace_id} mode=bounded"); + let dropped_candidates = bundle + .candidates + .as_deref() + .unwrap_or_default() + .iter() + .filter(|candidate| !candidate_is_selected(&selected_candidate_keys, candidate)) + .collect::>(); + let selected_cap = if !dropped_candidates.is_empty() && limit > 1 { + limit as usize - 1 + } else { + limit as usize + }; + let mut rows = Vec::new(); + + for item in bundle.items.iter().take(selected_cap) { + let source = source_refs.get(&item.note_id); + + rows.push(RecallDebugRow { + layer: "memory_notes".to_string(), + item_ref: serde_json::json!({ + "trace_id": trace_id, + "result_handle": item.result_handle, + "note_id": item.note_id, + "chunk_id": item.chunk_id, + }), + selection_state: "selected".to_string(), + authority_layer: "memory_note".to_string(), + freshness_state: freshness_from_note_source(source), + source_refs: source_ref_from_note_source(source), + score: Some(item.explain.ranking.final_score), + rank: Some(item.rank), + rationale: Some("final ranked search result".to_string()), + stage_reason: last_stage_name(bundle.stages.as_slice()) + .or_else(|| Some("final_ranking".to_string())), + replay_command: Some(replay_command.clone()), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({ + "ranking_explain": item.explain, + "note_updated_at": source.map(|row| row.updated_at), + }), + }); + } + + let dropped_cap = limit.saturating_sub(rows.len() as u32) as usize; + + for candidate in dropped_candidates.into_iter().take(dropped_cap) { + rows.push(candidate_debug_row( + trace_id, + candidate, + source_refs.get(&candidate.note_id), + replay_command.as_str(), + )); + } + + Ok(layer_from_rows( + "memory_notes", + "pass", + Some(trace_id.to_string()), + "Search trace bundle with selected results and replay candidates.", + rows, + )) + } + + async fn recall_docs_layer( + &self, + req: &RecallDebugPanelRequest, + docs_query: Option<&str>, + limit: u32, + ) -> Result { + let Some(query) = docs_query else { + return Ok(not_requested_layer( + "source_documents", + "Supply query or docs_query to show Source Library document candidates.", + )); + }; + let effective_limit = limit.min(MAX_RECALL_DEBUG_DOCS_LIMIT); + let response = self + .docs_search_l0(DocsSearchL0Request { + tenant_id: req.tenant_id.clone(), + project_id: req.project_id.clone(), + caller_agent_id: req.agent_id.clone(), + read_profile: req.read_profile.clone(), + query: query.to_string(), + scope: None, + status: Some("active".to_string()), + doc_type: None, + sparse_mode: None, + domain: None, + repo: None, + agent_id: None, + thread_id: None, + updated_after: None, + updated_before: None, + ts_gte: None, + ts_lte: None, + top_k: Some(effective_limit), + candidate_k: Some(effective_limit.saturating_mul(3).max(effective_limit)), + explain: Some(true), + }) + .await?; + let rows = response + .items + .into_iter() + .enumerate() + .map(|(index, item)| RecallDebugRow { + layer: "source_documents".to_string(), + item_ref: serde_json::json!({ + "trace_id": response.trace_id, + "doc_id": item.doc_id, + "chunk_id": item.chunk_id, + "pointer": item.pointer, + }), + selection_state: "selected".to_string(), + authority_layer: "source_library".to_string(), + freshness_state: "active".to_string(), + source_refs: serde_json::json!([{ + "schema": "source_ref/v1", + "resolver": "elf_doc_ext/v1", + "doc_id": item.doc_id, + "chunk_id": item.chunk_id, + "content_hash": item.content_hash, + "chunk_hash": item.chunk_hash, + "doc_updated_at": item.updated_at, + }]), + score: Some(item.score), + rank: Some(index as u32 + 1), + rationale: Some("docs_search_l0 selected chunk".to_string()), + stage_reason: response + .trajectory + .as_ref() + .and_then(|trajectory| trajectory.stages.last()) + .map(|stage| stage.stage_name.clone()) + .or(Some("docs_search_l0".to_string())), + replay_command: Some(format!("elf_docs_search_l0 query={query:?} explain=true")), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({ + "doc_type": item.doc_type, + "scope": item.scope, + "snippet": item.snippet, + "trajectory": response.trajectory, + "requested_limit": limit, + "effective_limit": effective_limit, + }), + }) + .collect(); + let summary = if effective_limit < limit { + format!( + "Source Library search rows selected by docs_search_l0; effective docs cap is {effective_limit}." + ) + } else { + "Source Library search rows selected by docs_search_l0.".to_string() + }; + + Ok(layer_from_rows("source_documents", "pass", Some(query.to_string()), &summary, rows)) + } + + async fn recall_knowledge_layer( + &self, + req: &RecallDebugPanelRequest, + knowledge_query: Option<&str>, + limit: u32, + ) -> Result { + let Some(query) = knowledge_query else { + return Ok(not_requested_layer( + "knowledge_pages", + "Supply query or knowledge_query to show Knowledge Workspace page candidates.", + )); + }; + let response = self + .knowledge_pages_search(KnowledgePageSearchRequest { + tenant_id: req.tenant_id.clone(), + project_id: req.project_id.clone(), + query: query.to_string(), + page_kind: None, + limit: Some(limit), + }) + .await?; + let rows = response + .items + .into_iter() + .enumerate() + .map(|(index, item)| RecallDebugRow { + layer: "knowledge_pages".to_string(), + item_ref: serde_json::json!({ + "page_id": item.page_id, + "section_id": item.section_id, + "page_kind": item.page_kind, + "page_key": item.page_key, + }), + selection_state: "selected".to_string(), + authority_layer: "derived_knowledge_page".to_string(), + freshness_state: knowledge_freshness(&item), + source_refs: serde_json::json!({ + "source_coverage": item.source_coverage, + "section_source_ref_count": item.source_ref_count, + "citation_count": item.citation_count, + "source_refs": item.source_refs, + }), + score: None, + rank: Some(index as u32 + 1), + rationale: Some("knowledge_pages_search selected section".to_string()), + stage_reason: Some("knowledge_page_search".to_string()), + replay_command: Some(format!( + "elf_recall_debug_panel knowledge_query={query:?} layer=knowledge_pages" + )), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({ + "title": item.title, + "heading": item.heading, + "lint_summary": item.lint_summary, + "trust_state": item.trust_state, + "repair_guidance": item.repair_guidance, + "snippet": item.snippet, + }), + }) + .collect(); + + Ok(layer_from_rows( + "knowledge_pages", + "pass", + Some(query.to_string()), + "Knowledge Workspace sections selected by page search.", + rows, + )) + } + + async fn recall_graph_layer( + &self, + req: &RecallDebugPanelRequest, + limit: u32, + ) -> Result { + let Some(subject) = req.graph_subject.clone() else { + return Ok(not_requested_layer( + "graph_facts", + "Supply graph_subject to show graph fact candidates and temporal status.", + )); + }; + let response = self + .graph_report(GraphReportRequest { + tenant_id: req.tenant_id.clone(), + project_id: req.project_id.clone(), + agent_id: req.agent_id.clone(), + read_profile: req.read_profile.clone(), + subject, + predicate: req.graph_predicate.clone(), + scopes: None, + as_of: None, + limit: Some(limit), + explain: Some(true), + }) + .await?; + let subject_anchor = response.subject.canonical.clone(); + let replay_command = graph_replay_command(&subject_anchor, req.graph_predicate.as_ref()); + let rows = response + .facts + .into_iter() + .enumerate() + .map(|(index, fact)| RecallDebugRow { + layer: "graph_facts".to_string(), + item_ref: serde_json::json!({ + "fact_id": fact.fact_id, + "subject": subject_anchor, + "predicate": fact.predicate, + "object": fact.object, + }), + selection_state: "available".to_string(), + authority_layer: "graph_fact".to_string(), + freshness_state: graph_temporal_status(fact.temporal_status), + source_refs: serde_json::json!({ + "evidence_note_ids": fact.evidence_note_ids, + "supersedes_fact_ids": fact.supersedes_fact_ids, + "superseded_by_fact_ids": fact.superseded_by_fact_ids, + }), + score: None, + rank: Some(index as u32 + 1), + rationale: Some("graph_report returned source-backed fact".to_string()), + stage_reason: Some(fact.status_markers.join(",")), + replay_command: Some(replay_command.clone()), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({ + "scope": fact.scope, + "actor": fact.actor, + "valid_from": fact.valid_from, + "valid_to": fact.valid_to, + "status_markers": fact.status_markers, + }), + }) + .collect(); + + Ok(layer_from_rows( + "graph_facts", + "pass", + Some(subject_anchor), + "Graph facts from source-backed graph report.", + rows, + )) + } + + async fn recall_dreaming_layer( + &self, + req: &RecallDebugPanelRequest, + include_dreaming: bool, + limit: u32, + ) -> Result { + if !include_dreaming { + return Ok(not_requested_layer( + "dreaming_proposals", + "Set include_dreaming=true to show reviewable Dreaming proposals.", + )); + } + + let response = self + .dreaming_review_queue(DreamingReviewQueueRequest { + tenant_id: req.tenant_id.clone(), + project_id: req.project_id.clone(), + run_id: None, + review_state: None, + limit: Some(limit), + }) + .await?; + let rows = response + .items + .into_iter() + .enumerate() + .map(|(index, item)| RecallDebugRow { + layer: "dreaming_proposals".to_string(), + item_ref: serde_json::json!({ + "proposal_id": item.proposal_id, + "run_id": item.run_id, + "queue_variant": item.queue_variant, + "target_ref": item.target_ref, + }), + selection_state: "reviewable".to_string(), + authority_layer: "reviewable_dreaming_proposal".to_string(), + freshness_state: item.review_state.clone(), + source_refs: serde_json::json!({ + "source_refs": item.source_refs, + "source_snapshot": item.source_snapshot, + "affected_refs": item.affected_refs, + }), + score: Some(item.confidence), + rank: Some(index as u32 + 1), + rationale: Some(item.policy.reason.clone()), + stage_reason: Some(format!( + "review_state={}, auto_apply_allowed={}", + item.review_state, item.policy.auto_apply_allowed + )), + replay_command: Some("elf_dreaming_review_queue limit=".to_string()), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({ + "policy": item.policy, + "unsupported_claim_flags": item.unsupported_claim_flags, + "contradiction_markers": item.contradiction_markers, + "staleness_markers": item.staleness_markers, + "diff": item.diff, + "review_audit": item.review_audit, + }), + }) + .collect(); + + Ok(layer_from_rows( + "dreaming_proposals", + "pass", + None, + "Dreaming review queue proposals available for reviewer action.", + rows, + )) + } + + async fn load_memory_note_debug_sources( + &self, + req: &RecallDebugPanelRequest, + note_ids: Vec, + ) -> Result> { + if note_ids.is_empty() { + return Ok(BTreeMap::new()); + } + + let rows = sqlx::query_as::<_, NoteDebugSourceRow>( + "\ +SELECT note_id, status, source_ref, updated_at +FROM memory_notes + WHERE tenant_id = $1 + AND note_id = ANY($3::uuid[]) + AND ( + project_id = $2 + OR (project_id = $4 AND scope = 'org_shared') + )", + ) + .bind(req.tenant_id.as_str()) + .bind(req.project_id.as_str()) + .bind(note_ids) + .bind(ORG_PROJECT_ID) + .fetch_all(&self.db.pool) + .await?; + + Ok(rows.into_iter().map(|row| (row.note_id, row)).collect()) + } +} + +fn candidate_debug_row( + trace_id: Uuid, + candidate: &TraceReplayCandidate, + source: Option<&NoteDebugSourceRow>, + replay_command: &str, +) -> RecallDebugRow { + let selected_by_diversity = candidate.diversity_selected.unwrap_or(false); + let skipped_reason = candidate.diversity_skipped_reason.clone().or_else(|| { + if selected_by_diversity { + candidate.diversity_selected_reason.clone() + } else { + Some("not_in_final_top_k".to_string()) + } + }); + + RecallDebugRow { + layer: "memory_notes".to_string(), + item_ref: serde_json::json!({ + "trace_id": trace_id, + "note_id": candidate.note_id, + "chunk_id": candidate.chunk_id, + "chunk_index": candidate.chunk_index, + }), + selection_state: "dropped".to_string(), + authority_layer: "memory_note".to_string(), + freshness_state: freshness_from_note_source(source), + source_refs: source_ref_from_note_source(source), + score: candidate.retrieval_score, + rank: Some(candidate.retrieval_rank), + rationale: Some( + "candidate captured for replay but not selected in final result set".to_string(), + ), + stage_reason: skipped_reason, + replay_command: Some(replay_command.to_string()), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({ + "snippet": candidate.snippet, + "rerank_score": candidate.rerank_score, + "note_scope": candidate.note_scope, + "diversity_selected": candidate.diversity_selected, + "diversity_selected_rank": candidate.diversity_selected_rank, + "diversity_nearest_selected_note_id": candidate.diversity_nearest_selected_note_id, + "diversity_similarity": candidate.diversity_similarity, + "diversity_mmr_score": candidate.diversity_mmr_score, + "diversity_missing_embedding": candidate.diversity_missing_embedding, + }), + } +} + +fn summarize_layers(layers: &[RecallDebugLayer]) -> RecallDebugPanelSummary { + let mut summary = RecallDebugPanelSummary { layer_count: layers.len(), ..Default::default() }; + + for layer in layers { + summary.row_count += layer.row_count; + summary.selected_count += layer.selected_count; + summary.dropped_count += layer.dropped_count; + summary.available_count += layer.available_count; + + if layer.evidence_class == "not_requested" { + summary.not_requested_layer_count += 1; + } + if matches!(layer.evidence_class.as_str(), "incomplete" | "blocked" | "wrong_result") { + summary.incomplete_layer_count += 1; + } + if layer.raw_sql_needed { + summary.raw_sql_needed_count += 1; + } + + summary.replay_command_count += layer + .rows + .iter() + .filter(|row| row.replay_command.as_ref().is_some_and(|value| !value.is_empty())) + .count(); + *summary.evidence_class_counts.entry(layer.evidence_class.clone()).or_default() += 1; + } + + summary +} + +fn layer_from_rows( + layer: &str, + evidence_class: &str, + anchor: Option, + summary: &str, + rows: Vec, +) -> RecallDebugLayer { + let selected_count = rows.iter().filter(|row| row.selection_state == "selected").count(); + let dropped_count = rows.iter().filter(|row| row.selection_state == "dropped").count(); + let available_count = rows + .iter() + .filter(|row| matches!(row.selection_state.as_str(), "available" | "reviewable")) + .count(); + let replayable = rows.iter().any(|row| row.replay_command.is_some()); + + RecallDebugLayer { + layer: layer.to_string(), + evidence_class: evidence_class.to_string(), + summary: summary.to_string(), + anchor, + row_count: rows.len(), + selected_count, + dropped_count, + available_count, + raw_sql_needed: false, + replayable, + rows, + } +} + +fn not_requested_layer(layer: &str, summary: &str) -> RecallDebugLayer { + RecallDebugLayer { + layer: layer.to_string(), + evidence_class: "not_requested".to_string(), + summary: summary.to_string(), + anchor: None, + row_count: 0, + selected_count: 0, + dropped_count: 0, + available_count: 0, + raw_sql_needed: false, + replayable: false, + rows: Vec::new(), + } +} + +fn blocked_layer( + layer: &str, + anchor: Option, + summary: &str, + err: &Error, +) -> RecallDebugLayer { + RecallDebugLayer { + layer: layer.to_string(), + evidence_class: "blocked".to_string(), + summary: format!("{summary} error_class={}", public_error_class(err)), + anchor, + row_count: 0, + selected_count: 0, + dropped_count: 0, + available_count: 0, + raw_sql_needed: false, + replayable: false, + rows: Vec::new(), + } +} + +fn public_error_class(err: &Error) -> &'static str { + match err { + Error::NonEnglishInput { .. } => "validation_non_english_input", + Error::InvalidRequest { .. } => "validation_invalid_request", + Error::ScopeDenied { .. } => "scope_denied", + Error::NotFound { .. } => "not_found", + Error::Conflict { .. } => "conflict", + Error::Provider { .. } => "provider_unavailable", + Error::Storage { .. } => "storage_unavailable", + Error::Qdrant { .. } => "vector_store_unavailable", + } +} + +fn json_anchor(value: &T) -> Option +where + T: Serialize + ?Sized, +{ + serde_json::to_value(value).ok().map(|value| value.to_string()) +} + +fn search_item_candidate_key(item: &SearchExplainItem) -> Option<(Uuid, Uuid)> { + item.chunk_id.map(|chunk_id| candidate_identity(item.note_id, chunk_id)) +} + +fn candidate_identity(note_id: Uuid, chunk_id: Uuid) -> (Uuid, Uuid) { + (note_id, chunk_id) +} + +fn candidate_is_selected( + selected_candidate_keys: &BTreeSet<(Uuid, Uuid)>, + candidate: &TraceReplayCandidate, +) -> bool { + selected_candidate_keys.contains(&candidate_identity(candidate.note_id, candidate.chunk_id)) +} + +fn graph_replay_command(subject: &str, predicate: Option<&GraphQueryPredicateRef>) -> String { + if let Some(predicate) = predicate.and_then(json_anchor) { + format!("elf_graph_report subject={subject} predicate={predicate} explain=true") + } else { + format!("elf_graph_report subject={subject} explain=true") + } +} + +fn freshness_from_note_source(source: Option<&NoteDebugSourceRow>) -> String { + source.map(|row| row.status.clone()).unwrap_or_else(|| "unknown".to_string()) +} + +fn source_ref_from_note_source(source: Option<&NoteDebugSourceRow>) -> Value { + source.map(|row| serde_json::json!([row.source_ref])).unwrap_or_else(|| serde_json::json!([])) +} + +fn last_stage_name(stages: &[SearchTrajectoryStage]) -> Option { + stages.last().map(|stage| stage.stage_name.clone()) +} + +fn knowledge_freshness(item: &KnowledgePageSearchItem) -> String { + if item.lint_summary.error_count > 0 { + "lint_error".to_string() + } else if item.lint_summary.warning_count > 0 { + "lint_warning".to_string() + } else if item.trust_state != "clean" { + item.trust_state.clone() + } else { + item.status.clone() + } +} + +fn graph_temporal_status(status: crate::RelationTemporalStatus) -> String { + match status { + crate::RelationTemporalStatus::Future => "future", + crate::RelationTemporalStatus::Current => "current", + crate::RelationTemporalStatus::Historical => "historical", + } + .to_string() +} + +#[cfg(test)] +mod tests { + use crate::{ + RecallDebugRow, + recall_debug::{self, BTreeSet, Error, Uuid}, + }; + + #[test] + fn summary_preserves_not_requested_and_replay_counts() { + let layers = vec![ + recall_debug::not_requested_layer("graph_facts", "missing graph subject"), + recall_debug::layer_from_rows( + "memory_notes", + "pass", + Some("trace".to_string()), + "trace rows", + vec![ + RecallDebugRow { + layer: "memory_notes".to_string(), + item_ref: serde_json::json!({"note_id": "n1"}), + selection_state: "selected".to_string(), + authority_layer: "memory_note".to_string(), + freshness_state: "active".to_string(), + source_refs: serde_json::json!([]), + score: Some(1.0), + rank: Some(1), + rationale: None, + stage_reason: None, + replay_command: Some("elf_admin_trace_bundle_get".to_string()), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({}), + }, + RecallDebugRow { + layer: "memory_notes".to_string(), + item_ref: serde_json::json!({"note_id": "n2"}), + selection_state: "dropped".to_string(), + authority_layer: "memory_note".to_string(), + freshness_state: "active".to_string(), + source_refs: serde_json::json!([]), + score: Some(0.5), + rank: Some(2), + rationale: None, + stage_reason: Some("not_in_final_top_k".to_string()), + replay_command: Some("elf_admin_trace_bundle_get".to_string()), + evidence_class: "pass".to_string(), + debug_artifacts: serde_json::json!({}), + }, + ], + ), + ]; + let summary = recall_debug::summarize_layers(&layers); + + assert_eq!(summary.layer_count, 2); + assert_eq!(summary.row_count, 2); + assert_eq!(summary.selected_count, 1); + assert_eq!(summary.dropped_count, 1); + assert_eq!(summary.not_requested_layer_count, 1); + assert_eq!(summary.replay_command_count, 2); + assert_eq!(summary.evidence_class_counts.get("pass"), Some(&1)); + assert_eq!(summary.evidence_class_counts.get("not_requested"), Some(&1)); + } + + #[test] + fn not_requested_layers_never_require_raw_sql() { + let layer = recall_debug::not_requested_layer("source_documents", "missing query"); + + assert_eq!(layer.evidence_class, "not_requested"); + assert_eq!(layer.row_count, 0); + assert!(!layer.raw_sql_needed); + assert!(!layer.replayable); + } + + #[test] + fn blocked_layers_are_counted_as_incomplete_evidence() { + let layer = recall_debug::blocked_layer( + "source_documents", + Some("alpha".to_string()), + "docs search failed", + &Error::Storage { message: "database unavailable".to_string() }, + ); + let summary = recall_debug::summarize_layers(&[layer]); + + assert_eq!(summary.layer_count, 1); + assert_eq!(summary.incomplete_layer_count, 1); + assert_eq!(summary.evidence_class_counts.get("blocked"), Some(&1)); + } + + #[test] + fn blocked_layer_does_not_expose_raw_backend_errors() { + let layer = recall_debug::blocked_layer( + "graph_facts", + None, + "graph report failed", + &Error::Storage { message: "password=secret host=db.internal".to_string() }, + ); + + assert!(layer.summary.contains("error_class=storage_unavailable")); + assert!(!layer.summary.contains("password=secret")); + assert!(!layer.summary.contains("db.internal")); + } + + #[test] + fn selected_candidate_filter_is_chunk_level() { + let note_id = Uuid::new_v4(); + let selected_chunk_id = Uuid::new_v4(); + let dropped_chunk_id = Uuid::new_v4(); + let selected = + BTreeSet::from([recall_debug::candidate_identity(note_id, selected_chunk_id)]); + + assert!(selected.contains(&recall_debug::candidate_identity(note_id, selected_chunk_id))); + assert!(!selected.contains(&recall_debug::candidate_identity(note_id, dropped_chunk_id))); + } +}