Distributed Private Diagnostics JSON

Cargo.toml

@@ -56,6 +56,7 @@ serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
 distributed_macros = { workspace = true }
 sqlx = { version = "0.9", default-features = false, optional = true }
+time = { version = "0.3", features = ["formatting"] }
 tonic = { version = "0.14", default-features = false, features = ["router", "transport", "codegen"], optional = true }
 tonic-prost = { version = "0.14", optional = true }
 prost = { version = "0.14", optional = true }

docs/diagnostics.md

@@ -0,0 +1,56 @@
+# Diagnostics
+
+Distributed diagnostics are a private JSON snapshot for operators and agents.
+They summarize service inventory, readiness, telemetry capabilities, outbox
+backlog, recent sanitized failures, causal hints, and suggested next checks.
+
+Diagnostics are disabled by default. `microsvc::router(...)` and
+`microsvc::serve(...)` do not expose the endpoint. A service must explicitly
+compose it:
+
+```rust
+let app = distributed::microsvc::router_with_diagnostics(
+    service,
+    distributed::diagnostics::DiagnosticsOptions::new()
+        .with_bearer_token("management-token"),
+);
+```
+
+The default path is `GET /_distributed/diagnostics`. Responses always include
+`Cache-Control: no-store`, even though snapshots are internally cached for a
+short TTL.
+
+## Privacy Boundary
+
+Treat diagnostics as reconnaissance-grade private data. It includes command and
+event names, transport names, readiness state, backlog summaries, recent failure
+timing, correlation IDs, trace IDs, and runbook hints. Do not expose it on the
+same unauthenticated scrape surface as `/metrics`.
+
+Use one of these controls:
+
+- localhost or management-only side port;
+- cluster-internal service with NetworkPolicy;
+- mTLS or service-mesh authorization;
+- trusted proxy authentication;
+- the `DiagnosticsOptions::with_access_check(...)` hook or
+  `with_bearer_token(...)` helper.
+
+The snapshot intentionally omits payloads, decoded handler input, session
+variables, arbitrary metadata, auth headers, cookies, environment variables, DB
+URLs, and secrets. Recent failures are sanitized before entering the in-memory
+ring buffer and are bounded by capacity, per-response limits, dropped counts,
+and truncation metadata.
+
+## Backlog Provider
+
+Diagnostics can include outbox backlog state when the service supplies an
+outbox store:
+
+```rust
+let diagnostics = distributed::diagnostics::DiagnosticsOptions::new()
+    .with_outbox_store(repo.outbox_store());
+```
+
+The snapshot uses `OutboxStore::backlog_stats`, so it reports count and oldest
+age summaries rather than raw rows.

docs/metrics.md

@@ -93,6 +93,10 @@ labels, and numeric samples only; diagnostics must not add payloads, metadata,
 trace ids, aggregate ids, user ids, raw HTTP targets, or request ids to that
 shape.
 
+Private diagnostics are a separate opt-in JSON endpoint, not a Prometheus
+scrape route. Do not expose `/_distributed/diagnostics` with the unauthenticated
+`/metrics` posture; see [`diagnostics.md`](diagnostics.md).
+
 ## Boundaries
 
 The `metrics` feature owns Prometheus text exposition for framework metrics. It

docs/observability.md

@@ -177,3 +177,10 @@ Future `logs` or private `diagnostics` features should build on the same
 bounded telemetry vocabulary. Logs may carry structured failure context, but
 must not log payloads or secrets by default. Diagnostics should read typed
 snapshots of framework state rather than parsing public Prometheus text.
+
+Private diagnostics now live behind explicit composition of
+`microsvc::router_with_diagnostics(...)` at `/_distributed/diagnostics`. Treat
+that JSON as reconnaissance-grade private data; it must use localhost,
+management-only networking, mTLS, trusted-proxy auth, or an access hook and must
+not follow `/metrics` unauthenticated exposure semantics. See
+[`diagnostics.md`](diagnostics.md).

src/bus/runner.rs

@@ -69,7 +69,16 @@ where
         // same as a permanent dispatch failure, so it is dead-lettered/parked.
         if let Some(error) = received.decode_error() {
             let action = options.failure_policy.resolve(error);
-            record_transport_failure(service, transport, error.kind(), action);
+            let message_context =
+                crate::diagnostics::FailureMessageContext::from_message(received.message());
+            record_transport_failure(
+                service,
+                transport,
+                Some(&message_context),
+                error.kind(),
+                action,
+                Some(error.message()),
+            );
             let kind = received.message().kind;
             match action {
                 FailureAction::Nack => {
@@ -154,7 +163,16 @@ where
             }
             Err(error) => match options.failure_policy.resolve(&error) {
                 action @ FailureAction::Nack => {
-                    record_transport_failure(service, transport, error.kind(), action);
+                    let message_context =
+                        crate::diagnostics::FailureMessageContext::from_message(received.message());
+                    record_transport_failure(
+                        service,
+                        transport,
+                        Some(&message_context),
+                        error.kind(),
+                        action,
+                        Some(error.message()),
+                    );
                     let reason = error.to_string();
                     settle_and_record(
                         service,
@@ -167,7 +185,16 @@ where
                     .await?;
                 }
                 action @ FailureAction::DeadLetter => {
-                    record_transport_failure(service, transport, error.kind(), action);
+                    let message_context =
+                        crate::diagnostics::FailureMessageContext::from_message(received.message());
+                    record_transport_failure(
+                        service,
+                        transport,
+                        Some(&message_context),
+                        error.kind(),
+                        action,
+                        Some(error.message()),
+                    );
                     let reason = error.to_string();
                     settle_and_record(
                         service,
@@ -180,7 +207,16 @@ where
                     .await?;
                 }
                 action @ FailureAction::Park => {
-                    record_transport_failure(service, transport, error.kind(), action);
+                    let message_context =
+                        crate::diagnostics::FailureMessageContext::from_message(received.message());
+                    record_transport_failure(
+                        service,
+                        transport,
+                        Some(&message_context),
+                        error.kind(),
+                        action,
+                        Some(error.message()),
+                    );
                     let reason = error.to_string();
                     settle_and_record(
                         service,
@@ -196,8 +232,12 @@ where
                     record_transport_failure(
                         service,
                         transport,
+                        Some(&crate::diagnostics::FailureMessageContext::from_message(
+                            received.message(),
+                        )),
                         error.kind(),
                         FailureAction::LogAndAck,
+                        Some(error.message()),
                     );
                     eprintln!(
                         "[bus::runner] dropping message '{}' after permanent failure: {error}",
@@ -214,7 +254,16 @@ where
                     .await?;
                 }
                 FailureAction::Stop => {
-                    record_transport_failure(service, transport, error.kind(), FailureAction::Stop);
+                    let message_context =
+                        crate::diagnostics::FailureMessageContext::from_message(received.message());
+                    record_transport_failure(
+                        service,
+                        transport,
+                        Some(&message_context),
+                        error.kind(),
+                        FailureAction::Stop,
+                        Some(error.message()),
+                    );
                     return Err(error);
                 }
             },
@@ -244,8 +293,10 @@ where
             record_transport_failure(
                 service,
                 transport,
+                None,
                 error.kind(),
                 crate::telemetry::settle_failure_action(settle_action),
+                Some(error.message()),
             );
             Err(error)
         }
@@ -263,8 +314,10 @@ async fn recv_next<S: MessageSource>(
             record_transport_failure(
                 service,
                 transport,
+                None,
                 error.kind(),
                 crate::telemetry::failure_action::RECV_ERROR,
+                Some(error.message()),
             );
             Err(error)
         }
@@ -329,23 +382,33 @@ fn record_transport_message(
 fn record_transport_failure<A>(
     service: Option<&str>,
     transport: &str,
+    message: Option<&crate::diagnostics::FailureMessageContext>,
     kind: TransportErrorKind,
     action: A,
+    error_summary: Option<&str>,
 ) where
     A: IntoFailureActionLabel,
 {
+    let action = action.into_failure_action_label();
     #[cfg(feature = "metrics")]
     crate::metrics::record_transport_failure(
         service,
         transport,
         crate::telemetry::transport_failure_class(kind),
-        action.into_failure_action_label(),
+        action,
     );
     #[cfg(not(feature = "metrics"))]
     {
         let _ = (service, transport, kind);
-        let _ = action.into_failure_action_label();
     }
+    crate::diagnostics::record_transport_failure(
+        service,
+        transport,
+        message,
+        kind,
+        action,
+        error_summary,
+    );
 }
 
 trait IntoFailureActionLabel {