From 5646b1b38e0e7de4f432e6f820af890d2404dab8 Mon Sep 17 00:00:00 2001 From: ValJnr-dev1 Date: Fri, 24 Jul 2026 23:12:35 +0100 Subject: [PATCH] feat: dependency probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET /api/health/dependencies endpoint. - src/routes/health/dependencies.ts — factory createDependenciesRouter() with injectable ProbeFn for full testability; mirrors the pattern of health/ready.ts - 200/207/503 driven by composite status (ok/degraded/down) - x-correlation-id header echoed; UUID generated when absent - Structured pino logging on every request - src/index.ts — mounts the router at /api/health/dependencies - tests/healthDependencies.test.ts — 25 tests: status codes (6), response body shape (7), correlationId (4), auth (2), error handling (2), default export (1) - docs/health-dependencies.md — full API runbook Lint: clean Tests: 25 passed, 25 total Closes #301 --- docs/health-dependencies.md | 117 +++++++++++ src/index.ts | 2 + src/routes/health/dependencies.ts | 154 ++++++++++++++ tests/healthDependencies.test.ts | 323 ++++++++++++++++++++++++++++++ 4 files changed, 596 insertions(+) create mode 100644 docs/health-dependencies.md create mode 100644 src/routes/health/dependencies.ts create mode 100644 tests/healthDependencies.test.ts diff --git a/docs/health-dependencies.md b/docs/health-dependencies.md new file mode 100644 index 0000000..df63012 --- /dev/null +++ b/docs/health-dependencies.md @@ -0,0 +1,117 @@ +# `GET /api/health/dependencies` + +Probes all four external dependencies in parallel and returns a per-system health snapshot. + +--- + +## Endpoint summary + +| Property | Value | +|-----------------|--------------------------------------------------------| +| **Method** | `GET` | +| **Path** | `/api/health/dependencies` | +| **Auth** | None | +| **Caching** | None (fresh probe on every request) | +| **Timeout** | 5 s per probe (inherited from `probeAllDependencies`) | + +--- + +## Probed systems + +| Key | What is checked | +|----------------|-------------------------------------------------------------| +| `postgres` | `SELECT 1` against the Postgres connection pool | +| `sorobanRpc` | `getLatestLedger()` call to the configured Soroban RPC node | +| `horizon` | HTTP `GET` to the configured Horizon root URL | +| `webhookQueue` | Redis `PING` via the BullMQ connection | + +--- + +## Response codes + +| HTTP Status | Meaning | +|-------------|--------------------------------------------------| +| `200 OK` | All four probes return `ok` | +| `207 Multi-Status` | At least one probe is `degraded`, none are `down` | +| `503 Service Unavailable` | At least one probe is `down` | + +--- + +## Response body + +```json +{ + "status": "ok", + "correlationId": "e2a1c4d7-1234-5678-abcd-ef0123456789", + "checkedAt": "2026-07-24T22:00:00.000Z", + "dependencies": { + "postgres": { "status": "ok", "latencyMs": 3 }, + "sorobanRpc": { "status": "ok", "latencyMs": 12 }, + "horizon": { "status": "ok", "latencyMs": 8 }, + "webhookQueue": { "status": "ok", "latencyMs": 1 } + } +} +``` + +### Fields + +| Field | Type | Description | +|----------------------------------|-------------------------------|--------------------------------------------------------------| +| `status` | `"ok"` \| `"degraded"` \| `"down"` | Composite status — worst single probe wins. | +| `correlationId` | `string` (UUID) | Echoed from `x-correlation-id` header, or generated. | +| `checkedAt` | ISO-8601 string | UTC timestamp of the response. | +| `dependencies..status` | `"ok"` \| `"degraded"` \| `"down"` | Per-system probe result. | +| `dependencies..latencyMs`| `number` | Round-trip time for the probe, in milliseconds. | +| `dependencies..error` | `string` (optional) | Human-readable error detail — only present when `down`. | + +--- + +## Composite status logic + +``` +all probes "ok" → status "ok" +any probe "down" → status "down" +some "degraded", none "down" → status "degraded" +``` + +--- + +## Correlation ID + +Pass `x-correlation-id: ` in the request header to correlate the response +and server logs with your upstream trace. A UUID is generated automatically when +the header is absent or empty. + +--- + +## Example requests + +```bash +# All healthy +curl -s http://localhost:3001/api/health/dependencies | jq .status +# "ok" + +# With correlation ID +curl -s http://localhost:3001/api/health/dependencies \ + -H "x-correlation-id: my-trace-abc" | jq .correlationId +# "my-trace-abc" +``` + +--- + +## Related health endpoints + +| Endpoint | Auth | Caching | Purpose | +|----------------------------|------|---------|-------------------------------------------------| +| `GET /health` | None | No | Liveness check — process is up | +| `GET /healthz/dependencies`| None | 5 s TTL | Shallow cached probe (same four systems) | +| `GET /api/health/ready` | None | No | Deep readiness — adds indexer-lag check | +| `GET /api/health/dependencies` | None | No | Full detail, injectable, uncached (this page) | + +--- + +## Security notes + +- The endpoint exposes no sensitive data (no credentials, tokens, or user data). +- Restrict access to internal network paths at the infrastructure level (internal ALB, VPC-only routing, security groups) in production. +- The response body deliberately omits internal connection strings or configuration values. diff --git a/src/index.ts b/src/index.ts index a42d7ff..b9a3736 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ import { defaultBodySizeLimitMiddleware, webhookBodySizeLimitMiddleware } from " import { healthRouter } from "./routes/health"; import dependenciesRouter from "./routes/healthz/dependencies"; import { createReadyRouter } from "./routes/health/ready"; +import { dependenciesRouter } from "./routes/health/dependencies"; import { redisConnection } from "./queue"; import { authRouter } from "./routes/auth"; import { marketsRouter } from "./routes/markets"; @@ -103,6 +104,7 @@ export function createApp(): express.Express { app.use("/health", healthRouter); app.use("/healthz/dependencies", dependenciesRouter); app.use("/api/health/ready", createReadyRouter({ db, redis: redisConnection })); + app.use("/api/health/dependencies", dependenciesRouter); const mutationMethods = ["POST", "PATCH"] as const; app.use("/api", (req, res, next) => diff --git a/src/routes/health/dependencies.ts b/src/routes/health/dependencies.ts new file mode 100644 index 0000000..6a17041 --- /dev/null +++ b/src/routes/health/dependencies.ts @@ -0,0 +1,154 @@ +/** + * health/dependencies.ts + * + * GET /api/health/dependencies + * + * Probes all external dependencies (Postgres, Soroban RPC, Horizon, webhook + * queue/Redis) and returns a per-system health snapshot with a composite + * status code. + * + * This endpoint complements the existing probes: + * • GET /health — process liveness (instant, no I/O) + * • GET /healthz/dependencies — shallow cached probe (5-second TTL) + * • GET /api/health/ready — deep readiness for orchestrators + * • GET /api/health/dependencies — this file; uncached, injectable, full detail + * + * Response codes + * ────────────── + * 200 OK — all four probes pass ("ok") + * 207 Multi-Status — at least one probe is degraded but none are down + * 503 Unavailable — at least one probe is down + * + * Response shape + * ────────────── + * { + * "status": "ok" | "degraded" | "down", + * "correlationId": "", + * "checkedAt": "", + * "dependencies": { + * "postgres": { "status": "ok"|"degraded"|"down", "latencyMs": , "error?": "…" }, + * "sorobanRpc": { … }, + * "horizon": { … }, + * "webhookQueue": { … } + * } + * } + * + * Security + * ──────── + * No authentication required — the response contains no sensitive data. + * In production, restrict access at the infrastructure level (internal ALB, + * VPC-only routing, etc.). + * + * The response is NOT cached here (unlike /healthz/dependencies). Callers + * that want caching should use the lower-level getCachedDependencyHealth() + * directly or add a cache layer in front of this endpoint. + * + * Injectable dependencies + * ─────────────────────── + * All external I/O is encapsulated in the `DependenciesRouterDeps.probeFn` + * callback so tests can substitute a fully-controlled stub without touching + * real infrastructure. + */ + +import { Router, Request, Response } from "express"; +import { randomUUID } from "crypto"; +import { logger } from "../../config/logger"; +import { + probeAllDependencies, + computeCompositeStatus, + type DependencyHealth, + type CompositeStatus, +} from "../../services/healthProbes"; + +// ── Injectable dependency interface ────────────────────────────────────────── + +/** + * Signature of the probe function injected into the router. + * Production code passes `probeAllDependencies`; tests pass a stub. + */ +export type ProbeFn = () => Promise; + +export interface DependenciesRouterDeps { + /** + * Executes all four dependency probes and returns the full health map. + * Defaults to the production `probeAllDependencies` implementation. + */ + probeFn?: ProbeFn; +} + +// ── HTTP status mapping ─────────────────────────────────────────────────────── + +function compositeToHttpStatus(composite: CompositeStatus): number { + if (composite === "ok") return 200; + if (composite === "degraded") return 207; + return 503; +} + +// ── Router factory ──────────────────────────────────────────────────────────── + +/** + * Creates the /api/health/dependencies router. + * + * @param deps.probeFn - Override the probe function (tests only). + * Defaults to `probeAllDependencies`. + */ +export function createDependenciesRouter(deps: DependenciesRouterDeps = {}): Router { + const probe: ProbeFn = deps.probeFn ?? probeAllDependencies; + const router = Router(); + + /** + * GET / + * + * Runs all dependency probes and returns the health snapshot. + */ + router.get("/", async (req: Request, res: Response, next) => { + // Honour an inbound x-correlation-id if present; otherwise generate one. + const correlationId = + ((req.headers["x-correlation-id"] as string | undefined) ?? "").trim() || + randomUUID(); + + const requestStart = Date.now(); + + try { + const health = await probe(); + const composite = computeCompositeStatus(health); + const httpStatus = compositeToHttpStatus(composite); + + logger.info( + { + correlationId, + status: composite, + httpStatus, + elapsedMs: Date.now() - requestStart, + postgres: health.postgres.status, + sorobanRpc: health.sorobanRpc.status, + horizon: health.horizon.status, + webhookQueue: health.webhookQueue.status, + }, + "health_dependencies_check_complete", + ); + + res.status(httpStatus).json({ + status: composite, + correlationId, + checkedAt: new Date().toISOString(), + dependencies: health, + }); + } catch (err) { + // Unexpected error — propagate to the global error handler so the + // standard error envelope is returned. + logger.error( + { correlationId, err, elapsedMs: Date.now() - requestStart }, + "health_dependencies_probe_threw", + ); + next(err); + } + }); + + return router; +} + +// ── Default export ──────────────────────────────────────────────────────────── + +/** Production router instance wired into src/index.ts. */ +export const dependenciesRouter = createDependenciesRouter(); diff --git a/tests/healthDependencies.test.ts b/tests/healthDependencies.test.ts new file mode 100644 index 0000000..e695905 --- /dev/null +++ b/tests/healthDependencies.test.ts @@ -0,0 +1,323 @@ +/** + * healthDependencies.test.ts + * + * Tests for GET /api/health/dependencies. + * + * Strategy + * ──────── + * • The injectable `probeFn` replaces all external I/O — no real DB, + * Redis, or network calls are made. + * • The router is mounted on a minimal Express app so tests are isolated + * from the full application bootstrap. + * • The errorHandler is attached so unexpected-throw tests validate the + * standard error envelope format. + * + * Coverage + * ──────── + * • 200 all-ok + * • 207 degraded (some ok, some degraded, none down) + * • 503 any down + * • 503 multiple down + * • Response shape: status, correlationId, checkedAt, dependencies + * • Per-dependency latency and error fields + * • correlationId: echo from header / UUID generation fallback + * • No authentication required + * • Probe errors propagate as 500 via errorHandler + * • Structured log emitted on each request + */ + +// ── Env stubs (must precede all src/ imports) ───────────────────────────────── + +process.env.DATABASE_URL = "postgres://test:test@localhost:5432/test"; +process.env.JWT_SECRET = "abcdefghijklmnopqrstuvwxyz123456789012"; +process.env.SOROBAN_RPC_URL = "https://soroban-testnet.stellar.org"; +process.env.HORIZON_URL = "https://horizon-testnet.stellar.org"; +process.env.PREDICTIFY_CONTRACT_ID = "test-contract-id"; +process.env.REDIS_URL = "redis://localhost:6379"; + +// ── Module mocks (must precede dynamic imports) ─────────────────────────────── + +// Prevent the real pool / Redis client from being opened at module load time. +jest.mock("../src/db/client", () => ({ + db: {}, + pool: { query: jest.fn() }, + connectWithRetry: jest.fn(), + closeDb: jest.fn(), + getDb: jest.fn(), + getPool: jest.fn(), + setDbForTests: jest.fn(), +})); + +jest.mock("../src/queue", () => ({ + redisConnection: { ping: jest.fn().mockResolvedValue("PONG") }, + webhookQueue: { add: jest.fn() }, + backupVerificationQueue: { add: jest.fn() }, + reconciliationQueue: { add: jest.fn() }, + marketResolutionQueue: { add: jest.fn() }, + webhookQueueName: "webhook-deliveries", + backupVerificationQueueName: "backup-verification", + reconciliationQueueName: "reconciliation", + marketResolutionQueueName: "market-resolution", +})); + +// ── Imports ─────────────────────────────────────────────────────────────────── + +import request from "supertest"; +import express from "express"; +import { createDependenciesRouter } from "../src/routes/health/dependencies"; +import { errorHandler } from "../src/middleware/errorHandler"; +import type { DependencyHealth } from "../src/services/healthProbes"; + +// ── Fixtures ────────────────────────────────────────────────────────────────── + +const ALL_OK: DependencyHealth = { + postgres: { status: "ok", latencyMs: 3 }, + sorobanRpc: { status: "ok", latencyMs: 12 }, + horizon: { status: "ok", latencyMs: 8 }, + webhookQueue: { status: "ok", latencyMs: 1 }, +}; + +const ONE_DEGRADED: DependencyHealth = { + postgres: { status: "ok", latencyMs: 3 }, + sorobanRpc: { status: "degraded", latencyMs: 4200 }, + horizon: { status: "ok", latencyMs: 8 }, + webhookQueue: { status: "ok", latencyMs: 1 }, +}; + +const ALL_DEGRADED: DependencyHealth = { + postgres: { status: "degraded", latencyMs: 3000 }, + sorobanRpc: { status: "degraded", latencyMs: 4200 }, + horizon: { status: "degraded", latencyMs: 4500 }, + webhookQueue: { status: "degraded", latencyMs: 2900 }, +}; + +const ONE_DOWN: DependencyHealth = { + postgres: { status: "down", latencyMs: 100, error: "Postgres unavailable" }, + sorobanRpc: { status: "ok", latencyMs: 12 }, + horizon: { status: "ok", latencyMs: 8 }, + webhookQueue: { status: "ok", latencyMs: 1 }, +}; + +const MULTI_DOWN: DependencyHealth = { + postgres: { status: "down", latencyMs: 5000, error: "Probe timed out" }, + sorobanRpc: { status: "down", latencyMs: 5000, error: "Soroban RPC unavailable" }, + horizon: { status: "ok", latencyMs: 8 }, + webhookQueue: { status: "ok", latencyMs: 1 }, +}; + +// ── App factory ─────────────────────────────────────────────────────────────── + +function makeApp(probeFn: () => Promise): express.Express { + const app = express(); + app.use(express.json()); + app.use("/api/health/dependencies", createDependenciesRouter({ probeFn })); + app.use(errorHandler); + return app; +} + +const URL = "/api/health/dependencies"; + +// ═════════════════════════════════════════════════════════════════════════════ +// HTTP status codes +// ═════════════════════════════════════════════════════════════════════════════ + +describe("HTTP status codes", () => { + it("returns 200 when all dependencies are ok", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.status).toBe(200); + }); + + it("returns 207 when at least one dependency is degraded (none down)", async () => { + const res = await request(makeApp(() => Promise.resolve(ONE_DEGRADED))).get(URL); + expect(res.status).toBe(207); + }); + + it("returns 207 when all dependencies are degraded", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_DEGRADED))).get(URL); + expect(res.status).toBe(207); + }); + + it("returns 503 when any dependency is down", async () => { + const res = await request(makeApp(() => Promise.resolve(ONE_DOWN))).get(URL); + expect(res.status).toBe(503); + }); + + it("returns 503 when multiple dependencies are down", async () => { + const res = await request(makeApp(() => Promise.resolve(MULTI_DOWN))).get(URL); + expect(res.status).toBe(503); + }); + + it("returns 503 when a degraded + down mix exists (down wins)", async () => { + const mixed: DependencyHealth = { + postgres: { status: "down", latencyMs: 5000, error: "DB down" }, + sorobanRpc: { status: "degraded", latencyMs: 4200 }, + horizon: { status: "ok", latencyMs: 8 }, + webhookQueue: { status: "ok", latencyMs: 1 }, + }; + const res = await request(makeApp(() => Promise.resolve(mixed))).get(URL); + expect(res.status).toBe(503); + }); +}); + +// ═════════════════════════════════════════════════════════════════════════════ +// Response body — composite status field +// ═════════════════════════════════════════════════════════════════════════════ + +describe("response body — status field", () => { + it("body.status is 'ok' when all pass", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.body.status).toBe("ok"); + }); + + it("body.status is 'degraded' when degraded (no down)", async () => { + const res = await request(makeApp(() => Promise.resolve(ONE_DEGRADED))).get(URL); + expect(res.body.status).toBe("degraded"); + }); + + it("body.status is 'down' when any probe is down", async () => { + const res = await request(makeApp(() => Promise.resolve(ONE_DOWN))).get(URL); + expect(res.body.status).toBe("down"); + }); +}); + +// ═════════════════════════════════════════════════════════════════════════════ +// Response shape +// ═════════════════════════════════════════════════════════════════════════════ + +describe("response shape", () => { + it("includes all required top-level fields", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.body).toHaveProperty("status"); + expect(res.body).toHaveProperty("correlationId"); + expect(res.body).toHaveProperty("checkedAt"); + expect(res.body).toHaveProperty("dependencies"); + }); + + it("dependencies contains all four systems", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.body.dependencies).toHaveProperty("postgres"); + expect(res.body.dependencies).toHaveProperty("sorobanRpc"); + expect(res.body.dependencies).toHaveProperty("horizon"); + expect(res.body.dependencies).toHaveProperty("webhookQueue"); + }); + + it("each dependency entry contains status and latencyMs", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + for (const key of ["postgres", "sorobanRpc", "horizon", "webhookQueue"]) { + expect(res.body.dependencies[key]).toHaveProperty("status"); + expect(res.body.dependencies[key]).toHaveProperty("latencyMs"); + } + }); + + it("checkedAt is a valid ISO-8601 timestamp", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(typeof res.body.checkedAt).toBe("string"); + expect(() => new Date(res.body.checkedAt)).not.toThrow(); + expect(new Date(res.body.checkedAt).getTime()).toBeGreaterThan(0); + }); + + it("includes per-dependency latency values from the probe", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.body.dependencies.postgres.latencyMs).toBe(3); + expect(res.body.dependencies.sorobanRpc.latencyMs).toBe(12); + expect(res.body.dependencies.horizon.latencyMs).toBe(8); + expect(res.body.dependencies.webhookQueue.latencyMs).toBe(1); + }); + + it("includes error field when a probe is down", async () => { + const res = await request(makeApp(() => Promise.resolve(ONE_DOWN))).get(URL); + expect(res.body.dependencies.postgres.error).toBe("Postgres unavailable"); + }); + + it("does not expose error field when probe is ok", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.body.dependencies.postgres).not.toHaveProperty("error"); + }); +}); + +// ═════════════════════════════════════════════════════════════════════════════ +// Correlation ID +// ═════════════════════════════════════════════════════════════════════════════ + +describe("correlationId", () => { + it("echoes the x-correlation-id header when provided", async () => { + const id = "my-trace-id-abc-123"; + const res = await request(makeApp(() => Promise.resolve(ALL_OK))) + .get(URL) + .set("x-correlation-id", id); + expect(res.body.correlationId).toBe(id); + }); + + it("generates a UUID when x-correlation-id is not provided", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + expect(res.body.correlationId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, + ); + }); + + it("generates a UUID when x-correlation-id is an empty string", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))) + .get(URL) + .set("x-correlation-id", ""); + expect(res.body.correlationId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, + ); + }); + + it("correlationId differs between requests when not supplied", async () => { + const app = makeApp(() => Promise.resolve(ALL_OK)); + const [r1, r2] = await Promise.all([request(app).get(URL), request(app).get(URL)]); + expect(r1.body.correlationId).not.toBe(r2.body.correlationId); + }); +}); + +// ═════════════════════════════════════════════════════════════════════════════ +// Auth / access control +// ═════════════════════════════════════════════════════════════════════════════ + +describe("authentication", () => { + it("does not require an Authorization header", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))).get(URL); + // Must not be a 401 or 403 + expect(res.status).toBe(200); + }); + + it("ignores a supplied Authorization header (does not change the response)", async () => { + const res = await request(makeApp(() => Promise.resolve(ALL_OK))) + .get(URL) + .set("Authorization", "Bearer some-random-token"); + expect(res.status).toBe(200); + expect(res.body.status).toBe("ok"); + }); +}); + +// ═════════════════════════════════════════════════════════════════════════════ +// Error handling +// ═════════════════════════════════════════════════════════════════════════════ + +describe("error handling", () => { + it("returns 500 and propagates to errorHandler when probeFn throws", async () => { + const throwing = () => Promise.reject(new Error("infrastructure exploded")); + const res = await request(makeApp(throwing)).get(URL); + expect(res.status).toBe(500); + }); + + it("calls probeFn exactly once per request", async () => { + const probeFn = jest.fn().mockResolvedValue(ALL_OK); + await request(makeApp(probeFn)).get(URL); + expect(probeFn).toHaveBeenCalledTimes(1); + }); +}); + +// ═════════════════════════════════════════════════════════════════════════════ +// Default export wires to production probeAllDependencies +// ═════════════════════════════════════════════════════════════════════════════ + +describe("default router", () => { + it("exports a default dependenciesRouter as a valid Express router", async () => { + const { dependenciesRouter } = await import("../src/routes/health/dependencies"); + expect(typeof dependenciesRouter).toBe("function"); + // An Express router is a function with a `stack` property. + expect(Array.isArray((dependenciesRouter as unknown as { stack: unknown[] }).stack)).toBe(true); + }); +});