From 9a0c8096c6eefacce24a1f775b2d597c023a9557 Mon Sep 17 00:00:00 2001 From: Ajibose Date: Sat, 18 Jul 2026 17:58:24 +0300 Subject: [PATCH] docs(health): document all health endpoint response fields and states Expands docs/health-endpoints.md to cover every response field for all four health endpoints (liveness, readiness, detailed, indexer heartbeat) with healthy/degraded/unhealthy values, liveness vs readiness distinction, and explicit non-200 trigger conditions per endpoint. Adds health.response-schema.test.ts with 70 tests that act as a living contract between the code and the documentation, covering field types, allowed enum values, and HTTP status mapping for every documented state. Fixes pre-existing setHeader TypeError in health.controllers.test.ts and health.controllers.integration.test.ts that caused 6 tests to fail. --- docs/health-endpoints.md | 306 +++++++- .../health.controllers.integration.test.ts | 1 + src/modules/health/health.controllers.test.ts | 2 + .../health/health.response-schema.test.ts | 651 ++++++++++++++++++ 4 files changed, 924 insertions(+), 36 deletions(-) create mode 100644 src/modules/health/health.response-schema.test.ts diff --git a/docs/health-endpoints.md b/docs/health-endpoints.md index 0a05dd9..c507052 100644 --- a/docs/health-endpoints.md +++ b/docs/health-endpoints.md @@ -1,17 +1,55 @@ # Health and Readiness Endpoints -The health module exposes three endpoints, each with a different contract. +The health module exposes four endpoints with different contracts and intended consumers. + +## Liveness vs Readiness + +| Endpoint | Purpose | Consumer | +| ----------------------------- | ------------------------------------------------------------ | ----------------------------------------- | +| `GET /api/v1/health` | Liveness — confirms the process is alive | Load balancers, uptime monitors | +| `GET /api/v1/health/ready` | Readiness — confirms all critical dependencies are reachable | Kubernetes `readinessProbe`, deploy gates | +| `GET /api/v1/health/detailed` | Diagnostics — full system snapshot | Operators, dashboards | +| `GET /api/v1/health/indexer` | Worker heartbeat — confirms the indexer is running | Alerting, internal monitoring | + +**Liveness** never probes dependencies. It responds `200` as long as the event loop is processing requests. Use it wherever a dependency failure should not pull the instance from rotation (e.g., when a downstream DB blip should not make the server appear down). + +**Readiness** actively pings each critical dependency. A single `fail` result flips the response to `503`, signalling to the orchestrator that this instance should stop receiving traffic until the dependency recovers. + +--- ## `GET /api/v1/health` — liveness -A minimal "the process is up" check. Always `200` while the event loop is -healthy. Safe for load balancers and uptime monitors that should not fan out -to dependencies. +A minimal "the process is up" check. Always `200` while the event loop is healthy. No dependency probing. + +### Response shape + +```json +{ + "success": true, + "message": "OK", + "timestamp": "2026-04-28T16:00:00.000Z" +} +``` + +### Fields + +| Field | Type | Description | +| ----------- | ----------------- | -------------------------------------------------- | +| `success` | boolean | Always `true`. | +| `message` | string | Human-readable confirmation string. Always `"OK"`. | +| `timestamp` | string (ISO-8601) | When the response was built. | + +### HTTP status codes + +| Code | Condition | +| ----- | ---------------------------------------- | +| `200` | Always — liveness never returns non-200. | + +--- ## `GET /api/v1/health/ready` — readiness -Probes critical dependencies (database, cache config). Returns `200` when -every probe passes and `503` otherwise. +Probes critical dependencies (database, cache config). Returns `200` when every probe passes and `503` otherwise. ### Response shape @@ -29,27 +67,172 @@ every probe passes and `503` otherwise. ### Fields -| Field | Type | Notes | -| ----------- | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `ready` | boolean | `true` only when every check is `ok`. Maps directly to the HTTP status (`200` vs `503`). | -| `timestamp` | string (ISO-8601) | When the response was built. | -| `latencyMs` | number | **Total** wall-clock duration of the readiness probe — sum of every check plus orchestration overhead. Useful for dashboards and SLO tracking. | -| `checks` | array | Per-dependency results, each with its own `name`, `status`, optional `latencyMs`, and optional `error`. | +| Field | Type | Description | +| ----------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `ready` | boolean | `true` only when every check returns `"ok"`. **This field determines the HTTP status** — `true` → `200`, `false` → `503`. | +| `timestamp` | string (ISO-8601) | When the response was built. | +| `latencyMs` | number | Total wall-clock duration of the entire readiness probe in milliseconds. Useful for dashboards and SLO tracking. | +| `checks` | array | Per-dependency probe results. See the checks table below. | + +#### `checks` array entry fields + +| Field | Type | Present when | Description | +| ----------- | ------ | ------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `name` | string | Always | Dependency name. Current values: `"database"`, `"cache"`. | +| `status` | string | Always | `"ok"` — probe passed; `"fail"` — probe failed. | +| `latencyMs` | number | `status: "ok"` and check measured a latency | Round-trip time for this probe in milliseconds. | +| `error` | string | `status: "fail"` | Human-readable reason for the failure. No internal hostnames, connection strings, or stack traces. | -The payload is intentionally public-safe: no internal hostnames, -connection strings, or stack traces are included even when a check fails. +#### Probe descriptions + +| Probe name | What is checked | Healthy value | Unhealthy value | +| ---------- | --------------------------------------------------------------------- | ---------------------------- | -------------------------- | +| `database` | Issues `SELECT 1` against the primary database. | `"ok"` + `latencyMs` present | `"fail"` + `error` present | +| `cache` | Verifies that the HTTP cache layer config constant is a valid number. | `"ok"` | `"fail"` + `error` present | + +### Non-200 trigger + +The `ready` field is the sole trigger. When **any** check in the `checks` array has `status: "fail"`, `ready` is `false` and the HTTP status is `503`. + +The payload is intentionally public-safe: no internal hostnames, connection strings, or stack traces are included even when a check fails. + +--- ## `GET /api/v1/health/detailed` — diagnostics -Full system snapshot including memory, uptime, system info, database -response time, chain-sync lag and per-service health flags. Intended for -operators rather than load balancers — cheaper liveness/readiness paths -should be preferred for automated probing. +Full system snapshot including memory, uptime, system info, database response time, chain-sync lag, and per-service health flags. Intended for operators and dashboards — use the cheaper liveness and readiness paths for automated probing. + +### Response shape + +```json +{ + "success": true, + "message": "Access Layer server is running", + "timestamp": "2026-04-28T16:00:00.000Z", + "version": "1.0.0", + "environment": "production", + "uptime": 3600.5, + "memory": { + "used": 48.32, + "total": 64.0 + }, + "system": { + "platform": "linux", + "nodeVersion": "v20.11.0" + }, + "timeouts": { + "database_timeout_ms": 5000, + "cache_timeout_ms": 300000 + }, + "database": { + "status": "connected", + "responseTime": 4 + }, + "syncing": { + "status": "in-sync", + "latestIndexedLedger": 12345, + "observedHeadLedger": 12400, + "syncLagLedgers": 55 + }, + "services": [ + { "name": "API Server", "status": "healthy" }, + { "name": "Database", "status": "healthy" }, + { "name": "Chain Sync", "status": "healthy" } + ] +} +``` + +### Top-level fields + +| Field | Type | Description | +| ------------- | ----------------- | --------------------------------------------------------------------------------------------------- | +| `success` | boolean | Always `true` in a normal response. `false` only on an unexpected exception. | +| `message` | string | Human-readable status string. `"Access Layer server is running"` when healthy. | +| `timestamp` | string (ISO-8601) | When the response was built. | +| `version` | string | Application version string. | +| `environment` | string | Deployment environment. Mirrors `MODE` env var (e.g. `"production"`, `"staging"`, `"development"`). | +| `uptime` | number (seconds) | Seconds since the Node.js process started (`process.uptime()`). Always ≥ 0. | +| `memory` | object | JVM heap snapshot. See below. | +| `system` | object | Runtime identity. See below. | +| `timeouts` | object | Public-safe dependency timeout values. See below. | +| `database` | object | Database connectivity status. See below. | +| `syncing` | object \| absent | Chain indexer sync lag. Absent when the sync status check itself errors. | +| `services` | array | Rolled-up health flags for each major component. See below. | + +### `memory` fields + +| Field | Type | Description | +| ------- | ------ | ----------------------------------------------------------------------- | +| `used` | number | Node.js heap memory currently used, in **megabytes** (rounded to 2 dp). | +| `total` | number | Node.js heap memory allocated, in **megabytes** (rounded to 2 dp). | + +### `system` fields + +| Field | Type | Description | +| ------------- | ------ | -------------------------------------------------------------- | +| `platform` | string | Operating system platform string (e.g. `"linux"`, `"darwin"`). | +| `nodeVersion` | string | Node.js version string (e.g. `"v20.11.0"`). | -### Field order and grouping +### `timeouts` fields -The detailed health payload should preserve this top-level order to keep -JSON snapshots predictable for contributors and dashboards: +Public-safe dependency timeout configuration. No connection strings, hostnames, or credentials are included. + +| Field | Type | Description | +| --------------------- | ------ | -------------------------------------------------- | +| `database_timeout_ms` | number | Configured database query timeout in milliseconds. | +| `cache_timeout_ms` | number | Configured public HTTP cache TTL in milliseconds. | + +### `database` fields + +| Field | Type | Present when | Description | +| -------------- | ------ | --------------------- | ---------------------------------------------------------------------- | +| `status` | string | Always | `"connected"` — `SELECT 1` succeeded; `"disconnected"` — query failed. | +| `responseTime` | number | `status: "connected"` | Round-trip time for the DB probe in milliseconds. | + +#### Database states + +| `status` | Meaning | HTTP impact | +| ---------------- | ------------------------------------------- | ----------------------------------------------------- | +| `"connected"` | Database is reachable and responding. | Healthy. | +| `"disconnected"` | Database probe threw an error or timed out. | **503** in `production`; `200` in other environments. | + +### `syncing` fields + +Present when the chain indexer sync status is available. Absent if the sync check itself errors. + +| Field | Type | Description | +| --------------------- | ------ | ----------------------------------------------------------------------------------- | +| `status` | string | `"in-sync"` — lag is within threshold; `"degraded"` — lag exceeds the threshold. | +| `latestIndexedLedger` | number | Most recent ledger the indexer has processed. | +| `observedHeadLedger` | number | Latest ledger observed on the chain. | +| `syncLagLedgers` | number | Difference: `observedHeadLedger − latestIndexedLedger`. Threshold: **100 ledgers**. | + +#### Sync states + +| `status` | Condition | `syncLagLedgers` | +| ------------ | ---------------------------------- | ---------------- | +| `"in-sync"` | Lag is within the 100-ledger limit | 0 – 100 | +| `"degraded"` | Lag exceeds 100 ledgers | > 100 | + +The sync state does **not** affect the HTTP status code of this endpoint. Use `GET /api/v1/health/indexer` for a dedicated alerting-friendly probe. + +### `services` array + +A rolled-up health summary of each major component. + +| `name` | `status: "healthy"` condition | `status: "unhealthy"` condition | +| -------------- | ------------------------------------------------- | ------------------------------------- | +| `"API Server"` | Always healthy (present means the server is up). | Never unhealthy in normal operation. | +| `"Database"` | `database.status === "connected"`. | `database.status === "disconnected"`. | +| `"Chain Sync"` | `syncing.status !== "degraded"` (or sync absent). | `syncing.status === "degraded"`. | + +### Non-200 trigger + +`database.status === "disconnected"` **and** `environment === "production"` triggers a `503`. In non-production environments the endpoint always returns `200` regardless of database connectivity. + +### Field order + +The top-level JSON keys are guaranteed to appear in this order: 1. `success` 2. `message` @@ -59,24 +242,75 @@ JSON snapshots predictable for contributors and dashboards: 6. `uptime` 7. `memory` 8. `system` -9. `database` -10. `syncing` -11. `services` +9. `timeouts` +10. `database` +11. `syncing` +12. `services` + +--- + +## `GET /api/v1/health/indexer` — worker heartbeat -Nested grouping follows the same convention: +Reports the liveness state of the indexer background worker. The indexer calls `POST /api/v1/health/indexer/heartbeat` after each successful run; this endpoint exposes the resulting state. -- `memory`: `used`, `total` -- `system`: `platform`, `nodeVersion` -- `database`: `status`, `responseTime` (when connected) -- `syncing`: `status`, `latestIndexedLedger`, `observedHeadLedger`, `syncLagLedgers` -- `services`: ordered as `API Server`, `Database`, `Chain Sync` -- `timeouts`: `database_timeout_ms`, `cache_timeout_ms` +### Response shape + +```json +{ + "success": true, + "data": { + "service": "indexer", + "status": "healthy", + "lastSuccessfulRun": "2026-04-28T15:59:00.000Z", + "staleSinceMs": null + } +} +``` + +### Fields + +| Field | Type | Description | +| ------------------------ | -------------- | ---------------------------------------------------------------------------- | +| `success` | boolean | Always `true`. | +| `data.service` | string | Always `"indexer"`. | +| `data.status` | string | Current worker state. See status table below. | +| `data.lastSuccessfulRun` | string \| null | ISO-8601 timestamp of the most recent heartbeat, or `null` if none recorded. | +| `data.staleSinceMs` | number \| null | Milliseconds since the heartbeat became stale, or `null` when not stale. | + +#### Worker status values -### Detailed health fields +| `data.status` | Condition | `lastSuccessfulRun` | `staleSinceMs` | HTTP status | +| ------------- | ------------------------------------------------------------------------------------- | ------------------- | --------------- | ----------- | +| `"unknown"` | No heartbeat has ever been recorded (fresh deploy or restart). | `null` | `null` | `200` | +| `"healthy"` | Last heartbeat is within the stale threshold. | ISO-8601 string | `null` | `200` | +| `"degraded"` | Last heartbeat exceeded the stale threshold (`INDEXER_HEARTBEAT_STALE_THRESHOLD_MS`). | ISO-8601 string | positive number | `503` | -The `/api/v1/health/detailed` response also includes an explicit `timeouts` object with public-safe dependency timeout values: +### Non-200 trigger -- `database_timeout_ms` (number) — configured database query timeout in milliseconds. -- `cache_timeout_ms` (number) — configured public cache timeout in milliseconds. +`data.status === "degraded"` is the sole trigger for `503`. Both `"unknown"` and `"healthy"` return `200`. + +--- + +## `POST /api/v1/health/indexer/heartbeat` — record worker run + +Called by the indexer worker after each successful run to reset the stale timer. + +### Response shape + +```json +{ + "success": true, + "data": { + "recorded": true, + "timestamp": "2026-04-28T16:00:00.000Z" + }, + "message": "Heartbeat recorded" +} +``` -These values are intentionally numeric-only and do not include connection strings, hostnames, credentials, or other internal topology details. +| Field | Type | Description | +| ---------------- | ----------------- | -------------------------------------------- | +| `success` | boolean | Always `true`. | +| `data.recorded` | boolean | Always `true` when the heartbeat was stored. | +| `data.timestamp` | string (ISO-8601) | The time the heartbeat was recorded. | +| `message` | string | Always `"Heartbeat recorded"`. | diff --git a/src/modules/health/health.controllers.integration.test.ts b/src/modules/health/health.controllers.integration.test.ts index 5998055..baeea69 100644 --- a/src/modules/health/health.controllers.integration.test.ts +++ b/src/modules/health/health.controllers.integration.test.ts @@ -43,6 +43,7 @@ function mockResponse(): Response & { statusCode: number; body: any } { res.body = payload; return res; }; + res.setHeader = () => res; return res; } diff --git a/src/modules/health/health.controllers.test.ts b/src/modules/health/health.controllers.test.ts index 048c65e..94b1214 100644 --- a/src/modules/health/health.controllers.test.ts +++ b/src/modules/health/health.controllers.test.ts @@ -62,6 +62,8 @@ function mockResponse(): Response & { statusCode: number; body: any } { return res; }; + res.setHeader = () => res; + return res; } diff --git a/src/modules/health/health.response-schema.test.ts b/src/modules/health/health.response-schema.test.ts new file mode 100644 index 0000000..a5e927a --- /dev/null +++ b/src/modules/health/health.response-schema.test.ts @@ -0,0 +1,651 @@ +// src/modules/health/health.response-schema.test.ts +// +// Verifies that every documented response field exists, carries the right type, +// and takes only the values listed in docs/health-endpoints.md. +// These tests act as a living contract between the code and the documentation. + +jest.mock('../../config', () => ({ + envConfig: { + MODE: 'test', + PORT: 3000, + INDEXER_HEARTBEAT_STALE_THRESHOLD_MS: 300000, + DB_QUERY_TIMEOUT_MS: 5000, + }, + appConfig: { + allowedOrigins: [], + }, +})); + +jest.mock('../../utils/prisma.utils', () => ({ + prisma: { + $queryRaw: jest.fn(), + }, +})); + +jest.mock('../../utils/indexer-cursor-staleness.utils', () => ({ + checkIndexerCursorStalenessFromStore: jest.fn().mockResolvedValue(undefined), +})); + +import { Request, Response } from 'express'; +import { + healthCheck, + indexerHeartbeatCheck, + readinessCheck, + recordIndexerHeartbeat, + simpleHealthCheck, +} from './health.controllers'; +import { indexerHeartbeat } from '../../utils/heartbeat.service'; +import { prisma } from '../../utils/prisma.utils'; + +const queryRawMock = prisma.$queryRaw as unknown as jest.Mock; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function mockResponse(): Response & { statusCode: number; body: any } { + const res = { statusCode: 0, body: undefined as any } as any; + res.status = (code: number) => { + res.statusCode = code; + return res; + }; + res.json = (payload: any) => { + res.body = payload; + return res; + }; + res.setHeader = () => res; + return res; +} + +function mockRequest(): Request { + return {} as Request; +} + +// --------------------------------------------------------------------------- +// GET /api/v1/health — liveness +// --------------------------------------------------------------------------- + +describe('GET /health — liveness response schema', () => { + it('always returns HTTP 200', () => { + const res = mockResponse(); + simpleHealthCheck(mockRequest(), res); + expect(res.statusCode).toBe(200); + }); + + it('has success field always set to true', () => { + const res = mockResponse(); + simpleHealthCheck(mockRequest(), res); + expect(res.body.success).toBe(true); + }); + + it('has message field set to "OK"', () => { + const res = mockResponse(); + simpleHealthCheck(mockRequest(), res); + expect(res.body.message).toBe('OK'); + }); + + it('has timestamp field as a valid ISO-8601 string', () => { + const res = mockResponse(); + simpleHealthCheck(mockRequest(), res); + expect(typeof res.body.timestamp).toBe('string'); + expect(new Date(res.body.timestamp).toISOString()).toBe(res.body.timestamp); + }); + + it('response contains exactly the documented fields', () => { + const res = mockResponse(); + simpleHealthCheck(mockRequest(), res); + expect(Object.keys(res.body).sort()).toEqual(['message', 'success', 'timestamp'].sort()); + }); +}); + +// --------------------------------------------------------------------------- +// GET /api/v1/health/ready — readiness: healthy path +// --------------------------------------------------------------------------- + +describe('GET /health/ready — readiness response schema (all checks pass)', () => { + beforeEach(() => { + queryRawMock.mockResolvedValue([{ '?column?': 1 }]); + }); + + afterEach(() => { + queryRawMock.mockReset(); + }); + + it('returns HTTP 200 when all checks pass', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(res.statusCode).toBe(200); + }); + + it('has ready field set to true', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(res.body.ready).toBe(true); + }); + + it('has timestamp as a valid ISO-8601 string', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(typeof res.body.timestamp).toBe('string'); + expect(new Date(res.body.timestamp).toISOString()).toBe(res.body.timestamp); + }); + + it('has latencyMs as a non-negative number', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(typeof res.body.latencyMs).toBe('number'); + expect(res.body.latencyMs).toBeGreaterThanOrEqual(0); + }); + + it('has checks as an array', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(Array.isArray(res.body.checks)).toBe(true); + }); + + it('includes a "database" check with status "ok"', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + const dbCheck = res.body.checks.find((c: any) => c.name === 'database'); + expect(dbCheck).toBeDefined(); + expect(dbCheck.status).toBe('ok'); + }); + + it('includes latencyMs on the database check when it passes', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + const dbCheck = res.body.checks.find((c: any) => c.name === 'database'); + expect(typeof dbCheck.latencyMs).toBe('number'); + expect(dbCheck.latencyMs).toBeGreaterThanOrEqual(0); + }); + + it('includes a "cache" check with status "ok"', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + const cacheCheck = res.body.checks.find((c: any) => c.name === 'cache'); + expect(cacheCheck).toBeDefined(); + expect(cacheCheck.status).toBe('ok'); + }); + + it('check status values are only "ok" or "fail"', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + for (const check of res.body.checks) { + expect(['ok', 'fail']).toContain(check.status); + } + }); +}); + +// --------------------------------------------------------------------------- +// GET /api/v1/health/ready — readiness: database failure path +// --------------------------------------------------------------------------- + +describe('GET /health/ready — readiness response schema (database failure)', () => { + beforeEach(() => { + queryRawMock.mockRejectedValue(new Error('connection refused')); + }); + + afterEach(() => { + queryRawMock.mockReset(); + }); + + it('returns HTTP 503 when a check fails', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(res.statusCode).toBe(503); + }); + + it('sets ready to false — the sole non-200 trigger', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(res.body.ready).toBe(false); + }); + + it('includes error string on the failed check — no stack trace or hostnames', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + const dbCheck = res.body.checks.find((c: any) => c.name === 'database'); + expect(dbCheck.status).toBe('fail'); + expect(typeof dbCheck.error).toBe('string'); + expect(dbCheck.error.length).toBeGreaterThan(0); + }); + + it('does not include latencyMs on the failed database check', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + const dbCheck = res.body.checks.find((c: any) => c.name === 'database'); + expect(dbCheck.latencyMs).toBeUndefined(); + }); + + it('still includes latencyMs at the top level even when a check fails', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + expect(typeof res.body.latencyMs).toBe('number'); + expect(res.body.latencyMs).toBeGreaterThanOrEqual(0); + }); + + it('cache check still passes independently of the database check', async () => { + const res = mockResponse(); + await readinessCheck(mockRequest(), res); + const cacheCheck = res.body.checks.find((c: any) => c.name === 'cache'); + expect(cacheCheck).toBeDefined(); + expect(cacheCheck.status).toBe('ok'); + }); +}); + +// --------------------------------------------------------------------------- +// GET /api/v1/health/detailed — diagnostics: healthy path +// --------------------------------------------------------------------------- + +describe('GET /health/detailed — diagnostics response schema (DB connected)', () => { + beforeEach(() => { + queryRawMock.mockResolvedValue([{ '?column?': 1 }]); + }); + + afterEach(() => { + queryRawMock.mockReset(); + }); + + it('returns HTTP 200', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(res.statusCode).toBe(200); + }); + + it('has success set to true', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(res.body.success).toBe(true); + }); + + it('has message as a non-empty string', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.message).toBe('string'); + expect(res.body.message.length).toBeGreaterThan(0); + }); + + it('has timestamp as a valid ISO-8601 string', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.timestamp).toBe('string'); + expect(new Date(res.body.timestamp).toISOString()).toBe(res.body.timestamp); + }); + + it('has version as a non-empty string', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.version).toBe('string'); + expect(res.body.version.length).toBeGreaterThan(0); + }); + + it('has environment as a non-empty string', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.environment).toBe('string'); + expect(res.body.environment.length).toBeGreaterThan(0); + }); + + it('has uptime as a non-negative number (seconds)', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.uptime).toBe('number'); + expect(res.body.uptime).toBeGreaterThanOrEqual(0); + }); + + describe('memory object', () => { + it('has memory.used as a non-negative number (megabytes)', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.memory.used).toBe('number'); + expect(res.body.memory.used).toBeGreaterThanOrEqual(0); + }); + + it('has memory.total as a positive number (megabytes)', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.memory.total).toBe('number'); + expect(res.body.memory.total).toBeGreaterThan(0); + }); + + it('has memory.used <= memory.total', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(res.body.memory.used).toBeLessThanOrEqual(res.body.memory.total); + }); + }); + + describe('system object', () => { + it('has system.platform as a non-empty string', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.system.platform).toBe('string'); + expect(res.body.system.platform.length).toBeGreaterThan(0); + }); + + it('has system.nodeVersion as a string starting with "v"', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.system.nodeVersion).toBe('string'); + expect(res.body.system.nodeVersion).toMatch(/^v\d+/); + }); + }); + + describe('timeouts object', () => { + it('has timeouts.database_timeout_ms as a positive number', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.timeouts.database_timeout_ms).toBe('number'); + expect(res.body.timeouts.database_timeout_ms).toBeGreaterThan(0); + }); + + it('has timeouts.cache_timeout_ms as a positive number', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.timeouts.cache_timeout_ms).toBe('number'); + expect(res.body.timeouts.cache_timeout_ms).toBeGreaterThan(0); + }); + }); + + describe('database object', () => { + it('has database.status set to "connected" when DB responds', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(res.body.database.status).toBe('connected'); + }); + + it('database.status is only "connected" or "disconnected"', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(['connected', 'disconnected']).toContain(res.body.database.status); + }); + + it('has database.responseTime as a non-negative number when connected', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(typeof res.body.database.responseTime).toBe('number'); + expect(res.body.database.responseTime).toBeGreaterThanOrEqual(0); + }); + }); + + describe('syncing object', () => { + it('has syncing.status as either "in-sync" or "degraded"', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + if (res.body.syncing !== undefined) { + expect(['in-sync', 'degraded']).toContain(res.body.syncing.status); + } + }); + + it('has syncing.latestIndexedLedger as a non-negative integer when present', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + if (res.body.syncing !== undefined) { + expect(typeof res.body.syncing.latestIndexedLedger).toBe('number'); + expect(res.body.syncing.latestIndexedLedger).toBeGreaterThanOrEqual(0); + } + }); + + it('has syncing.observedHeadLedger as a non-negative integer when present', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + if (res.body.syncing !== undefined) { + expect(typeof res.body.syncing.observedHeadLedger).toBe('number'); + expect(res.body.syncing.observedHeadLedger).toBeGreaterThanOrEqual(0); + } + }); + + it('has syncing.syncLagLedgers equal to observedHead - latestIndexed', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + if (res.body.syncing !== undefined) { + expect(res.body.syncing.syncLagLedgers).toBe( + res.body.syncing.observedHeadLedger - res.body.syncing.latestIndexedLedger + ); + } + }); + }); + + describe('services array', () => { + it('has services as an array', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(Array.isArray(res.body.services)).toBe(true); + }); + + it('includes "API Server", "Database", and "Chain Sync" entries', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + const names = res.body.services.map((s: any) => s.name); + expect(names).toContain('API Server'); + expect(names).toContain('Database'); + expect(names).toContain('Chain Sync'); + }); + + it('each service status is only "healthy" or "unhealthy"', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + for (const svc of res.body.services) { + expect(['healthy', 'unhealthy']).toContain(svc.status); + } + }); + + it('"API Server" is always healthy', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + const apiServer = res.body.services.find((s: any) => s.name === 'API Server'); + expect(apiServer.status).toBe('healthy'); + }); + + it('"Database" service is "healthy" when database is connected', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + const dbService = res.body.services.find((s: any) => s.name === 'Database'); + expect(dbService.status).toBe('healthy'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// GET /api/v1/health/detailed — diagnostics: database failure in test mode +// --------------------------------------------------------------------------- + +describe('GET /health/detailed — diagnostics response schema (DB disconnected, non-production)', () => { + beforeEach(() => { + queryRawMock.mockRejectedValue(new Error('connection refused')); + }); + + afterEach(() => { + queryRawMock.mockReset(); + }); + + it('returns HTTP 200 in non-production even when DB is disconnected', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + // MODE is "test" in this suite's mock — non-production returns 200 + expect(res.statusCode).toBe(200); + }); + + it('has database.status set to "disconnected"', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(res.body.database.status).toBe('disconnected'); + }); + + it('does not include database.responseTime when disconnected', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + expect(res.body.database.responseTime).toBeUndefined(); + }); + + it('"Database" service is "unhealthy" when database is disconnected', async () => { + const res = mockResponse(); + await healthCheck(mockRequest(), res); + const dbService = res.body.services.find((s: any) => s.name === 'Database'); + expect(dbService.status).toBe('unhealthy'); + }); +}); + +// --------------------------------------------------------------------------- +// GET /api/v1/health/indexer — worker heartbeat: all three status values +// --------------------------------------------------------------------------- + +describe('GET /health/indexer — indexer heartbeat response schema', () => { + beforeEach(() => { + indexerHeartbeat.reset(); + }); + + it('has success set to true in all states', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.success).toBe(true); + }); + + it('has data.service set to "indexer"', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.service).toBe('indexer'); + }); + + it('data.status is only "healthy", "degraded", or "unknown"', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(['healthy', 'degraded', 'unknown']).toContain(res.body.data.status); + }); + + describe('"unknown" state — no heartbeat ever recorded', () => { + it('returns HTTP 200', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.statusCode).toBe(200); + }); + + it('has data.status set to "unknown"', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.status).toBe('unknown'); + }); + + it('has data.lastSuccessfulRun as null', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.lastSuccessfulRun).toBeNull(); + }); + + it('has data.staleSinceMs as null', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.staleSinceMs).toBeNull(); + }); + }); + + describe('"healthy" state — recent heartbeat recorded', () => { + beforeEach(() => { + indexerHeartbeat.recordHeartbeat(); + }); + + it('returns HTTP 200', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.statusCode).toBe(200); + }); + + it('has data.status set to "healthy"', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.status).toBe('healthy'); + }); + + it('has data.lastSuccessfulRun as a valid ISO-8601 string', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(typeof res.body.data.lastSuccessfulRun).toBe('string'); + expect(new Date(res.body.data.lastSuccessfulRun).toISOString()).toBe( + res.body.data.lastSuccessfulRun + ); + }); + + it('has data.staleSinceMs as null when healthy', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.staleSinceMs).toBeNull(); + }); + }); + + describe('"degraded" state — stale heartbeat', () => { + beforeEach(() => { + indexerHeartbeat.recordHeartbeat(); + const longAgo = new Date(Date.now() - 10 * 60 * 1000); // 10 min ago + (indexerHeartbeat as unknown as { lastSuccessfulRun: Date }).lastSuccessfulRun = longAgo; + }); + + it('returns HTTP 503 — sole non-200 trigger for this endpoint', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.statusCode).toBe(503); + }); + + it('has data.status set to "degraded"', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(res.body.data.status).toBe('degraded'); + }); + + it('has data.lastSuccessfulRun as a valid ISO-8601 string', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(typeof res.body.data.lastSuccessfulRun).toBe('string'); + expect(new Date(res.body.data.lastSuccessfulRun).toISOString()).toBe( + res.body.data.lastSuccessfulRun + ); + }); + + it('has data.staleSinceMs as a positive number when degraded', () => { + const res = mockResponse(); + indexerHeartbeatCheck(mockRequest(), res); + expect(typeof res.body.data.staleSinceMs).toBe('number'); + expect(res.body.data.staleSinceMs).toBeGreaterThan(0); + }); + }); +}); + +// --------------------------------------------------------------------------- +// POST /api/v1/health/indexer/heartbeat — record worker run +// --------------------------------------------------------------------------- + +describe('POST /health/indexer/heartbeat — response schema', () => { + beforeEach(() => { + indexerHeartbeat.reset(); + }); + + it('returns HTTP 200', async () => { + const res = mockResponse(); + await recordIndexerHeartbeat(mockRequest(), res); + expect(res.statusCode).toBe(200); + }); + + it('has success set to true', async () => { + const res = mockResponse(); + await recordIndexerHeartbeat(mockRequest(), res); + expect(res.body.success).toBe(true); + }); + + it('has data.recorded set to true', async () => { + const res = mockResponse(); + await recordIndexerHeartbeat(mockRequest(), res); + expect(res.body.data.recorded).toBe(true); + }); + + it('has data.timestamp as a valid ISO-8601 string', async () => { + const res = mockResponse(); + await recordIndexerHeartbeat(mockRequest(), res); + expect(typeof res.body.data.timestamp).toBe('string'); + expect(new Date(res.body.data.timestamp).toISOString()).toBe(res.body.data.timestamp); + }); + + it('has message set to "Heartbeat recorded"', async () => { + const res = mockResponse(); + await recordIndexerHeartbeat(mockRequest(), res); + expect(res.body.message).toBe('Heartbeat recorded'); + }); +});