From 057fd647964bbd09186ff6d8c430dc4fe04dfdb7 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Sat, 15 Aug 2026 10:13:02 -0700 Subject: [PATCH] test(api): the documented error table is held to ERROR_RULES docs/api.md's "full table" is where a client author reads the code inventory, and nothing checked it against the code. It fell nineteen codes behind before anybody noticed; #533 synced it, and syncing alone was never the fix, because an unchecked mirror is a second spelling waiting to drift. Three tests, the same exact-correspondence construction the module already uses for ERROR_RULES against kernel/errors.py, with the markdown as the other side: - every rule appears in the table under its own status - every documented code still exists at that status, or is one of the five the framework and the auth guard raise - the 5xx opt-out table equals expose_message, exactly, both ways Codes are collected as (code, status) pairs rather than a mapping, so a code listed under two statuses arrives as two members instead of one silently overwriting the other, and a table that parses as empty raises rather than letting the second test pass on nothing. Also corrects errors.py's own docstring, which said four errors opt out of 5xx opacity when it has been six since the inference work landed. --- docs/api.md | 5 +++ src/visionset/server/errors.py | 2 +- tests/server/test_errors.py | 75 ++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index b8549d0..ab3aad3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -584,6 +584,11 @@ Every row but `VALIDATION_ERROR`, `NOT_FOUND`, `METHOD_NOT_ALLOWED`, `UNAUTHORIZ in `server/errors.py`, which `tests/server/test_errors.py` holds in exact correspondence with `kernel/errors.py`. A new kernel error fails that suite until somebody maps it. +**That same suite reads this table.** It parses the rows above and holds them to `ERROR_RULES` +in both directions — a code that ships without a row here fails, and so does a row naming a code +that no longer exists or sitting under the wrong status. The table is a mirror, and an unchecked +mirror drifts: this one was nineteen codes behind before anybody noticed (#524). + `CORRUPT_MEDIA` and `UNSUPPORTED_MEDIA` carry `detail.reason`. The file's *name* is deliberately absent from both the detail and the message: on the ingest path it is an absolute path inside a directory the operator, not the client, pointed at. diff --git a/src/visionset/server/errors.py b/src/visionset/server/errors.py index 938237d..ec07bec 100644 --- a/src/visionset/server/errors.py +++ b/src/visionset/server/errors.py @@ -21,7 +21,7 @@ - **422** — the payload itself is wrong. 5xx is opaque by default: the body carries a generic sentence and an -``incident_id``, and the real message and traceback go to the log. Four errors +``incident_id``, and the real message and traceback go to the log. Six errors opt out, each because its message *is* the operator's remedy — see ``expose_message`` below. """ diff --git a/tests/server/test_errors.py b/tests/server/test_errors.py index 9751284..20b7857 100644 --- a/tests/server/test_errors.py +++ b/tests/server/test_errors.py @@ -10,6 +10,7 @@ import inspect import re from collections.abc import Iterator +from pathlib import Path import pytest from fastapi import FastAPI @@ -234,6 +235,80 @@ def _rules_by_name() -> dict[str, ErrorRule]: return {cls.__name__: rule for cls, rule in ERROR_RULES.items()} +# --- the table, as the document publishes it ------------------------------ + +# ``docs/api.md`` is where a client author reads the inventory, and until now +# nothing held it to the code. It fell nineteen codes behind before anybody +# noticed, and syncing it was never the fix: an ungated mirror is a second +# spelling waiting to drift. These tests are the same exact-correspondence +# construction as ``test_the_status_and_code_of_every_error`` above, with the +# markdown as the other side — ``frontend/ui-core/src/tokens.test.ts`` is the +# precedent for holding a *document* this way. cf. #524. + +DOCS = Path(__file__).resolve().parents[2] / "docs" / "api.md" + +# The five codes the document lists that ``ERROR_RULES`` does not hold. None of +# them has a kernel class to map: FastAPI raises the first, Starlette's router +# the next two, the auth guard the fourth, and the last is what an exception no +# rule covers becomes. The document is the only place they are written down, +# which is why they are named here rather than derived. +FRAMEWORK_CODES = { + "VALIDATION_ERROR", + "NOT_FOUND", + "METHOD_NOT_ALLOWED", + "UNAUTHORIZED", + "INTERNAL_ERROR", +} + + +def test_every_error_rule_appears_in_the_documented_table() -> None: + documented = _documented_codes() + undocumented = {(rule.code, rule.status) for rule in ERROR_RULES.values()} - documented + assert undocumented == set() + + +def test_every_documented_code_still_exists() -> None: + mapped = {(rule.code, rule.status) for rule in ERROR_RULES.values()} + stale = {pair for pair in _documented_codes() if pair[0] not in FRAMEWORK_CODES} - mapped + assert stale == set() + + +def test_the_published_message_table_matches_expose_message() -> None: + published = set( + re.findall(r"^\|\s*`([A-Z][A-Z0-9_]+)`\s*\|", _section("The 5xx contract"), re.MULTILINE) + ) + assert published == {rule.code for rule in ERROR_RULES.values() if rule.expose_message} + + +def _section(heading: str) -> str: + """The body under a ``##`` heading, up to the next one of that level.""" + text = DOCS.read_text(encoding="utf-8") + marker = f"\n## {heading}\n" + if marker not in text: + raise AssertionError(f"docs/api.md no longer has a '## {heading}' section") + return re.split(r"^## ", text.split(marker, 1)[1], maxsplit=1, flags=re.MULTILINE)[0] + + +def _documented_codes() -> set[tuple[str, int]]: + """``(code, status)`` pairs read off the status rows of the full table. + + Pairs rather than a mapping, so one code listed under two statuses arrives + as two members and is caught, instead of one silently overwriting the other. + """ + pairs: set[tuple[str, int]] = set() + for line in _section("The full table").splitlines(): + row = re.match(r"\|\s*\*\*(\d{3})\*\*\s*\|(.+)\|", line) + if row is None: + continue + status = int(row.group(1)) + pairs.update((code, status) for code in re.findall(r"`([A-Z][A-Z0-9_]+)`", row.group(2))) + # An empty parse would make the second test vacuous — a deleted or reshaped + # table must be a failure, not a gate that quietly stops guarding anything. + if not pairs: + raise AssertionError("the full table in docs/api.md parsed as no codes at all") + return pairs + + # --- the handlers ---------------------------------------------------------