fix(seo): close the gaps an independent audit found

CHANGELOG.md

@@ -232,6 +232,21 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 
 ### Fixed
 
+- **A database outage would have reopened the hole #10453 closed** — with no catalogue to check
+  against, the bot routes answered 200 with a fabricated page for any string, self-canonicalising,
+  exactly the defect that PR removed. Degraded pages now carry `noindex`. The path is unreachable in
+  production, where the database is configured, but it is one misconfiguration away from indexable.
+  `bot_fetch` also moved from a router dependency to a middleware and gained a `status` property: a
+  dependency runs before the handler and cannot see the response, so every 404 was being recorded as
+  a successful page read (#10479).
+
+- **Documentation that still described the superseded crawler policy** — `docs/reference/seo.md`
+  said `gptbot`, `meta-externalagent` and `amazonbot` were "declined in robots.txt" and
+  `app/nginx.conf` carried the same claim in a comment. Both were written about an hour before
+  #10474 opened the policy and never reconciled, so the page contradicted itself. The measured
+  edge-state table was stale for the opposite reason: the dashboard unblock it prescribed had since
+  been done. A docstring also pointed at `app/src/router.tsx`, which does not exist — the routing
+  lives in `app/src/routes/index.tsx` (#10479).
 - **The bot analytics recorded nothing at all** — Plausible identifies crawler user agents and
   discards their events, so forwarding the real one guaranteed an empty dashboard. Verified against
   the live API: the same event sent as `Claude-User` never appears, sent as a browser agent it does,

api/analytics.py

@@ -307,8 +307,8 @@ def track_og_image(
     task.add_done_callback(_handle_task_exception)
 
 
-def track_bot_fetch(request: Request, path: str) -> None:
-    """Record an AI or search agent reading a page (fire-and-forget).
+def track_bot_fetch(request: Request, path: str, status: int) -> None:
+    """Record an AI or search agent requesting a page (fire-and-forget).
 
     Recorded against BOT_DOMAIN, never the main site: every Plausible event
     creates a visitor, and mixing these in is what made the human numbers
@@ -322,7 +322,11 @@ def track_bot_fetch(request: Request, path: str) -> None:
 
     Args:
         request: FastAPI request, for the UA and forwarded IP
-        path: Public path being read, e.g. "/box-basic/python/matplotlib"
+        path: Public path being requested, e.g. "/box-basic/python/matplotlib"
+        status: Response status. Recorded rather than filtered on: an assistant
+            asking for a URL that no longer exists is a signal worth having, and
+            counting it as a successful read would be a lie. Filter on it in the
+            dashboard.
     """
     user_agent = request.headers.get("user-agent", "")
     detected = detect_ai_agent(user_agent)
@@ -335,7 +339,7 @@ def track_bot_fetch(request: Request, path: str) -> None:
     # visitor's, and the rate limiter deliberately returns the rightmost
     # forwarded entry — ours. See api/request_context.py for why the two differ.
     client_ip = visitor_ip(request)
-    props = {"assistant": assistant, "kind": kind, "path": path}
+    props = {"assistant": assistant, "kind": kind, "path": path, "status": str(status)}
     url = f"https://anyplot.ai{path}"
 
     task = asyncio.create_task(_send_plausible_event(user_agent, client_ip, "bot_fetch", url, props, domain=BOT_DOMAIN))

api/main.py

@@ -16,6 +16,7 @@
 from fastapi.middleware.cors import CORSMiddleware  # noqa: E402
 from starlette.middleware.gzip import GZipMiddleware  # noqa: E402
 
+from api.analytics import track_bot_fetch  # noqa: E402
 from api.cache import cache_key, set_cache  # noqa: E402
 from api.exceptions import (  # noqa: E402
     AnyplotException,
@@ -155,6 +156,27 @@ async def lifespan(app: FastAPI):
 )
 
 
+# Record which AI or search agent requested which catalogue page.
+#
+# A middleware rather than a router dependency: a dependency runs BEFORE the
+# handler and cannot see the response, so every 404 was recorded as a
+# successful read. The status matters — an assistant asking for a URL that no
+# longer exists is a signal worth keeping, it just is not a page view.
+@app.middleware("http")
+async def record_bot_fetch(request: Request, call_next):
+    """Report AI/search agent page requests to the bot analytics site.
+
+    Requests, not reads: the status is recorded rather than filtered on, so a
+    404 is visible as a miss instead of counted as a page view.
+    """
+    response: Response = await call_next(request)
+    path = request.url.path
+    if path.startswith("/seo-proxy"):
+        # The public URL, never this router's internal prefix
+        track_bot_fetch(request, path.removeprefix("/seo-proxy") or "/", response.status_code)
+    return response
+
+
 # Add cache headers middleware
 @app.middleware("http")
 async def add_cache_headers(request: Request, call_next):

api/routers/seo.py

@@ -10,7 +10,6 @@
 from fastapi.responses import HTMLResponse, RedirectResponse, Response
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from api.analytics import track_bot_fetch
 from api.cache import cache_key, get_cache, get_or_set_cache, set_cache
 from api.dependencies import optional_db
 from core.config import settings
@@ -20,21 +19,7 @@
 from core.utils import strip_noqa_comments
 
 
-async def _record_bot_fetch(request: Request) -> None:
-    """Router dependency: record which agent read which page.
-
-    Attached to the router rather than to each handler — there are a dozen and
-    the next one added would silently go unrecorded. Only /seo-proxy paths are
-    page reads; robots.txt and sitemap.xml also live on this router and are not.
-    """
-    path = request.url.path
-    if not path.startswith("/seo-proxy"):
-        return
-    # Report the public URL, not this router's internal prefix
-    track_bot_fetch(request, path.removeprefix("/seo-proxy") or "/")
-
-
-router = APIRouter(tags=["seo"], dependencies=[Depends(_record_bot_fetch)])
+router = APIRouter(tags=["seo"])
 
 # Canonical spec-id shape — lowercase alphanumerics with hyphen separators.
 # Same pattern enforced in automation/scripts/sync_to_postgres.py. Used here to
@@ -122,7 +107,7 @@ async def _refresh_sitemap() -> str:
     <meta name="twitter:title" content="{title}" />
     <meta name="twitter:description" content="{description}" />
     <meta name="twitter:image" content="{image}" />
-    <link rel="canonical" href="{url}" />{jsonld}
+    <link rel="canonical" href="{url}" />{robots}{jsonld}
 </head>
 <body>
 {body}
@@ -280,6 +265,7 @@ def _render_bot_html(
     og_url: str | None = None,
     body: str = "",
     jsonld: dict | None = None,
+    noindex: bool = False,
 ) -> str:
     """Render a bot-serving page.
 
@@ -304,6 +290,7 @@ def _render_bot_html(
         image=image,
         url=url,
         og_url=og_url if og_url is not None else url,
+        robots='\n    <meta name="robots" content="noindex" />' if noindex else "",
         jsonld=_jsonld_script(jsonld) if jsonld else "",
         body=f"{body or f'<h1>{title}</h1><p>{description}</p>'}\n{_BOT_NAV_HTML}",
     )
@@ -750,12 +737,17 @@ async def seo_stats():
 async def seo_spec_hub(spec_id: str, db: AsyncSession | None = Depends(optional_db)):
     """Bot-optimized cross-language spec hub."""
     if db is None:
+        # Degraded mode: without the catalogue we cannot tell a real spec from
+        # an invented one, so this page must not be indexable — serving it
+        # otherwise recreates the defect #10453 removed, an indexable
+        # near-duplicate for any string anyone tries.
         return HTMLResponse(
             _render_bot_html(
                 title=f"{html.escape(spec_id)} | anyplot.ai",
                 description=DEFAULT_DESCRIPTION,
                 image=DEFAULT_HOME_IMAGE,
                 url=f"https://anyplot.ai/{html.escape(spec_id)}",
+                noindex=True,
             )
         )
 
@@ -783,7 +775,7 @@ async def seo_spec_language(spec_id: str, language: str):
 
     The /{spec_id}/{language} tier was consolidated into /{spec_id} to eliminate
     duplicate content. Bots following this endpoint get a 301 to the public hub
-    URL; humans get the SPA redirect configured in app/src/router.tsx. The
+    URL; humans get the SPA redirect configured in app/src/routes/index.tsx. The
     `language` query parameter is dropped because the hub's canonical tag does
     not include it — Google should consolidate the page, not a filtered variant.
     """
@@ -820,12 +812,14 @@ async def seo_spec_implementation(
 ):
     """Bot-optimized implementation detail."""
     if db is None:
+        # Same reasoning as the hub route — unverifiable, so not indexable.
         return HTMLResponse(
             _render_bot_html(
                 title=f"{html.escape(spec_id)} - {html.escape(library)} | anyplot.ai",
                 description=DEFAULT_DESCRIPTION,
                 image=DEFAULT_HOME_IMAGE,
                 url=f"https://anyplot.ai/{html.escape(spec_id)}/{html.escape(language)}/{html.escape(library)}",
+                noindex=True,
             )
         )
 

app/nginx.conf

@@ -34,9 +34,10 @@ map $http_user_agent $is_bot {
     # their assistant to open the page — and matter most.
     # This map only decides WHAT a crawler gets once it arrives, never whether
     # it may: the allow/deny policy is robots.txt plus the Cloudflare AI Crawl
-    # Control zone setting (docs/reference/seo.md). GPTBot is listed even
-    # though robots.txt declines it, so the rendering is right if that call is
-    # ever reversed.
+    # Control zone setting (docs/reference/seo.md). Under the current policy
+    # everything except Bytespider may crawl, so an agent missing from this map
+    # is not thereby declined — it just gets the empty SPA shell, which is the
+    # failure this list exists to prevent.
     ~*claudebot 1;
     ~*claude-user 1;
     ~*claude-searchbot 1;

docs/reference/plausible.md

@@ -240,7 +240,7 @@ events arrive server-side through the Events API, which is why it shows
 
 | Event | Properties | Source | Description |
 |-------|-----------|--------|-------------|
-| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Always recorded on `bots.anyplot.ai`. |
+| `bot_fetch` | `assistant`, `kind`, `path`, `status` | `api/main.py` (middleware) | An AI assistant or search crawler requested a catalogue page. Always recorded on `bots.anyplot.ai`. |
 | `og_image_view` | `page`, `platform`, `spec`?, `language`?, `library`?, `filter_*`?, plus `assistant` + `kind` when machine-fetched | `api/routers/og_images.py` | A preview image was fetched. **Split by audience**, see below. |
 
 #### og:image is split by who fetched it
@@ -290,7 +290,16 @@ them:
 |----------|-------------|---------|
 | `assistant` | Vendor (`claude`, `chatgpt`, `gemini`, `mistral`, `perplexity`, `meta`, `amazon`, `duckduckgo`, `grok`, `google`, `bing`, …) | `bot_fetch` |
 | `kind` | Why it fetched — see the table below | `bot_fetch` |
-| `path` | Public path that was read, e.g. `/box-basic/python/matplotlib` | `bot_fetch` |
+| `path` | Public path that was requested, e.g. `/box-basic/python/matplotlib` | `bot_fetch` |
+| `status` | Response status as a string (`200`, `404`, …) | `bot_fetch` |
+
+Filter on `status` before reading anything else. The event records the
+*request*, not a successful read: an assistant asking for a URL that no longer
+exists is a signal worth keeping — it is how a library migration announces
+itself — but counting it as a page view would be a lie. This is also why the
+recording lives in a middleware rather than a router dependency: a dependency
+runs before the handler and cannot see the response, so every miss was recorded
+as a read.
 
 Machine-side `og_image_view` events carry the main site's image properties as
 well, so register these on `bots.anyplot.ai` too: `page`, `platform`, `spec`,

docs/reference/seo.md

@@ -104,18 +104,20 @@ Three vendor splits are easy to get backwards, so they are spelled out:
 
 - **Meta**: `meta-externalfetcher` (user-directed) and `meta-webindexer` (AI
   search index) are served; `meta-externalagent` is the training crawler and is
-  declined in robots.txt.
+  not mapped.
 - **Mistral**: `mistralai-user` and `mistralai-index` are served — Mistral
   documents both as never used for generative-AI training; `mistralai-training`
   is the training crawler and is not mapped.
 - **Amazon**: `amzn-searchbot` and `amzn-user` are the sanctioned retrieval
   agents, both documented as not crawling for model training; plain `amazonbot`
-  is the training crawler and is declined.
+  is the training crawler and is not mapped.
 
 Mapping is not permission. This map decides only *what* an agent receives once
 it arrives; whether it may crawl at all is robots.txt plus Cloudflare's AI Crawl
-Control. That is why `gptbot` appears here while robots.txt declines it, and why
-mapping community-reported tokens costs nothing.
+Control — which is why mapping community-reported tokens costs nothing, and why
+an unmapped agent is not thereby declined. Under the current policy everything
+except `Bytespider` may crawl, so the mapping question is only ever about
+rendering.
 
 **Search Engines:**
 | Bot | User-Agent Pattern |
@@ -135,7 +137,7 @@ mapping community-reported tokens costs nothing.
 | Anthropic crawler | `claudebot` | index/crawl |
 | Claude user fetch | `claude-user` | a human asked Claude to open the page |
 | Claude search | `claude-searchbot` | citation index |
-| OpenAI crawler | `gptbot` | training (declined in robots.txt, mapped anyway — see below) |
+| OpenAI crawler | `gptbot` | training crawler; allowed and mapped |
 | ChatGPT search | `oai-searchbot` | citation index |
 | ChatGPT user fetch | `chatgpt-user` | a human asked ChatGPT to open the page |
 | Perplexity | `perplexitybot`, `perplexity-user` | citation index / user fetch |
@@ -380,19 +382,26 @@ in step.
 Measured on the live zone 2026-08-18 (zone `anyplot.ai` → **AI Crawl Control** →
 Security):
 
+The dashboard was brought in line with the open policy on 2026-08-18. State
+after that change:
+
 | State | Agents |
 |---|---|
-| Blocked at the edge | `GPTBot`, `CCBot`, `Amazonbot`, `meta-externalagent`, `Bytespider`, `Google-CloudVertexBot`, `PetalBot`, `Anchor Browser`, `Arquivo Web Crawler` |
-| Passing | `Googlebot`, `bingbot`, `Baidu`, `Applebot`, `Claude-User`, `ClaudeBot`, `Claude-SearchBot`, `ChatGPT-User`, `OAI-SearchBot`, `PerplexityBot`, `Perplexity-User`, `DuckAssistBot`, `MistralAI-User`, `Meta-ExternalFetcher`, `archive.org_bot` |
+| Blocked at the edge | `Bytespider`, `TikTok Spider`, `Anchor Browser`, `Novellum AI Crawl`, `Timpibot` |
+| Passing | everything else, including `GPTBot`, `CCBot`, `Amazonbot`, `meta-externalagent`, `Google-CloudVertexBot`, `PetalBot`, `FacebookBot`, `Arquivo Web Crawler`, and every retrieval agent |
+
+`Bytespider` and `TikTok Spider` are both ByteDance; the first is documented as
+ignoring robots.txt, and the second shares its operator. The remaining three are
+blocked for a weaker reason — no vendor documentation of their crawling
+behaviour could be found, so their rule-compliance is unverified rather than
+disproven. Revisit if that changes.
 
-Two things to note. `Google-CloudVertexBot` is blocked at the edge and appears in
-no repo-side policy — an undocumented decision that only exists as a dashboard
-toggle. And unlike the 2026-07-25 measurement, Cloudflare is **no longer
-prepending a managed robots.txt block**: the live file is byte-identical to this
-repo's, and enforcement is purely the 403.
+Cloudflare is **not** prepending a managed robots.txt block: the live file is
+byte-identical to this repo's, and enforcement is purely the 403. That was
+different at the 2026-07-25 measurement, so re-check rather than assume.
 
-To bring the edge in line with the policy above, unblock everything except
-`Bytespider` in the dashboard.
+`bot-serving-check` deliberately tests the Cloud Run origin, not the edge, so it
+will never catch a dashboard change. Edge drift has to be checked by hand.
 
 Verify afterwards:
 

tests/unit/api/test_analytics.py

@@ -376,7 +376,7 @@ async def test_records_against_the_bot_site_not_the_main_one(self) -> None:
             mock_client = AsyncMock()
             mock_client_class.return_value.__aenter__.return_value = mock_client
 
-            track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic")
+            track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic", 200)
             await asyncio.sleep(0)  # let the fire-and-forget task run
 
             assert mock_client.post.call_args[1]["json"]["domain"] == BOT_DOMAIN
@@ -387,7 +387,7 @@ async def test_carries_assistant_kind_and_public_path(self) -> None:
             mock_client = AsyncMock()
             mock_client_class.return_value.__aenter__.return_value = mock_client
 
-            track_bot_fetch(self._request("MistralAI-User/1.0"), "/box-basic/python/matplotlib")
+            track_bot_fetch(self._request("MistralAI-User/1.0"), "/box-basic/python/matplotlib", 200)
             await asyncio.sleep(0)
 
             payload = mock_client.post.call_args[1]["json"]
@@ -398,6 +398,7 @@ async def test_carries_assistant_kind_and_public_path(self) -> None:
                 "assistant": "mistral",
                 "kind": "user_directed",
                 "path": "/box-basic/python/matplotlib",
+                "status": "200",
             }
 
     @pytest.mark.asyncio
@@ -422,7 +423,7 @@ async def test_reports_the_visitor_not_our_own_infrastructure(self) -> None:
             mock_client = AsyncMock()
             mock_client_class.return_value.__aenter__.return_value = mock_client
 
-            track_bot_fetch(request, "/box-basic")
+            track_bot_fetch(request, "/box-basic", 200)
             await asyncio.sleep(0)
 
             assert mock_client.post.call_args[1]["headers"]["X-Forwarded-For"] == "203.0.113.7"
@@ -433,7 +434,7 @@ async def test_sends_nothing_for_a_human(self) -> None:
             mock_client = AsyncMock()
             mock_client_class.return_value.__aenter__.return_value = mock_client
 
-            track_bot_fetch(self._request("Mozilla/5.0 (X11; Linux) Chrome/126.0 Safari/537.36"), "/")
+            track_bot_fetch(self._request("Mozilla/5.0 (X11; Linux) Chrome/126.0 Safari/537.36"), "/", 200)
             await asyncio.sleep(0)
 
             mock_client.post.assert_not_called()
@@ -509,7 +510,7 @@ async def test_bot_fetch_does_not_forward_the_crawler_agent(self) -> None:
             mock_client = AsyncMock()
             mock_client_class.return_value.__aenter__.return_value = mock_client
 
-            track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic")
+            track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic", 200)
             await asyncio.sleep(0)
 
             call = mock_client.post.call_args[1]
@@ -573,3 +574,25 @@ def test_skips_tokens_that_are_not_addresses(self, headers: dict[str, str], expe
         from api.request_context import visitor_ip
 
         assert visitor_ip(self._request(headers)) == expected
+
+
+class TestBotFetchRecordsTheStatus:
+    """A miss is a signal, but it is not a page read — so record it, don't hide it."""
+
+    @staticmethod
+    def _request() -> MagicMock:
+        request = MagicMock()
+        request.headers = {"user-agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"}
+        request.client.host = "203.0.113.7"
+        return request
+
+    @pytest.mark.asyncio
+    async def test_a_miss_is_recorded_as_such(self) -> None:
+        with patch("api.analytics.httpx.AsyncClient") as mock_client_class:
+            mock_client = AsyncMock()
+            mock_client_class.return_value.__aenter__.return_value = mock_client
+
+            track_bot_fetch(self._request(), "/bar-basic/python/highcharts", 404)
+            await asyncio.sleep(0)
+
+            assert mock_client.post.call_args[1]["json"]["props"]["status"] == "404"