{description}
'}\n{_BOT_NAV_HTML}", ) @@ -750,12 +737,17 @@ async def seo_stats(): async def seo_spec_hub(spec_id: str, db: AsyncSession | None = Depends(optional_db)): """Bot-optimized cross-language spec hub.""" if db is None: + # Degraded mode: without the catalogue we cannot tell a real spec from + # an invented one, so this page must not be indexable — serving it + # otherwise recreates the defect #10453 removed, an indexable + # near-duplicate for any string anyone tries. return HTMLResponse( _render_bot_html( title=f"{html.escape(spec_id)} | anyplot.ai", description=DEFAULT_DESCRIPTION, image=DEFAULT_HOME_IMAGE, url=f"https://anyplot.ai/{html.escape(spec_id)}", + noindex=True, ) ) @@ -783,7 +775,7 @@ async def seo_spec_language(spec_id: str, language: str): The /{spec_id}/{language} tier was consolidated into /{spec_id} to eliminate duplicate content. Bots following this endpoint get a 301 to the public hub - URL; humans get the SPA redirect configured in app/src/router.tsx. The + URL; humans get the SPA redirect configured in app/src/routes/index.tsx. The `language` query parameter is dropped because the hub's canonical tag does not include it — Google should consolidate the page, not a filtered variant. """ @@ -820,12 +812,14 @@ async def seo_spec_implementation( ): """Bot-optimized implementation detail.""" if db is None: + # Same reasoning as the hub route — unverifiable, so not indexable. return HTMLResponse( _render_bot_html( title=f"{html.escape(spec_id)} - {html.escape(library)} | anyplot.ai", description=DEFAULT_DESCRIPTION, image=DEFAULT_HOME_IMAGE, url=f"https://anyplot.ai/{html.escape(spec_id)}/{html.escape(language)}/{html.escape(library)}", + noindex=True, ) ) diff --git a/app/nginx.conf b/app/nginx.conf index d64e572806..1aa88a8f38 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -34,9 +34,10 @@ map $http_user_agent $is_bot { # their assistant to open the page — and matter most. # This map only decides WHAT a crawler gets once it arrives, never whether # it may: the allow/deny policy is robots.txt plus the Cloudflare AI Crawl - # Control zone setting (docs/reference/seo.md). GPTBot is listed even - # though robots.txt declines it, so the rendering is right if that call is - # ever reversed. + # Control zone setting (docs/reference/seo.md). Under the current policy + # everything except Bytespider may crawl, so an agent missing from this map + # is not thereby declined — it just gets the empty SPA shell, which is the + # failure this list exists to prevent. ~*claudebot 1; ~*claude-user 1; ~*claude-searchbot 1; diff --git a/docs/reference/plausible.md b/docs/reference/plausible.md index ebb385e02e..fef539a819 100644 --- a/docs/reference/plausible.md +++ b/docs/reference/plausible.md @@ -240,7 +240,7 @@ events arrive server-side through the Events API, which is why it shows | Event | Properties | Source | Description | |-------|-----------|--------|-------------| -| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Always recorded on `bots.anyplot.ai`. | +| `bot_fetch` | `assistant`, `kind`, `path`, `status` | `api/main.py` (middleware) | An AI assistant or search crawler requested a catalogue page. Always recorded on `bots.anyplot.ai`. | | `og_image_view` | `page`, `platform`, `spec`?, `language`?, `library`?, `filter_*`?, plus `assistant` + `kind` when machine-fetched | `api/routers/og_images.py` | A preview image was fetched. **Split by audience**, see below. | #### og:image is split by who fetched it @@ -290,7 +290,16 @@ them: |----------|-------------|---------| | `assistant` | Vendor (`claude`, `chatgpt`, `gemini`, `mistral`, `perplexity`, `meta`, `amazon`, `duckduckgo`, `grok`, `google`, `bing`, …) | `bot_fetch` | | `kind` | Why it fetched — see the table below | `bot_fetch` | -| `path` | Public path that was read, e.g. `/box-basic/python/matplotlib` | `bot_fetch` | +| `path` | Public path that was requested, e.g. `/box-basic/python/matplotlib` | `bot_fetch` | +| `status` | Response status as a string (`200`, `404`, …) | `bot_fetch` | + +Filter on `status` before reading anything else. The event records the +*request*, not a successful read: an assistant asking for a URL that no longer +exists is a signal worth keeping — it is how a library migration announces +itself — but counting it as a page view would be a lie. This is also why the +recording lives in a middleware rather than a router dependency: a dependency +runs before the handler and cannot see the response, so every miss was recorded +as a read. Machine-side `og_image_view` events carry the main site's image properties as well, so register these on `bots.anyplot.ai` too: `page`, `platform`, `spec`, diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 19a493ea01..6406e1e120 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -104,18 +104,20 @@ Three vendor splits are easy to get backwards, so they are spelled out: - **Meta**: `meta-externalfetcher` (user-directed) and `meta-webindexer` (AI search index) are served; `meta-externalagent` is the training crawler and is - declined in robots.txt. + not mapped. - **Mistral**: `mistralai-user` and `mistralai-index` are served — Mistral documents both as never used for generative-AI training; `mistralai-training` is the training crawler and is not mapped. - **Amazon**: `amzn-searchbot` and `amzn-user` are the sanctioned retrieval agents, both documented as not crawling for model training; plain `amazonbot` - is the training crawler and is declined. + is the training crawler and is not mapped. Mapping is not permission. This map decides only *what* an agent receives once it arrives; whether it may crawl at all is robots.txt plus Cloudflare's AI Crawl -Control. That is why `gptbot` appears here while robots.txt declines it, and why -mapping community-reported tokens costs nothing. +Control — which is why mapping community-reported tokens costs nothing, and why +an unmapped agent is not thereby declined. Under the current policy everything +except `Bytespider` may crawl, so the mapping question is only ever about +rendering. **Search Engines:** | Bot | User-Agent Pattern | @@ -135,7 +137,7 @@ mapping community-reported tokens costs nothing. | Anthropic crawler | `claudebot` | index/crawl | | Claude user fetch | `claude-user` | a human asked Claude to open the page | | Claude search | `claude-searchbot` | citation index | -| OpenAI crawler | `gptbot` | training (declined in robots.txt, mapped anyway — see below) | +| OpenAI crawler | `gptbot` | training crawler; allowed and mapped | | ChatGPT search | `oai-searchbot` | citation index | | ChatGPT user fetch | `chatgpt-user` | a human asked ChatGPT to open the page | | Perplexity | `perplexitybot`, `perplexity-user` | citation index / user fetch | @@ -380,19 +382,26 @@ in step. Measured on the live zone 2026-08-18 (zone `anyplot.ai` → **AI Crawl Control** → Security): +The dashboard was brought in line with the open policy on 2026-08-18. State +after that change: + | State | Agents | |---|---| -| Blocked at the edge | `GPTBot`, `CCBot`, `Amazonbot`, `meta-externalagent`, `Bytespider`, `Google-CloudVertexBot`, `PetalBot`, `Anchor Browser`, `Arquivo Web Crawler` | -| Passing | `Googlebot`, `bingbot`, `Baidu`, `Applebot`, `Claude-User`, `ClaudeBot`, `Claude-SearchBot`, `ChatGPT-User`, `OAI-SearchBot`, `PerplexityBot`, `Perplexity-User`, `DuckAssistBot`, `MistralAI-User`, `Meta-ExternalFetcher`, `archive.org_bot` | +| Blocked at the edge | `Bytespider`, `TikTok Spider`, `Anchor Browser`, `Novellum AI Crawl`, `Timpibot` | +| Passing | everything else, including `GPTBot`, `CCBot`, `Amazonbot`, `meta-externalagent`, `Google-CloudVertexBot`, `PetalBot`, `FacebookBot`, `Arquivo Web Crawler`, and every retrieval agent | + +`Bytespider` and `TikTok Spider` are both ByteDance; the first is documented as +ignoring robots.txt, and the second shares its operator. The remaining three are +blocked for a weaker reason — no vendor documentation of their crawling +behaviour could be found, so their rule-compliance is unverified rather than +disproven. Revisit if that changes. -Two things to note. `Google-CloudVertexBot` is blocked at the edge and appears in -no repo-side policy — an undocumented decision that only exists as a dashboard -toggle. And unlike the 2026-07-25 measurement, Cloudflare is **no longer -prepending a managed robots.txt block**: the live file is byte-identical to this -repo's, and enforcement is purely the 403. +Cloudflare is **not** prepending a managed robots.txt block: the live file is +byte-identical to this repo's, and enforcement is purely the 403. That was +different at the 2026-07-25 measurement, so re-check rather than assume. -To bring the edge in line with the policy above, unblock everything except -`Bytespider` in the dashboard. +`bot-serving-check` deliberately tests the Cloud Run origin, not the edge, so it +will never catch a dashboard change. Edge drift has to be checked by hand. Verify afterwards: diff --git a/tests/unit/api/test_analytics.py b/tests/unit/api/test_analytics.py index b5423ddfd7..2ffbfa7813 100644 --- a/tests/unit/api/test_analytics.py +++ b/tests/unit/api/test_analytics.py @@ -376,7 +376,7 @@ async def test_records_against_the_bot_site_not_the_main_one(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic") + track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic", 200) await asyncio.sleep(0) # let the fire-and-forget task run assert mock_client.post.call_args[1]["json"]["domain"] == BOT_DOMAIN @@ -387,7 +387,7 @@ async def test_carries_assistant_kind_and_public_path(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("MistralAI-User/1.0"), "/box-basic/python/matplotlib") + track_bot_fetch(self._request("MistralAI-User/1.0"), "/box-basic/python/matplotlib", 200) await asyncio.sleep(0) payload = mock_client.post.call_args[1]["json"] @@ -398,6 +398,7 @@ async def test_carries_assistant_kind_and_public_path(self) -> None: "assistant": "mistral", "kind": "user_directed", "path": "/box-basic/python/matplotlib", + "status": "200", } @pytest.mark.asyncio @@ -422,7 +423,7 @@ async def test_reports_the_visitor_not_our_own_infrastructure(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(request, "/box-basic") + track_bot_fetch(request, "/box-basic", 200) await asyncio.sleep(0) assert mock_client.post.call_args[1]["headers"]["X-Forwarded-For"] == "203.0.113.7" @@ -433,7 +434,7 @@ async def test_sends_nothing_for_a_human(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("Mozilla/5.0 (X11; Linux) Chrome/126.0 Safari/537.36"), "/") + track_bot_fetch(self._request("Mozilla/5.0 (X11; Linux) Chrome/126.0 Safari/537.36"), "/", 200) await asyncio.sleep(0) mock_client.post.assert_not_called() @@ -509,7 +510,7 @@ async def test_bot_fetch_does_not_forward_the_crawler_agent(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic") + track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic", 200) await asyncio.sleep(0) call = mock_client.post.call_args[1] @@ -573,3 +574,25 @@ def test_skips_tokens_that_are_not_addresses(self, headers: dict[str, str], expe from api.request_context import visitor_ip assert visitor_ip(self._request(headers)) == expected + + +class TestBotFetchRecordsTheStatus: + """A miss is a signal, but it is not a page read — so record it, don't hide it.""" + + @staticmethod + def _request() -> MagicMock: + request = MagicMock() + request.headers = {"user-agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"} + request.client.host = "203.0.113.7" + return request + + @pytest.mark.asyncio + async def test_a_miss_is_recorded_as_such(self) -> None: + with patch("api.analytics.httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + + track_bot_fetch(self._request(), "/bar-basic/python/highcharts", 404) + await asyncio.sleep(0) + + assert mock_client.post.call_args[1]["json"]["props"]["status"] == "404" diff --git a/tests/unit/api/test_routers.py b/tests/unit/api/test_routers.py index 0d73ee4744..54002fc5c3 100644 --- a/tests/unit/api/test_routers.py +++ b/tests/unit/api/test_routers.py @@ -708,21 +708,22 @@ def test_seo_home_with_db_counts_specs(self, db_client, mock_spec) -> None: def test_seo_proxy_records_the_agent_that_read_the_page(self, client: TestClient) -> None: """The router dependency reports the PUBLIC path, not its own prefix.""" - with patch(DB_CONFIG_PATCH, return_value=False), patch("api.routers.seo.track_bot_fetch") as track: + with patch(DB_CONFIG_PATCH, return_value=False), patch("api.main.track_bot_fetch") as track: client.get("/seo-proxy/bar-basic", headers={"User-Agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"}) track.assert_called_once() assert track.call_args.args[1] == "/bar-basic" + assert track.call_args.args[2] == 200 # the status the handler produced def test_seo_proxy_home_records_root(self, client: TestClient) -> None: """The proxy root maps to "/", and the hook fires exactly once per request.""" - with patch(DB_CONFIG_PATCH, return_value=False), patch("api.routers.seo.track_bot_fetch") as track: + with patch(DB_CONFIG_PATCH, return_value=False), patch("api.main.track_bot_fetch") as track: client.get("/seo-proxy/", headers={"User-Agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"}) track.assert_called_once() assert track.call_args.args[1] == "/" def test_robots_and_sitemap_are_not_page_reads(self, client: TestClient) -> None: """Both live on this router but are machine files, not catalogue pages.""" - with patch("api.routers.seo.track_bot_fetch") as track: + with patch("api.main.track_bot_fetch") as track: client.get("/robots.txt", headers={"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)"}) client.get("/sitemap.xml", headers={"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)"}) track.assert_not_called() @@ -801,6 +802,10 @@ def test_seo_spec_overview_without_db(self, client: TestClient) -> None: with patch(DB_CONFIG_PATCH, return_value=False): response = client.get("/seo-proxy/scatter-basic") assert response.status_code == 200 + # Degraded mode cannot tell a real spec from an invented one, so the + # page must not be indexable — otherwise an outage reopens the hole + # #10453 closed, one thin near-duplicate per string anyone tries. + assert '' in response.text assert "og:title" in response.text assert "scatter-basic" in response.text assert "api.anyplot.ai/og/home.png" in response.text # Default image via API @@ -821,6 +826,14 @@ def test_seo_spec_overview_with_db(self, db_client, mock_spec) -> None: # Legacy /python/{spec} prefix must NOT appear assert "https://anyplot.ai/python/scatter-basic" not in response.text + def test_normal_pages_are_not_noindex(self, db_client, mock_spec) -> None: + """The guard must apply only to degraded mode, never to real pages.""" + client, _ = db_client + mock_spec_repo = MagicMock() + mock_spec_repo.get_by_id = AsyncMock(return_value=mock_spec) + with patch("api.routers.seo.SpecRepository", return_value=mock_spec_repo): + assert "noindex" not in client.get("/seo-proxy/scatter-basic").text + def test_seo_spec_overview_not_found(self, db_client) -> None: """SEO spec overview should return 404 when spec not found.""" client, _ = db_client @@ -871,6 +884,7 @@ def test_seo_spec_implementation_without_db(self, client: TestClient) -> None: with patch(DB_CONFIG_PATCH, return_value=False): response = client.get("/seo-proxy/scatter-basic/python/matplotlib") assert response.status_code == 200 + assert '' in response.text assert "og:title" in response.text assert "scatter-basic" in response.text assert "matplotlib" in response.text