From f2ef4900323e92d7419b739c7b7c19321147983c Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:56:22 +0200 Subject: [PATCH 1/2] fix(seo): close the gaps an independent audit found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from a fresh-context audit of today's twelve PRs, none of which I had spotted. Documentation contradicting itself. docs/reference/seo.md's bot-map section still said gptbot, meta-externalagent and amazonbot were "declined in robots.txt", and app/nginx.conf repeated it in a comment. Both were written about an hour before #10474 opened the policy, and neither was reconciled — so the page asserted the old policy three screens from the section declaring the new one. The measured edge-state table was stale the other way round: the dashboard unblock it prescribed had since been carried out. A database outage reopening #10453. With no catalogue to check against, the bot routes answered 200 with a fabricated, self-canonicalising page for any string — the precise defect that PR removed, surviving in degraded mode. Those pages now carry noindex. It is unreachable in production, where the database is configured, but "unreachable" here means one misconfiguration away from indexable, and the tests lean on this path heavily enough that a 503 broke eleven of them; noindex keeps the behaviour and removes the risk. 404s counted as page reads. bot_fetch ran as a router dependency, which executes before the handler and cannot see the response. It has moved to a middleware and gained a status property — a miss is worth recording, since it is how a library migration announces itself, but recording it as a successful read is a lie. Also corrects a docstring pointing at app/src/router.tsx, which does not exist; routing lives in app/src/routes/index.tsx. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 16 ++++++++++++++ api/analytics.py | 8 +++++-- api/main.py | 18 ++++++++++++++++ api/routers/seo.py | 30 +++++++++++--------------- app/nginx.conf | 7 +++--- docs/reference/plausible.md | 13 +++++++++-- docs/reference/seo.md | 37 ++++++++++++++++++++------------ tests/unit/api/test_analytics.py | 31 ++++++++++++++++++++++---- tests/unit/api/test_routers.py | 20 ++++++++++++++--- 9 files changed, 134 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83c2fc8df5..bedba76cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -232,6 +232,22 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed +- **A database outage would have reopened the hole #10453 closed** — with no catalogue to check + against, the bot routes answered 200 with a fabricated page for any string, self-canonicalising, + exactly the defect that PR removed. Degraded pages now carry `noindex`. The path is unreachable in + production, where the database is configured, but it is one misconfiguration away from indexable. + `bot_fetch` also moved from a router dependency to a middleware and gained a `status` property: a + dependency runs before the handler and cannot see the response, so every 404 was being recorded as + a successful page read (#10479). + +- **Documentation that still described the superseded crawler policy** — `docs/reference/seo.md` + said `gptbot`, `meta-externalagent` and `amazonbot` were "declined in robots.txt" and + `app/nginx.conf` carried the same claim in a comment. Both were written about an hour before + #10474 opened the policy and never reconciled, so the page contradicted itself. The measured + edge-state table was stale for the opposite reason: the dashboard unblock it prescribed had since + been done. A docstring also pointed at `app/src/router.tsx`, which does not exist — the routing + lives in `app/src/routes/index.tsx` (#10479). + - **The trailing-slash redirect leaked the internal port** — #10473 removed a redirect to `http://api.anyplot.ai/seo-proxy/…` and replaced it with `http://anyplot.ai:8080/…`: a smaller version of the same defect, since nginx builds a `permanent` rewrite's Location from diff --git a/api/analytics.py b/api/analytics.py index 88982f9594..19d4fda286 100644 --- a/api/analytics.py +++ b/api/analytics.py @@ -293,7 +293,7 @@ def track_og_image( task.add_done_callback(_handle_task_exception) -def track_bot_fetch(request: Request, path: str) -> None: +def track_bot_fetch(request: Request, path: str, status: int) -> None: """Record an AI or search agent reading a page (fire-and-forget). Recorded against BOT_DOMAIN, never the main site: every Plausible event @@ -309,6 +309,10 @@ def track_bot_fetch(request: Request, path: str) -> None: Args: request: FastAPI request, for the UA and forwarded IP path: Public path being read, e.g. "/box-basic/python/matplotlib" + status: Response status. Recorded rather than filtered on: an assistant + asking for a URL that no longer exists is a signal worth having, and + counting it as a successful read would be a lie. Filter on it in the + dashboard. """ user_agent = request.headers.get("user-agent", "") detected = detect_ai_agent(user_agent) @@ -323,7 +327,7 @@ def track_bot_fetch(request: Request, path: str) -> None: # cf-connecting-ip and takes the rightmost entry, which is the one a # client cannot forge. client_ip = resolve_client_ip(request) - props = {"assistant": assistant, "kind": kind, "path": path} + props = {"assistant": assistant, "kind": kind, "path": path, "status": str(status)} url = f"https://anyplot.ai{path}" task = asyncio.create_task(_send_plausible_event(user_agent, client_ip, "bot_fetch", url, props, domain=BOT_DOMAIN)) diff --git a/api/main.py b/api/main.py index 33c0f4eae7..e42cf7a3a5 100644 --- a/api/main.py +++ b/api/main.py @@ -16,6 +16,7 @@ from fastapi.middleware.cors import CORSMiddleware # noqa: E402 from starlette.middleware.gzip import GZipMiddleware # noqa: E402 +from api.analytics import track_bot_fetch # noqa: E402 from api.cache import cache_key, set_cache # noqa: E402 from api.exceptions import ( # noqa: E402 AnyplotException, @@ -155,6 +156,23 @@ async def lifespan(app: FastAPI): ) +# Record which AI or search agent read which catalogue page. +# +# A middleware rather than a router dependency: a dependency runs BEFORE the +# handler and cannot see the response, so every 404 was recorded as a +# successful read. The status matters — an assistant asking for a URL that no +# longer exists is a signal worth keeping, it just is not a page view. +@app.middleware("http") +async def record_bot_fetch(request: Request, call_next): + """Report AI/search agent page reads to the bot analytics site.""" + response: Response = await call_next(request) + path = request.url.path + if path.startswith("/seo-proxy"): + # The public URL, never this router's internal prefix + track_bot_fetch(request, path.removeprefix("/seo-proxy") or "/", response.status_code) + return response + + # Add cache headers middleware @app.middleware("http") async def add_cache_headers(request: Request, call_next): diff --git a/api/routers/seo.py b/api/routers/seo.py index 950fbc1e5e..189121d95d 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -10,7 +10,6 @@ from fastapi.responses import HTMLResponse, RedirectResponse, Response from sqlalchemy.ext.asyncio import AsyncSession -from api.analytics import track_bot_fetch from api.cache import cache_key, get_cache, get_or_set_cache, set_cache from api.dependencies import optional_db from core.config import settings @@ -20,21 +19,7 @@ from core.utils import strip_noqa_comments -async def _record_bot_fetch(request: Request) -> None: - """Router dependency: record which agent read which page. - - Attached to the router rather than to each handler — there are a dozen and - the next one added would silently go unrecorded. Only /seo-proxy paths are - page reads; robots.txt and sitemap.xml also live on this router and are not. - """ - path = request.url.path - if not path.startswith("/seo-proxy"): - return - # Report the public URL, not this router's internal prefix - track_bot_fetch(request, path.removeprefix("/seo-proxy") or "/") - - -router = APIRouter(tags=["seo"], dependencies=[Depends(_record_bot_fetch)]) +router = APIRouter(tags=["seo"]) # Canonical spec-id shape — lowercase alphanumerics with hyphen separators. # Same pattern enforced in automation/scripts/sync_to_postgres.py. Used here to @@ -122,7 +107,7 @@ async def _refresh_sitemap() -> str: - {jsonld} + {robots}{jsonld} {body} @@ -280,6 +265,7 @@ def _render_bot_html( og_url: str | None = None, body: str = "", jsonld: dict | None = None, + noindex: bool = False, ) -> str: """Render a bot-serving page. @@ -304,6 +290,7 @@ def _render_bot_html( image=image, url=url, og_url=og_url if og_url is not None else url, + robots='\n ' if noindex else "", jsonld=_jsonld_script(jsonld) if jsonld else "", body=f"{body or f'

{title}

{description}

'}\n{_BOT_NAV_HTML}", ) @@ -750,12 +737,17 @@ async def seo_stats(): async def seo_spec_hub(spec_id: str, db: AsyncSession | None = Depends(optional_db)): """Bot-optimized cross-language spec hub.""" if db is None: + # Degraded mode: without the catalogue we cannot tell a real spec from + # an invented one, so this page must not be indexable — serving it + # otherwise recreates the defect #10453 removed, an indexable + # near-duplicate for any string anyone tries. return HTMLResponse( _render_bot_html( title=f"{html.escape(spec_id)} | anyplot.ai", description=DEFAULT_DESCRIPTION, image=DEFAULT_HOME_IMAGE, url=f"https://anyplot.ai/{html.escape(spec_id)}", + noindex=True, ) ) @@ -783,7 +775,7 @@ async def seo_spec_language(spec_id: str, language: str): The /{spec_id}/{language} tier was consolidated into /{spec_id} to eliminate duplicate content. Bots following this endpoint get a 301 to the public hub - URL; humans get the SPA redirect configured in app/src/router.tsx. The + URL; humans get the SPA redirect configured in app/src/routes/index.tsx. The `language` query parameter is dropped because the hub's canonical tag does not include it — Google should consolidate the page, not a filtered variant. """ @@ -820,12 +812,14 @@ async def seo_spec_implementation( ): """Bot-optimized implementation detail.""" if db is None: + # Same reasoning as the hub route — unverifiable, so not indexable. return HTMLResponse( _render_bot_html( title=f"{html.escape(spec_id)} - {html.escape(library)} | anyplot.ai", description=DEFAULT_DESCRIPTION, image=DEFAULT_HOME_IMAGE, url=f"https://anyplot.ai/{html.escape(spec_id)}/{html.escape(language)}/{html.escape(library)}", + noindex=True, ) ) diff --git a/app/nginx.conf b/app/nginx.conf index d64e572806..1aa88a8f38 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -34,9 +34,10 @@ map $http_user_agent $is_bot { # their assistant to open the page — and matter most. # This map only decides WHAT a crawler gets once it arrives, never whether # it may: the allow/deny policy is robots.txt plus the Cloudflare AI Crawl - # Control zone setting (docs/reference/seo.md). GPTBot is listed even - # though robots.txt declines it, so the rendering is right if that call is - # ever reversed. + # Control zone setting (docs/reference/seo.md). Under the current policy + # everything except Bytespider may crawl, so an agent missing from this map + # is not thereby declined — it just gets the empty SPA shell, which is the + # failure this list exists to prevent. ~*claudebot 1; ~*claude-user 1; ~*claude-searchbot 1; diff --git a/docs/reference/plausible.md b/docs/reference/plausible.md index 44db34498a..4a9a5068b7 100644 --- a/docs/reference/plausible.md +++ b/docs/reference/plausible.md @@ -240,7 +240,7 @@ events arrive server-side through the Events API, which is why it shows | Event | Properties | Source | Description | |-------|-----------|--------|-------------| -| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Always recorded on `bots.anyplot.ai`. | +| `bot_fetch` | `assistant`, `kind`, `path`, `status` | `api/main.py` (middleware) | An AI assistant or search crawler requested a catalogue page. Always recorded on `bots.anyplot.ai`. | | `og_image_view` | `page`, `platform`, `spec`?, `language`?, `library`?, `filter_*`?, plus `assistant` + `kind` when machine-fetched | `api/routers/og_images.py` | A preview image was fetched. **Split by audience**, see below. | #### og:image is split by who fetched it @@ -265,7 +265,16 @@ them: |----------|-------------|---------| | `assistant` | Vendor (`claude`, `chatgpt`, `gemini`, `mistral`, `perplexity`, `meta`, `amazon`, `duckduckgo`, `grok`, `google`, `bing`, …) | `bot_fetch` | | `kind` | Why it fetched — see the table below | `bot_fetch` | -| `path` | Public path that was read, e.g. `/box-basic/python/matplotlib` | `bot_fetch` | +| `path` | Public path that was requested, e.g. `/box-basic/python/matplotlib` | `bot_fetch` | +| `status` | Response status as a string (`200`, `404`, …) | `bot_fetch` | + +Filter on `status` before reading anything else. The event records the +*request*, not a successful read: an assistant asking for a URL that no longer +exists is a signal worth keeping — it is how a library migration announces +itself — but counting it as a page view would be a lie. This is also why the +recording lives in a middleware rather than a router dependency: a dependency +runs before the handler and cannot see the response, so every miss was recorded +as a read. Machine-side `og_image_view` events carry the main site's image properties as well, so register these on `bots.anyplot.ai` too: `page`, `platform`, `spec`, diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 19a493ea01..6406e1e120 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -104,18 +104,20 @@ Three vendor splits are easy to get backwards, so they are spelled out: - **Meta**: `meta-externalfetcher` (user-directed) and `meta-webindexer` (AI search index) are served; `meta-externalagent` is the training crawler and is - declined in robots.txt. + not mapped. - **Mistral**: `mistralai-user` and `mistralai-index` are served — Mistral documents both as never used for generative-AI training; `mistralai-training` is the training crawler and is not mapped. - **Amazon**: `amzn-searchbot` and `amzn-user` are the sanctioned retrieval agents, both documented as not crawling for model training; plain `amazonbot` - is the training crawler and is declined. + is the training crawler and is not mapped. Mapping is not permission. This map decides only *what* an agent receives once it arrives; whether it may crawl at all is robots.txt plus Cloudflare's AI Crawl -Control. That is why `gptbot` appears here while robots.txt declines it, and why -mapping community-reported tokens costs nothing. +Control — which is why mapping community-reported tokens costs nothing, and why +an unmapped agent is not thereby declined. Under the current policy everything +except `Bytespider` may crawl, so the mapping question is only ever about +rendering. **Search Engines:** | Bot | User-Agent Pattern | @@ -135,7 +137,7 @@ mapping community-reported tokens costs nothing. | Anthropic crawler | `claudebot` | index/crawl | | Claude user fetch | `claude-user` | a human asked Claude to open the page | | Claude search | `claude-searchbot` | citation index | -| OpenAI crawler | `gptbot` | training (declined in robots.txt, mapped anyway — see below) | +| OpenAI crawler | `gptbot` | training crawler; allowed and mapped | | ChatGPT search | `oai-searchbot` | citation index | | ChatGPT user fetch | `chatgpt-user` | a human asked ChatGPT to open the page | | Perplexity | `perplexitybot`, `perplexity-user` | citation index / user fetch | @@ -380,19 +382,26 @@ in step. Measured on the live zone 2026-08-18 (zone `anyplot.ai` → **AI Crawl Control** → Security): +The dashboard was brought in line with the open policy on 2026-08-18. State +after that change: + | State | Agents | |---|---| -| Blocked at the edge | `GPTBot`, `CCBot`, `Amazonbot`, `meta-externalagent`, `Bytespider`, `Google-CloudVertexBot`, `PetalBot`, `Anchor Browser`, `Arquivo Web Crawler` | -| Passing | `Googlebot`, `bingbot`, `Baidu`, `Applebot`, `Claude-User`, `ClaudeBot`, `Claude-SearchBot`, `ChatGPT-User`, `OAI-SearchBot`, `PerplexityBot`, `Perplexity-User`, `DuckAssistBot`, `MistralAI-User`, `Meta-ExternalFetcher`, `archive.org_bot` | +| Blocked at the edge | `Bytespider`, `TikTok Spider`, `Anchor Browser`, `Novellum AI Crawl`, `Timpibot` | +| Passing | everything else, including `GPTBot`, `CCBot`, `Amazonbot`, `meta-externalagent`, `Google-CloudVertexBot`, `PetalBot`, `FacebookBot`, `Arquivo Web Crawler`, and every retrieval agent | + +`Bytespider` and `TikTok Spider` are both ByteDance; the first is documented as +ignoring robots.txt, and the second shares its operator. The remaining three are +blocked for a weaker reason — no vendor documentation of their crawling +behaviour could be found, so their rule-compliance is unverified rather than +disproven. Revisit if that changes. -Two things to note. `Google-CloudVertexBot` is blocked at the edge and appears in -no repo-side policy — an undocumented decision that only exists as a dashboard -toggle. And unlike the 2026-07-25 measurement, Cloudflare is **no longer -prepending a managed robots.txt block**: the live file is byte-identical to this -repo's, and enforcement is purely the 403. +Cloudflare is **not** prepending a managed robots.txt block: the live file is +byte-identical to this repo's, and enforcement is purely the 403. That was +different at the 2026-07-25 measurement, so re-check rather than assume. -To bring the edge in line with the policy above, unblock everything except -`Bytespider` in the dashboard. +`bot-serving-check` deliberately tests the Cloud Run origin, not the edge, so it +will never catch a dashboard change. Edge drift has to be checked by hand. Verify afterwards: diff --git a/tests/unit/api/test_analytics.py b/tests/unit/api/test_analytics.py index 6677be04c1..5e274d8b69 100644 --- a/tests/unit/api/test_analytics.py +++ b/tests/unit/api/test_analytics.py @@ -375,7 +375,7 @@ async def test_records_against_the_bot_site_not_the_main_one(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic") + track_bot_fetch(self._request("Mozilla/5.0 (compatible; Claude-User/1.0)"), "/box-basic", 200) await asyncio.sleep(0) # let the fire-and-forget task run assert mock_client.post.call_args[1]["json"]["domain"] == BOT_DOMAIN @@ -386,7 +386,7 @@ async def test_carries_assistant_kind_and_public_path(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("MistralAI-User/1.0"), "/box-basic/python/matplotlib") + track_bot_fetch(self._request("MistralAI-User/1.0"), "/box-basic/python/matplotlib", 200) await asyncio.sleep(0) payload = mock_client.post.call_args[1]["json"] @@ -397,6 +397,7 @@ async def test_carries_assistant_kind_and_public_path(self) -> None: "assistant": "mistral", "kind": "user_directed", "path": "/box-basic/python/matplotlib", + "status": "200", } @pytest.mark.asyncio @@ -418,7 +419,7 @@ async def test_resolves_a_forwarded_for_chain_to_one_address(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(request, "/box-basic") + track_bot_fetch(request, "/box-basic", 200) await asyncio.sleep(0) assert mock_client.post.call_args[1]["headers"]["X-Forwarded-For"] == "10.0.0.9" @@ -429,7 +430,7 @@ async def test_sends_nothing_for_a_human(self) -> None: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - track_bot_fetch(self._request("Mozilla/5.0 (X11; Linux) Chrome/126.0 Safari/537.36"), "/") + track_bot_fetch(self._request("Mozilla/5.0 (X11; Linux) Chrome/126.0 Safari/537.36"), "/", 200) await asyncio.sleep(0) mock_client.post.assert_not_called() @@ -486,3 +487,25 @@ async def test_a_crawler_fetch_goes_to_the_bot_site(self, user_agent: str, assis assert payload["domain"] == BOT_DOMAIN assert payload["props"]["assistant"] == assistant assert payload["props"]["kind"] == kind + + +class TestBotFetchRecordsTheStatus: + """A miss is a signal, but it is not a page read — so record it, don't hide it.""" + + @staticmethod + def _request() -> MagicMock: + request = MagicMock() + request.headers = {"user-agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"} + request.client.host = "203.0.113.7" + return request + + @pytest.mark.asyncio + async def test_a_miss_is_recorded_as_such(self) -> None: + with patch("api.analytics.httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + + track_bot_fetch(self._request(), "/bar-basic/python/highcharts", 404) + await asyncio.sleep(0) + + assert mock_client.post.call_args[1]["json"]["props"]["status"] == "404" diff --git a/tests/unit/api/test_routers.py b/tests/unit/api/test_routers.py index 0d73ee4744..54002fc5c3 100644 --- a/tests/unit/api/test_routers.py +++ b/tests/unit/api/test_routers.py @@ -708,21 +708,22 @@ def test_seo_home_with_db_counts_specs(self, db_client, mock_spec) -> None: def test_seo_proxy_records_the_agent_that_read_the_page(self, client: TestClient) -> None: """The router dependency reports the PUBLIC path, not its own prefix.""" - with patch(DB_CONFIG_PATCH, return_value=False), patch("api.routers.seo.track_bot_fetch") as track: + with patch(DB_CONFIG_PATCH, return_value=False), patch("api.main.track_bot_fetch") as track: client.get("/seo-proxy/bar-basic", headers={"User-Agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"}) track.assert_called_once() assert track.call_args.args[1] == "/bar-basic" + assert track.call_args.args[2] == 200 # the status the handler produced def test_seo_proxy_home_records_root(self, client: TestClient) -> None: """The proxy root maps to "/", and the hook fires exactly once per request.""" - with patch(DB_CONFIG_PATCH, return_value=False), patch("api.routers.seo.track_bot_fetch") as track: + with patch(DB_CONFIG_PATCH, return_value=False), patch("api.main.track_bot_fetch") as track: client.get("/seo-proxy/", headers={"User-Agent": "Mozilla/5.0 (compatible; Claude-User/1.0)"}) track.assert_called_once() assert track.call_args.args[1] == "/" def test_robots_and_sitemap_are_not_page_reads(self, client: TestClient) -> None: """Both live on this router but are machine files, not catalogue pages.""" - with patch("api.routers.seo.track_bot_fetch") as track: + with patch("api.main.track_bot_fetch") as track: client.get("/robots.txt", headers={"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)"}) client.get("/sitemap.xml", headers={"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)"}) track.assert_not_called() @@ -801,6 +802,10 @@ def test_seo_spec_overview_without_db(self, client: TestClient) -> None: with patch(DB_CONFIG_PATCH, return_value=False): response = client.get("/seo-proxy/scatter-basic") assert response.status_code == 200 + # Degraded mode cannot tell a real spec from an invented one, so the + # page must not be indexable — otherwise an outage reopens the hole + # #10453 closed, one thin near-duplicate per string anyone tries. + assert '' in response.text assert "og:title" in response.text assert "scatter-basic" in response.text assert "api.anyplot.ai/og/home.png" in response.text # Default image via API @@ -821,6 +826,14 @@ def test_seo_spec_overview_with_db(self, db_client, mock_spec) -> None: # Legacy /python/{spec} prefix must NOT appear assert "https://anyplot.ai/python/scatter-basic" not in response.text + def test_normal_pages_are_not_noindex(self, db_client, mock_spec) -> None: + """The guard must apply only to degraded mode, never to real pages.""" + client, _ = db_client + mock_spec_repo = MagicMock() + mock_spec_repo.get_by_id = AsyncMock(return_value=mock_spec) + with patch("api.routers.seo.SpecRepository", return_value=mock_spec_repo): + assert "noindex" not in client.get("/seo-proxy/scatter-basic").text + def test_seo_spec_overview_not_found(self, db_client) -> None: """SEO spec overview should return 404 when spec not found.""" client, _ = db_client @@ -871,6 +884,7 @@ def test_seo_spec_implementation_without_db(self, client: TestClient) -> None: with patch(DB_CONFIG_PATCH, return_value=False): response = client.get("/seo-proxy/scatter-basic/python/matplotlib") assert response.status_code == 200 + assert '' in response.text assert "og:title" in response.text assert "scatter-basic" in response.text assert "matplotlib" in response.text From 4265fe797b54afa60d05c663ee748c03e722db70 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:21:58 +0200 Subject: [PATCH 2/2] docs(analytics): say 'request', not 'read', where the status is recorded From the Copilot review on #10479. The middleware and track_bot_fetch both still described 'page reads' after the event started recording the request status, misses included. A 404 is a request worth keeping and not a read, and the wording now matches what the code does and what docs/reference/plausible.md already said. Co-Authored-By: Claude Opus 5 (1M context) --- api/analytics.py | 4 ++-- api/main.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/analytics.py b/api/analytics.py index 19d4fda286..2cfec072c7 100644 --- a/api/analytics.py +++ b/api/analytics.py @@ -294,7 +294,7 @@ def track_og_image( def track_bot_fetch(request: Request, path: str, status: int) -> None: - """Record an AI or search agent reading a page (fire-and-forget). + """Record an AI or search agent requesting a page (fire-and-forget). Recorded against BOT_DOMAIN, never the main site: every Plausible event creates a visitor, and mixing these in is what made the human numbers @@ -308,7 +308,7 @@ def track_bot_fetch(request: Request, path: str, status: int) -> None: Args: request: FastAPI request, for the UA and forwarded IP - path: Public path being read, e.g. "/box-basic/python/matplotlib" + path: Public path being requested, e.g. "/box-basic/python/matplotlib" status: Response status. Recorded rather than filtered on: an assistant asking for a URL that no longer exists is a signal worth having, and counting it as a successful read would be a lie. Filter on it in the diff --git a/api/main.py b/api/main.py index e42cf7a3a5..3ed94929f1 100644 --- a/api/main.py +++ b/api/main.py @@ -156,7 +156,7 @@ async def lifespan(app: FastAPI): ) -# Record which AI or search agent read which catalogue page. +# Record which AI or search agent requested which catalogue page. # # A middleware rather than a router dependency: a dependency runs BEFORE the # handler and cannot see the response, so every 404 was recorded as a @@ -164,7 +164,11 @@ async def lifespan(app: FastAPI): # longer exists is a signal worth keeping, it just is not a page view. @app.middleware("http") async def record_bot_fetch(request: Request, call_next): - """Report AI/search agent page reads to the bot analytics site.""" + """Report AI/search agent page requests to the bot analytics site. + + Requests, not reads: the status is recorded rather than filtered on, so a + 404 is visible as a miss instead of counted as a page view. + """ response: Response = await call_next(request) path = request.url.path if path.startswith("/seo-proxy"):