From 4570641aea31de06be3cb2345095d9c21bcd4127 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:04:43 +0200 Subject: [PATCH 1/2] feat(analytics): split og:image tracking by who fetched the image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A social or messenger preview means a human shared a link. That is a product signal and belongs with the human numbers, so those events stay on anyplot.ai. A search or AI crawler fetching the same image is not a share. The distinction only starts mattering now. robots.txt on api.anyplot.ai was a blanket Disallow until #10472, so crawlers were never allowed to fetch /og/ at all — every og_image_view came from a link preview, and recording them all on the main site was right. Opening /og/ changes that: crawler fetches of 3,913 preview images would arrive in volume, drown the sharing signal, and inflate visitor counts on the site whose bot inflation audit 2026-07-08 already had to remove once. The split reuses detect_ai_agent rather than inventing a second rule: if it recognises the agent the fetch is machine-side and goes to bots.anyplot.ai, carrying the same assistant and kind props as bot_fetch so both events slice alike. Anything it does not recognise — Twitter, Facebook, Slack, WhatsApp — is a share and stays put. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 8 +++++ api/analytics.py | 15 ++++++++- docs/reference/plausible.md | 20 +++++++++++- tests/unit/api/test_analytics.py | 54 ++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ce0f38cb..1b012d7882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,14 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Changed +- **`og_image_view` is split by who fetched the image** — a social or messenger preview means a + human shared a link, which is a product signal and stays on the main site; a search or AI crawler + fetching the same image is not a share and now goes to `bots.anyplot.ai`. The distinction matters + from now on rather than in principle: `robots.txt` only began permitting `/og/` in the same + release, so crawler fetches of the 3,913 preview images would otherwise have arrived in volume on + the main site, drowning the sharing signal and inflating visitor counts. Machine-side events carry + the same `assistant` and `kind` props as `bot_fetch`, so both slice alike. + - **Repository prose now follows the Google developer documentation style guide** — the `write-docs` skill gains a "Writing style" section anchoring [Google style](https://developers.google.com/style) as the baseline for `docs/`, `README.md`, diff --git a/api/analytics.py b/api/analytics.py index df25da6f65..88982f9594 100644 --- a/api/analytics.py +++ b/api/analytics.py @@ -259,7 +259,20 @@ def track_og_image( # Fallback: missing spec for a spec-based page url = "https://anyplot.ai/" + # Where this is recorded depends on WHO fetched it. A social or messenger + # preview means a human shared a link — that is a product signal and belongs + # with the human numbers. A search or AI crawler fetching the same image is + # not a share, and now that robots.txt permits /og/ (#10472) those will + # arrive in volume: recording them on the main site would drown the sharing + # signal in crawler traffic and inflate visitor counts. detect_ai_agent + # recognises exactly the machine side, so its verdict makes the split. + agent = detect_ai_agent(user_agent) + domain = BOT_DOMAIN if agent else DOMAIN + props: dict[str, str] = {"page": page, "platform": platform} + if agent: + # Same shape as bot_fetch, so the bot site slices both events alike + props["assistant"], props["kind"] = agent if spec: props["spec"] = spec if language: @@ -274,7 +287,7 @@ def track_og_image( # Fire-and-forget: create task without awaiting, but add exception handler. # Track via _BACKGROUND_TASKS so the GC cannot collect the task before it runs. - task = asyncio.create_task(_send_plausible_event(user_agent, client_ip, "og_image_view", url, props)) + task = asyncio.create_task(_send_plausible_event(user_agent, client_ip, "og_image_view", url, props, domain=domain)) _BACKGROUND_TASKS.add(task) task.add_done_callback(_BACKGROUND_TASKS.discard) task.add_done_callback(_handle_task_exception) diff --git a/docs/reference/plausible.md b/docs/reference/plausible.md index c100543d42..44db34498a 100644 --- a/docs/reference/plausible.md +++ b/docs/reference/plausible.md @@ -240,7 +240,21 @@ events arrive server-side through the Events API, which is why it shows | Event | Properties | Source | Description | |-------|-----------|--------|-------------| -| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Recorded on `bots.anyplot.ai`. | +| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Always recorded on `bots.anyplot.ai`. | +| `og_image_view` | `page`, `platform`, `spec`?, `language`?, `library`?, `filter_*`?, plus `assistant` + `kind` when machine-fetched | `api/routers/og_images.py` | A preview image was fetched. **Split by audience**, see below. | + +#### og:image is split by who fetched it + +A social or messenger preview means a human shared a link — a product signal +that belongs with the human numbers, so those stay on `anyplot.ai`. A search or +AI crawler fetching the same image is not a share, and since `robots.txt` began +permitting `/og/` those arrive in volume: recording them on the main site would +drown the sharing signal in crawler traffic and inflate visitor counts. + +The split uses `detect_ai_agent` — if it recognises the user agent, the event is +machine-side and goes to `bots.anyplot.ai` carrying the same `assistant` and +`kind` props as `bot_fetch`, so both events slice alike. Anything it does not +recognise (Twitter, Facebook, Slack, WhatsApp, …) is a share and stays put. Register the three properties on the **`bots.anyplot.ai`** site, not on `anyplot.ai` — property registration is per site, and without it the events @@ -253,6 +267,10 @@ them: | `kind` | Why it fetched — see the table below | `bot_fetch` | | `path` | Public path that was read, e.g. `/box-basic/python/matplotlib` | `bot_fetch` | +Machine-side `og_image_view` events carry the main site's image properties as +well, so register these on `bots.anyplot.ai` too: `page`, `platform`, `spec`, +`language`, `library`, and the `filter_*` family. + `kind` is the property worth filtering on: | `kind` | Meaning | diff --git a/tests/unit/api/test_analytics.py b/tests/unit/api/test_analytics.py index 75877cfec6..6677be04c1 100644 --- a/tests/unit/api/test_analytics.py +++ b/tests/unit/api/test_analytics.py @@ -7,6 +7,7 @@ from api.analytics import ( BOT_DOMAIN, + DOMAIN, PLATFORM_PATTERNS, _detect_whatsapp_variant, detect_ai_agent, @@ -432,3 +433,56 @@ async def test_sends_nothing_for_a_human(self) -> None: await asyncio.sleep(0) mock_client.post.assert_not_called() + + +class TestOgImageAudienceSplit: + """A shared link and a crawler fetch are different things and are recorded apart.""" + + @staticmethod + def _request(user_agent: str) -> MagicMock: + request = MagicMock() + request.headers = {"user-agent": user_agent} + request.client.host = "203.0.113.7" + return request + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "user_agent", + ["Twitterbot/1.0", "facebookexternalhit/1.1", "Slackbot-LinkExpanding 1.0", "WhatsApp/2.23.18.78 i"], + ) + async def test_a_shared_link_stays_with_the_human_numbers(self, user_agent: str) -> None: + """A preview fetch means someone shared the page — that is a product signal.""" + with patch("api.analytics.httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + + track_og_image(self._request(user_agent), page="spec_detail", spec="box-basic") + await asyncio.sleep(0) + + payload = mock_client.post.call_args[1]["json"] + assert payload["domain"] == DOMAIN + assert "assistant" not in payload["props"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("user_agent", "assistant", "kind"), + [ + ("Mozilla/5.0 (compatible; Googlebot/2.1)", "google", "search"), + ("Mozilla/5.0 (compatible; Claude-User/1.0)", "claude", "user_directed"), + ("Mozilla/5.0 (compatible; GPTBot/1.4)", "chatgpt", "training"), + ], + ) + async def test_a_crawler_fetch_goes_to_the_bot_site(self, user_agent: str, assistant: str, kind: str) -> None: + """Now that /og/ is crawlable these arrive in volume; they must not + drown the sharing signal or inflate visitors on the main site.""" + with patch("api.analytics.httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + + track_og_image(self._request(user_agent), page="spec_detail", spec="box-basic") + await asyncio.sleep(0) + + payload = mock_client.post.call_args[1]["json"] + assert payload["domain"] == BOT_DOMAIN + assert payload["props"]["assistant"] == assistant + assert payload["props"]["kind"] == kind From 97d478cd660c7f5806adab618dc6023136a5c94b Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:16:44 +0200 Subject: [PATCH 2/2] docs(changelog): add this PR's reference Same convention fix as #10474, applied to this entry. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21bfadd72f..b1d7c37945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,7 +143,7 @@ aggregate instead: an italic *Catalog* line at the end of the version section an from now on rather than in principle: `robots.txt` only began permitting `/og/` in the same release, so crawler fetches of the 3,913 preview images would otherwise have arrived in volume on the main site, drowning the sharing signal and inflating visitor counts. Machine-side events carry - the same `assistant` and `kind` props as `bot_fetch`, so both slice alike. + the same `assistant` and `kind` props as `bot_fetch`, so both slice alike (#10475). - **The crawler policy is open to every operator** — retrieval, citation, search indexing and model training are now permitted for all of them, replacing the retrieval-yes / training-no split from #9633. That split was never coherent for an MIT-licensed catalogue published to be reused: