feat(analytics): split og:image tracking by who fetched the image

CHANGELOG.md

@@ -137,6 +137,13 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 
 ### Changed
 
+- **`og_image_view` is split by who fetched the image** — a social or messenger preview means a
+  human shared a link, which is a product signal and stays on the main site; a search or AI crawler
+  fetching the same image is not a share and now goes to `bots.anyplot.ai`. The distinction matters
+  from now on rather than in principle: `robots.txt` only began permitting `/og/` in the same
+  release, so crawler fetches of the 3,913 preview images would otherwise have arrived in volume on
+  the main site, drowning the sharing signal and inflating visitor counts. Machine-side events carry
+  the same `assistant` and `kind` props as `bot_fetch`, so both slice alike (#10475).
 - **The crawler policy is open to every operator** — retrieval, citation, search indexing and model
   training are now permitted for all of them, replacing the retrieval-yes / training-no split from
   #9633. That split was never coherent for an MIT-licensed catalogue published to be reused:

api/analytics.py

@@ -259,7 +259,20 @@ def track_og_image(
         # Fallback: missing spec for a spec-based page
         url = "https://anyplot.ai/"
 
+    # Where this is recorded depends on WHO fetched it. A social or messenger
+    # preview means a human shared a link — that is a product signal and belongs
+    # with the human numbers. A search or AI crawler fetching the same image is
+    # not a share, and now that robots.txt permits /og/ (#10472) those will
+    # arrive in volume: recording them on the main site would drown the sharing
+    # signal in crawler traffic and inflate visitor counts. detect_ai_agent
+    # recognises exactly the machine side, so its verdict makes the split.
+    agent = detect_ai_agent(user_agent)
+    domain = BOT_DOMAIN if agent else DOMAIN
+
     props: dict[str, str] = {"page": page, "platform": platform}
+    if agent:
+        # Same shape as bot_fetch, so the bot site slices both events alike
+        props["assistant"], props["kind"] = agent
     if spec:
         props["spec"] = spec
     if language:
@@ -274,7 +287,7 @@ def track_og_image(
 
     # Fire-and-forget: create task without awaiting, but add exception handler.
     # Track via _BACKGROUND_TASKS so the GC cannot collect the task before it runs.
-    task = asyncio.create_task(_send_plausible_event(user_agent, client_ip, "og_image_view", url, props))
+    task = asyncio.create_task(_send_plausible_event(user_agent, client_ip, "og_image_view", url, props, domain=domain))
     _BACKGROUND_TASKS.add(task)
     task.add_done_callback(_BACKGROUND_TASKS.discard)
     task.add_done_callback(_handle_task_exception)

docs/reference/plausible.md

@@ -240,7 +240,21 @@ events arrive server-side through the Events API, which is why it shows
 
 | Event | Properties | Source | Description |
 |-------|-----------|--------|-------------|
-| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Recorded on `bots.anyplot.ai`. |
+| `bot_fetch` | `assistant`, `kind`, `path` | `api/routers/seo.py` (router dependency) | An AI assistant or search crawler read a catalogue page. Always recorded on `bots.anyplot.ai`. |
+| `og_image_view` | `page`, `platform`, `spec`?, `language`?, `library`?, `filter_*`?, plus `assistant` + `kind` when machine-fetched | `api/routers/og_images.py` | A preview image was fetched. **Split by audience**, see below. |
+
+#### og:image is split by who fetched it
+
+A social or messenger preview means a human shared a link — a product signal
+that belongs with the human numbers, so those stay on `anyplot.ai`. A search or
+AI crawler fetching the same image is not a share, and since `robots.txt` began
+permitting `/og/` those arrive in volume: recording them on the main site would
+drown the sharing signal in crawler traffic and inflate visitor counts.
+
+The split uses `detect_ai_agent` — if it recognises the user agent, the event is
+machine-side and goes to `bots.anyplot.ai` carrying the same `assistant` and
+`kind` props as `bot_fetch`, so both events slice alike. Anything it does not
+recognise (Twitter, Facebook, Slack, WhatsApp, …) is a share and stays put.
 
 Register the three properties on the **`bots.anyplot.ai`** site, not on
 `anyplot.ai` — property registration is per site, and without it the events
@@ -253,6 +267,10 @@ them:
 | `kind` | Why it fetched — see the table below | `bot_fetch` |
 | `path` | Public path that was read, e.g. `/box-basic/python/matplotlib` | `bot_fetch` |
 
+Machine-side `og_image_view` events carry the main site's image properties as
+well, so register these on `bots.anyplot.ai` too: `page`, `platform`, `spec`,
+`language`, `library`, and the `filter_*` family.
+
 `kind` is the property worth filtering on:
 
 | `kind` | Meaning |

tests/unit/api/test_analytics.py

@@ -7,6 +7,7 @@
 
 from api.analytics import (
     BOT_DOMAIN,
+    DOMAIN,
     PLATFORM_PATTERNS,
     _detect_whatsapp_variant,
     detect_ai_agent,
@@ -432,3 +433,56 @@ async def test_sends_nothing_for_a_human(self) -> None:
             await asyncio.sleep(0)
 
             mock_client.post.assert_not_called()
+
+
+class TestOgImageAudienceSplit:
+    """A shared link and a crawler fetch are different things and are recorded apart."""
+
+    @staticmethod
+    def _request(user_agent: str) -> MagicMock:
+        request = MagicMock()
+        request.headers = {"user-agent": user_agent}
+        request.client.host = "203.0.113.7"
+        return request
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "user_agent",
+        ["Twitterbot/1.0", "facebookexternalhit/1.1", "Slackbot-LinkExpanding 1.0", "WhatsApp/2.23.18.78 i"],
+    )
+    async def test_a_shared_link_stays_with_the_human_numbers(self, user_agent: str) -> None:
+        """A preview fetch means someone shared the page — that is a product signal."""
+        with patch("api.analytics.httpx.AsyncClient") as mock_client_class:
+            mock_client = AsyncMock()
+            mock_client_class.return_value.__aenter__.return_value = mock_client
+
+            track_og_image(self._request(user_agent), page="spec_detail", spec="box-basic")
+            await asyncio.sleep(0)
+
+            payload = mock_client.post.call_args[1]["json"]
+            assert payload["domain"] == DOMAIN
+            assert "assistant" not in payload["props"]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        ("user_agent", "assistant", "kind"),
+        [
+            ("Mozilla/5.0 (compatible; Googlebot/2.1)", "google", "search"),
+            ("Mozilla/5.0 (compatible; Claude-User/1.0)", "claude", "user_directed"),
+            ("Mozilla/5.0 (compatible; GPTBot/1.4)", "chatgpt", "training"),
+        ],
+    )
+    async def test_a_crawler_fetch_goes_to_the_bot_site(self, user_agent: str, assistant: str, kind: str) -> None:
+        """Now that /og/ is crawlable these arrive in volume; they must not
+        drown the sharing signal or inflate visitors on the main site."""
+        with patch("api.analytics.httpx.AsyncClient") as mock_client_class:
+            mock_client = AsyncMock()
+            mock_client_class.return_value.__aenter__.return_value = mock_client
+
+            track_og_image(self._request(user_agent), page="spec_detail", spec="box-basic")
+            await asyncio.sleep(0)
+
+            payload = mock_client.post.call_args[1]["json"]
+            assert payload["domain"] == BOT_DOMAIN
+            assert payload["props"]["assistant"] == assistant
+            assert payload["props"]["kind"] == kind