From 7d33e13d749cb2d2e46b2d50650eb8edcc428395 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:52:37 +0200 Subject: [PATCH 1/2] fix(seo): make lastmod describe the page, not the row behind it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured against Search Console: Google last fetched /heatmap-annotated/python/altair on 2026-07-29 and /bar-error/python/matplotlib on 2026-08-07, and last read the sitemap on 2026-08-14 — all of it before today's work. Everything shipped since is invisible to anything grounding on Google's index, which is exactly the symptom the owner reported: Mistral, which fetches live, works; Gemini, which does not, still describes the old page. The sitemap was actively working against that. lastmod came from an implementation's `updated` column, and today changed the rendering rather than the data — new render markup, both themes, the interactive version, a rewritten meta description — so every date stayed put and the file told Google nothing had changed. It is the one signal that asks a crawler to come back, and it was saying "don't bother". lastmod is now the later of the record's own date and TEMPLATE_LAST_CHANGED. That constant is to be bumped only when the rendered page genuinely changes for every URL: it asserts that ~3,900 pages changed at once, and making that claim casually is how a site teaches Google to stop trusting its lastmod. A missing `updated` no longer yields a missing lastmod either. The page was last modified at least when its template was, and saying nothing was the same lost signal in a smaller form. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 9 +++++++ api/routers/seo.py | 23 ++++++++++++++++-- docs/reference/seo.md | 18 ++++++++++++++ tests/unit/api/test_seo_helpers.py | 39 ++++++++++++++++++------------ 4 files changed, 72 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65f9971dd2..9f61da0eff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -246,6 +246,15 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed +- **The sitemap was telling Google nothing had changed** — `lastmod` came from an implementation's + `updated` column, which does not move when the page's *rendering* changes. So after a day of work + that gave every implementation page the real render, both themes, the interactive version and a + rewritten description, the sitemap still reported the old dates, and Google — last seen fetching + some of those pages three weeks earlier — had no reason to return. `lastmod` is now the later of + the record's date and `TEMPLATE_LAST_CHANGED`, a constant to bump only when the rendered page + changes for every URL; claiming 3,900 changes casually is how a site teaches Google to ignore its + `lastmod` (#10483). + - **CodeQL alert #103** — an ECharts tooltip called `.replace("\n", " ")` with a string argument, which replaces only the first occurrence (`plots/bar-heart-rate-zones/implementations/javascript/echarts.js`). Nothing renders differently diff --git a/api/routers/seo.py b/api/routers/seo.py index eb082e209c..38a6e4a3c4 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -27,9 +27,28 @@ _SPEC_ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +# The date the bot-facing page template last changed materially. `lastmod` +# describes the PAGE, not the row behind it, and those drift apart: on +# 2026-08-18 every implementation page gained the real render, both themes, the +# interactive version and a rewritten meta description, while no `updated` +# column moved. The sitemap consequently told Google nothing had changed, and +# Google — which had last fetched some of these pages three weeks earlier — had +# no reason to come back and see any of it. +# +# Bump this ONLY when the rendered page genuinely changes for every URL. It is +# a claim to search engines that ~3,900 pages changed at once; making it +# casually is how a site teaches Google to stop trusting its lastmod. +TEMPLATE_LAST_CHANGED = datetime(2026, 8, 18) + + def _lastmod(dt: datetime | None) -> str: - """Format datetime as XML element, or empty string if None.""" - return f"{dt.strftime('%Y-%m-%d')}" if dt else "" + """Format the later of the record's own date and the template's, as . + + Empty when neither is known — an absent lastmod is honest, a wrong one is + not. + """ + latest = max(dt, TEMPLATE_LAST_CHANGED) if dt else TEMPLATE_LAST_CHANGED + return f"{latest.strftime('%Y-%m-%d')}" def _build_sitemap_xml(specs: list) -> str: diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 8fc00805f9..c82ac7d92f 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -481,6 +481,24 @@ filtering is served as `/{spec_id}?language={language}` (the hub with a filter query param, same canonical as the unfiltered hub), so listing it would create duplicate-content entries for Google. +### `lastmod` describes the page, not the row + +`lastmod` is the later of an implementation's own `updated` date and +`TEMPLATE_LAST_CHANGED` in `api/routers/seo.py`. The two drift apart, and the +drift matters: on 2026-08-18 every implementation page gained the real render, +both themes, the interactive version and a rewritten meta description, while no +`updated` column moved. The sitemap accordingly told Google nothing had changed +— and Google, which had last fetched some of those pages three weeks earlier, +had no reason to come back and see any of it. + +Bump `TEMPLATE_LAST_CHANGED` **only** when the rendered page genuinely changes +for every URL. It asserts to search engines that ~3,900 pages changed at once; +making that claim casually is how a site teaches Google to ignore its `lastmod` +altogether. + +A sitemap resubmission in Search Console pairs with the bump — the file is only +re-read every few days on its own. + ### Included URLs 1. Home page (`/`) diff --git a/tests/unit/api/test_seo_helpers.py b/tests/unit/api/test_seo_helpers.py index e8ca241372..08724d18d7 100644 --- a/tests/unit/api/test_seo_helpers.py +++ b/tests/unit/api/test_seo_helpers.py @@ -12,6 +12,7 @@ from api.routers.seo import ( _HOME_JSONLD, _META_DESCRIPTION_LIMIT, + TEMPLATE_LAST_CHANGED, _build_home_body, _build_impl_html, _build_sitemap_xml, @@ -55,20 +56,21 @@ def _mock_spec(impls: list) -> MagicMock: class TestLastmod: - """Tests for _lastmod helper.""" + """lastmod describes the page, which is not the same as the row behind it.""" - def test_with_datetime(self) -> None: - dt = datetime(2025, 3, 15) - result = _lastmod(dt) - assert result == "2025-03-15" + def test_a_record_newer_than_the_template_wins(self) -> None: + dt = datetime(2099, 3, 15) + assert _lastmod(dt) == "2099-03-15" - def test_with_none(self) -> None: - assert _lastmod(None) == "" + def test_an_older_record_is_lifted_to_the_template_date(self) -> None: + """The row has not moved, but the page it renders into has.""" + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert _lastmod(datetime(2024, 12, 1, 10, 30, 0)) == f"{stamp}" - def test_with_different_date(self) -> None: - dt = datetime(2024, 12, 1, 10, 30, 0) - result = _lastmod(dt) - assert result == "2024-12-01" + def test_without_a_record_date_the_template_date_still_applies(self) -> None: + """The page was last modified at least when its template was.""" + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert _lastmod(None) == f"{stamp}" class TestBuildSitemapXml: @@ -115,8 +117,12 @@ def test_spec_with_impls(self) -> None: assert "https://anyplot.ai/scatter-basic/python" not in result # Legacy /python/{spec} path must NOT appear assert "https://anyplot.ai/python/scatter-basic" not in result - assert "2025-03-14" in result - assert "2025-03-15" in result + # The record's own date is older than the template's, so the page's + # lastmod is the template's — the page changed even though the row did not. + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert f"{stamp}" in result + assert "2025-03-14" not in result + assert "2025-03-15" not in result def test_spec_without_impls_excluded(self) -> None: spec = MagicMock() @@ -223,8 +229,11 @@ def test_spec_with_none_updated(self) -> None: spec.updated = None result = _build_sitemap_xml([spec]) - # Should not have lastmod when updated is None - assert "https://anyplot.ai/scatter-basic" in result + # A missing `updated` no longer means a missing lastmod: the page was + # last modified at least when its template was, and saying nothing left + # Google with no reason to recrawl pages whose rendering had changed. + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert f"https://anyplot.ai/scatter-basic{stamp}" in result class TestRenderBotHtml: From 1c33b164fafb6984d5c0ddcaf68947569104e831 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:56:38 +0200 Subject: [PATCH 2/2] docs(seo): correct two claims the review caught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both from Copilot, both cases of prose outrunning the code. _lastmod's docstring still said it could return empty. It cannot: the template date is always defined, so every URL now gets an element — which is the point of the change, and the tests assert it. The sitemap section described lastmod as coming from an implementation's updated date, while hub URLs use spec.updated. Now says 'the record's', naming both tiers. Co-Authored-By: Claude Opus 5 (1M context) --- api/routers/seo.py | 5 +++-- docs/reference/seo.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/routers/seo.py b/api/routers/seo.py index 38a6e4a3c4..40177cfe2e 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -44,8 +44,9 @@ def _lastmod(dt: datetime | None) -> str: """Format the later of the record's own date and the template's, as . - Empty when neither is known — an absent lastmod is honest, a wrong one is - not. + Always returns an element. A record without its own date still has a page, + and that page was last modified at least when its template was — emitting + nothing there was the same lost recrawl signal in a smaller form. """ latest = max(dt, TEMPLATE_LAST_CHANGED) if dt else TEMPLATE_LAST_CHANGED return f"{latest.strftime('%Y-%m-%d')}" diff --git a/docs/reference/seo.md b/docs/reference/seo.md index c82ac7d92f..95345ffe4c 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -483,8 +483,9 @@ duplicate-content entries for Google. ### `lastmod` describes the page, not the row -`lastmod` is the later of an implementation's own `updated` date and -`TEMPLATE_LAST_CHANGED` in `api/routers/seo.py`. The two drift apart, and the +`lastmod` is the later of the record's own `updated` date — `spec.updated` for a +hub URL, `impl.updated` for an implementation URL — and `TEMPLATE_LAST_CHANGED` +in `api/routers/seo.py`. The two drift apart, and the drift matters: on 2026-08-18 every implementation page gained the real render, both themes, the interactive version and a rewritten meta description, while no `updated` column moved. The sitemap accordingly told Google nothing had changed