diff --git a/CHANGELOG.md b/CHANGELOG.md index 65f9971dd2..9f61da0eff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -246,6 +246,15 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed +- **The sitemap was telling Google nothing had changed** — `lastmod` came from an implementation's + `updated` column, which does not move when the page's *rendering* changes. So after a day of work + that gave every implementation page the real render, both themes, the interactive version and a + rewritten description, the sitemap still reported the old dates, and Google — last seen fetching + some of those pages three weeks earlier — had no reason to return. `lastmod` is now the later of + the record's date and `TEMPLATE_LAST_CHANGED`, a constant to bump only when the rendered page + changes for every URL; claiming 3,900 changes casually is how a site teaches Google to ignore its + `lastmod` (#10483). + - **CodeQL alert #103** — an ECharts tooltip called `.replace("\n", " ")` with a string argument, which replaces only the first occurrence (`plots/bar-heart-rate-zones/implementations/javascript/echarts.js`). Nothing renders differently diff --git a/api/routers/seo.py b/api/routers/seo.py index eb082e209c..40177cfe2e 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -27,9 +27,29 @@ _SPEC_ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +# The date the bot-facing page template last changed materially. `lastmod` +# describes the PAGE, not the row behind it, and those drift apart: on +# 2026-08-18 every implementation page gained the real render, both themes, the +# interactive version and a rewritten meta description, while no `updated` +# column moved. The sitemap consequently told Google nothing had changed, and +# Google — which had last fetched some of these pages three weeks earlier — had +# no reason to come back and see any of it. +# +# Bump this ONLY when the rendered page genuinely changes for every URL. It is +# a claim to search engines that ~3,900 pages changed at once; making it +# casually is how a site teaches Google to stop trusting its lastmod. +TEMPLATE_LAST_CHANGED = datetime(2026, 8, 18) + + def _lastmod(dt: datetime | None) -> str: - """Format datetime as XML element, or empty string if None.""" - return f"{dt.strftime('%Y-%m-%d')}" if dt else "" + """Format the later of the record's own date and the template's, as . + + Always returns an element. A record without its own date still has a page, + and that page was last modified at least when its template was — emitting + nothing there was the same lost recrawl signal in a smaller form. + """ + latest = max(dt, TEMPLATE_LAST_CHANGED) if dt else TEMPLATE_LAST_CHANGED + return f"{latest.strftime('%Y-%m-%d')}" def _build_sitemap_xml(specs: list) -> str: diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 8fc00805f9..95345ffe4c 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -481,6 +481,25 @@ filtering is served as `/{spec_id}?language={language}` (the hub with a filter query param, same canonical as the unfiltered hub), so listing it would create duplicate-content entries for Google. +### `lastmod` describes the page, not the row + +`lastmod` is the later of the record's own `updated` date — `spec.updated` for a +hub URL, `impl.updated` for an implementation URL — and `TEMPLATE_LAST_CHANGED` +in `api/routers/seo.py`. The two drift apart, and the +drift matters: on 2026-08-18 every implementation page gained the real render, +both themes, the interactive version and a rewritten meta description, while no +`updated` column moved. The sitemap accordingly told Google nothing had changed +— and Google, which had last fetched some of those pages three weeks earlier, +had no reason to come back and see any of it. + +Bump `TEMPLATE_LAST_CHANGED` **only** when the rendered page genuinely changes +for every URL. It asserts to search engines that ~3,900 pages changed at once; +making that claim casually is how a site teaches Google to ignore its `lastmod` +altogether. + +A sitemap resubmission in Search Console pairs with the bump — the file is only +re-read every few days on its own. + ### Included URLs 1. Home page (`/`) diff --git a/tests/unit/api/test_seo_helpers.py b/tests/unit/api/test_seo_helpers.py index e8ca241372..08724d18d7 100644 --- a/tests/unit/api/test_seo_helpers.py +++ b/tests/unit/api/test_seo_helpers.py @@ -12,6 +12,7 @@ from api.routers.seo import ( _HOME_JSONLD, _META_DESCRIPTION_LIMIT, + TEMPLATE_LAST_CHANGED, _build_home_body, _build_impl_html, _build_sitemap_xml, @@ -55,20 +56,21 @@ def _mock_spec(impls: list) -> MagicMock: class TestLastmod: - """Tests for _lastmod helper.""" + """lastmod describes the page, which is not the same as the row behind it.""" - def test_with_datetime(self) -> None: - dt = datetime(2025, 3, 15) - result = _lastmod(dt) - assert result == "2025-03-15" + def test_a_record_newer_than_the_template_wins(self) -> None: + dt = datetime(2099, 3, 15) + assert _lastmod(dt) == "2099-03-15" - def test_with_none(self) -> None: - assert _lastmod(None) == "" + def test_an_older_record_is_lifted_to_the_template_date(self) -> None: + """The row has not moved, but the page it renders into has.""" + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert _lastmod(datetime(2024, 12, 1, 10, 30, 0)) == f"{stamp}" - def test_with_different_date(self) -> None: - dt = datetime(2024, 12, 1, 10, 30, 0) - result = _lastmod(dt) - assert result == "2024-12-01" + def test_without_a_record_date_the_template_date_still_applies(self) -> None: + """The page was last modified at least when its template was.""" + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert _lastmod(None) == f"{stamp}" class TestBuildSitemapXml: @@ -115,8 +117,12 @@ def test_spec_with_impls(self) -> None: assert "https://anyplot.ai/scatter-basic/python" not in result # Legacy /python/{spec} path must NOT appear assert "https://anyplot.ai/python/scatter-basic" not in result - assert "2025-03-14" in result - assert "2025-03-15" in result + # The record's own date is older than the template's, so the page's + # lastmod is the template's — the page changed even though the row did not. + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert f"{stamp}" in result + assert "2025-03-14" not in result + assert "2025-03-15" not in result def test_spec_without_impls_excluded(self) -> None: spec = MagicMock() @@ -223,8 +229,11 @@ def test_spec_with_none_updated(self) -> None: spec.updated = None result = _build_sitemap_xml([spec]) - # Should not have lastmod when updated is None - assert "https://anyplot.ai/scatter-basic" in result + # A missing `updated` no longer means a missing lastmod: the page was + # last modified at least when its template was, and saying nothing left + # Google with no reason to recrawl pages whose rendering had changed. + stamp = TEMPLATE_LAST_CHANGED.strftime("%Y-%m-%d") + assert f"https://anyplot.ai/scatter-basic{stamp}" in result class TestRenderBotHtml: