fix(evergreen-tracks): authenticate Hub tag reads — anonymous pagination now blocked past offset 1000

.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/cli.py

@@ -10,24 +10,94 @@
 import os
 import sys
 
-from .calver import newest, parse_release
+from .calver import age_days, newest, parse_release
 from .executor import delete_tag, hub_login, point_tag
 from .markers import TRACKS, held_tracks, hold_tag, tainted_versions, taint_tag
 from .planner import TrackState, plan
 from .registry import list_tags
 
 log = logging.getLogger("evergreen_tracks")
 
+# How many months the narrow read path walks back before giving up and full-scanning.
+# Releases ship weekly, so promotion needs one or two months; the extra headroom is for
+# a quiet stretch or a track parked on something old by a hold.
+_MONTH_WALK_LIMIT = 6
+
 
 def _state(repo: str):
-    """Return (releases, tainted, held, name->digest) read from the registry."""
+    """Return (releases, tainted, held, name->digest) read from the registry.
+
+    The full ~79-page listing. Still the right tool for `admin` (rare, human-triggered,
+    and it addresses arbitrarily old versions) and the terminal fallback for promote.
+    """
     tags = list_tags(repo)
     names = [t.name for t in tags]
     digests = {t.name: t.digest for t in tags}
     releases = [r for r in (parse_release(n) for n in names) if r is not None]
     return releases, tainted_versions(names), held_tracks(names), digests
 
 
+def _month_str(date: dt.date) -> str:
+    return f"{date.year % 100:02d}.{date.month:02d}"
+
+
+def _prev_month(ym: str) -> str:
+    yy, mm = int(ym[:2]), int(ym[3:5])
+    return f"{yy - 1:02d}.12" if mm == 1 else f"{yy:02d}.{mm - 1:02d}"
+
+
+def _promote_state(repo: str, wanted: list[str], max_days: int, today: dt.date):
+    """`_state`, assembled from a handful of filtered reads instead of the full listing.
+
+    Moving a floating tag needs a few GA CalVer tags and three marker names, but the
+    full listing pages the whole repo — 7.8k tags, 79 calls, ~45s, against a Hub budget
+    of 180 requests/60s per IP. Hub's `name=` substring filter turns that into:
+
+      * one call per requested track  -> the `<track>` tag's digest, plus its
+        `<track>_hold` marker if one exists (both match the track name as a substring)
+      * one call per month of history -> that month's GA releases AND their
+        `<version>_tainted` markers, which share the version's month prefix
+
+    A month is one page (6-28 tags), so the daily standard+trailing promote is 3-4 calls.
+
+    Months are fetched newest-first and CONTIGUOUSLY, which is what makes the narrowed
+    pool safe for `planner.plan`'s forward-only guard. That guard only matters when a
+    track already sits on something NEWER than the newest eligible release — and any
+    such version lives in a month at or above the one the walk stopped on, so it has
+    necessarily already been read. A track on something OLDER resolves to "move
+    forward", which is the same decision the full listing produces. Nothing here can
+    move a track backwards that the full scan would have held.
+
+    Returns None if the walk window turns up nothing promotable at all, so the caller
+    can fall back to the full listing rather than silently reporting "no moves" — a
+    quiet stall is exactly how a broken read path would look.
+    """
+    digests: dict[str, str] = {}
+    names: list[str] = []
+
+    def absorb(tags) -> None:
+        for tag in tags:
+            digests.setdefault(tag.name, tag.digest)
+            names.append(tag.name)
+
+    # Track tags and their hold markers. `name=standard` also matches `standard_hold`;
+    # `name=latest` additionally drags in a dozen `*_latest_SNAPSHOT` build tags, which
+    # parse as neither releases nor markers and are simply ignored.
+    for track in wanted:
+        absorb(list_tags(repo, name_filter=track))
+
+    month = _month_str(today)
+    for walked in range(_MONTH_WALK_LIMIT):
+        absorb(list_tags(repo, name_filter=month))
+        releases = [r for r in (parse_release(n) for n in names) if r is not None]
+        tainted = tainted_versions(names)
+        if any(age_days(r, today) >= max_days and r.version not in tainted for r in releases):
+            log.info("read registry state in %d filtered call(s)", len(wanted) + walked + 1)
+            return releases, tainted, held_tracks(names), digests
+        month = _prev_month(month)
+    return None
+
+
 def _current_version(track: str, digests: dict[str, str], releases) -> str | None:
     """Which GA version the floating <track> tag currently points at, by digest match."""
     track_digest = digests.get(track)
@@ -40,27 +110,39 @@ def _current_version(track: str, digests: dict[str, str], releases) -> str | Non
 
 
 def cmd_promote(args: argparse.Namespace) -> int:
-    releases, tainted, held, digests = _state(args.repo)
-    tracks = [
-        TrackState("latest", args.latest_days, _current_version("latest", digests, releases)),
-        TrackState("standard", args.standard_days, _current_version("standard", digests, releases)),
-        TrackState("trailing", args.trailing_days, _current_version("trailing", digests, releases)),
-    ]
-
     # Optional subset (e.g. --tracks latest): the release pipeline invokes this
     # engine on-demand to move only `latest` the instant a GA ships, while an
     # operator manually dispatches a full promote to age standard/trailing.
     # One engine, two triggers.
+    #
+    # Resolved BEFORE any registry read, so an unwanted track costs no calls at all.
+    wanted = set(TRACKS)
     if args.tracks:
         wanted = {t.strip() for t in args.tracks.split(",") if t.strip()}
         unknown = wanted - set(TRACKS)
         if unknown:
             log.error("unknown track(s): %s", ", ".join(sorted(unknown)))
             return 2
-        tracks = [t for t in tracks if t.name in wanted]
-        held = held & wanted
+    thresholds = {
+        "latest": args.latest_days,
+        "standard": args.standard_days,
+        "trailing": args.trailing_days,
+    }
+    ordered = [t for t in TRACKS if t in wanted]      # stable, TRACKS order
+    today = dt.date.today()
+
+    state = _promote_state(args.repo, ordered, max(thresholds[t] for t in ordered), today)
+    if state is None:
+        log.info("no promotable release in the filtered window; reading the full tag listing")
+        state = _state(args.repo)
+    releases, tainted, held, digests = state
+    held = held & wanted
+
+    tracks = [
+        TrackState(t, thresholds[t], _current_version(t, digests, releases)) for t in ordered
+    ]
 
-    moves = plan(releases, tainted, held, tracks, today=dt.date.today())
+    moves = plan(releases, tainted, held, tracks, today=today)
 
     # Held tracks are frozen against promotion; instead reconcile the floating
     # <track> tag to its <track>_hold marker digest so a divergence self-heals.

.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/registry.py

@@ -1,10 +1,21 @@
-"""Read side: list tags and digests from the Docker Hub API. Public repos need no auth."""
+"""Read side: list tags and digests from the Docker Hub API.
+
+Reads are authenticated when DOCKER_USERNAME/DOCKER_TOKEN are set. Hub refuses
+ANONYMOUS pagination past offset 1000 ("pagination offset too large for anonymous
+requests; sign in to page further"), and dotcms/dotcms is at ~7.8k tags / 79 pages,
+so an unauthenticated walk 403s on page 11. Auth is therefore required in practice
+for the real repos; it stays optional so tests and small repos need no creds.
+"""
 from __future__ import annotations
 
+import os
 from dataclasses import dataclass
+from functools import lru_cache
 
 import requests
 
+from .executor import hub_login
+
 _HUB = "https://hub.docker.com/v2"
 _TIMEOUT = 30
 
@@ -24,13 +35,58 @@ def _digest_of(result: dict) -> str | None:
     return None
 
 
-def list_tags(repo: str) -> list[Tag]:
-    """All tags in the repo with their manifest digests, following pagination."""
+def _hub_message(resp: requests.Response) -> str:
+    """Hub's own error text, when the body carries one."""
+    try:
+        return (resp.json() or {}).get("message") or ""
+    except ValueError:
+        return ""
+
+
+@lru_cache(maxsize=4)
+def _jwt(username: str, password: str) -> str:
+    """Cached per credential pair: one login per process, not one per request."""
+    return hub_login(username, password)
+
+
+def _auth_headers() -> dict[str, str]:
+    """Hub JWT header, or {} when no creds are in the environment."""
+    username = os.environ.get("DOCKER_USERNAME")
+    password = os.environ.get("DOCKER_TOKEN")
+    if not username or not password:
+        return {}
+    return {"Authorization": f"JWT {_jwt(username, password)}"}
+
+
+def list_tags(repo: str, *, name_filter: str = "") -> list[Tag]:
+    """Tags in the repo with their manifest digests, following pagination.
+
+    `name_filter` applies Hub's SUBSTRING filter (`?name=`), which is what keeps the
+    common path off the ~79-page full listing. It is observed behaviour rather than a
+    documented contract; if Hub ever drops it the filtered queries simply return the
+    whole repo and callers still get a correct — merely slower — answer.
+    """
     namespace, name = repo.split("/", 1)
     url = f"{_HUB}/namespaces/{namespace}/repositories/{name}/tags?page_size=100"
+    if name_filter:
+        url += f"&name={name_filter}"
+    # One login per walk, not per page: the JWT outlives a full 79-page listing.
+    headers = _auth_headers()
     out: list[Tag] = []
     while url:
-        resp = requests.get(url, timeout=_TIMEOUT)
+        resp = requests.get(url, headers=headers, timeout=_TIMEOUT)
+        if resp.status_code == 403 and not headers:
+            # Quote Hub's own reason rather than asserting one. The offset cap is the
+            # only anonymous 403 observed here (a missing repo or namespace 404s), but
+            # the cap keys off the OFFSET alone — even a repo with 3 tags 403s on
+            # page 11 — so "this repo has too many tags" would be an invented cause.
+            # Credentials are the remedy either way.
+            raise RuntimeError(
+                f"Docker Hub refused an anonymous read of {repo} at {url} — "
+                f"{_hub_message(resp) or f'HTTP {resp.status_code}'}. "
+                "Set DOCKER_USERNAME and DOCKER_TOKEN: anonymous requests cannot page "
+                "past offset 1000."
+            )
         resp.raise_for_status()
         body = resp.json()
         for result in body.get("results", []):

.github/actions/core-cicd/evergreen-tracks/tests/conftest.py

@@ -0,0 +1,29 @@
+"""Shared test isolation for the whole suite."""
+from __future__ import annotations
+
+import pytest
+
+from evergreen_tracks.registry import _jwt
+
+
+@pytest.fixture(autouse=True)
+def _no_ambient_hub_creds(monkeypatch):
+    """Neutralise DOCKER_USERNAME / DOCKER_TOKEN inherited from the shell.
+
+    `registry.list_tags` reads them to decide whether to authenticate, so a developer or
+    runner with Docker creds exported would send the anonymous-path tests through a real
+    `hub_login()` POST that `responses` never registered — turning deterministic tests
+    into environment-dependent ones. Tests that WANT auth set the vars themselves.
+
+    In conftest rather than one test module so it covers every module, including ones
+    not written yet: the trap is invisible until someone's shell happens to have creds.
+
+    Also drops the memoised JWT. `_jwt` is deliberately cached — one Hub login per
+    process rather than one per request — but that cache outlives a test, so without
+    this a later test using the same credentials silently reuses an earlier test's
+    token. Verified: two tests with identical creds and different tokens, and the
+    second one saw the first one's JWT.
+    """
+    monkeypatch.delenv("DOCKER_USERNAME", raising=False)
+    monkeypatch.delenv("DOCKER_TOKEN", raising=False)
+    _jwt.cache_clear()