From 48891dd56a1649311493de1942023855910fdab7 Mon Sep 17 00:00:00 2001 From: Stephen Freudenthaler Date: Tue, 11 Aug 2026 18:50:18 -0400 Subject: [PATCH 1/2] perf(evergreen-tracks): plan promotions from filtered reads, not a full tag scan Moving one floating tag paged the entire repo: 7,832 tags, page_size capped at 100 by Hub, so 79 calls and ~45s against a budget of 180 requests/60s per IP. The promoter has no 429 handling, so one rate-limited page is a hard failure and a Slack page. Hub's `name=` substring filter narrows it to what promotion actually needs: * one call per requested track -> the `` digest, and its `_hold` marker for free (both match the track name as a substring) * one call per month of history -> that month's GA releases AND their `_tainted` markers, which share the version's month prefix A month is one page (6-28 tags). Measured against dotcms/dotcms: the daily standard+trailing promote is 4 calls, `--tracks latest` is 2, all three is 5 -- down from 79, and the plan is byte-identical to the full scan's (`standard -> 26.07.28-01`, same digest). `--tracks` is now resolved BEFORE any registry read, so the release pipeline's latest-only promote no longer pays to resolve standard and trailing. Correctness rests on fetching months newest-first and CONTIGUOUSLY. planner's forward-only guard only bites when a track sits on something NEWER than the newest eligible release, and any such version is in a month at or above the one the walk stopped on -- already read. A track on something older resolves to "move forward", the same decision the full listing makes. A mutation that starts the walk at the threshold cutoff instead of today (skipping newer months) is caught by test_promote_narrow_reads_still_refuse_to_move_a_track_backwards. Falls back to the full listing when the walk window holds nothing promotable, so a broken read path surfaces as the expensive right answer instead of a quiet "no moves" stall. `admin` keeps the full scan: rare, human-triggered, and it addresses arbitrarily old versions. Closes: #37028 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PqQqV1iv96Db8AzBSWTXV4 --- .../src/evergreen_tracks/cli.py | 106 +++++++++++-- .../src/evergreen_tracks/registry.py | 21 ++- .../evergreen-tracks/tests/test_cli.py | 145 ++++++++++++++++++ 3 files changed, 257 insertions(+), 15 deletions(-) diff --git a/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/cli.py b/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/cli.py index cfc8f47fd56e..3c2d8d78c5bd 100644 --- a/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/cli.py +++ b/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/cli.py @@ -10,7 +10,7 @@ import os import sys -from .calver import newest, parse_release +from .calver import age_days, newest, parse_release from .executor import delete_tag, hub_login, point_tag from .markers import TRACKS, held_tracks, hold_tag, tainted_versions, taint_tag from .planner import TrackState, plan @@ -18,9 +18,18 @@ log = logging.getLogger("evergreen_tracks") +# How many months the narrow read path walks back before giving up and full-scanning. +# Releases ship weekly, so promotion needs one or two months; the extra headroom is for +# a quiet stretch or a track parked on something old by a hold. +_MONTH_WALK_LIMIT = 6 + def _state(repo: str): - """Return (releases, tainted, held, name->digest) read from the registry.""" + """Return (releases, tainted, held, name->digest) read from the registry. + + The full ~79-page listing. Still the right tool for `admin` (rare, human-triggered, + and it addresses arbitrarily old versions) and the terminal fallback for promote. + """ tags = list_tags(repo) names = [t.name for t in tags] digests = {t.name: t.digest for t in tags} @@ -28,6 +37,67 @@ def _state(repo: str): return releases, tainted_versions(names), held_tracks(names), digests +def _month_str(date: dt.date) -> str: + return f"{date.year % 100:02d}.{date.month:02d}" + + +def _prev_month(ym: str) -> str: + yy, mm = int(ym[:2]), int(ym[3:5]) + return f"{yy - 1:02d}.12" if mm == 1 else f"{yy:02d}.{mm - 1:02d}" + + +def _promote_state(repo: str, wanted: list[str], max_days: int, today: dt.date): + """`_state`, assembled from a handful of filtered reads instead of the full listing. + + Moving a floating tag needs a few GA CalVer tags and three marker names, but the + full listing pages the whole repo — 7.8k tags, 79 calls, ~45s, against a Hub budget + of 180 requests/60s per IP. Hub's `name=` substring filter turns that into: + + * one call per requested track -> the `` tag's digest, plus its + `_hold` marker if one exists (both match the track name as a substring) + * one call per month of history -> that month's GA releases AND their + `_tainted` markers, which share the version's month prefix + + A month is one page (6-28 tags), so the daily standard+trailing promote is 3-4 calls. + + Months are fetched newest-first and CONTIGUOUSLY, which is what makes the narrowed + pool safe for `planner.plan`'s forward-only guard. That guard only matters when a + track already sits on something NEWER than the newest eligible release — and any + such version lives in a month at or above the one the walk stopped on, so it has + necessarily already been read. A track on something OLDER resolves to "move + forward", which is the same decision the full listing produces. Nothing here can + move a track backwards that the full scan would have held. + + Returns None if the walk window turns up nothing promotable at all, so the caller + can fall back to the full listing rather than silently reporting "no moves" — a + quiet stall is exactly how a broken read path would look. + """ + digests: dict[str, str] = {} + names: list[str] = [] + + def absorb(tags) -> None: + for tag in tags: + digests.setdefault(tag.name, tag.digest) + names.append(tag.name) + + # Track tags and their hold markers. `name=standard` also matches `standard_hold`; + # `name=latest` additionally drags in a dozen `*_latest_SNAPSHOT` build tags, which + # parse as neither releases nor markers and are simply ignored. + for track in wanted: + absorb(list_tags(repo, name_filter=track)) + + month = _month_str(today) + for walked in range(_MONTH_WALK_LIMIT): + absorb(list_tags(repo, name_filter=month)) + releases = [r for r in (parse_release(n) for n in names) if r is not None] + tainted = tainted_versions(names) + if any(age_days(r, today) >= max_days and r.version not in tainted for r in releases): + log.info("read registry state in %d filtered call(s)", len(wanted) + walked + 1) + return releases, tainted, held_tracks(names), digests + month = _prev_month(month) + return None + + def _current_version(track: str, digests: dict[str, str], releases) -> str | None: """Which GA version the floating tag currently points at, by digest match.""" track_digest = digests.get(track) @@ -40,27 +110,39 @@ def _current_version(track: str, digests: dict[str, str], releases) -> str | Non def cmd_promote(args: argparse.Namespace) -> int: - releases, tainted, held, digests = _state(args.repo) - tracks = [ - TrackState("latest", args.latest_days, _current_version("latest", digests, releases)), - TrackState("standard", args.standard_days, _current_version("standard", digests, releases)), - TrackState("trailing", args.trailing_days, _current_version("trailing", digests, releases)), - ] - # Optional subset (e.g. --tracks latest): the release pipeline invokes this # engine on-demand to move only `latest` the instant a GA ships, while an # operator manually dispatches a full promote to age standard/trailing. # One engine, two triggers. + # + # Resolved BEFORE any registry read, so an unwanted track costs no calls at all. + wanted = set(TRACKS) if args.tracks: wanted = {t.strip() for t in args.tracks.split(",") if t.strip()} unknown = wanted - set(TRACKS) if unknown: log.error("unknown track(s): %s", ", ".join(sorted(unknown))) return 2 - tracks = [t for t in tracks if t.name in wanted] - held = held & wanted + thresholds = { + "latest": args.latest_days, + "standard": args.standard_days, + "trailing": args.trailing_days, + } + ordered = [t for t in TRACKS if t in wanted] # stable, TRACKS order + today = dt.date.today() + + state = _promote_state(args.repo, ordered, max(thresholds[t] for t in ordered), today) + if state is None: + log.info("no promotable release in the filtered window; reading the full tag listing") + state = _state(args.repo) + releases, tainted, held, digests = state + held = held & wanted + + tracks = [ + TrackState(t, thresholds[t], _current_version(t, digests, releases)) for t in ordered + ] - moves = plan(releases, tainted, held, tracks, today=dt.date.today()) + moves = plan(releases, tainted, held, tracks, today=today) # Held tracks are frozen against promotion; instead reconcile the floating # tag to its _hold marker digest so a divergence self-heals. diff --git a/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/registry.py b/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/registry.py index a02156ed5b55..a6b805c50ce7 100644 --- a/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/registry.py +++ b/.github/actions/core-cicd/evergreen-tracks/src/evergreen_tracks/registry.py @@ -10,6 +10,7 @@ import os from dataclasses import dataclass +from functools import lru_cache import requests @@ -42,19 +43,33 @@ def _hub_message(resp: requests.Response) -> str: return "" +@lru_cache(maxsize=4) +def _jwt(username: str, password: str) -> str: + """Cached per credential pair: one login per process, not one per request.""" + return hub_login(username, password) + + def _auth_headers() -> dict[str, str]: """Hub JWT header, or {} when no creds are in the environment.""" username = os.environ.get("DOCKER_USERNAME") password = os.environ.get("DOCKER_TOKEN") if not username or not password: return {} - return {"Authorization": f"JWT {hub_login(username, password)}"} + return {"Authorization": f"JWT {_jwt(username, password)}"} + +def list_tags(repo: str, *, name_filter: str = "") -> list[Tag]: + """Tags in the repo with their manifest digests, following pagination. -def list_tags(repo: str) -> list[Tag]: - """All tags in the repo with their manifest digests, following pagination.""" + `name_filter` applies Hub's SUBSTRING filter (`?name=`), which is what keeps the + common path off the ~79-page full listing. It is observed behaviour rather than a + documented contract; if Hub ever drops it the filtered queries simply return the + whole repo and callers still get a correct — merely slower — answer. + """ namespace, name = repo.split("/", 1) url = f"{_HUB}/namespaces/{namespace}/repositories/{name}/tags?page_size=100" + if name_filter: + url += f"&name={name_filter}" # One login per walk, not per page: the JWT outlives a full 79-page listing. headers = _auth_headers() out: list[Tag] = [] diff --git a/.github/actions/core-cicd/evergreen-tracks/tests/test_cli.py b/.github/actions/core-cicd/evergreen-tracks/tests/test_cli.py index f6e0be381a26..122529f0eb86 100644 --- a/.github/actions/core-cicd/evergreen-tracks/tests/test_cli.py +++ b/.github/actions/core-cicd/evergreen-tracks/tests/test_cli.py @@ -601,3 +601,148 @@ def test_promote_tracks_unknown_returns_2(mock_list_tags, mock_point_tag): rc = cmd_promote(args) assert rc == 2 mock_point_tag.assert_not_called() + + +# --------------------------------------------------------------------------- +# Narrow (filtered) registry reads — see cli._promote_state +# --------------------------------------------------------------------------- + +def _fake_registry(tags): + """Stand-in for registry.list_tags that honours Hub's `name=` SUBSTRING filter. + + An empty filter returns everything, which is exactly what the real full listing + does — so `"" in calls` is the assertion for "fell back to the 79-page scan". + """ + calls: list[str] = [] + + def fake(repo, *, name_filter=""): + calls.append(name_filter) + return [t for t in tags if name_filter in t.name] + + return fake, calls + + +def _registry_tags(standard_digest="sha256:r0705", trailing_digest="sha256:r0602"): + """Four GA releases plus two floating track tags pointing into that history.""" + return [ + Tag("26.06.02-01", "sha256:r0602"), + Tag("26.07.05-01", "sha256:r0705"), + Tag("26.07.28-01", "sha256:r0728"), + Tag("26.08.10-01", "sha256:r0810"), + Tag("standard", standard_digest), + Tag("trailing", trailing_digest), + ] + + +@patch("evergreen_tracks.cli.point_tag") +@patch("evergreen_tracks.cli.list_tags") +def test_promote_uses_filtered_reads_not_the_full_listing(mock_list_tags, mock_point_tag): + """The daily promote must plan from a handful of filtered calls, never the full scan. + + This is the whole point of the narrow path: the full listing is 79 calls against a + 180-req/60s per-IP Hub budget. + """ + fake, calls = _fake_registry(_registry_tags()) + mock_list_tags.side_effect = fake + args = build_parser().parse_args( + ["promote", "--repo", "dotcms/dotcms-test", "--tracks", "standard,trailing", + "--standard-days", "14", "--trailing-days", "28"] + ) + with patch("evergreen_tracks.cli.dt") as mock_dt: + mock_dt.date.today.return_value = dt.date(2026, 8, 11) + rc = cmd_promote(args) + + assert rc == 0 + assert "" not in calls, f"fell back to the unfiltered full listing: {calls}" + assert len(calls) <= 6, f"expected a handful of filtered calls, got {calls}" + # Newest release ≥14d is 26.07.28-01; newest ≥28d is 26.07.05-01. + moved = {c.args[1]: c.args[2] for c in mock_point_tag.call_args_list} + assert moved == {"standard": "sha256:r0728", "trailing": "sha256:r0705"} + + +@patch("evergreen_tracks.cli.point_tag") +@patch("evergreen_tracks.cli.list_tags") +def test_promote_walks_back_months_for_an_eligible_release(mock_list_tags, mock_point_tag): + """The threshold cutoff can sit in an earlier month than today's, so walk 1 must + keep fetching months until a candidate actually clears the bar.""" + fake, calls = _fake_registry(_registry_tags()) + mock_list_tags.side_effect = fake + args = build_parser().parse_args( + ["promote", "--repo", "dotcms/dotcms-test", "--tracks", "trailing", + "--trailing-days", "28"] + ) + with patch("evergreen_tracks.cli.dt") as mock_dt: + mock_dt.date.today.return_value = dt.date(2026, 8, 11) + cmd_promote(args) + + # 26.08 holds nothing ≥28 days old, so it must have walked back to 26.07. + assert "26.08" in calls and "26.07" in calls + assert "" not in calls + + +@patch("evergreen_tracks.cli.point_tag") +@patch("evergreen_tracks.cli.list_tags") +def test_promote_narrow_reads_still_refuse_to_move_a_track_backwards( + mock_list_tags, mock_point_tag +): + """Forward-only must survive the narrowed pool. + + `standard` sits on 26.08.10-01, newer than anything 14 days old. If the filtered + reads failed to name the current version, `plan` would skip its forward-only guard + and demote the track to 26.07.28-01. Nothing may move here. + """ + fake, calls = _fake_registry(_registry_tags(standard_digest="sha256:r0810")) + mock_list_tags.side_effect = fake + args = build_parser().parse_args( + ["promote", "--repo", "dotcms/dotcms-test", "--tracks", "standard", + "--standard-days", "14"] + ) + with patch("evergreen_tracks.cli.dt") as mock_dt: + mock_dt.date.today.return_value = dt.date(2026, 8, 11) + rc = cmd_promote(args) + + assert rc == 0 + assert "" not in calls + mock_point_tag.assert_not_called() + + +@patch("evergreen_tracks.cli.point_tag") +@patch("evergreen_tracks.cli.list_tags") +def test_promote_falls_back_to_full_listing_when_the_window_is_empty( + mock_list_tags, mock_point_tag +): + """Nothing promotable in the walked months must fall back to the full listing. + + "No moves" is indistinguishable from a broken read path, and a quiet stall is how + tracks would silently stop advancing — so the expensive answer is worth having here. + """ + fake, calls = _fake_registry(_registry_tags()) + mock_list_tags.side_effect = fake + args = build_parser().parse_args( + ["promote", "--repo", "dotcms/dotcms-test", "--tracks", "standard", + "--standard-days", "14"] + ) + # Far enough ahead that the 6-month walk window holds no release at all. + with patch("evergreen_tracks.cli.dt") as mock_dt: + mock_dt.date.today.return_value = dt.date(2027, 6, 11) + rc = cmd_promote(args) + + assert rc == 0 + assert "" in calls, f"expected the terminal full-listing fallback, got {calls}" + + +@patch("evergreen_tracks.cli.point_tag") +@patch("evergreen_tracks.cli.list_tags") +def test_promote_scoped_to_latest_never_reads_other_tracks(mock_list_tags, mock_point_tag): + """--tracks latest (the release pipeline's path) must not spend calls on the + tracks it was told to leave alone.""" + fake, calls = _fake_registry(_registry_tags()) + mock_list_tags.side_effect = fake + args = build_parser().parse_args( + ["promote", "--repo", "dotcms/dotcms-test", "--tracks", "latest", "--latest-days", "0"] + ) + with patch("evergreen_tracks.cli.dt") as mock_dt: + mock_dt.date.today.return_value = dt.date(2026, 8, 11) + cmd_promote(args) + + assert "standard" not in calls and "trailing" not in calls, calls From 11ab662148e377de49c0259d21ddc771299dc770 Mon Sep 17 00:00:00 2001 From: Stephen Freudenthaler Date: Wed, 12 Aug 2026 12:05:08 -0400 Subject: [PATCH 2/2] test(evergreen-tracks): clear the memoised JWT between tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_jwt`'s lru_cache is process-global, so it outlives a test. Proven with two tests using identical credentials and different tokens: the second saw the first's JWT. Harmless today (only one test authenticates) but a trap for the next auth test, and the same class of non-determinism as the ambient-creds issue already fixed in conftest — so it belongs in the same fixture. The cache itself stays: in production the CLI is one-shot, so one login per process is exactly right, and it is what keeps a full-listing fallback from re-authenticating on all 79 pages. Closes: #37028 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PqQqV1iv96Db8AzBSWTXV4 --- .../actions/core-cicd/evergreen-tracks/tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/core-cicd/evergreen-tracks/tests/conftest.py b/.github/actions/core-cicd/evergreen-tracks/tests/conftest.py index 9043ce44b710..5cf0881c81c1 100644 --- a/.github/actions/core-cicd/evergreen-tracks/tests/conftest.py +++ b/.github/actions/core-cicd/evergreen-tracks/tests/conftest.py @@ -3,6 +3,8 @@ import pytest +from evergreen_tracks.registry import _jwt + @pytest.fixture(autouse=True) def _no_ambient_hub_creds(monkeypatch): @@ -15,6 +17,13 @@ def _no_ambient_hub_creds(monkeypatch): In conftest rather than one test module so it covers every module, including ones not written yet: the trap is invisible until someone's shell happens to have creds. + + Also drops the memoised JWT. `_jwt` is deliberately cached — one Hub login per + process rather than one per request — but that cache outlives a test, so without + this a later test using the same credentials silently reuses an earlier test's + token. Verified: two tests with identical creds and different tokens, and the + second one saw the first one's JWT. """ monkeypatch.delenv("DOCKER_USERNAME", raising=False) monkeypatch.delenv("DOCKER_TOKEN", raising=False) + _jwt.cache_clear()