From 5980f34d911faf26cd8db617d3d75e62f39b42bf Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 11:55:49 +0700 Subject: [PATCH 01/34] fix(release): enforce Debian 13 image boundary Signed-off-by: Jeremi Joslin --- .github/workflows/ci.yml | 6 + .../scripts/check-registryctl-tutorials.sh | 2 +- release/scripts/check-debian13-images.py | 195 ++++++++++++++++-- release/scripts/check-gates-inventory.py | 8 + release/scripts/test_check_debian13_images.py | 162 +++++++++++++++ release/scripts/test_check_gates_inventory.py | 17 ++ release/scripts/test_registry_release.py | 42 ---- 7 files changed, 377 insertions(+), 55 deletions(-) create mode 100644 release/scripts/test_check_debian13_images.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6991b07f..c88c9d65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,9 @@ jobs: echo "registryctl_tutorial=${registryctl_tutorial}" } >> "${GITHUB_OUTPUT}" + - name: Check Debian 13 image contract + run: python3 release/scripts/check-debian13-images.py + secrets: name: Secret scan runs-on: ubuntu-24.04 @@ -507,6 +510,9 @@ jobs: - name: Test release helper run: python3 -m unittest release/scripts/test_registry_release.py + - name: Test Debian 13 image contract checker + run: python3 -m unittest release/scripts/test_check_debian13_images.py + - name: Test release planning commands run: python3 -m unittest release/scripts/test_registry_release_plans.py diff --git a/docs/site/scripts/check-registryctl-tutorials.sh b/docs/site/scripts/check-registryctl-tutorials.sh index 77e70304..de8c556b 100644 --- a/docs/site/scripts/check-registryctl-tutorials.sh +++ b/docs/site/scripts/check-registryctl-tutorials.sh @@ -15,7 +15,7 @@ REPO_ROOT="$(cd "$SITE_ROOT/../.." && pwd)" HELPER="$SITE_ROOT/scripts/registryctl-tutorial.mjs" RELAY_TUTORIAL="$SITE_ROOT/src/content/docs/tutorials/publish-spreadsheet-secured-registry-api.mdx" NOTARY_TUTORIAL="$SITE_ROOT/src/content/docs/tutorials/verify-claim-registry-api.mdx" -BUILDER_IMAGE="rust:1.95-bookworm@sha256:4c2fd73ef19c5ef9d54bee03b06b2839a392604fbfcd578ed948b71b37c1d7fb" +BUILDER_IMAGE="rust:1.95-trixie@sha256:f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3" LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64" CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home" NATIVE_TARGET="$REPO_ROOT/target/registryctl-tutorial-native" diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index b6aac87c..35bfc9da 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import re import sys from pathlib import Path @@ -27,7 +28,7 @@ "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) -DOCKERFILES = ( +PRODUCT_DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), Path("crates/registry-relay/Dockerfile.demo"), Path("products/notary/Dockerfile"), @@ -35,9 +36,10 @@ Path("release/docker/Dockerfile.registry-relay"), ) -# These are the maintained image and image-policy surfaces. Historical release -# notes are immutable evidence and intentionally are not rewritten by this gate. -MAINTAINED_TEXT_PATHS = DOCKERFILES + ( +# Product-specific assertions below require these surfaces. The Debian +# generation and immutable-pin boundary is broader and is discovered from the +# repository instead of relying on this tuple. +REQUIRED_PRODUCT_SURFACES = PRODUCT_DOCKERFILES + ( Path(".github/workflows/release.yml"), Path("release/scripts/build-release-binaries.sh"), Path("crates/registry-relay/docs/ops.md"), @@ -47,8 +49,24 @@ Path("products/notary/docs/security-assurance.md"), ) -RUST_BUILDER_DOCKERFILES = DOCKERFILES[:3] -PREPARATION_DOCKERFILES = DOCKERFILES[3:] +EXCLUDED_DIRECTORY_NAMES = { + ".git", + ".repo-docs-cache", + ".research", + ".venv", + "__pycache__", + "dist", + "node_modules", + "target", +} +# Historical release notes and .research preserve observations, not active policy. +EXCLUDED_PATH_PREFIXES = (Path("release/notes"),) +MARKDOWN_SUFFIXES = {".md", ".mdx"} +SCRIPT_SUFFIXES = {".bash", ".js", ".mjs", ".py", ".sh", ".ts"} +YAML_SUFFIXES = {".yaml", ".yml"} + +RUST_BUILDER_DOCKERFILES = PRODUCT_DOCKERFILES[:3] +PREPARATION_DOCKERFILES = PRODUCT_DOCKERFILES[3:] RELAY_DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), Path("crates/registry-relay/Dockerfile.demo"), @@ -59,8 +77,121 @@ Path("release/docker/Dockerfile.registry-notary"), ) -FROM_RE = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)", re.MULTILINE) +FROM_RE = re.compile( + r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", + re.IGNORECASE | re.MULTILINE, +) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") +CONTAINER_REFERENCE_RE = re.compile( + r"(?" + r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*" + r"[A-Za-z0-9._-]+:[A-Za-z0-9._-]+" + r"(?:@sha256:[0-9a-f]{64})?" + r")" + r"(?![A-Za-z0-9._/@+-])", + re.IGNORECASE, +) +SCRIPT_IMAGE_CONTEXT_RE = re.compile( + r"\b(?:docker|podman|container|image)\b" + r"|\b[A-Za-z_][A-Za-z0-9_]*(?:image|base)[A-Za-z0-9_]*\s*=", + re.IGNORECASE, +) + + +def is_excluded(relative: Path) -> bool: + if ".research" in relative.parts: + return True + return any( + relative == prefix or prefix in relative.parents + for prefix in EXCLUDED_PATH_PREFIXES + ) + + +def is_dockerfile(relative: Path) -> bool: + return ( + relative.name == "Dockerfile" + or relative.name.startswith("Dockerfile.") + or relative.name.endswith(".Dockerfile") + ) + + +def is_active_script(relative: Path) -> bool: + return relative.suffix in {".bash", ".sh"} or ( + "scripts" in relative.parts + and (relative.suffix in SCRIPT_SUFFIXES or not relative.suffix) + ) + + +def is_workflow(relative: Path) -> bool: + return relative.parts[:2] == (".github", "workflows") + + +def is_maintained_text_surface(relative: Path) -> bool: + return ( + is_dockerfile(relative) + or is_active_script(relative) + or is_workflow(relative) + or relative.suffix in MARKDOWN_SUFFIXES + or relative.suffix in YAML_SUFFIXES + ) + + +def is_image_reference_surface(relative: Path) -> bool: + return ( + is_dockerfile(relative) + or is_active_script(relative) + or relative.suffix in YAML_SUFFIXES + ) + + +def discover_maintained_surfaces(root: Path) -> tuple[Path, ...]: + """Discover maintained image policy surfaces without walking build output.""" + + discovered: set[Path] = set() + for directory, directory_names, file_names in os.walk(root): + directory_path = Path(directory) + directory_names[:] = sorted( + name + for name in directory_names + if name not in EXCLUDED_DIRECTORY_NAMES + and not is_excluded((directory_path / name).relative_to(root)) + ) + for name in file_names: + relative = (directory_path / name).relative_to(root) + if is_excluded(relative) or not is_maintained_text_surface(relative): + continue + discovered.add(relative) + return tuple(sorted(discovered)) + + +def is_debian_derived(reference: str) -> bool: + unpinned = reference.split("@", 1)[0] + repository, tag = unpinned.rsplit(":", 1) + image_name = repository.rsplit("/", 1)[-1].casefold() + lowered_tag = tag.casefold() + generation_tags = ("trixie", "book" + "worm", "bullseye", "buster") + return ( + image_name == "debian" + or "debian" in image_name + or any(marker in lowered_tag for marker in generation_tags) + or re.search(r"(?:^|[-_.])debian-?1[0-9](?:$|[-_.])", lowered_tag) + is not None + ) + + +def script_reference_is_consumed( + relative: Path, + text: str, + match: re.Match[str], +) -> bool: + if relative.suffix not in {".js", ".mjs", ".py", ".ts"}: + return True + line_start = text.rfind("\n", 0, match.start()) + 1 + line_end = text.find("\n", match.end()) + if line_end < 0: + line_end = len(text) + return SCRIPT_IMAGE_CONTEXT_RE.search(text[line_start:line_end]) is not None def read(root: Path, relative: Path, failures: list[str]) -> str: @@ -70,6 +201,9 @@ def read(root: Path, relative: Path, failures: list[str]) -> str: except FileNotFoundError: failures.append(f"missing maintained image surface: {relative}") return "" + except UnicodeDecodeError: + failures.append(f"maintained image surface is not UTF-8 text: {relative}") + return "" def require( @@ -91,13 +225,16 @@ def runtime_stage(text: str) -> str: def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] + maintained_paths = discover_maintained_surfaces(root) + all_paths = tuple(sorted(set(maintained_paths) | set(REQUIRED_PRODUCT_SURFACES))) texts = { relative: read(root, relative, failures) - for relative in MAINTAINED_TEXT_PATHS + for relative in all_paths } retired_markers = ("book" + "worm", "debian" + "12") - for relative, text in texts.items(): + for relative in maintained_paths: + text = texts[relative] lowered = text.casefold() for marker in retired_markers: if marker in lowered: @@ -105,18 +242,52 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{relative}: retired Debian image generation marker remains: {marker}" ) - for relative in DOCKERFILES: + dockerfiles = tuple( + relative for relative in maintained_paths if is_dockerfile(relative) + ) + if not dockerfiles: + failures.append("no maintained Dockerfiles discovered") + + for relative in dockerfiles: text = texts[relative] bases = FROM_RE.findall(text) if not bases: failures.append(f"{relative}: no FROM instruction found") continue - for base in bases: - if not DIGEST_PIN_RE.search(base): + stage_names: set[str] = set() + for base, stage_name in bases: + internal_stage = base.casefold() in stage_names + if ( + base.casefold() != "scratch" + and not internal_stage + and not DIGEST_PIN_RE.search(base) + ): failures.append( f"{relative}: upstream base is not pinned by immutable digest: {base}" ) + if stage_name: + stage_names.add(stage_name.casefold()) + + for relative in maintained_paths: + if not is_image_reference_surface(relative): + continue + text = texts[relative] + for match in CONTAINER_REFERENCE_RE.finditer(text): + reference = match.group("reference") + if ( + not script_reference_is_consumed(relative, text, match) + or not is_debian_derived(reference) + or DIGEST_PIN_RE.search(reference) + ): + continue + line = text.count("\n", 0, match.start()) + 1 + failures.append( + f"{relative}:{line}: Debian-derived image reference is not pinned " + f"by immutable digest: {reference}" + ) + for relative in PRODUCT_DOCKERFILES: + text = texts[relative] require( text, f"FROM {DISTROLESS_RUNTIME} AS runtime", diff --git a/release/scripts/check-gates-inventory.py b/release/scripts/check-gates-inventory.py index 3d4837d6..2b2d6ed8 100644 --- a/release/scripts/check-gates-inventory.py +++ b/release/scripts/check-gates-inventory.py @@ -32,6 +32,10 @@ "Other workflow path classification", ".github/workflows/*)\n mark_all\n ;;", ), + ( + "Debian 13 image contract", + "run: python3 release/scripts/check-debian13-images.py", + ), ("Cargo metadata", "run: cargo metadata --locked --format-version 1"), ( "Manifest profile validation", @@ -112,6 +116,10 @@ ("Relay OpenAPI command", "run: just openapi-contract"), ("Relay exposure check", "name: Relay exposure check"), ("Release helper tests", "run: python3 -m unittest release/scripts/test_registry_release.py"), + ( + "Debian 13 image contract checker tests", + "run: python3 -m unittest release/scripts/test_check_debian13_images.py", + ), ( "Release planning command tests", "run: python3 -m unittest release/scripts/test_registry_release_plans.py", diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py new file mode 100644 index 00000000..88868bad --- /dev/null +++ b/release/scripts/test_check_debian13_images.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import shutil +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "release/scripts/check-debian13-images.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("check_debian13_images", SCRIPT) + if spec is None or spec.loader is None: + raise ImportError(f"could not load module spec from {SCRIPT}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Debian13ImageCheckTest(unittest.TestCase): + def setUp(self) -> None: + self.module = load_module() + + def copy_required_surfaces(self, root: Path) -> None: + for relative in self.module.REQUIRED_PRODUCT_SURFACES: + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / relative, destination) + + def test_real_repository_follows_debian13_contract(self) -> None: + self.assertEqual([], self.module.check_repository(ROOT)) + + def test_discovery_covers_dockerfiles_and_active_scripts(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dockerfile = Path("Dockerfile.worker") + script = Path("docs/site/scripts/build-example.sh") + note = Path("release/notes/v0.1.0.md") + research = Path("docs/site/.research/container-notes.md") + for relative in (dockerfile, script, note, research): + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("fixture\n", encoding="utf-8") + + discovered = self.module.discover_maintained_surfaces(root) + + self.assertIn(dockerfile, discovered) + self.assertIn(script, discovered) + self.assertNotIn(note, discovered) + self.assertNotIn(research, discovered) + + def test_discovered_dockerfile_rejects_retired_unpinned_base(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + dockerfile = root / "products/example/Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text( + "FROM rust:1.95-" + "book" + "worm\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "retired Debian image generation marker" in failure + for failure in failures + ), + failures, + ) + self.assertTrue( + any( + "upstream base is not pinned by immutable digest" in failure + for failure in failures + ), + failures, + ) + + def test_discovered_script_rejects_unpinned_debian_derived_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker run --rm rust:1.95-" + "trixie cargo build --locked\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "Debian-derived image reference is not pinned by immutable digest" + in failure + and "docs/site/scripts/build-example.sh:2" in failure + for failure in failures + ), + failures, + ) + + def test_dockerfile_internal_stage_reference_needs_no_digest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + dockerfile = root / "products/example/Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text( + f"FROM {self.module.RUST_BUILDER} AS builder\n" + "FROM builder AS runtime\n", + encoding="utf-8", + ) + + self.assertEqual([], self.module.check_repository(root)) + + def test_discovered_python_script_rejects_unpinned_builder_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "products/example/scripts/build_image.py" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + 'BUILDER_IMAGE = "rust:1.95-' + 'trixie"\n', + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "Debian-derived image reference is not pinned by immutable digest" + in failure + and "products/example/scripts/build_image.py:1" in failure + for failure in failures + ), + failures, + ) + + def test_history_and_research_are_outside_the_maintained_boundary(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + retired_reference = "Debian " + "12 and Book" + "worm\n" + for relative in ( + Path("release/notes/v0.1.0.md"), + Path("docs/site/.research/container-notes.md"), + ): + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(retired_reference, encoding="utf-8") + + self.assertEqual([], self.module.check_repository(root)) + + +if __name__ == "__main__": + unittest.main() diff --git a/release/scripts/test_check_gates_inventory.py b/release/scripts/test_check_gates_inventory.py index a050580f..a82e3e20 100644 --- a/release/scripts/test_check_gates_inventory.py +++ b/release/scripts/test_check_gates_inventory.py @@ -146,6 +146,23 @@ def test_missing_release_planning_command_tests_are_reported(self) -> None: ) self.assertIn("Release planning command tests", self.module.missing_gates(text)) + def test_missing_debian13_image_contract_is_reported(self) -> None: + text = self.workflow.replace( + "run: python3 release/scripts/check-debian13-images.py", + "run: python3 release/scripts/skip-debian13-images.py", + ) + self.assertIn("Debian 13 image contract", self.module.missing_gates(text)) + + def test_missing_debian13_checker_tests_are_reported(self) -> None: + text = self.workflow.replace( + "run: python3 -m unittest release/scripts/test_check_debian13_images.py", + "run: true", + ) + self.assertIn( + "Debian 13 image contract checker tests", + self.module.missing_gates(text), + ) + def test_missing_release_image_oci_checker_tests_are_reported(self) -> None: text = self.workflow.replace( "run: python3 -m unittest release/scripts/test_check_release_image_oci_labels.py", diff --git a/release/scripts/test_registry_release.py b/release/scripts/test_registry_release.py index 19e0ffab..8e12d44d 100755 --- a/release/scripts/test_registry_release.py +++ b/release/scripts/test_registry_release.py @@ -19,49 +19,7 @@ IMAGE_DIGEST_REF = f"ghcr.io/registrystack/registry-notary@{IMAGE_DIGEST}" -def load_debian13_image_check(): - path = ROOT / "release/scripts/check-debian13-images.py" - spec = importlib.util.spec_from_file_location("check_debian13_images", path) - if spec is None or spec.loader is None: - raise ImportError(f"could not load module spec from {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - class RegistryReleaseTest(unittest.TestCase): - def test_maintained_images_follow_debian13_contract(self) -> None: - module = load_debian13_image_check() - self.assertEqual([], module.check_repository(ROOT)) - - def test_debian13_contract_rejects_retired_base_and_unpinned_base(self) -> None: - module = load_debian13_image_check() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - for relative in module.MAINTAINED_TEXT_PATHS: - destination = root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text( - (ROOT / relative).read_text(encoding="utf-8"), - encoding="utf-8", - ) - - relay_dockerfile = root / "crates/registry-relay/Dockerfile" - text = relay_dockerfile.read_text(encoding="utf-8") - text = text.replace( - module.RUST_BUILDER, - "rust:1.95-" + "book" + "worm", - 1, - ) - relay_dockerfile.write_text(text, encoding="utf-8") - - failures = module.check_repository(root) - self.assertTrue( - any("retired Debian image generation marker" in failure for failure in failures) - ) - self.assertTrue( - any("not pinned by immutable digest" in failure for failure in failures) - ) def test_contributing_documents_major_functionality_test_policy(self) -> None: text = (ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") From a9e8aad3fe978fe74ca2421694d78f996de7fc1a Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 12:17:08 +0700 Subject: [PATCH 02/34] fix(release): reject untagged Debian image refs Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 131 ++++++++++++++++++ release/scripts/test_check_debian13_images.py | 71 ++++++++++ 2 files changed, 202 insertions(+) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 35bfc9da..c7d4904d 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -5,6 +5,7 @@ import os import re +import shlex import sys from pathlib import Path @@ -97,6 +98,40 @@ r"|\b[A-Za-z_][A-Za-z0-9_]*(?:image|base)[A-Za-z0-9_]*\s*=", re.IGNORECASE, ) +IMAGE_ASSIGNMENT_RE = re.compile( + r"^\s*(?:export\s+)?" + r"(?P[A-Za-z_][A-Za-z0-9_-]*)\s*(?:=|:)\s*" + r"(?P[^#\s]+)", +) +UNTAGGED_IMAGE_REFERENCE_RE = re.compile( + r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+", +) +DEBIAN_DEFAULT_IMAGE_NAMES = { + "buildpack-deps", + "debian", + "golang", + "node", + "python", + "rust", +} +CONTAINER_COMMANDS = {"create", "pull", "run"} +CONTAINER_BOOLEAN_OPTIONS = { + "--detach", + "--init", + "--interactive", + "--privileged", + "--publish-all", + "--quiet", + "--read-only", + "--rm", + "--tty", + "-d", + "-i", + "-it", + "-P", + "-q", + "-t", +} def is_excluded(relative: Path) -> bool: @@ -180,6 +215,97 @@ def is_debian_derived(reference: str) -> bool: ) +def is_untagged_debian_derived(reference: str) -> bool: + if UNTAGGED_IMAGE_REFERENCE_RE.fullmatch(reference) is None: + return False + image_name = reference.rsplit("/", 1)[-1].casefold() + return "debian" in image_name or image_name in DEBIAN_DEFAULT_IMAGE_NAMES + + +def is_image_assignment(name: str) -> bool: + tokens = re.split(r"[_-]+", name.casefold()) + return "image" in tokens or ("base" in tokens and len(tokens) > 1) + + +def logical_lines(text: str) -> list[tuple[int, str]]: + lines: list[tuple[int, str]] = [] + start = 0 + parts: list[str] = [] + for line_number, raw_line in enumerate(text.splitlines(), 1): + if not parts: + start = line_number + stripped = raw_line.rstrip() + continued = stripped.endswith("\\") + parts.append(stripped[:-1] if continued else stripped) + if not continued: + lines.append((start, " ".join(parts))) + parts = [] + if parts: + lines.append((start, " ".join(parts))) + return lines + + +def command_image_reference(command: str) -> str | None: + try: + tokens = shlex.split(command, comments=True, posix=True) + except ValueError: + return None + for container_index, token in enumerate(tokens): + if token not in {"docker", "podman"}: + continue + prefix = tokens[:container_index] + if any( + item not in {"-", "command", "env", "sudo"} and "=" not in item + for item in prefix + ): + continue + action_index = container_index + 1 + if action_index >= len(tokens): + continue + action = tokens[action_index] + if action == "image" and action_index + 1 < len(tokens): + action_index += 1 + action = tokens[action_index] + if action not in CONTAINER_COMMANDS: + continue + index = action_index + 1 + while index < len(tokens): + candidate = tokens[index] + if candidate == "--": + index += 1 + break + if not candidate.startswith("-"): + break + if "=" in candidate or candidate in CONTAINER_BOOLEAN_OPTIONS: + index += 1 + else: + index += 2 + if index < len(tokens): + return tokens[index] + return None + + +def untagged_debian_references( + relative: Path, + text: str, +) -> list[tuple[int, str]]: + references: set[tuple[int, str]] = set() + for line_number, line in enumerate(text.splitlines(), 1): + assignment = IMAGE_ASSIGNMENT_RE.match(line) + if assignment is None or not is_image_assignment(assignment.group("name")): + continue + reference = assignment.group("reference").strip("\"'") + if is_untagged_debian_derived(reference): + references.add((line_number, reference)) + + if relative.suffix in {".bash", ".sh", ".yaml", ".yml"} or not relative.suffix: + for line_number, command in logical_lines(text): + reference = command_image_reference(command) + if reference is not None and is_untagged_debian_derived(reference): + references.add((line_number, reference)) + return sorted(references) + + def script_reference_is_consumed( relative: Path, text: str, @@ -285,6 +411,11 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{relative}:{line}: Debian-derived image reference is not pinned " f"by immutable digest: {reference}" ) + for line, reference in untagged_debian_references(relative, text): + failures.append( + f"{relative}:{line}: Debian-derived image reference is not pinned " + f"by immutable digest: {reference}" + ) for relative in PRODUCT_DOCKERFILES: text = texts[relative] diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 88868bad..1dd66e9b 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -105,6 +105,77 @@ def test_discovered_script_rejects_unpinned_debian_derived_image(self) -> None: failures, ) + def test_discovered_script_rejects_untagged_debian_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker run --rm debian apt-get update\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "Debian-derived image reference is not pinned by immutable digest" + in failure + and "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + + def test_discovered_yaml_rejects_untagged_debian_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + compose = root / "products/example/compose.yaml" + compose.parent.mkdir(parents=True, exist_ok=True) + compose.write_text( + "services:\n" + " helper:\n" + " image: debian\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "Debian-derived image reference is not pinned by immutable digest" + in failure + and "products/example/compose.yaml:3" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + + def test_untagged_detection_ignores_comments_and_yaml_prose(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/explain-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "# docker run --rm debian apt-get update\n", + encoding="utf-8", + ) + metadata = root / "products/example/metadata.yaml" + metadata.parent.mkdir(parents=True, exist_ok=True) + metadata.write_text( + 'description: "docker run --rm debian is an unsafe example"\n' + "base: debian\n", + encoding="utf-8", + ) + + self.assertEqual([], self.module.check_repository(root)) + def test_dockerfile_internal_stage_reference_needs_no_digest(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From db99552c98a5c41443c54ffc056c80c0511975b8 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 12:28:47 +0700 Subject: [PATCH 03/34] fix(release): reject untagged workflow containers Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 6 +++- release/scripts/test_check_debian13_images.py | 28 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index c7d4904d..23a6697e 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -224,7 +224,11 @@ def is_untagged_debian_derived(reference: str) -> bool: def is_image_assignment(name: str) -> bool: tokens = re.split(r"[_-]+", name.casefold()) - return "image" in tokens or ("base" in tokens and len(tokens) > 1) + return ( + name.casefold() == "container" + or "image" in tokens + or ("base" in tokens and len(tokens) > 1) + ) def logical_lines(text: str) -> list[tuple[int, str]]: diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 1dd66e9b..7950cc26 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -170,12 +170,38 @@ def test_untagged_detection_ignores_comments_and_yaml_prose(self) -> None: metadata.parent.mkdir(parents=True, exist_ok=True) metadata.write_text( 'description: "docker run --rm debian is an unsafe example"\n' - "base: debian\n", + "base: debian\n" + "container_note: debian\n", encoding="utf-8", ) self.assertEqual([], self.module.check_repository(root)) + def test_discovered_workflow_rejects_untagged_container_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + workflow = root / ".github/workflows/example.yml" + workflow.parent.mkdir(parents=True, exist_ok=True) + workflow.write_text( + "jobs:\n" + " test:\n" + " runs-on: ubuntu-latest\n" + " container: debian\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + ".github/workflows/example.yml:4" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + def test_dockerfile_internal_stage_reference_needs_no_digest(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 5b769f0fefae7efc98b79e3209f383d6136690c7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 21:49:47 +0700 Subject: [PATCH 04/34] fix(release): tighten Debian image boundary checks Signed-off-by: Jeremi Joslin --- .../scripts/check-registryctl-tutorials.sh | 5 +- release/scripts/check-debian13-images.py | 229 +++++++++++++----- release/scripts/test_check_debian13_images.py | 178 ++++++++++++++ 3 files changed, 348 insertions(+), 64 deletions(-) diff --git a/docs/site/scripts/check-registryctl-tutorials.sh b/docs/site/scripts/check-registryctl-tutorials.sh index de8c556b..16bf2ec7 100644 --- a/docs/site/scripts/check-registryctl-tutorials.sh +++ b/docs/site/scripts/check-registryctl-tutorials.sh @@ -16,8 +16,9 @@ HELPER="$SITE_ROOT/scripts/registryctl-tutorial.mjs" RELAY_TUTORIAL="$SITE_ROOT/src/content/docs/tutorials/publish-spreadsheet-secured-registry-api.mdx" NOTARY_TUTORIAL="$SITE_ROOT/src/content/docs/tutorials/verify-claim-registry-api.mdx" BUILDER_IMAGE="rust:1.95-trixie@sha256:f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3" -LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64" -CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home" +BUILDER_CACHE_KEY="${BUILDER_IMAGE##*@sha256:}" +LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY" +CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY" NATIVE_TARGET="$REPO_ROOT/target/registryctl-tutorial-native" RUN_ID="${GITHUB_RUN_ID:-local}-$$-$(date -u +%s)" RELAY_IMAGE="registryctl-tutorial-relay:$RUN_ID" diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 23a6697e..5cabae13 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -28,6 +28,7 @@ "docker/dockerfile:1.7@sha256:" "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) +REGISTRYCTL_TUTORIAL_SCRIPT = Path("docs/site/scripts/check-registryctl-tutorials.sh") PRODUCT_DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), @@ -47,6 +48,7 @@ Path("crates/registry-relay/docs/security-assurance.md"), Path("crates/registry-relay/scripts/check_docker_build_contract.py"), Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + REGISTRYCTL_TUTORIAL_SCRIPT, Path("products/notary/docs/security-assurance.md"), ) @@ -63,7 +65,10 @@ # Historical release notes and .research preserve observations, not active policy. EXCLUDED_PATH_PREFIXES = (Path("release/notes"),) MARKDOWN_SUFFIXES = {".md", ".mdx"} -SCRIPT_SUFFIXES = {".bash", ".js", ".mjs", ".py", ".sh", ".ts"} +SHELL_SUFFIXES = {".bash", ".sh"} +PYTHON_SUFFIXES = {".py"} +JS_TS_SUFFIXES = {".js", ".mjs", ".ts"} +SCRIPT_SUFFIXES = SHELL_SUFFIXES | PYTHON_SUFFIXES | JS_TS_SUFFIXES YAML_SUFFIXES = {".yaml", ".yml"} RUST_BUILDER_DOCKERFILES = PRODUCT_DOCKERFILES[:3] @@ -93,19 +98,26 @@ r"(?![A-Za-z0-9._/@+-])", re.IGNORECASE, ) -SCRIPT_IMAGE_CONTEXT_RE = re.compile( - r"\b(?:docker|podman|container|image)\b" - r"|\b[A-Za-z_][A-Za-z0-9_]*(?:image|base)[A-Za-z0-9_]*\s*=", - re.IGNORECASE, -) IMAGE_ASSIGNMENT_RE = re.compile( r"^\s*(?:export\s+)?" r"(?P[A-Za-z_][A-Za-z0-9_-]*)\s*(?:=|:)\s*" r"(?P[^#\s]+)", ) +PY_IMAGE_ASSIGNMENT_RE = re.compile( + r"^\s*(?P[A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=\s*(?P.*)$" +) +JS_TS_IMAGE_ASSIGNMENT_RE = re.compile( + r"^\s*(?:export\s+)?(?:const|let|var)\s+" + r"(?P[A-Za-z_$][A-Za-z0-9_$]*)" + r"\s*(?::[^=]+)?=\s*(?P.*)$" +) +STRING_LITERAL_RE = re.compile( + r"\"(?P(?:\\.|[^\"\\])*)\"|'(?P(?:\\.|[^'\\])*)'" +) UNTAGGED_IMAGE_REFERENCE_RE = re.compile( r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+", ) +VERSION_ONLY_TAG_RE = re.compile(r"^[0-9]+(?:[._][0-9]+)*$") DEBIAN_DEFAULT_IMAGE_NAMES = { "buildpack-deps", "debian", @@ -119,6 +131,8 @@ "--detach", "--init", "--interactive", + "--no-healthcheck", + "--oom-kill-disable", "--privileged", "--publish-all", "--quiet", @@ -215,20 +229,30 @@ def is_debian_derived(reference: str) -> bool: ) +def is_debian_default_version_only(reference: str) -> bool: + unpinned = reference.split("@", 1)[0] + if ":" not in unpinned: + return False + repository, tag = unpinned.rsplit(":", 1) + image_name = repository.rsplit("/", 1)[-1].casefold() + return ( + image_name in DEBIAN_DEFAULT_IMAGE_NAMES + and VERSION_ONLY_TAG_RE.fullmatch(tag) is not None + ) + + def is_untagged_debian_derived(reference: str) -> bool: if UNTAGGED_IMAGE_REFERENCE_RE.fullmatch(reference) is None: return False + if ":" in reference or "@" in reference: + return False image_name = reference.rsplit("/", 1)[-1].casefold() return "debian" in image_name or image_name in DEBIAN_DEFAULT_IMAGE_NAMES def is_image_assignment(name: str) -> bool: - tokens = re.split(r"[_-]+", name.casefold()) - return ( - name.casefold() == "container" - or "image" in tokens - or ("base" in tokens and len(tokens) > 1) - ) + lowered = name.casefold().replace("$", "") + return lowered in {"container", "image"} or lowered.endswith("image") def logical_lines(text: str) -> list[tuple[int, str]]: @@ -259,7 +283,9 @@ def command_image_reference(command: str) -> str | None: continue prefix = tokens[:container_index] if any( - item not in {"-", "command", "env", "sudo"} and "=" not in item + item not in {"-", "command", "env", "sudo"} + and "=" not in item + and not item.endswith(":") for item in prefix ): continue @@ -267,7 +293,7 @@ def command_image_reference(command: str) -> str | None: if action_index >= len(tokens): continue action = tokens[action_index] - if action == "image" and action_index + 1 < len(tokens): + if action in {"container", "image"} and action_index + 1 < len(tokens): action_index += 1 action = tokens[action_index] if action not in CONTAINER_COMMANDS: @@ -289,39 +315,111 @@ def command_image_reference(command: str) -> str | None: return None -def untagged_debian_references( - relative: Path, - text: str, -) -> list[tuple[int, str]]: +def decode_string_literal(value: str) -> str: + return value.replace(r"\/", "/").replace(r"\"", '"').replace(r"\'", "'") + + +def string_literals(line: str) -> list[str]: + values: list[str] = [] + for match in STRING_LITERAL_RE.finditer(line): + raw = match.group("double") if match.group("double") is not None else match.group("single") + values.append(decode_string_literal(raw)) + return values + + +def reference_candidates(value: str) -> list[str]: + stripped = value.strip().strip(";").strip(",").strip("\"'") + candidates = [match.group("reference") for match in CONTAINER_REFERENCE_RE.finditer(value)] + if stripped and is_untagged_debian_derived(stripped): + candidates.append(stripped) + return candidates + + +def assignment_match(relative: Path, line: str) -> tuple[str, str] | None: + if relative.suffix in PYTHON_SUFFIXES: + match = PY_IMAGE_ASSIGNMENT_RE.match(line) + if match is None: + return None + return match.group("name"), match.group("value") + if relative.suffix in JS_TS_SUFFIXES: + match = JS_TS_IMAGE_ASSIGNMENT_RE.match(line) + if match is None: + return None + return match.group("name"), match.group("value") + match = IMAGE_ASSIGNMENT_RE.match(line) + if match is None: + return None + return match.group("name"), match.group("reference") + + +def assignment_continues(relative: Path, value: str) -> bool: + stripped = value.strip() + if relative.suffix not in PYTHON_SUFFIXES | JS_TS_SUFFIXES: + return False + if stripped.endswith("\\") or stripped in {"", "(", "["}: + return True + if stripped.count("(") > stripped.count(")"): + return True + if relative.suffix in JS_TS_SUFFIXES and not stripped.endswith(";"): + return not string_literals(stripped) + return False + + +def image_assignment_references(relative: Path, text: str) -> list[tuple[int, str]]: references: set[tuple[int, str]] = set() + active = False for line_number, line in enumerate(text.splitlines(), 1): - assignment = IMAGE_ASSIGNMENT_RE.match(line) - if assignment is None or not is_image_assignment(assignment.group("name")): + if active: + for literal in string_literals(line): + for reference in reference_candidates(literal): + references.add((line_number, reference)) + stripped = line.strip() + if stripped.endswith((")", "];", ";")) or stripped == ")": + active = False continue - reference = assignment.group("reference").strip("\"'") - if is_untagged_debian_derived(reference): - references.add((line_number, reference)) - if relative.suffix in {".bash", ".sh", ".yaml", ".yml"} or not relative.suffix: - for line_number, command in logical_lines(text): - reference = command_image_reference(command) - if reference is not None and is_untagged_debian_derived(reference): + matched = assignment_match(relative, line) + if matched is None: + continue + name, value = matched + if not is_image_assignment(name): + continue + literals = string_literals(value) + if literals: + for literal in literals: + for reference in reference_candidates(literal): + references.add((line_number, reference)) + else: + for reference in reference_candidates(value): references.add((line_number, reference)) + active = assignment_continues(relative, value) return sorted(references) -def script_reference_is_consumed( - relative: Path, - text: str, - match: re.Match[str], -) -> bool: - if relative.suffix not in {".js", ".mjs", ".py", ".ts"}: - return True - line_start = text.rfind("\n", 0, match.start()) + 1 - line_end = text.find("\n", match.end()) - if line_end < 0: - line_end = len(text) - return SCRIPT_IMAGE_CONTEXT_RE.search(text[line_start:line_end]) is not None +def command_image_references(relative: Path, text: str) -> list[tuple[int, str]]: + if relative.suffix not in SHELL_SUFFIXES | YAML_SUFFIXES and relative.suffix: + return [] + references: set[tuple[int, str]] = set() + for line_number, command in logical_lines(text): + reference = command_image_reference(command) + if reference is not None: + references.add((line_number, reference)) + return sorted(references) + + +def image_references(relative: Path, text: str) -> list[tuple[int, str]]: + return sorted( + set(image_assignment_references(relative, text)) + | set(command_image_references(relative, text)) + ) + + +def reference_requires_digest(reference: str) -> bool: + if DIGEST_PIN_RE.search(reference): + return False + if ":" in reference: + return is_debian_derived(reference) or is_debian_default_version_only(reference) + return is_untagged_debian_derived(reference) def read(root: Path, relative: Path, failures: list[str]) -> str: @@ -362,16 +460,6 @@ def check_repository(root: Path = ROOT) -> list[str]: for relative in all_paths } - retired_markers = ("book" + "worm", "debian" + "12") - for relative in maintained_paths: - text = texts[relative] - lowered = text.casefold() - for marker in retired_markers: - if marker in lowered: - failures.append( - f"{relative}: retired Debian image generation marker remains: {marker}" - ) - dockerfiles = tuple( relative for relative in maintained_paths if is_dockerfile(relative) ) @@ -387,6 +475,12 @@ def check_repository(root: Path = ROOT) -> list[str]: stage_names: set[str] = set() for base, stage_name in bases: internal_stage = base.casefold() in stage_names + lowered_base = base.casefold() + for marker in ("book" + "worm", "debian" + "12"): + if not internal_stage and marker in lowered_base: + failures.append( + f"{relative}: retired Debian image generation marker remains: {marker}" + ) if ( base.casefold() != "scratch" and not internal_stage @@ -402,20 +496,9 @@ def check_repository(root: Path = ROOT) -> list[str]: if not is_image_reference_surface(relative): continue text = texts[relative] - for match in CONTAINER_REFERENCE_RE.finditer(text): - reference = match.group("reference") - if ( - not script_reference_is_consumed(relative, text, match) - or not is_debian_derived(reference) - or DIGEST_PIN_RE.search(reference) - ): + for line, reference in image_references(relative, text): + if not reference_requires_digest(reference): continue - line = text.count("\n", 0, match.start()) + 1 - failures.append( - f"{relative}:{line}: Debian-derived image reference is not pinned " - f"by immutable digest: {reference}" - ) - for line, reference in untagged_debian_references(relative, text): failures.append( f"{relative}:{line}: Debian-derived image reference is not pinned " f"by immutable digest: {reference}" @@ -579,6 +662,28 @@ def check_repository(root: Path = ROOT) -> list[str]: "pinned Debian 13 live-journey builder", failures, ) + tutorial_checker = texts[REGISTRYCTL_TUTORIAL_SCRIPT] + require( + tutorial_checker, + 'BUILDER_CACHE_KEY="${BUILDER_IMAGE##*@sha256:}"', + REGISTRYCTL_TUTORIAL_SCRIPT, + "Debian 13 builder identity cache key", + failures, + ) + require( + tutorial_checker, + 'registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY', + REGISTRYCTL_TUTORIAL_SCRIPT, + "builder-keyed linux target cache", + failures, + ) + require( + tutorial_checker, + 'registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY', + REGISTRYCTL_TUTORIAL_SCRIPT, + "builder-keyed Cargo home cache", + failures, + ) return failures diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 7950cc26..26d8055c 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -105,6 +105,79 @@ def test_discovered_script_rejects_unpinned_debian_derived_image(self) -> None: failures, ) + def test_discovered_script_rejects_version_only_debian_default_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker run --rm node:22 npm test\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "Debian-derived image reference is not pinned by immutable digest" + in failure + and "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": node:22") + for failure in failures + ), + failures, + ) + + def test_discovered_script_parses_docker_container_run(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker container run --rm rust:1.95-trixie cargo build --locked\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "Debian-derived image reference is not pinned by immutable digest" + in failure + and "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": rust:1.95-trixie") + for failure in failures + ), + failures, + ) + + def test_discovered_script_treats_docker_boolean_run_flags_as_valueless(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker run --rm --init --no-healthcheck --oom-kill-disable debian true\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + def test_discovered_script_rejects_untagged_debian_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -202,6 +275,111 @@ def test_discovered_workflow_rejects_untagged_container_image(self) -> None: failures, ) + def test_discovered_js_ts_rejects_bare_image_declarations(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/images.ts" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + 'const image = "debian";\n' + "let builderImage: string = 'node:22';\n" + 'var container = "python:3.12";\n', + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue(any("images.ts:1" in failure for failure in failures), failures) + self.assertTrue( + any( + "images.ts:2" in failure and failure.endswith(": node:22") + for failure in failures + ), + failures, + ) + self.assertTrue( + any( + "images.ts:3" in failure and failure.endswith(": python:3.12") + for failure in failures + ), + failures, + ) + + def test_wrapped_python_and_ts_image_assignments_keep_context(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + python_script = root / "products/example/scripts/build_image.py" + python_script.parent.mkdir(parents=True, exist_ok=True) + python_script.write_text( + "BUILDER_IMAGE = (\n" + " 'rust:1.95-trixie'\n" + ")\n", + encoding="utf-8", + ) + ts_script = root / "docs/site/scripts/images.ts" + ts_script.parent.mkdir(parents=True, exist_ok=True) + ts_script.write_text( + "const builderImage =\n" + " 'node:22';\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "products/example/scripts/build_image.py:2" in failure + and failure.endswith(": rust:1.95-trixie") + for failure in failures + ), + failures, + ) + self.assertTrue( + any( + "docs/site/scripts/images.ts:2" in failure + and failure.endswith(": node:22") + for failure in failures + ), + failures, + ) + + def test_registryctl_tutorial_cache_key_must_include_builder_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + tutorial = root / self.module.REGISTRYCTL_TUTORIAL_SCRIPT + text = tutorial.read_text(encoding="utf-8") + tutorial.write_text( + text.replace( + 'BUILDER_CACHE_KEY="${BUILDER_IMAGE##*@sha256:}"\n', + "", + ).replace( + "registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY", + "registryctl-tutorial-linux-amd64", + ).replace( + "registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY", + "registryctl-tutorial-cargo-home", + ), + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any("Debian 13 builder identity cache key" in failure for failure in failures), + failures, + ) + self.assertTrue( + any("builder-keyed linux target cache" in failure for failure in failures), + failures, + ) + self.assertTrue( + any("builder-keyed Cargo home cache" in failure for failure in failures), + failures, + ) + def test_dockerfile_internal_stage_reference_needs_no_digest(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 6631ba6072337cfad0f182956e4c3522076a2bf1 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 21:55:40 +0700 Subject: [PATCH 05/34] fix(release): align tutorial cache boundary Signed-off-by: Jeremi Joslin --- .github/workflows/ci.yml | 4 +- .../scripts/check-registryctl-tutorials.sh | 5 +- release/scripts/check-debian13-images.py | 61 +++++++++++-- release/scripts/test_check_debian13_images.py | 90 ++++++++++++++++--- 4 files changed, 133 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c88c9d65..2b400629 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -650,9 +650,7 @@ jobs: path: | target/registryctl-tutorial-cargo-home target/registryctl-tutorial-linux-amd64 - key: registryctl-tutorial-${{ runner.os }}-rust-1.95.0-${{ hashFiles('Cargo.lock') }} - restore-keys: | - registryctl-tutorial-${{ runner.os }}-rust-1.95.0- + key: registryctl-tutorial-${{ runner.os }}-${{ hashFiles('docs/site/scripts/check-registryctl-tutorials.sh') }}-${{ hashFiles('Cargo.lock') }} - name: Execute registryctl tutorials from source working-directory: docs/site diff --git a/docs/site/scripts/check-registryctl-tutorials.sh b/docs/site/scripts/check-registryctl-tutorials.sh index 16bf2ec7..de8c556b 100644 --- a/docs/site/scripts/check-registryctl-tutorials.sh +++ b/docs/site/scripts/check-registryctl-tutorials.sh @@ -16,9 +16,8 @@ HELPER="$SITE_ROOT/scripts/registryctl-tutorial.mjs" RELAY_TUTORIAL="$SITE_ROOT/src/content/docs/tutorials/publish-spreadsheet-secured-registry-api.mdx" NOTARY_TUTORIAL="$SITE_ROOT/src/content/docs/tutorials/verify-claim-registry-api.mdx" BUILDER_IMAGE="rust:1.95-trixie@sha256:f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3" -BUILDER_CACHE_KEY="${BUILDER_IMAGE##*@sha256:}" -LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY" -CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY" +LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64" +CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home" NATIVE_TARGET="$REPO_ROOT/target/registryctl-tutorial-native" RUN_ID="${GITHUB_RUN_ID:-local}-$$-$(date -u +%s)" RELAY_IMAGE="registryctl-tutorial-relay:$RUN_ID" diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 5cabae13..4a3b046c 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -28,6 +28,7 @@ "docker/dockerfile:1.7@sha256:" "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) +CI_WORKFLOW = Path(".github/workflows/ci.yml") REGISTRYCTL_TUTORIAL_SCRIPT = Path("docs/site/scripts/check-registryctl-tutorials.sh") PRODUCT_DOCKERFILES = ( @@ -42,6 +43,7 @@ # generation and immutable-pin boundary is broader and is discovered from the # repository instead of relying on this tuple. REQUIRED_PRODUCT_SURFACES = PRODUCT_DOCKERFILES + ( + CI_WORKFLOW, Path(".github/workflows/release.yml"), Path("release/scripts/build-release-binaries.sh"), Path("crates/registry-relay/docs/ops.md"), @@ -229,6 +231,14 @@ def is_debian_derived(reference: str) -> bool: ) +def retired_generation_marker(reference: str) -> str | None: + lowered = reference.casefold() + for marker in ("book" + "worm", "debian" + "12"): + if marker in lowered: + return marker + return None + + def is_debian_default_version_only(reference: str) -> bool: unpinned = reference.split("@", 1)[0] if ":" not in unpinned: @@ -451,6 +461,14 @@ def runtime_stage(text: str) -> str: return text[offset:] if offset >= 0 else "" +def registryctl_tutorial_cache_step(text: str) -> str: + start = text.find("- name: Cache source-under-test Cargo build") + if start < 0: + return "" + end = text.find("\n - name: Execute registryctl tutorials from source", start) + return text[start:] if end < 0 else text[start:end] + + def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] maintained_paths = discover_maintained_surfaces(root) @@ -497,6 +515,12 @@ def check_repository(root: Path = ROOT) -> list[str]: continue text = texts[relative] for line, reference in image_references(relative, text): + marker = retired_generation_marker(reference) + if marker is not None: + failures.append( + f"{relative}:{line}: retired Debian image generation marker " + f"remains in image reference: {marker}: {reference}" + ) if not reference_requires_digest(reference): continue failures.append( @@ -635,6 +659,7 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{relative}: vendor PKCS#11 modules must remain external read-only mounts" ) + ci_workflow = texts[CI_WORKFLOW] workflow = texts[Path(".github/workflows/release.yml")] binary_recipe = texts[Path("release/scripts/build-release-binaries.sh")] require( @@ -663,27 +688,47 @@ def check_repository(root: Path = ROOT) -> list[str]: failures, ) tutorial_checker = texts[REGISTRYCTL_TUTORIAL_SCRIPT] + tutorial_cache = registryctl_tutorial_cache_step(ci_workflow) require( tutorial_checker, - 'BUILDER_CACHE_KEY="${BUILDER_IMAGE##*@sha256:}"', + 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', REGISTRYCTL_TUTORIAL_SCRIPT, - "Debian 13 builder identity cache key", + "registryctl tutorial linux target path matching container target", failures, ) require( tutorial_checker, - 'registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY', + 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', REGISTRYCTL_TUTORIAL_SCRIPT, - "builder-keyed linux target cache", + "registryctl tutorial Cargo home path matching container Cargo home", failures, ) require( - tutorial_checker, - 'registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY', - REGISTRYCTL_TUTORIAL_SCRIPT, - "builder-keyed Cargo home cache", + tutorial_cache, + "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", + CI_WORKFLOW, + "registryctl tutorial cache key including builder-bearing script", + failures, + ) + require( + tutorial_cache, + "target/registryctl-tutorial-linux-amd64", + CI_WORKFLOW, + "registryctl tutorial linux target cache path", failures, ) + require( + tutorial_cache, + "target/registryctl-tutorial-cargo-home", + CI_WORKFLOW, + "registryctl tutorial Cargo home cache path", + failures, + ) + if "restore-keys:" in tutorial_cache: + failures.append( + f"{CI_WORKFLOW}: registryctl tutorial cache must not restore from " + "pre-builder-identity keys" + ) return failures diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 26d8055c..098f21e9 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -241,8 +241,11 @@ def test_untagged_detection_ignores_comments_and_yaml_prose(self) -> None: ) metadata = root / "products/example/metadata.yaml" metadata.parent.mkdir(parents=True, exist_ok=True) + retired_bookworm = f"rust:1.95-{'book' + 'worm'}@sha256:{'a' * 64}" metadata.write_text( 'description: "docker run --rm debian is an unsafe example"\n' + f"# docker run --rm {retired_bookworm} cargo build\n" + f"notes: {retired_bookworm}\n" "base: debian\n" "container_note: debian\n", encoding="utf-8", @@ -275,6 +278,32 @@ def test_discovered_workflow_rejects_untagged_container_image(self) -> None: failures, ) + def test_discovered_script_rejects_retired_digest_pinned_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + retired_bookworm = f"rust:1.95-{'book' + 'worm'}@sha256:{'a' * 64}" + script.write_text( + "#!/usr/bin/env bash\n" + f"docker run --rm {retired_bookworm} cargo build --locked\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "retired Debian image generation marker remains in image reference" + in failure + and "docs/site/scripts/build-example.sh:2" in failure + and "bookworm" in failure + for failure in failures + ), + failures, + ) + def test_discovered_js_ts_rejects_bare_image_declarations(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -346,6 +375,42 @@ def test_wrapped_python_and_ts_image_assignments_keep_context(self) -> None: ) def test_registryctl_tutorial_cache_key_must_include_builder_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + workflow = root / self.module.CI_WORKFLOW + text = workflow.read_text(encoding="utf-8") + workflow.write_text( + text.replace( + "key: registryctl-tutorial-${{ runner.os }}-${{ " + "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh') " + "}}-${{ hashFiles('Cargo.lock') }}", + "key: registryctl-tutorial-${{ runner.os }}-rust-1.95.0-" + "${{ hashFiles('Cargo.lock') }}\n" + " restore-keys: |\n" + " registryctl-tutorial-${{ runner.os }}-rust-1.95.0-", + ), + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "cache key including builder-bearing script" in failure + for failure in failures + ), + failures, + ) + self.assertTrue( + any( + "must not restore from pre-builder-identity keys" in failure + for failure in failures + ), + failures, + ) + + def test_registryctl_tutorial_host_paths_must_match_container_paths(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.copy_required_surfaces(root) @@ -353,14 +418,11 @@ def test_registryctl_tutorial_cache_key_must_include_builder_identity(self) -> N text = tutorial.read_text(encoding="utf-8") tutorial.write_text( text.replace( - 'BUILDER_CACHE_KEY="${BUILDER_IMAGE##*@sha256:}"\n', - "", - ).replace( - "registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY", - "registryctl-tutorial-linux-amd64", + 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', + 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY"', ).replace( - "registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY", - "registryctl-tutorial-cargo-home", + 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', + 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY"', ), encoding="utf-8", ) @@ -368,15 +430,17 @@ def test_registryctl_tutorial_cache_key_must_include_builder_identity(self) -> N failures = self.module.check_repository(root) self.assertTrue( - any("Debian 13 builder identity cache key" in failure for failure in failures), - failures, - ) - self.assertTrue( - any("builder-keyed linux target cache" in failure for failure in failures), + any( + "linux target path matching container target" in failure + for failure in failures + ), failures, ) self.assertTrue( - any("builder-keyed Cargo home cache" in failure for failure in failures), + any( + "Cargo home path matching container Cargo home" in failure + for failure in failures + ), failures, ) From f6b4c580df39653bd1b519fafe609eb608b8b4f8 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 22:47:26 +0700 Subject: [PATCH 06/34] fix(release): cover image scanner parser gaps Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 308 +++++++++++++++--- release/scripts/test_check_debian13_images.py | 277 +++++++++++++++- 2 files changed, 532 insertions(+), 53 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 4a3b046c..2cb97b98 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -101,7 +101,7 @@ re.IGNORECASE, ) IMAGE_ASSIGNMENT_RE = re.compile( - r"^\s*(?:export\s+)?" + r"^\s*(?:-\s*)?(?:(?:export|local|readonly)(?:\s+-[A-Za-z]+\s+|\s+))?" r"(?P[A-Za-z_][A-Za-z0-9_-]*)\s*(?:=|:)\s*" r"(?P[^#\s]+)", ) @@ -116,6 +116,11 @@ STRING_LITERAL_RE = re.compile( r"\"(?P(?:\\.|[^\"\\])*)\"|'(?P(?:\\.|[^'\\])*)'" ) +GITHUB_ACTIONS_DOCKER_USES_RE = re.compile( + r"^\s*(?:-\s*)?uses:\s*docker://(?P[^#\s]+)", + re.IGNORECASE, +) +COPY_RE = re.compile(r"^\s*COPY\b(?P.*)$", re.IGNORECASE) UNTAGGED_IMAGE_REFERENCE_RE = re.compile( r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+", ) @@ -129,6 +134,18 @@ "rust", } CONTAINER_COMMANDS = {"create", "pull", "run"} +DOCKER_GLOBAL_BOOLEAN_OPTIONS = { + "--debug", + "--tls", + "-D", +} +DOCKER_GLOBAL_VALUE_OPTIONS = { + "--config", + "--context", + "--host", + "--log-level", + "-H", +} CONTAINER_BOOLEAN_OPTIONS = { "--detach", "--init", @@ -140,6 +157,7 @@ "--quiet", "--read-only", "--rm", + "--sig-proxy", "--tty", "-d", "-i", @@ -148,6 +166,10 @@ "-q", "-t", } +SHELL_COMMAND_SEPARATORS = {";", "&&", "||", "|"} +MARKDOWN_SHELL_FENCE_LANGS = {"bash", "console", "sh", "shell", "terminal", "zsh"} +MARKDOWN_YAML_FENCE_LANGS = {"yaml", "yml"} +MARKDOWN_DOCKERFILE_FENCE_LANGS = {"dockerfile"} def is_excluded(relative: Path) -> bool: @@ -193,6 +215,7 @@ def is_image_reference_surface(relative: Path) -> bool: is_dockerfile(relative) or is_active_script(relative) or relative.suffix in YAML_SUFFIXES + or relative.suffix in MARKDOWN_SUFFIXES ) @@ -216,9 +239,26 @@ def discover_maintained_surfaces(root: Path) -> tuple[Path, ...]: return tuple(sorted(discovered)) +def normalize_reference(reference: str) -> str: + stripped = reference.strip().strip(",;").strip("\"'") + if stripped.startswith("docker://"): + stripped = stripped.removeprefix("docker://") + return stripped + + +def repository_and_tag(reference: str) -> tuple[str, str | None]: + unpinned = normalize_reference(reference).split("@", 1)[0] + slash = unpinned.rfind("/") + colon = unpinned.rfind(":") + if colon > slash: + return unpinned[:colon], unpinned[colon + 1:] + return unpinned, None + + def is_debian_derived(reference: str) -> bool: - unpinned = reference.split("@", 1)[0] - repository, tag = unpinned.rsplit(":", 1) + repository, tag = repository_and_tag(reference) + if tag is None: + return False image_name = repository.rsplit("/", 1)[-1].casefold() lowered_tag = tag.casefold() generation_tags = ("trixie", "book" + "worm", "bullseye", "buster") @@ -240,10 +280,9 @@ def retired_generation_marker(reference: str) -> str | None: def is_debian_default_version_only(reference: str) -> bool: - unpinned = reference.split("@", 1)[0] - if ":" not in unpinned: + repository, tag = repository_and_tag(reference) + if tag is None: return False - repository, tag = unpinned.rsplit(":", 1) image_name = repository.rsplit("/", 1)[-1].casefold() return ( image_name in DEBIAN_DEFAULT_IMAGE_NAMES @@ -254,9 +293,10 @@ def is_debian_default_version_only(reference: str) -> bool: def is_untagged_debian_derived(reference: str) -> bool: if UNTAGGED_IMAGE_REFERENCE_RE.fullmatch(reference) is None: return False - if ":" in reference or "@" in reference: + repository, tag = repository_and_tag(reference) + if tag is not None or "@" in reference: return False - image_name = reference.rsplit("/", 1)[-1].casefold() + image_name = repository.rsplit("/", 1)[-1].casefold() return "debian" in image_name or image_name in DEBIAN_DEFAULT_IMAGE_NAMES @@ -283,23 +323,82 @@ def logical_lines(text: str) -> list[tuple[int, str]]: return lines -def command_image_reference(command: str) -> str | None: +def shell_tokens(command: str) -> list[str]: try: - tokens = shlex.split(command, comments=True, posix=True) + lexer = shlex.shlex(command, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + lexer.commenters = "#" + return list(lexer) except ValueError: - return None + return [] + + +def shell_command_segments(tokens: list[str]) -> list[list[str]]: + segments: list[list[str]] = [] + segment: list[str] = [] + for token in tokens: + if token in SHELL_COMMAND_SEPARATORS: + if segment: + segments.append(segment) + segment = [] + continue + segment.append(token) + if segment: + segments.append(segment) + return segments + + +def is_allowed_command_prefix(token: str) -> bool: + return ( + token in {"$", "-", "command", "env", "sudo"} + or "=" in token + or token.endswith(":") + ) + + +def skip_options( + tokens: list[str], + index: int, + *, + boolean_options: set[str], + value_options: set[str], +) -> int: + while index < len(tokens): + candidate = tokens[index] + if candidate == "--": + return index + 1 + if not candidate.startswith("-"): + return index + if ( + "=" in candidate + or candidate in boolean_options + or any( + candidate.startswith(f"{option}=") + for option in value_options + if option.startswith("--") + ) + ): + index += 1 + elif candidate in value_options: + index += 2 + else: + index += 2 + return index + + +def command_segment_image_reference(tokens: list[str]) -> str | None: for container_index, token in enumerate(tokens): if token not in {"docker", "podman"}: continue prefix = tokens[:container_index] - if any( - item not in {"-", "command", "env", "sudo"} - and "=" not in item - and not item.endswith(":") - for item in prefix - ): + if any(not is_allowed_command_prefix(item) for item in prefix): continue - action_index = container_index + 1 + action_index = skip_options( + tokens, + container_index + 1, + boolean_options=DOCKER_GLOBAL_BOOLEAN_OPTIONS, + value_options=DOCKER_GLOBAL_VALUE_OPTIONS, + ) if action_index >= len(tokens): continue action = tokens[action_index] @@ -309,22 +408,26 @@ def command_image_reference(command: str) -> str | None: if action not in CONTAINER_COMMANDS: continue index = action_index + 1 - while index < len(tokens): - candidate = tokens[index] - if candidate == "--": - index += 1 - break - if not candidate.startswith("-"): - break - if "=" in candidate or candidate in CONTAINER_BOOLEAN_OPTIONS: - index += 1 - else: - index += 2 + index = skip_options( + tokens, + index, + boolean_options=CONTAINER_BOOLEAN_OPTIONS, + value_options=set(), + ) if index < len(tokens): - return tokens[index] + return normalize_reference(tokens[index]) return None +def command_image_references_in_command(command: str) -> list[str]: + references: list[str] = [] + for segment in shell_command_segments(shell_tokens(command)): + reference = command_segment_image_reference(segment) + if reference is not None: + references.append(reference) + return references + + def decode_string_literal(value: str) -> str: return value.replace(r"\/", "/").replace(r"\"", '"').replace(r"\'", "'") @@ -338,8 +441,11 @@ def string_literals(line: str) -> list[str]: def reference_candidates(value: str) -> list[str]: - stripped = value.strip().strip(";").strip(",").strip("\"'") - candidates = [match.group("reference") for match in CONTAINER_REFERENCE_RE.finditer(value)] + stripped = normalize_reference(value) + candidates = [ + normalize_reference(match.group("reference")) + for match in CONTAINER_REFERENCE_RE.finditer(value) + ] if stripped and is_untagged_debian_derived(stripped): candidates.append(stripped) return candidates @@ -379,6 +485,12 @@ def image_assignment_references(relative: Path, text: str) -> list[tuple[int, st references: set[tuple[int, str]] = set() active = False for line_number, line in enumerate(text.splitlines(), 1): + if relative.suffix in YAML_SUFFIXES: + match = GITHUB_ACTIONS_DOCKER_USES_RE.match(line) + if match is not None: + for reference in reference_candidates(match.group("reference")): + references.add((line_number, reference)) + if active: for literal in string_literals(line): for reference in reference_candidates(literal): @@ -411,13 +523,71 @@ def command_image_references(relative: Path, text: str) -> list[tuple[int, str]] return [] references: set[tuple[int, str]] = set() for line_number, command in logical_lines(text): - reference = command_image_reference(command) - if reference is not None: + for reference in command_image_references_in_command(command): references.add((line_number, reference)) return sorted(references) +def markdown_code_blocks(text: str) -> list[tuple[int, str, str]]: + blocks: list[tuple[int, str, str]] = [] + block_lines: list[str] = [] + block_start = 0 + language = "" + in_block = False + for line_number, line in enumerate(text.splitlines(), 1): + stripped = line.strip() + if stripped.startswith("```"): + if in_block: + blocks.append((block_start, language, "\n".join(block_lines))) + block_lines = [] + in_block = False + else: + block_start = line_number + 1 + info = stripped.removeprefix("```").strip().split(maxsplit=1) + language = info[0].casefold() if info else "" + in_block = True + continue + if in_block: + block_lines.append(line) + return blocks + + +def dockerfile_reference_lines(text: str) -> list[tuple[int, str]]: + references: list[tuple[int, str]] = [] + stage_names: set[str] = set() + for match in FROM_RE.finditer(text): + base = normalize_reference(match.group(1)) + line = text.count("\n", 0, match.start()) + 1 + if base.casefold() != "scratch" and base.casefold() not in stage_names: + references.append((line, base)) + stage_name = match.group(2) + if stage_name: + stage_names.add(stage_name.casefold()) + references.extend(copy_from_references(text, stage_names)) + return references + + +def markdown_image_references(text: str) -> list[tuple[int, str]]: + references: set[tuple[int, str]] = set() + for start_line, language, block in markdown_code_blocks(text): + if language in MARKDOWN_SHELL_FENCE_LANGS: + block_relative = Path("example.sh") + block_references = command_image_references(block_relative, block) + elif language in MARKDOWN_YAML_FENCE_LANGS: + block_relative = Path("example.yaml") + block_references = image_references(block_relative, block) + elif language in MARKDOWN_DOCKERFILE_FENCE_LANGS: + block_references = dockerfile_reference_lines(block) + else: + continue + for line_number, reference in block_references: + references.add((start_line + line_number - 1, reference)) + return sorted(references) + + def image_references(relative: Path, text: str) -> list[tuple[int, str]]: + if relative.suffix in MARKDOWN_SUFFIXES: + return markdown_image_references(text) return sorted( set(image_assignment_references(relative, text)) | set(command_image_references(relative, text)) @@ -427,7 +597,8 @@ def image_references(relative: Path, text: str) -> list[tuple[int, str]]: def reference_requires_digest(reference: str) -> bool: if DIGEST_PIN_RE.search(reference): return False - if ":" in reference: + _, tag = repository_and_tag(reference) + if tag is not None: return is_debian_derived(reference) or is_debian_default_version_only(reference) return is_untagged_debian_derived(reference) @@ -455,6 +626,58 @@ def require( failures.append(f"{relative}: missing {detail}: {needle!r}") +def copy_from_references(text: str, stage_names: set[str]) -> list[tuple[int, str]]: + references: list[tuple[int, str]] = [] + for line_number, line in logical_lines(text): + match = COPY_RE.match(line) + if match is None: + continue + try: + tokens = shlex.split(match.group("args"), comments=True, posix=True) + except ValueError: + continue + index = 0 + while index < len(tokens): + token = tokens[index] + source: str | None = None + if token == "--from" and index + 1 < len(tokens): + source = tokens[index + 1] + index += 2 + elif token.startswith("--from="): + source = token.split("=", 1)[1] + index += 1 + elif token.startswith("--"): + index += 1 + else: + break + if source is None: + continue + normalized = normalize_reference(source) + if normalized.casefold() in stage_names or normalized.isdigit(): + continue + references.append((line_number, normalized)) + return references + + +def append_reference_failures( + failures: list[str], + relative: Path, + line: int, + reference: str, +) -> None: + marker = retired_generation_marker(reference) + if marker is not None: + failures.append( + f"{relative}:{line}: retired Debian image generation marker " + f"remains in image reference: {marker}: {reference}" + ) + if reference_requires_digest(reference): + failures.append( + f"{relative}:{line}: Debian-derived image reference is not pinned " + f"by immutable digest: {reference}" + ) + + def runtime_stage(text: str) -> str: marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" offset = text.find(marker) @@ -509,24 +732,15 @@ def check_repository(root: Path = ROOT) -> list[str]: ) if stage_name: stage_names.add(stage_name.casefold()) + for line, reference in copy_from_references(text, stage_names): + append_reference_failures(failures, relative, line, reference) for relative in maintained_paths: if not is_image_reference_surface(relative): continue text = texts[relative] for line, reference in image_references(relative, text): - marker = retired_generation_marker(reference) - if marker is not None: - failures.append( - f"{relative}:{line}: retired Debian image generation marker " - f"remains in image reference: {marker}: {reference}" - ) - if not reference_requires_digest(reference): - continue - failures.append( - f"{relative}:{line}: Debian-derived image reference is not pinned " - f"by immutable digest: {reference}" - ) + append_reference_failures(failures, relative, line, reference) for relative in PRODUCT_DOCKERFILES: text = texts[relative] diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 098f21e9..70325bdc 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -81,6 +81,30 @@ def test_discovered_dockerfile_rejects_retired_unpinned_base(self) -> None: failures, ) + def test_discovered_dockerfile_rejects_external_copy_from_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + dockerfile = root / "products/example/Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text( + "FROM scratch\n" + "COPY --from=debian:book" + "worm@sha256:" + "a" * 64 + + " /etc/os-release /os-release\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "products/example/Dockerfile:2" in failure + and "retired Debian image generation marker" in failure + for failure in failures + ), + failures, + ) + def test_discovered_script_rejects_unpinned_debian_derived_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -163,7 +187,101 @@ def test_discovered_script_treats_docker_boolean_run_flags_as_valueless(self) -> script.parent.mkdir(parents=True, exist_ok=True) script.write_text( "#!/usr/bin/env bash\n" - "docker run --rm --init --no-healthcheck --oom-kill-disable debian true\n", + "docker run --rm --init --no-healthcheck " + "--oom-kill-disable --sig-proxy debian true\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + + def test_discovered_script_rejects_private_registry_untagged_debian_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker run --rm localhost:5000/debian true\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": localhost:5000/debian") + for failure in failures + ), + failures, + ) + + def test_discovered_script_strips_same_line_shell_separator_after_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker run --rm debian; echo ok\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + + def test_discovered_script_parses_docker_global_options(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "docker --context ci run --rm debian true\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + + def test_discovered_script_scans_after_shell_list_operators(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + 'cd "$PWD" && docker run --rm debian true\n' + "echo ok; podman run --rm node:22 true\n", encoding="utf-8", ) @@ -177,6 +295,14 @@ def test_discovered_script_treats_docker_boolean_run_flags_as_valueless(self) -> ), failures, ) + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:3" in failure + and failure.endswith(": node:22") + for failure in failures + ), + failures, + ) def test_discovered_script_rejects_untagged_debian_image(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -229,6 +355,29 @@ def test_discovered_yaml_rejects_untagged_debian_image(self) -> None: failures, ) + def test_discovered_yaml_rejects_list_item_image_key(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + compose = root / "products/example/compose.yaml" + compose.parent.mkdir(parents=True, exist_ok=True) + compose.write_text( + "containers:\n" + " - image: debian\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "products/example/compose.yaml:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + def test_untagged_detection_ignores_comments_and_yaml_prose(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -278,6 +427,31 @@ def test_discovered_workflow_rejects_untagged_container_image(self) -> None: failures, ) + def test_discovered_workflow_rejects_docker_image_action(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + workflow = root / ".github/workflows/example.yml" + workflow.parent.mkdir(parents=True, exist_ok=True) + workflow.write_text( + "jobs:\n" + " test:\n" + " steps:\n" + " - uses: docker://debian:book" + "worm@sha256:" + "a" * 64 + "\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + ".github/workflows/example.yml:4" in failure + and "retired Debian image generation marker" in failure + for failure in failures + ), + failures, + ) + def test_discovered_script_rejects_retired_digest_pinned_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -335,6 +509,38 @@ def test_discovered_js_ts_rejects_bare_image_declarations(self) -> None: failures, ) + def test_discovered_shell_rejects_prefixed_image_declarations(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + script = root / "docs/site/scripts/build-example.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "#!/usr/bin/env bash\n" + "local BUILDER_IMAGE=debian\n" + "readonly RUNTIME_IMAGE=rust:1.95-book" + "worm\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:2" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + self.assertTrue( + any( + "docs/site/scripts/build-example.sh:3" in failure + and "retired Debian image generation marker" in failure + for failure in failures + ), + failures, + ) + def test_wrapped_python_and_ts_image_assignments_keep_context(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -416,13 +622,19 @@ def test_registryctl_tutorial_host_paths_must_match_container_paths(self) -> Non self.copy_required_surfaces(root) tutorial = root / self.module.REGISTRYCTL_TUTORIAL_SCRIPT text = tutorial.read_text(encoding="utf-8") + linux_target = ( + 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"' + ) + cargo_home = ( + 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"' + ) tutorial.write_text( text.replace( - 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', - 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64-$BUILDER_CACHE_KEY"', + linux_target, + linux_target.removesuffix('"') + '-$BUILDER_CACHE_KEY"', ).replace( - 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', - 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home-$BUILDER_CACHE_KEY"', + cargo_home, + cargo_home.removesuffix('"') + '-$BUILDER_CACHE_KEY"', ), encoding="utf-8", ) @@ -452,7 +664,8 @@ def test_dockerfile_internal_stage_reference_needs_no_digest(self) -> None: dockerfile.parent.mkdir(parents=True, exist_ok=True) dockerfile.write_text( f"FROM {self.module.RUST_BUILDER} AS builder\n" - "FROM builder AS runtime\n", + "FROM builder AS runtime\n" + "COPY --from=builder /usr/local/bin/tool /usr/local/bin/tool\n", encoding="utf-8", ) @@ -481,6 +694,58 @@ def test_discovered_python_script_rejects_unpinned_builder_image(self) -> None: failures, ) + def test_discovered_markdown_rejects_executable_code_block_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + doc = root / "crates/registry-relay/docs/example.md" + doc.parent.mkdir(parents=True, exist_ok=True) + doc.write_text( + "Run the example:\n" + "```bash\n" + "docker run --rm debian true\n" + "```\n" + "```console\n" + "$ docker run --rm node:22 true\n" + "```\n", + encoding="utf-8", + ) + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "crates/registry-relay/docs/example.md:3" in failure + and failure.endswith(": debian") + for failure in failures + ), + failures, + ) + self.assertTrue( + any( + "crates/registry-relay/docs/example.md:6" in failure + and failure.endswith(": node:22") + for failure in failures + ), + failures, + ) + + def test_markdown_scan_ignores_prose_and_nonexecutable_fences(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + doc = root / "crates/registry-relay/docs/example.md" + doc.parent.mkdir(parents=True, exist_ok=True) + doc.write_text( + "A sentence can discuss docker run --rm debian true.\n" + "```text\n" + "docker run --rm debian true\n" + "```\n", + encoding="utf-8", + ) + + self.assertEqual([], self.module.check_repository(root)) + def test_history_and_research_are_outside_the_maintained_boundary(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 0401069842f4c50aeb2b6288b5db8fbb33922c98 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 23:01:55 +0700 Subject: [PATCH 07/34] fix(release): parse shell control prefixes Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 46 ++++++++++++++++--- release/scripts/test_check_debian13_images.py | 43 +++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 2cb97b98..6432faf8 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -167,6 +167,21 @@ "-t", } SHELL_COMMAND_SEPARATORS = {";", "&&", "||", "|"} +SHELL_COMMAND_CONTROL_PREFIXES = { + "!", + "(", + "{", + "do", + "elif", + "else", + "if", + "then", + "until", + "while", +} +SHELL_COMMAND_WRAPPER_PREFIXES = {"command", "env", "sudo"} +SHELL_TIME_OPTIONS = {"--", "-p"} +YAML_COMMAND_PREFIXES = {"command:", "run:", "script:"} MARKDOWN_SHELL_FENCE_LANGS = {"bash", "console", "sh", "shell", "terminal", "zsh"} MARKDOWN_YAML_FENCE_LANGS = {"yaml", "yml"} MARKDOWN_DOCKERFILE_FENCE_LANGS = {"dockerfile"} @@ -348,12 +363,29 @@ def shell_command_segments(tokens: list[str]) -> list[list[str]]: return segments -def is_allowed_command_prefix(token: str) -> bool: - return ( - token in {"$", "-", "command", "env", "sudo"} - or "=" in token - or token.endswith(":") - ) +def is_allowed_command_prefix(tokens: list[str]) -> bool: + index = 0 + if index < len(tokens) and tokens[index] in {"$", "-"}: + index += 1 + if index < len(tokens) and tokens[index].casefold() in YAML_COMMAND_PREFIXES: + index += 1 + + while index < len(tokens): + token = tokens[index] + if ( + token in SHELL_COMMAND_CONTROL_PREFIXES + or token in SHELL_COMMAND_WRAPPER_PREFIXES + or "=" in token + ): + index += 1 + continue + if token == "time": + index += 1 + if index < len(tokens) and tokens[index] in SHELL_TIME_OPTIONS: + index += 1 + continue + return False + return True def skip_options( @@ -391,7 +423,7 @@ def command_segment_image_reference(tokens: list[str]) -> str | None: if token not in {"docker", "podman"}: continue prefix = tokens[:container_index] - if any(not is_allowed_command_prefix(item) for item in prefix): + if not is_allowed_command_prefix(prefix): continue action_index = skip_options( tokens, diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 70325bdc..652d6de0 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -304,6 +304,49 @@ def test_discovered_script_scans_after_shell_list_operators(self) -> None: failures, ) + def test_shell_control_prefixes_find_image_references(self) -> None: + cases = { + "if docker run --rm debian true; then": ["debian"], + "while podman run --rm node:22 true; do": ["node:22"], + "until docker run --rm python:3.12 true; do": ["python:3.12"], + "! docker run --rm rust:1.95-trixie true": ["rust:1.95-trixie"], + "time docker run --rm debian true": ["debian"], + "time -p docker run --rm node:22 true": ["node:22"], + "time -- podman run --rm python:3.12 true": ["python:3.12"], + "{ docker run --rm debian true; }": ["debian"], + "(podman run --rm node:22 true)": ["node:22"], + "if ! time -p docker run --rm rust:1.95-trixie true; then": [ + "rust:1.95-trixie" + ], + "if true; then docker run --rm debian true; fi": ["debian"], + "while true; do podman run --rm node:22 true; done": ["node:22"], + "run: if docker run --rm debian true; then": ["debian"], + "- script: time -p podman run --rm node:22 true": ["node:22"], + } + + for command, expected in cases.items(): + with self.subTest(command=command): + self.assertEqual( + expected, + self.module.command_image_references_in_command(command), + ) + + def test_command_scan_ignores_prose_prefixes(self) -> None: + commands = ( + "echo if docker run --rm debian true", + "printf '%s\\n' time docker run --rm node:22 true", + "description: if docker run --rm debian true", + "notes: time -p docker run --rm node:22 true", + "a sentence about { docker run --rm debian true; }", + ) + + for command in commands: + with self.subTest(command=command): + self.assertEqual( + [], + self.module.command_image_references_in_command(command), + ) + def test_discovered_script_rejects_untagged_debian_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 8eddb6cfcde10fe44a0b991bddf360a58cf9c0cf Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 23 Jul 2026 23:31:51 +0700 Subject: [PATCH 08/34] fix(release): scope executable image scanning Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 422 +++++++++++++++--- release/scripts/test_check_debian13_images.py | 145 +++++- 2 files changed, 504 insertions(+), 63 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 6432faf8..b700ff18 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -7,8 +7,12 @@ import re import shlex import sys +from functools import lru_cache from pathlib import Path +import yaml +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + ROOT = Path(__file__).resolve().parents[2] @@ -116,11 +120,8 @@ STRING_LITERAL_RE = re.compile( r"\"(?P(?:\\.|[^\"\\])*)\"|'(?P(?:\\.|[^'\\])*)'" ) -GITHUB_ACTIONS_DOCKER_USES_RE = re.compile( - r"^\s*(?:-\s*)?uses:\s*docker://(?P[^#\s]+)", - re.IGNORECASE, -) COPY_RE = re.compile(r"^\s*COPY\b(?P.*)$", re.IGNORECASE) +SHELL_ASSIGNMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=.*$") UNTAGGED_IMAGE_REFERENCE_RE = re.compile( r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+", ) @@ -168,9 +169,6 @@ } SHELL_COMMAND_SEPARATORS = {";", "&&", "||", "|"} SHELL_COMMAND_CONTROL_PREFIXES = { - "!", - "(", - "{", "do", "elif", "else", @@ -179,9 +177,48 @@ "until", "while", } -SHELL_COMMAND_WRAPPER_PREFIXES = {"command", "env", "sudo"} -SHELL_TIME_OPTIONS = {"--", "-p"} -YAML_COMMAND_PREFIXES = {"command:", "run:", "script:"} +SHELL_OPEN_GROUP_PREFIXES = {"(", "{"} +# Only these YAML values are command surfaces. Other scalars may contain +# shell-looking documentation or data and must not be interpreted as executable. +YAML_EXECUTABLE_KEYS = {"command", "run", "script"} +COMMAND_BOOLEAN_OPTIONS = {"-p"} +ENV_BOOLEAN_OPTIONS = {"--debug", "--ignore-environment", "-i", "-v"} +ENV_VALUE_OPTIONS = {"--chdir", "--split-string", "--unset", "-C", "-S", "-u"} +SUDO_BOOLEAN_OPTIONS = { + "--background", + "--non-interactive", + "--preserve-env", + "--set-home", + "--stdin", + "-E", + "-H", + "-S", + "-b", + "-k", + "-n", +} +SUDO_VALUE_OPTIONS = { + "--chdir", + "--chroot", + "--close-from", + "--command-timeout", + "--group", + "--host", + "--prompt", + "--role", + "--type", + "--user", + "-C", + "-D", + "-g", + "-h", + "-p", + "-r", + "-R", + "-t", + "-T", + "-u", +} MARKDOWN_SHELL_FENCE_LANGS = {"bash", "console", "sh", "shell", "terminal", "zsh"} MARKDOWN_YAML_FENCE_LANGS = {"yaml", "yml"} MARKDOWN_DOCKERFILE_FENCE_LANGS = {"dockerfile"} @@ -320,11 +357,13 @@ def is_image_assignment(name: str) -> bool: return lowered in {"container", "image"} or lowered.endswith("image") -def logical_lines(text: str) -> list[tuple[int, str]]: +def numbered_logical_lines( + numbered_lines: list[tuple[int, str]], +) -> list[tuple[int, str]]: lines: list[tuple[int, str]] = [] start = 0 parts: list[str] = [] - for line_number, raw_line in enumerate(text.splitlines(), 1): + for line_number, raw_line in numbered_lines: if not parts: start = line_number stripped = raw_line.rstrip() @@ -338,6 +377,10 @@ def logical_lines(text: str) -> list[tuple[int, str]]: return lines +def logical_lines(text: str) -> list[tuple[int, str]]: + return numbered_logical_lines(list(enumerate(text.splitlines(), 1))) + + def shell_tokens(command: str) -> list[str]: try: lexer = shlex.shlex(command, posix=True, punctuation_chars=True) @@ -363,29 +406,145 @@ def shell_command_segments(tokens: list[str]) -> list[list[str]]: return segments -def is_allowed_command_prefix(tokens: list[str]) -> bool: +def skip_explicit_options( + tokens: list[str], + index: int, + *, + boolean_options: set[str], + value_options: set[str], + groupable_short_options: set[str], + attached_value_options: set[str], + optional_value_options: set[str] | None = None, +) -> int | None: + optional_value_options = optional_value_options or set() + while index < len(tokens): + candidate = tokens[index] + if candidate == "--": + return index + 1 + if not candidate.startswith("-") or candidate == "-": + return index + if candidate in boolean_options: + index += 1 + continue + if candidate in value_options: + if index + 1 >= len(tokens): + return None + index += 2 + continue + if candidate.startswith("--") and "=" in candidate: + option = candidate.split("=", 1)[0] + if option in value_options | optional_value_options: + index += 1 + continue + return None + if any( + candidate.startswith(option) and len(candidate) > len(option) + for option in attached_value_options + ): + index += 1 + continue + if len(candidate) > 2 and all( + character in groupable_short_options for character in candidate[1:] + ): + index += 1 + continue + return None + return index + + +def skip_time_prefix(tokens: list[str], index: int) -> int: + index += 1 + if index < len(tokens) and tokens[index] == "-p": + index += 1 + if index < len(tokens) and tokens[index] == "--": + index += 1 + return index + + +def simple_command_index(tokens: list[str]) -> int | None: + """Locate a container CLI only through the supported shell prefix grammar.""" + index = 0 - if index < len(tokens) and tokens[index] in {"$", "-"}: + if index < len(tokens) and tokens[index] == "$": + index += 1 + + while index < len(tokens) and tokens[index] in SHELL_OPEN_GROUP_PREFIXES: + index += 1 + if index < len(tokens) and tokens[index] in SHELL_COMMAND_CONTROL_PREFIXES: index += 1 - if index < len(tokens) and tokens[index].casefold() in YAML_COMMAND_PREFIXES: + while index < len(tokens) and tokens[index] in SHELL_OPEN_GROUP_PREFIXES: index += 1 + seen_negation = False + seen_time = False while index < len(tokens): token = tokens[index] - if ( - token in SHELL_COMMAND_CONTROL_PREFIXES - or token in SHELL_COMMAND_WRAPPER_PREFIXES - or "=" in token - ): + if token == "!" and not seen_negation: + seen_negation = True index += 1 - continue - if token == "time": + elif token == "time" and not seen_time: + seen_time = True + index = skip_time_prefix(tokens, index) + elif token in SHELL_OPEN_GROUP_PREFIXES: index += 1 - if index < len(tokens) and tokens[index] in SHELL_TIME_OPTIONS: - index += 1 + else: + break + + while index < len(tokens): + while index < len(tokens) and SHELL_ASSIGNMENT_RE.fullmatch(tokens[index]): + index += 1 + if index >= len(tokens): + return None + + wrapper = tokens[index] + if wrapper == "time" and not seen_time: + seen_time = True + index = skip_time_prefix(tokens, index) continue - return False - return True + if wrapper == "command": + index = skip_explicit_options( + tokens, + index + 1, + boolean_options=COMMAND_BOOLEAN_OPTIONS, + value_options=set(), + groupable_short_options={"p"}, + attached_value_options=set(), + ) + elif wrapper == "env": + index = skip_explicit_options( + tokens, + index + 1, + boolean_options=ENV_BOOLEAN_OPTIONS, + value_options=ENV_VALUE_OPTIONS, + groupable_short_options={"i", "v"}, + attached_value_options={"-C", "-S", "-u"}, + ) + elif wrapper == "sudo": + index = skip_explicit_options( + tokens, + index + 1, + boolean_options=SUDO_BOOLEAN_OPTIONS, + value_options=SUDO_VALUE_OPTIONS, + groupable_short_options={"E", "H", "S", "b", "k", "n"}, + attached_value_options={ + "-C", + "-D", + "-g", + "-h", + "-p", + "-r", + "-R", + "-t", + "-T", + "-u", + }, + optional_value_options={"--preserve-env"}, + ) + else: + return index + if index is None: + return None + return index def skip_options( @@ -419,35 +578,36 @@ def skip_options( def command_segment_image_reference(tokens: list[str]) -> str | None: - for container_index, token in enumerate(tokens): - if token not in {"docker", "podman"}: - continue - prefix = tokens[:container_index] - if not is_allowed_command_prefix(prefix): - continue - action_index = skip_options( - tokens, - container_index + 1, - boolean_options=DOCKER_GLOBAL_BOOLEAN_OPTIONS, - value_options=DOCKER_GLOBAL_VALUE_OPTIONS, - ) - if action_index >= len(tokens): - continue + container_index = simple_command_index(tokens) + if ( + container_index is None + or container_index >= len(tokens) + or tokens[container_index] not in {"docker", "podman"} + ): + return None + action_index = skip_options( + tokens, + container_index + 1, + boolean_options=DOCKER_GLOBAL_BOOLEAN_OPTIONS, + value_options=DOCKER_GLOBAL_VALUE_OPTIONS, + ) + if action_index >= len(tokens): + return None + action = tokens[action_index] + if action in {"container", "image"} and action_index + 1 < len(tokens): + action_index += 1 action = tokens[action_index] - if action in {"container", "image"} and action_index + 1 < len(tokens): - action_index += 1 - action = tokens[action_index] - if action not in CONTAINER_COMMANDS: - continue - index = action_index + 1 - index = skip_options( - tokens, - index, - boolean_options=CONTAINER_BOOLEAN_OPTIONS, - value_options=set(), - ) - if index < len(tokens): - return normalize_reference(tokens[index]) + if action not in CONTAINER_COMMANDS: + return None + index = action_index + 1 + index = skip_options( + tokens, + index, + boolean_options=CONTAINER_BOOLEAN_OPTIONS, + value_options=set(), + ) + if index < len(tokens): + return normalize_reference(tokens[index]) return None @@ -514,15 +674,22 @@ def assignment_continues(relative: Path, value: str) -> bool: def image_assignment_references(relative: Path, text: str) -> list[tuple[int, str]]: + if relative.suffix in YAML_SUFFIXES: + references = set(yaml_declarative_image_references(text)) + for line_number, command in yaml_executable_commands(text): + matched = assignment_match(relative, command) + if matched is None: + continue + name, value = matched + if not is_image_assignment(name): + continue + for reference in reference_candidates(value): + references.add((line_number, reference)) + return sorted(references) + references: set[tuple[int, str]] = set() active = False for line_number, line in enumerate(text.splitlines(), 1): - if relative.suffix in YAML_SUFFIXES: - match = GITHUB_ACTIONS_DOCKER_USES_RE.match(line) - if match is not None: - for reference in reference_candidates(match.group("reference")): - references.add((line_number, reference)) - if active: for literal in string_literals(line): for reference in reference_candidates(literal): @@ -550,11 +717,144 @@ def image_assignment_references(relative: Path, text: str) -> list[tuple[int, st return sorted(references) +@lru_cache(maxsize=1) +def compose_yaml(text: str) -> Node | None: + try: + return yaml.compose(text, Loader=yaml.SafeLoader) + except yaml.YAMLError: + return None + + +def yaml_mapping_entries(root: Node) -> list[tuple[Node, Node]]: + entries: list[tuple[Node, Node]] = [] + stack = [root] + visited: set[int] = set() + while stack: + node = stack.pop() + if id(node) in visited: + continue + visited.add(id(node)) + if isinstance(node, MappingNode): + entries.extend(node.value) + stack.extend(value for _, value in node.value) + elif isinstance(node, SequenceNode): + stack.extend(node.value) + return entries + + +def yaml_scalar_commands( + node: ScalarNode, + lines: list[str], +) -> list[tuple[int, str]]: + if node.tag != "tag:yaml.org,2002:str" or not node.value: + return [] + if node.style != "|": + source_line = node.start_mark.line + 1 + if node.style == ">": + for index in range(node.start_mark.line + 1, node.end_mark.line): + if index < len(lines) and lines[index].strip(): + source_line = index + 1 + break + return [ + (source_line + line_number - 1, command) + for line_number, command in logical_lines(node.value) + ] + + content = [ + (index + 1, lines[index]) + for index in range( + node.start_mark.line + 1, + min(node.end_mark.line, len(lines)), + ) + ] + indents = [ + len(line) - len(line.lstrip(" ")) + for _, line in content + if line.strip() + ] + if not indents: + return [] + content_indent = min(indents) + return numbered_logical_lines( + [ + (line_number, line[content_indent:] if line.strip() else "") + for line_number, line in content + ] + ) + + +def yaml_executable_commands(text: str) -> list[tuple[int, str]]: + """Extract only values of YAML run, script, and command mapping keys.""" + + root = compose_yaml(text) + if root is None: + return [] + lines = text.splitlines() + commands: list[tuple[int, str]] = [] + for key_node, value_node in yaml_mapping_entries(root): + if ( + not isinstance(key_node, ScalarNode) + or key_node.tag != "tag:yaml.org,2002:str" + or key_node.value.casefold() not in YAML_EXECUTABLE_KEYS + ): + continue + key = key_node.value.casefold() + if isinstance(value_node, ScalarNode): + commands.extend(yaml_scalar_commands(value_node, lines)) + elif isinstance(value_node, SequenceNode): + scalar_nodes = [ + item + for item in value_node.value + if isinstance(item, ScalarNode) + and item.tag == "tag:yaml.org,2002:str" + ] + if len(scalar_nodes) != len(value_node.value): + continue + # Command sequences are exec-form argv; run/script sequences are + # lists of shell command strings. + if key == "command": + command = shlex.join([item.value for item in scalar_nodes]) + if command: + commands.append((value_node.start_mark.line + 1, command)) + else: + for item in scalar_nodes: + commands.extend(yaml_scalar_commands(item, lines)) + return commands + + +def yaml_declarative_image_references(text: str) -> list[tuple[int, str]]: + root = compose_yaml(text) + if root is None: + return [] + references: set[tuple[int, str]] = set() + for key_node, value_node in yaml_mapping_entries(root): + if ( + not isinstance(key_node, ScalarNode) + or key_node.tag != "tag:yaml.org,2002:str" + or not isinstance(value_node, ScalarNode) + or value_node.tag != "tag:yaml.org,2002:str" + ): + continue + key = key_node.value.casefold() + if key == "uses" and not value_node.value.casefold().startswith("docker://"): + continue + if key != "uses" and not is_image_assignment(key): + continue + for reference in reference_candidates(normalize_reference(value_node.value)): + references.add((key_node.start_mark.line + 1, reference)) + return sorted(references) + + def command_image_references(relative: Path, text: str) -> list[tuple[int, str]]: if relative.suffix not in SHELL_SUFFIXES | YAML_SUFFIXES and relative.suffix: return [] references: set[tuple[int, str]] = set() - for line_number, command in logical_lines(text): + commands = ( + yaml_executable_commands(text) + if relative.suffix in YAML_SUFFIXES + else logical_lines(text) + ) + for line_number, command in commands: for reference in command_image_references_in_command(command): references.add((line_number, reference)) return sorted(references) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 652d6de0..3bd6d2c3 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -320,8 +320,6 @@ def test_shell_control_prefixes_find_image_references(self) -> None: ], "if true; then docker run --rm debian true; fi": ["debian"], "while true; do podman run --rm node:22 true; done": ["node:22"], - "run: if docker run --rm debian true; then": ["debian"], - "- script: time -p podman run --rm node:22 true": ["node:22"], } for command, expected in cases.items(): @@ -331,6 +329,45 @@ def test_shell_control_prefixes_find_image_references(self) -> None: self.module.command_image_references_in_command(command), ) + def test_shell_wrapper_options_find_image_references(self) -> None: + cases = { + "sudo -E docker run --rm debian true": ["debian"], + "env -i podman run --rm node:22 true": ["node:22"], + "command -- docker run --rm python:3.12 true": ["python:3.12"], + "sudo -u root env -u EXAMPLE command -p " + "docker run --rm rust:1.95-trixie true": ["rust:1.95-trixie"], + "sudo --preserve-env=CI time -p " + "podman run --rm debian true": ["debian"], + } + + for command, expected in cases.items(): + with self.subTest(command=command): + self.assertEqual( + expected, + self.module.command_image_references_in_command(command), + ) + + def test_shell_parser_rejects_invalid_control_and_wrapper_prefixes(self) -> None: + commands = ( + "if then docker run --rm debian true", + "while until docker run --rm debian true", + "! if docker run --rm debian true", + "time while docker run --rm debian true", + "time -- -p docker run --rm debian true", + "sudo if docker run --rm debian true", + "sudo --unknown docker run --rm debian true", + "sudo -v docker run --rm debian true", + "env --unknown docker run --rm debian true", + "command -v docker run --rm debian true", + ) + + for command in commands: + with self.subTest(command=command): + self.assertEqual( + [], + self.module.command_image_references_in_command(command), + ) + def test_command_scan_ignores_prose_prefixes(self) -> None: commands = ( "echo if docker run --rm debian true", @@ -347,6 +384,110 @@ def test_command_scan_ignores_prose_prefixes(self) -> None: self.module.command_image_references_in_command(command), ) + def test_yaml_executable_command_forms_find_image_references(self) -> None: + cases = { + "inline scalar": ( + "run: sudo -E docker run --rm debian true\n", + [(1, "debian")], + ), + "quoted scalar": ( + 'script: "env -i podman run --rm node:22 true"\n', + [(1, "node:22")], + ), + "literal scalar": ( + "run: |\n" + " command -- docker run --rm python:3.12 true\n", + [(2, "python:3.12")], + ), + "folded scalar": ( + "script: >\n" + " sudo -E docker run --rm\n" + " rust:1.95-trixie true\n", + [(2, "rust:1.95-trixie")], + ), + "flow script list": ( + "script: [\"docker run --rm debian true\", " + "\"env -i podman run --rm node:22 true\"]\n", + [(1, "debian"), (1, "node:22")], + ), + "flow command argv": ( + "command: [command, --, docker, run, --rm, python:3.12]\n", + [(1, "python:3.12")], + ), + "multiline flow script list": ( + "script: [\n" + " \"sudo -E docker run --rm debian true\",\n" + " \"podman run --rm node:22 true\"\n" + "]\n", + [(2, "debian"), (3, "node:22")], + ), + "block script list": ( + "script:\n" + " # CI commands\n" + " - docker run --rm debian true\n" + " - env -i podman run --rm node:22 true\n", + [(3, "debian"), (4, "node:22")], + ), + "block command argv": ( + "command:\n" + " - command\n" + " - --\n" + " - docker\n" + " - run\n" + " - --rm\n" + " - python:3.12\n", + [(2, "python:3.12")], + ), + "shell image assignment": ( + "run: |\n" + " BUILDER_IMAGE=debian\n", + [(2, "debian")], + ), + } + + for name, (text, expected) in cases.items(): + with self.subTest(name=name): + self.assertEqual( + expected, + self.module.image_references(Path("example.yaml"), text), + ) + + def test_yaml_descriptive_scalars_and_lists_are_not_executable(self) -> None: + text = ( + "\"description\": |\n" + " docker run --rm debian true\n" + " run: podman run --rm node:22 true\n" + " image: debian\n" + " uses: docker://python:3.12\n" + "notes: >\n" + " command -- docker run --rm python:3.12 true\n" + "examples:\n" + " - |\n" + " script: docker run --rm rust:1.95-trixie true\n" + "summary: [\"docker run --rm debian true\"]\n" + "script:\n" + " file: adapter.rhai\n" + ) + + self.assertEqual( + [], + self.module.image_references(Path("example.yaml"), text), + ) + + def test_yaml_block_scalar_scope_ends_at_parent_indent(self) -> None: + text = ( + "steps:\n" + " - description: |\n" + " docker run --rm debian true\n" + " run: docker run --rm node:22 true\n" + " - run: docker run --rm python:3.12 true\n" + ) + + self.assertEqual( + [(4, "node:22"), (5, "python:3.12")], + self.module.command_image_references(Path("example.yaml"), text), + ) + def test_discovered_script_rejects_untagged_debian_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 2ccd55688ced46edbc8712cf550d365344c25908 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 00:03:39 +0700 Subject: [PATCH 09/34] fix(release): harden image surface parsing Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 483 ++++++++++++++---- release/scripts/test_check_debian13_images.py | 274 ++++++---- 2 files changed, 556 insertions(+), 201 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index b700ff18..c346bbbf 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -10,8 +10,15 @@ from functools import lru_cache from pathlib import Path -import yaml -from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode +try: + import yaml + from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode +except ImportError as error: + yaml = None + MappingNode = Node = ScalarNode = SequenceNode = None + YAML_IMPORT_ERROR = error +else: + YAML_IMPORT_ERROR = None ROOT = Path(__file__).resolve().parents[2] @@ -167,7 +174,7 @@ "-q", "-t", } -SHELL_COMMAND_SEPARATORS = {";", "&&", "||", "|"} +SHELL_COMMAND_SEPARATORS = {";", "&&", "||", "&", "|", "|&"} SHELL_COMMAND_CONTROL_PREFIXES = { "do", "elif", @@ -178,22 +185,37 @@ "while", } SHELL_OPEN_GROUP_PREFIXES = {"(", "{"} -# Only these YAML values are command surfaces. Other scalars may contain -# shell-looking documentation or data and must not be interpreted as executable. -YAML_EXECUTABLE_KEYS = {"command", "run", "script"} +SUPPORTED_SHELL_COMMANDS = {"bash", "dash", "ksh", "sh", "zsh"} +SHELL_COMMAND_SHORT_OPTIONS = set("abcefhklmnptuvxBCEHP") +MAX_SHELL_COMMAND_CHARS = 65_536 +MAX_SHELL_COMMAND_TOKENS = 2_048 +MAX_SHELL_RECURSION_DEPTH = 4 +MAX_YAML_TEXT_CHARS = 2_000_000 +MAX_YAML_DOCUMENTS = 128 +MAX_YAML_NODES = 100_000 +MAX_YAML_DEPTH = 64 +KUBERNETES_CONTAINER_COLLECTIONS = { + "containers", "ephemeralContainers", "initContainers" +} COMMAND_BOOLEAN_OPTIONS = {"-p"} ENV_BOOLEAN_OPTIONS = {"--debug", "--ignore-environment", "-i", "-v"} -ENV_VALUE_OPTIONS = {"--chdir", "--split-string", "--unset", "-C", "-S", "-u"} +ENV_VALUE_OPTIONS = {"--chdir", "--unset", "-C", "-u"} SUDO_BOOLEAN_OPTIONS = { + "--askpass", "--background", + "--login", "--non-interactive", "--preserve-env", + "--preserve-groups", "--set-home", "--stdin", + "-A", "-E", "-H", + "-P", "-S", "-b", + "-i", "-k", "-n", } @@ -224,6 +246,10 @@ MARKDOWN_DOCKERFILE_FENCE_LANGS = {"dockerfile"} +class ImageSurfaceError(RuntimeError): + """A maintained image surface could not be scanned conclusively.""" + + def is_excluded(relative: Path) -> bool: if ".research" in relative.parts: return True @@ -415,20 +441,45 @@ def skip_explicit_options( groupable_short_options: set[str], attached_value_options: set[str], optional_value_options: set[str] | None = None, -) -> int | None: + split_string_options: set[str] | None = None, +) -> tuple[int | None, str | None]: optional_value_options = optional_value_options or set() + split_string_options = split_string_options or set() while index < len(tokens): candidate = tokens[index] if candidate == "--": - return index + 1 + return index + 1, None if not candidate.startswith("-") or candidate == "-": - return index + return index, None + split_option = candidate.split("=", 1)[0] + attached_split = ( + "-S" + if "-S" in split_string_options + and candidate.startswith("-S") + and candidate != "-S" + else None + ) + if split_option in split_string_options or attached_split is not None: + if "=" in candidate: + payload = candidate.split("=", 1)[1] + suffix = tokens[index + 1 :] + elif candidate in split_string_options: + if index + 1 >= len(tokens): + return None, None + payload = tokens[index + 1] + suffix = tokens[index + 2 :] + else: + payload = candidate[len(attached_split) :] + suffix = tokens[index + 1 :] + return None, ( + payload if not suffix else f"{payload} {shlex.join(suffix)}" + ) if candidate in boolean_options: index += 1 continue if candidate in value_options: if index + 1 >= len(tokens): - return None + return None, None index += 2 continue if candidate.startswith("--") and "=" in candidate: @@ -436,7 +487,7 @@ def skip_explicit_options( if option in value_options | optional_value_options: index += 1 continue - return None + return None, None if any( candidate.startswith(option) and len(candidate) > len(option) for option in attached_value_options @@ -448,8 +499,8 @@ def skip_explicit_options( ): index += 1 continue - return None - return index + return None, None + return index, None def skip_time_prefix(tokens: list[str], index: int) -> int: @@ -461,7 +512,7 @@ def skip_time_prefix(tokens: list[str], index: int) -> int: return index -def simple_command_index(tokens: list[str]) -> int | None: +def simple_command(tokens: list[str]) -> tuple[int | None, str | None]: """Locate a container CLI only through the supported shell prefix grammar.""" index = 0 @@ -494,7 +545,7 @@ def simple_command_index(tokens: list[str]) -> int | None: while index < len(tokens) and SHELL_ASSIGNMENT_RE.fullmatch(tokens[index]): index += 1 if index >= len(tokens): - return None + return None, None wrapper = tokens[index] if wrapper == "time" and not seen_time: @@ -502,7 +553,7 @@ def simple_command_index(tokens: list[str]) -> int | None: index = skip_time_prefix(tokens, index) continue if wrapper == "command": - index = skip_explicit_options( + index, _ = skip_explicit_options( tokens, index + 1, boolean_options=COMMAND_BOOLEAN_OPTIONS, @@ -511,21 +562,24 @@ def simple_command_index(tokens: list[str]) -> int | None: attached_value_options=set(), ) elif wrapper == "env": - index = skip_explicit_options( + index, payload = skip_explicit_options( tokens, index + 1, boolean_options=ENV_BOOLEAN_OPTIONS, value_options=ENV_VALUE_OPTIONS, groupable_short_options={"i", "v"}, attached_value_options={"-C", "-S", "-u"}, + split_string_options={"-S", "--split-string"}, ) + if payload is not None: + return None, payload elif wrapper == "sudo": - index = skip_explicit_options( + index, _ = skip_explicit_options( tokens, index + 1, boolean_options=SUDO_BOOLEAN_OPTIONS, value_options=SUDO_VALUE_OPTIONS, - groupable_short_options={"E", "H", "S", "b", "k", "n"}, + groupable_short_options={"A", "E", "H", "P", "S", "b", "i", "k", "n"}, attached_value_options={ "-C", "-D", @@ -541,10 +595,10 @@ def simple_command_index(tokens: list[str]) -> int | None: optional_value_options={"--preserve-env"}, ) else: - return index + return index, None if index is None: - return None - return index + return None, None + return index, None def skip_options( @@ -577,11 +631,12 @@ def skip_options( return index -def command_segment_image_reference(tokens: list[str]) -> str | None: - container_index = simple_command_index(tokens) +def container_command_image_reference( + tokens: list[str], + container_index: int, +) -> str | None: if ( - container_index is None - or container_index >= len(tokens) + container_index >= len(tokens) or tokens[container_index] not in {"docker", "podman"} ): return None @@ -611,12 +666,75 @@ def command_segment_image_reference(tokens: list[str]) -> str | None: return None -def command_image_references_in_command(command: str) -> list[str]: +def shell_command_payload(tokens: list[str], command_index: int) -> str | None: + command = tokens[command_index].rsplit("/", 1)[-1] + if command not in SUPPORTED_SHELL_COMMANDS: + return None + index = command_index + 1 + while index < len(tokens): + option = tokens[index] + if option == "-c": + return tokens[index + 1] if index + 1 < len(tokens) else None + if ( + option.startswith("-") + and not option.startswith("--") + and len(option) > 2 + and all(character in SHELL_COMMAND_SHORT_OPTIONS for character in option[1:]) + ): + if "c" in option[1:]: + return tokens[index + 1] if index + 1 < len(tokens) else None + index += 1 + continue + if ( + len(option) == 2 + and option.startswith("-") + and option[1] in SHELL_COMMAND_SHORT_OPTIONS + ): + index += 1 + continue + return None + return None + + +def command_segment_image_references( + tokens: list[str], + depth: int, +) -> list[str]: + command_index, split_payload = simple_command(tokens) + if split_payload is not None: + return command_image_references_in_command(split_payload, depth=depth + 1) + if command_index is None or command_index >= len(tokens): + return [] + reference = container_command_image_reference(tokens, command_index) + if reference is not None: + return [reference] + payload = shell_command_payload(tokens, command_index) + if payload is not None: + return command_image_references_in_command(payload, depth=depth + 1) + return [] + + +def command_image_references_in_command( + command: str, + *, + depth: int = 0, +) -> list[str]: + if depth > MAX_SHELL_RECURSION_DEPTH: + raise ImageSurfaceError( + f"shell command nesting exceeds {MAX_SHELL_RECURSION_DEPTH} levels" + ) + if len(command) > MAX_SHELL_COMMAND_CHARS: + raise ImageSurfaceError( + f"shell command exceeds {MAX_SHELL_COMMAND_CHARS} characters" + ) + tokens = shell_tokens(command) + if len(tokens) > MAX_SHELL_COMMAND_TOKENS: + raise ImageSurfaceError( + f"shell command exceeds {MAX_SHELL_COMMAND_TOKENS} tokens" + ) references: list[str] = [] - for segment in shell_command_segments(shell_tokens(command)): - reference = command_segment_image_reference(segment) - if reference is not None: - references.append(reference) + for segment in shell_command_segments(tokens): + references.extend(command_segment_image_references(segment, depth)) return references @@ -675,8 +793,8 @@ def assignment_continues(relative: Path, value: str) -> bool: def image_assignment_references(relative: Path, text: str) -> list[tuple[int, str]]: if relative.suffix in YAML_SUFFIXES: - references = set(yaml_declarative_image_references(text)) - for line_number, command in yaml_executable_commands(text): + references = set(yaml_declarative_image_references(relative, text)) + for line_number, command in yaml_executable_commands(relative, text): matched = assignment_match(relative, command) if matched is None: continue @@ -717,44 +835,156 @@ def image_assignment_references(relative: Path, text: str) -> list[tuple[int, st return sorted(references) -@lru_cache(maxsize=1) -def compose_yaml(text: str) -> Node | None: - try: - return yaml.compose(text, Loader=yaml.SafeLoader) - except yaml.YAMLError: - return None +def require_yaml() -> None: + if yaml is None: + detail = f": {YAML_IMPORT_ERROR}" if YAML_IMPORT_ERROR is not None else "" + raise ImageSurfaceError( + "PyYAML is required to scan maintained YAML image surfaces" + detail + ) -def yaml_mapping_entries(root: Node) -> list[tuple[Node, Node]]: - entries: list[tuple[Node, Node]] = [] - stack = [root] - visited: set[int] = set() +@lru_cache(maxsize=4) +def compose_yaml_documents(text: str) -> tuple[Node, ...]: + require_yaml() + if len(text) > MAX_YAML_TEXT_CHARS: + raise ImageSurfaceError( + f"YAML input exceeds {MAX_YAML_TEXT_CHARS} characters" + ) + try: + documents: list[Node] = [] + document_count = 0 + for root in yaml.compose_all(text, Loader=yaml.SafeLoader): + document_count += 1 + if root is not None: + documents.append(root) + if document_count > MAX_YAML_DOCUMENTS: + raise ImageSurfaceError( + f"YAML input exceeds {MAX_YAML_DOCUMENTS} documents" + ) + return tuple(documents) + except RecursionError as error: + raise ImageSurfaceError("YAML parser recursion limit exceeded") from error + except yaml.YAMLError as error: + detail = str(error).splitlines()[0] if str(error) else type(error).__name__ + raise ImageSurfaceError(f"invalid YAML: {detail}") from error + + +def yaml_mapping_entries( + root: Node, +) -> list[tuple[tuple[str | int, ...], Node, Node]]: + entries: list[tuple[tuple[str | int, ...], Node, Node]] = [] + stack: list[tuple[Node, tuple[str | int, ...], int, tuple[int, ...]]] = [ + (root, (), 0, ()) + ] + visited_nodes = 0 while stack: - node = stack.pop() - if id(node) in visited: - continue - visited.add(id(node)) + node, path, depth, ancestors = stack.pop() + if depth > MAX_YAML_DEPTH: + raise ImageSurfaceError(f"YAML nesting exceeds {MAX_YAML_DEPTH} levels") + if id(node) in ancestors: + raise ImageSurfaceError("recursive YAML aliases are not supported") + visited_nodes += 1 + (len(node.value) if isinstance(node, MappingNode) else 0) + if visited_nodes > MAX_YAML_NODES: + raise ImageSurfaceError(f"YAML input exceeds {MAX_YAML_NODES} nodes") + child_ancestors = ancestors + (id(node),) if isinstance(node, MappingNode): - entries.extend(node.value) - stack.extend(value for _, value in node.value) + for key_node, value_node in reversed(node.value): + key = ( + key_node.value + if isinstance(key_node, ScalarNode) + and key_node.tag == "tag:yaml.org,2002:str" + else "" + ) + child_path = path + (key,) + entries.append((child_path, key_node, value_node)) + stack.append((value_node, child_path, depth + 1, child_ancestors)) elif isinstance(node, SequenceNode): - stack.extend(node.value) + for index, value_node in reversed(list(enumerate(node.value))): + stack.append( + (value_node, path + (index,), depth + 1, child_ancestors) + ) return entries +def yaml_document_context(relative: Path, root: Node) -> tuple[bool, bool, bool]: + fields = ( + { + key.value: value + for key, value in root.value + if isinstance(key, ScalarNode) + and key.tag == "tag:yaml.org,2002:str" + } + if isinstance(root, MappingNode) + else {} + ) + name = relative.name.casefold() + compose = ( + name in {"compose.yaml", "compose.yml"} + or name.startswith("docker-compose") + or isinstance(fields.get("services"), MappingNode) + and not {"registry", "starter", "integrations", "entities"} & set(fields) + ) + return ( + is_workflow(relative) or isinstance(fields.get("jobs"), MappingNode), + compose, + {"apiVersion", "kind"} <= set(fields), + ) + + +def yaml_field_context( + path: tuple[str | int, ...], + context: tuple[bool, bool, bool], +) -> str: + """Classify only GitHub Actions, Compose, Kubernetes, and image variables.""" + + workflow, compose, kubernetes = context + if ( + workflow + and len(path) == 5 + and path[:1] == ("jobs",) + and path[2] == "steps" + and path[4] in {"run", "uses"} + ): + return f"workflow_{path[4]}" + if workflow and path[:1] == ("jobs",) and ( + len(path) == 3 and path[2] == "container" + or len(path) == 4 and path[2:] == ("container", "image") + or len(path) == 5 and path[2] == "services" and path[4] == "image" + ): + return "image" + if compose and len(path) == 3 and path[0] == "services": + return f"compose_{path[2]}" + if kubernetes and len(path) >= 4 and ( + path[-3] in KUBERNETES_CONTAINER_COLLECTIONS + and isinstance(path[-2], int) + and "spec" in path[:-3] + ): + return f"kubernetes_{path[-1]}" + key = str(path[-1]) if path else "" + lowered = key.casefold().replace("-", "_") + return ( + "image" + if key == "IMAGE" + or lowered.endswith("_image") + or key != lowered and lowered.endswith("image") + else "" + ) + + def yaml_scalar_commands( node: ScalarNode, lines: list[str], ) -> list[tuple[int, str]]: if node.tag != "tag:yaml.org,2002:str" or not node.value: return [] + if node.style == ">": + source_line = node.start_mark.line + 1 + return [ + (source_line, command) + for _, command in logical_lines(node.value) + ] if node.style != "|": source_line = node.start_mark.line + 1 - if node.style == ">": - for index in range(node.start_mark.line + 1, node.end_mark.line): - if index < len(lines) and lines[index].strip(): - source_line = index + 1 - break return [ (source_line + line_number - 1, command) for line_number, command in logical_lines(node.value) @@ -783,66 +1013,90 @@ def yaml_scalar_commands( ) -def yaml_executable_commands(text: str) -> list[tuple[int, str]]: - """Extract only values of YAML run, script, and command mapping keys.""" +def yaml_sequence_argv(node: Node) -> list[str] | None: + if not isinstance(node, SequenceNode): + return None + values = [ + item.value + for item in node.value + if isinstance(item, ScalarNode) + and item.tag == "tag:yaml.org,2002:str" + ] + return values if len(values) == len(node.value) else None - root = compose_yaml(text) - if root is None: - return [] + +@lru_cache(maxsize=4) +def yaml_surface_values( + relative: Path, + text: str, +) -> tuple[tuple[tuple[int, str], ...], tuple[tuple[int, str], ...]]: + """Return commands and image references from the finite YAML support matrix.""" + + roots = compose_yaml_documents(text) lines = text.splitlines() commands: list[tuple[int, str]] = [] - for key_node, value_node in yaml_mapping_entries(root): - if ( - not isinstance(key_node, ScalarNode) - or key_node.tag != "tag:yaml.org,2002:str" - or key_node.value.casefold() not in YAML_EXECUTABLE_KEYS - ): - continue - key = key_node.value.casefold() - if isinstance(value_node, ScalarNode): - commands.extend(yaml_scalar_commands(value_node, lines)) - elif isinstance(value_node, SequenceNode): - scalar_nodes = [ - item - for item in value_node.value - if isinstance(item, ScalarNode) - and item.tag == "tag:yaml.org,2002:str" - ] - if len(scalar_nodes) != len(value_node.value): + references: set[tuple[int, str]] = set() + for root in roots: + document_context = yaml_document_context(relative, root) + kubernetes_argv: dict[ + tuple[str | int, ...], dict[str, tuple[int, list[str]]] + ] = {} + for path, key_node, value_node in yaml_mapping_entries(root): + context = yaml_field_context(path, document_context) + if context == "workflow_run" and isinstance(value_node, ScalarNode): + commands.extend(yaml_scalar_commands(value_node, lines)) continue - # Command sequences are exec-form argv; run/script sequences are - # lists of shell command strings. - if key == "command": - command = shlex.join([item.value for item in scalar_nodes]) - if command: - commands.append((value_node.start_mark.line + 1, command)) - else: - for item in scalar_nodes: - commands.extend(yaml_scalar_commands(item, lines)) - return commands + if context in {"compose_command", "compose_entrypoint"}: + if isinstance(value_node, ScalarNode): + commands.extend(yaml_scalar_commands(value_node, lines)) + else: + argv = yaml_sequence_argv(value_node) + if argv: + commands.append((key_node.start_mark.line + 1, shlex.join(argv))) + continue + if context in {"kubernetes_command", "kubernetes_args"}: + argv = yaml_sequence_argv(value_node) + if argv is not None: + kubernetes_argv.setdefault(path[:-1], {})[context] = ( + key_node.start_mark.line + 1, + argv, + ) + continue + if ( + context + not in { + "compose_image", + "image", + "kubernetes_image", + "workflow_uses", + } + or not isinstance(key_node, ScalarNode) + or not isinstance(value_node, ScalarNode) + or value_node.tag != "tag:yaml.org,2002:str" + ): + continue + value = value_node.value + if context == "workflow_uses" and not value.casefold().startswith("docker://"): + continue + for reference in reference_candidates(normalize_reference(value)): + references.add((key_node.start_mark.line + 1, reference)) + for fields in kubernetes_argv.values(): + command = fields.get("kubernetes_command") + args = fields.get("kubernetes_args", (0, []))[1] + if command is not None and command[1] + args: + commands.append((command[0], shlex.join(command[1] + args))) + return tuple(commands), tuple(sorted(references)) -def yaml_declarative_image_references(text: str) -> list[tuple[int, str]]: - root = compose_yaml(text) - if root is None: - return [] - references: set[tuple[int, str]] = set() - for key_node, value_node in yaml_mapping_entries(root): - if ( - not isinstance(key_node, ScalarNode) - or key_node.tag != "tag:yaml.org,2002:str" - or not isinstance(value_node, ScalarNode) - or value_node.tag != "tag:yaml.org,2002:str" - ): - continue - key = key_node.value.casefold() - if key == "uses" and not value_node.value.casefold().startswith("docker://"): - continue - if key != "uses" and not is_image_assignment(key): - continue - for reference in reference_candidates(normalize_reference(value_node.value)): - references.add((key_node.start_mark.line + 1, reference)) - return sorted(references) +def yaml_executable_commands(relative: Path, text: str) -> list[tuple[int, str]]: + return list(yaml_surface_values(relative, text)[0]) + + +def yaml_declarative_image_references( + relative: Path, + text: str, +) -> list[tuple[int, str]]: + return list(yaml_surface_values(relative, text)[1]) def command_image_references(relative: Path, text: str) -> list[tuple[int, str]]: @@ -850,7 +1104,7 @@ def command_image_references(relative: Path, text: str) -> list[tuple[int, str]] return [] references: set[tuple[int, str]] = set() commands = ( - yaml_executable_commands(text) + yaml_executable_commands(relative, text) if relative.suffix in YAML_SUFFIXES else logical_lines(text) ) @@ -1071,7 +1325,12 @@ def check_repository(root: Path = ROOT) -> list[str]: if not is_image_reference_surface(relative): continue text = texts[relative] - for line, reference in image_references(relative, text): + try: + references = image_references(relative, text) + except ImageSurfaceError as error: + failures.append(f"{relative}: image scan failed: {error}") + continue + for line, reference in references: append_reference_failures(failures, relative, line, reference) for relative in PRODUCT_DOCKERFILES: diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 3bd6d2c3..a022c7ab 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -6,6 +6,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -281,7 +282,9 @@ def test_discovered_script_scans_after_shell_list_operators(self) -> None: script.write_text( "#!/usr/bin/env bash\n" 'cd "$PWD" && docker run --rm debian true\n' - "echo ok; podman run --rm node:22 true\n", + "echo ok; podman run --rm node:22 true\n" + "echo ok & docker run --rm python:3.12 true\n" + "echo ok |& podman run --rm rust:1.95-trixie true\n", encoding="utf-8", ) @@ -303,6 +306,8 @@ def test_discovered_script_scans_after_shell_list_operators(self) -> None: ), failures, ) + self.assertTrue(any("build-example.sh:4" in item for item in failures)) + self.assertTrue(any("build-example.sh:5" in item for item in failures)) def test_shell_control_prefixes_find_image_references(self) -> None: cases = { @@ -338,6 +343,9 @@ def test_shell_wrapper_options_find_image_references(self) -> None: "docker run --rm rust:1.95-trixie true": ["rust:1.95-trixie"], "sudo --preserve-env=CI time -p " "podman run --rm debian true": ["debian"], + "sudo -A docker run --rm debian true": ["debian"], + "sudo -i podman run --rm node:22 true": ["node:22"], + "sudo -P docker run --rm python:3.12 true": ["python:3.12"], } for command, expected in cases.items(): @@ -347,6 +355,41 @@ def test_shell_wrapper_options_find_image_references(self) -> None: self.module.command_image_references_in_command(command), ) + def test_shell_launchers_are_unwrapped_with_a_depth_bound(self) -> None: + cases = { + 'sh -c "docker run --rm debian true"': ["debian"], + 'sudo -A bash -lc "podman run --rm node:22 true"': ["node:22"], + "env -S 'bash -c \"docker run --rm python:3.12 true\"'": [ + "python:3.12" + ], + "env --split-string='sh -c \"podman run --rm rust:1.95-trixie\"'": [ + "rust:1.95-trixie" + ], + "env -Ssh -c 'docker run --rm debian true'": ["debian"], + } + for command, expected in cases.items(): + with self.subTest(command=command): + self.assertEqual( + expected, + self.module.command_image_references_in_command(command), + ) + + nested = "docker run --rm debian true" + for _ in range(self.module.MAX_SHELL_RECURSION_DEPTH + 1): + nested = f"sh -c {self.module.shlex.quote(nested)}" + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "shell command nesting exceeds", + ): + self.module.command_image_references_in_command(nested) + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "shell command exceeds", + ): + self.module.command_image_references_in_command( + "x" * (self.module.MAX_SHELL_COMMAND_CHARS + 1) + ) + def test_shell_parser_rejects_invalid_control_and_wrapper_prefixes(self) -> None: commands = ( "if then docker run --rm debian true", @@ -384,110 +427,159 @@ def test_command_scan_ignores_prose_prefixes(self) -> None: self.module.command_image_references_in_command(command), ) - def test_yaml_executable_command_forms_find_image_references(self) -> None: + def test_yaml_supported_executable_contexts_find_image_references(self) -> None: cases = { - "inline scalar": ( - "run: sudo -E docker run --rm debian true\n", - [(1, "debian")], - ), - "quoted scalar": ( - 'script: "env -i podman run --rm node:22 true"\n', - [(1, "node:22")], - ), - "literal scalar": ( - "run: |\n" - " command -- docker run --rm python:3.12 true\n", - [(2, "python:3.12")], - ), - "folded scalar": ( - "script: >\n" - " sudo -E docker run --rm\n" - " rust:1.95-trixie true\n", - [(2, "rust:1.95-trixie")], - ), - "flow script list": ( - "script: [\"docker run --rm debian true\", " - "\"env -i podman run --rm node:22 true\"]\n", - [(1, "debian"), (1, "node:22")], - ), - "flow command argv": ( - "command: [command, --, docker, run, --rm, python:3.12]\n", - [(1, "python:3.12")], + "workflow literal": ( + Path(".github/workflows/example.yml"), + "jobs:\n" + " test:\n" + " steps:\n" + " - run: |\n" + " command -- docker run --rm python:3.12 true\n", + [(5, "python:3.12")], ), - "multiline flow script list": ( - "script: [\n" - " \"sudo -E docker run --rm debian true\",\n" - " \"podman run --rm node:22 true\"\n" - "]\n", - [(2, "debian"), (3, "node:22")], + "workflow folded reports scalar start": ( + Path(".github/workflows/example.yml"), + "jobs:\n" + " test:\n" + " steps:\n" + " - run: >\n" + " sudo -E docker run --rm\n" + " rust:1.95-trixie true\n", + [(4, "rust:1.95-trixie")], ), - "block script list": ( - "script:\n" - " # CI commands\n" - " - docker run --rm debian true\n" - " - env -i podman run --rm node:22 true\n", + "compose argv": ( + Path("compose.yaml"), + "services:\n" + " helper:\n" + " image: debian\n" + " command: [sh, -c, \"podman run --rm node:22 true\"]\n", [(3, "debian"), (4, "node:22")], ), - "block command argv": ( - "command:\n" - " - command\n" - " - --\n" - " - docker\n" - " - run\n" - " - --rm\n" - " - python:3.12\n", - [(2, "python:3.12")], + "Kubernetes command and args": ( + Path("pod.yaml"), + "apiVersion: v1\n" + "kind: Pod\n" + "spec:\n" + " containers:\n" + " - name: helper\n" + " image: debian\n" + " command: [sh, -c]\n" + " args: [\"docker run --rm python:3.12 true\"]\n", + [(6, "debian"), (7, "python:3.12")], ), - "shell image assignment": ( - "run: |\n" - " BUILDER_IMAGE=debian\n", - [(2, "debian")], + "image variable": ( + Path("images.yaml"), + "BUILDER_IMAGE: debian\n", + [(1, "debian")], ), } - for name, (text, expected) in cases.items(): + for name, (relative, text, expected) in cases.items(): with self.subTest(name=name): self.assertEqual( expected, - self.module.image_references(Path("example.yaml"), text), + self.module.image_references(relative, text), ) - def test_yaml_descriptive_scalars_and_lists_are_not_executable(self) -> None: + def test_yaml_domain_data_is_not_executable(self) -> None: text = ( - "\"description\": |\n" - " docker run --rm debian true\n" - " run: podman run --rm node:22 true\n" - " image: debian\n" - " uses: docker://python:3.12\n" - "notes: >\n" - " command -- docker run --rm python:3.12 true\n" - "examples:\n" - " - |\n" - " script: docker run --rm rust:1.95-trixie true\n" - "summary: [\"docker run --rm debian true\"]\n" - "script:\n" - " file: adapter.rhai\n" + "experiment:\n" + " run: docker run --rm debian true\n" + "capability:\n" + " script: docker run --rm node:22 true\n" + "metadata:\n" + " command: [docker, run, --rm, python:3.12]\n" ) - self.assertEqual( - [], - self.module.image_references(Path("example.yaml"), text), - ) + self.assertEqual([], self.module.image_references(Path("data.yaml"), text)) - def test_yaml_block_scalar_scope_ends_at_parent_indent(self) -> None: + def test_yaml_all_documents_are_scanned(self) -> None: text = ( - "steps:\n" - " - description: |\n" - " docker run --rm debian true\n" - " run: docker run --rm node:22 true\n" - " - run: docker run --rm python:3.12 true\n" + "experiment:\n" + " run: docker run --rm debian true\n" + "---\n" + "services:\n" + " helper:\n" + " image: node:22\n" + "---\n" + "jobs:\n" + " test:\n" + " steps:\n" + " - run: docker run --rm python:3.12 true\n" ) - self.assertEqual( - [(4, "node:22"), (5, "python:3.12")], - self.module.command_image_references(Path("example.yaml"), text), + [(6, "node:22"), (11, "python:3.12")], + self.module.image_references(Path("example.yaml"), text), ) + def test_malformed_yaml_fails_the_repository_check(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required_surfaces(root) + manifest = root / "products/example/broken.yaml" + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text("jobs:\n broken: [\n", encoding="utf-8") + + failures = self.module.check_repository(root) + + self.assertTrue( + any( + "products/example/broken.yaml: image scan failed: invalid YAML" + in failure + for failure in failures + ), + failures, + ) + + def test_yaml_parser_recursion_and_depth_fail_explicitly(self) -> None: + self.module.compose_yaml_documents.cache_clear() + with mock.patch.object( + self.module.yaml, + "compose_all", + side_effect=RecursionError, + ): + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "YAML parser recursion limit exceeded", + ): + self.module.image_references(Path("recursive.yaml"), "jobs: {}\n") + + deep = "".join( + f"{' ' * depth}level_{depth}:\n" for depth in range(6) + ) + " value: true\n" + with mock.patch.object(self.module, "MAX_YAML_DEPTH", 3): + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "YAML nesting exceeds", + ): + self.module.image_references(Path("deep.yaml"), deep) + with mock.patch.object(self.module, "MAX_YAML_DOCUMENTS", 2): + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "YAML input exceeds 2 documents", + ): + self.module.image_references( + Path("documents.yaml"), + "---\n---\n---\n", + ) + + def test_missing_pyyaml_fails_with_dependency_name(self) -> None: + self.module.compose_yaml_documents.cache_clear() + with ( + mock.patch.object(self.module, "yaml", None), + mock.patch.object( + self.module, + "YAML_IMPORT_ERROR", + ImportError("No module named yaml"), + ), + ): + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "PyYAML is required", + ): + self.module.image_references(Path("missing.yaml"), "jobs: {}\n") + def test_discovered_script_rejects_untagged_debian_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -539,15 +631,19 @@ def test_discovered_yaml_rejects_untagged_debian_image(self) -> None: failures, ) - def test_discovered_yaml_rejects_list_item_image_key(self) -> None: + def test_discovered_kubernetes_yaml_rejects_container_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.copy_required_surfaces(root) - compose = root / "products/example/compose.yaml" - compose.parent.mkdir(parents=True, exist_ok=True) - compose.write_text( - "containers:\n" - " - image: debian\n", + manifest = root / "products/example/pod.yaml" + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text( + "apiVersion: v1\n" + "kind: Pod\n" + "spec:\n" + " containers:\n" + " - name: helper\n" + " image: debian\n", encoding="utf-8", ) @@ -555,7 +651,7 @@ def test_discovered_yaml_rejects_list_item_image_key(self) -> None: self.assertTrue( any( - "products/example/compose.yaml:2" in failure + "products/example/pod.yaml:6" in failure and failure.endswith(": debian") for failure in failures ), From 7b77d1047c0c878fbf6da5a778e93c92b22501a4 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 01:08:47 +0700 Subject: [PATCH 10/34] fix(release): simplify image boundary scan Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 1808 ++++------------- release/scripts/test_check_debian13_images.py | 1264 +++--------- 2 files changed, 707 insertions(+), 2365 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index c346bbbf..8ee2e7a2 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -5,24 +5,11 @@ import os import re -import shlex +import subprocess import sys -from functools import lru_cache from pathlib import Path -try: - import yaml - from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode -except ImportError as error: - yaml = None - MappingNode = Node = ScalarNode = SequenceNode = None - YAML_IMPORT_ERROR = error -else: - YAML_IMPORT_ERROR = None - - ROOT = Path(__file__).resolve().parents[2] - RUST_BUILDER = ( "rust:1.95-trixie@sha256:" "f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3" @@ -39,9 +26,9 @@ "docker/dockerfile:1.7@sha256:" "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) -CI_WORKFLOW = Path(".github/workflows/ci.yml") -REGISTRYCTL_TUTORIAL_SCRIPT = Path("docs/site/scripts/check-registryctl-tutorials.sh") +CI_WORKFLOW = Path(".github/workflows/ci.yml") +TUTORIAL_SCRIPT = Path("docs/site/scripts/check-registryctl-tutorials.sh") PRODUCT_DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), Path("crates/registry-relay/Dockerfile.demo"), @@ -49,1495 +36,455 @@ Path("release/docker/Dockerfile.registry-notary"), Path("release/docker/Dockerfile.registry-relay"), ) - -# Product-specific assertions below require these surfaces. The Debian -# generation and immutable-pin boundary is broader and is discovered from the -# repository instead of relying on this tuple. REQUIRED_PRODUCT_SURFACES = PRODUCT_DOCKERFILES + ( CI_WORKFLOW, Path(".github/workflows/release.yml"), Path("release/scripts/build-release-binaries.sh"), - Path("crates/registry-relay/docs/ops.md"), - Path("crates/registry-relay/docs/security-assurance.md"), - Path("crates/registry-relay/scripts/check_docker_build_contract.py"), Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), - REGISTRYCTL_TUTORIAL_SCRIPT, - Path("products/notary/docs/security-assurance.md"), + TUTORIAL_SCRIPT, ) +RELAY_DOCKERFILES = ( + PRODUCT_DOCKERFILES[0], + PRODUCT_DOCKERFILES[1], + PRODUCT_DOCKERFILES[4], +) +NOTARY_DOCKERFILES = (PRODUCT_DOCKERFILES[2], PRODUCT_DOCKERFILES[3]) -EXCLUDED_DIRECTORY_NAMES = { - ".git", - ".repo-docs-cache", - ".research", - ".venv", - "__pycache__", - "dist", - "node_modules", - "target", +# Scan tracked UTF-8 text, excluding only historical evidence, third-party or +# generated material, build output, and this checker's negative fixtures. +EXCLUDED_DIRS = set( + ".git .repo-docs-cache .research .venv __pycache__ dist node_modules target".split() +) +EXCLUDED_PREFIXES = ("external/", "release/notes/") +EXCLUDED_EXACT = { + "release/scripts/check-debian13-images.py", + "release/scripts/test_check_debian13_images.py", } -# Historical release notes and .research preserve observations, not active policy. -EXCLUDED_PATH_PREFIXES = (Path("release/notes"),) -MARKDOWN_SUFFIXES = {".md", ".mdx"} -SHELL_SUFFIXES = {".bash", ".sh"} -PYTHON_SUFFIXES = {".py"} -JS_TS_SUFFIXES = {".js", ".mjs", ".ts"} -SCRIPT_SUFFIXES = SHELL_SUFFIXES | PYTHON_SUFFIXES | JS_TS_SUFFIXES -YAML_SUFFIXES = {".yaml", ".yml"} - -RUST_BUILDER_DOCKERFILES = PRODUCT_DOCKERFILES[:3] -PREPARATION_DOCKERFILES = PRODUCT_DOCKERFILES[3:] -RELAY_DOCKERFILES = ( - Path("crates/registry-relay/Dockerfile"), - Path("crates/registry-relay/Dockerfile.demo"), - Path("release/docker/Dockerfile.registry-relay"), +MAX_TRACKED_PATHS = 10_000 +MAX_TEXT_FILE_BYTES = 2_000_000 +MAX_TOTAL_TEXT_BYTES = 32_000_000 +MAX_LINE_CHARS = 131_072 + +RETIRED_MARKERS = ( + "book" + "worm", "bullseye", "buster", "debian" + "12", "debian-12", + "debian" + "11", "debian-11", "debian" + "10", "debian-10", ) -NOTARY_DOCKERFILES = ( - Path("products/notary/Dockerfile"), - Path("release/docker/Dockerfile.registry-notary"), +DEFAULT_DEBIAN_FAMILIES = {"golang", "node", "python", "rust"} +OCI_RE = re.compile( + r"(?(?:docker://)?" + r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+" + r"(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-fA-F]{64})" + r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" ) - +BARE_DEBIAN_RE = re.compile( + r"(?" - r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*" - r"[A-Za-z0-9._-]+:[A-Za-z0-9._-]+" - r"(?:@sha256:[0-9a-f]{64})?" - r")" - r"(?![A-Za-z0-9._/@+-])", - re.IGNORECASE, -) -IMAGE_ASSIGNMENT_RE = re.compile( - r"^\s*(?:-\s*)?(?:(?:export|local|readonly)(?:\s+-[A-Za-z]+\s+|\s+))?" - r"(?P[A-Za-z_][A-Za-z0-9_-]*)\s*(?:=|:)\s*" - r"(?P[^#\s]+)", +ASSIGN_RE = re.compile( + r"^\s*(?:-\s*)?(?:(?:export|local|readonly|const|let|var)\b[^A-Za-z_]*)?" + r"(?P[A-Za-z_][A-Za-z0-9_$-]*)(?:\s*:[^=]+)?" + r"\s*=\s*(?P.+)$" ) -PY_IMAGE_ASSIGNMENT_RE = re.compile( - r"^\s*(?P[A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=\s*(?P.*)$" +YAML_KEY_RE = re.compile( + r"^\s*(?:-\s*)?(?P[A-Za-z_][A-Za-z0-9_$-]*)\s*:\s*(?P.+)$" ) -JS_TS_IMAGE_ASSIGNMENT_RE = re.compile( - r"^\s*(?:export\s+)?(?:const|let|var)\s+" - r"(?P[A-Za-z_$][A-Za-z0-9_$]*)" - r"\s*(?::[^=]+)?=\s*(?P.*)$" +IMAGE_VAR_RE = re.compile( + r"\b(?P[A-Za-z_][A-Za-z0-9_$-]*(?:_image|Image)|IMAGE)\b", + re.IGNORECASE, ) -STRING_LITERAL_RE = re.compile( - r"\"(?P(?:\\.|[^\"\\])*)\"|'(?P(?:\\.|[^'\\])*)'" +CLI_RE = re.compile(r"(?:^|[\s;&|({])(?:\S*/)?(?:docker|podman)\b(?P[^\n]*)") +ACTION_RE = re.compile(r"\b(?:create|pull|run)\b") +NON_IMAGE_NAMESPACES = set("build buildx compose exec network volume".split()) +EXEMPTION_RE = re.compile( + r'' ) -COPY_RE = re.compile(r"^\s*COPY\b(?P.*)$", re.IGNORECASE) -SHELL_ASSIGNMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=.*$") -UNTAGGED_IMAGE_REFERENCE_RE = re.compile( - r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+", +MARKDOWN_SUFFIXES = {".md", ".mdx"} +CODE_SUFFIXES = set(".bash .js .mjs .py .sh .ts .yaml .yml".split()) +STRICT_ASSIGNMENT_SUFFIXES = CODE_SUFFIXES - {".yaml", ".yml"} +FENCE_LANGS = set( + "bash console dockerfile javascript js python sh shell terminal ts " + "typescript yaml yml zsh".split() ) -VERSION_ONLY_TAG_RE = re.compile(r"^[0-9]+(?:[._][0-9]+)*$") -DEBIAN_DEFAULT_IMAGE_NAMES = { - "buildpack-deps", - "debian", - "golang", - "node", - "python", - "rust", -} -CONTAINER_COMMANDS = {"create", "pull", "run"} -DOCKER_GLOBAL_BOOLEAN_OPTIONS = { - "--debug", - "--tls", - "-D", -} -DOCKER_GLOBAL_VALUE_OPTIONS = { - "--config", - "--context", - "--host", - "--log-level", - "-H", -} -CONTAINER_BOOLEAN_OPTIONS = { - "--detach", - "--init", - "--interactive", - "--no-healthcheck", - "--oom-kill-disable", - "--privileged", - "--publish-all", - "--quiet", - "--read-only", - "--rm", - "--sig-proxy", - "--tty", - "-d", - "-i", - "-it", - "-P", - "-q", - "-t", -} -SHELL_COMMAND_SEPARATORS = {";", "&&", "||", "&", "|", "|&"} -SHELL_COMMAND_CONTROL_PREFIXES = { - "do", - "elif", - "else", - "if", - "then", - "until", - "while", -} -SHELL_OPEN_GROUP_PREFIXES = {"(", "{"} -SUPPORTED_SHELL_COMMANDS = {"bash", "dash", "ksh", "sh", "zsh"} -SHELL_COMMAND_SHORT_OPTIONS = set("abcefhklmnptuvxBCEHP") -MAX_SHELL_COMMAND_CHARS = 65_536 -MAX_SHELL_COMMAND_TOKENS = 2_048 -MAX_SHELL_RECURSION_DEPTH = 4 -MAX_YAML_TEXT_CHARS = 2_000_000 -MAX_YAML_DOCUMENTS = 128 -MAX_YAML_NODES = 100_000 -MAX_YAML_DEPTH = 64 -KUBERNETES_CONTAINER_COLLECTIONS = { - "containers", "ephemeralContainers", "initContainers" -} -COMMAND_BOOLEAN_OPTIONS = {"-p"} -ENV_BOOLEAN_OPTIONS = {"--debug", "--ignore-environment", "-i", "-v"} -ENV_VALUE_OPTIONS = {"--chdir", "--unset", "-C", "-u"} -SUDO_BOOLEAN_OPTIONS = { - "--askpass", - "--background", - "--login", - "--non-interactive", - "--preserve-env", - "--preserve-groups", - "--set-home", - "--stdin", - "-A", - "-E", - "-H", - "-P", - "-S", - "-b", - "-i", - "-k", - "-n", -} -SUDO_VALUE_OPTIONS = { - "--chdir", - "--chroot", - "--close-from", - "--command-timeout", - "--group", - "--host", - "--prompt", - "--role", - "--type", - "--user", - "-C", - "-D", - "-g", - "-h", - "-p", - "-r", - "-R", - "-t", - "-T", - "-u", -} -MARKDOWN_SHELL_FENCE_LANGS = {"bash", "console", "sh", "shell", "terminal", "zsh"} -MARKDOWN_YAML_FENCE_LANGS = {"yaml", "yml"} -MARKDOWN_DOCKERFILE_FENCE_LANGS = {"dockerfile"} class ImageSurfaceError(RuntimeError): - """A maintained image surface could not be scanned conclusively.""" + """A maintained text surface exceeded a scanner boundary.""" - -def is_excluded(relative: Path) -> bool: - if ".research" in relative.parts: - return True - return any( - relative == prefix or prefix in relative.parents - for prefix in EXCLUDED_PATH_PREFIXES - ) - - -def is_dockerfile(relative: Path) -> bool: +def is_dockerfile(path: Path) -> bool: return ( - relative.name == "Dockerfile" - or relative.name.startswith("Dockerfile.") - or relative.name.endswith(".Dockerfile") - ) - - -def is_active_script(relative: Path) -> bool: - return relative.suffix in {".bash", ".sh"} or ( - "scripts" in relative.parts - and (relative.suffix in SCRIPT_SUFFIXES or not relative.suffix) + path.name == "Dockerfile" + or path.name.startswith("Dockerfile.") + or path.name.endswith(".Dockerfile") ) - -def is_workflow(relative: Path) -> bool: - return relative.parts[:2] == (".github", "workflows") - - -def is_maintained_text_surface(relative: Path) -> bool: +def is_excluded(path: Path) -> bool: + value = path.as_posix() return ( - is_dockerfile(relative) - or is_active_script(relative) - or is_workflow(relative) - or relative.suffix in MARKDOWN_SUFFIXES - or relative.suffix in YAML_SUFFIXES + value in EXCLUDED_EXACT + or any(part in EXCLUDED_DIRS for part in path.parts) + or any(value.startswith(prefix) for prefix in EXCLUDED_PREFIXES) + or path.name in {"CHANGELOG.md", "release-notes.md"} + or value == "docs/site/src/content/docs/changelog.mdx" + or "/resources/scalar/" in f"/{value}/" ) - -def is_image_reference_surface(relative: Path) -> bool: - return ( - is_dockerfile(relative) - or is_active_script(relative) - or relative.suffix in YAML_SUFFIXES - or relative.suffix in MARKDOWN_SUFFIXES - ) - - def discover_maintained_surfaces(root: Path) -> tuple[Path, ...]: - """Discover maintained image policy surfaces without walking build output.""" - - discovered: set[Path] = set() - for directory, directory_names, file_names in os.walk(root): - directory_path = Path(directory) - directory_names[:] = sorted( - name - for name in directory_names - if name not in EXCLUDED_DIRECTORY_NAMES - and not is_excluded((directory_path / name).relative_to(root)) + command = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z"], + capture_output=True, + check=False, + ) + if command.returncode == 0: + paths = [Path(item.decode()) for item in command.stdout.split(b"\0") if item] + else: + paths = [] + for directory, names, files in os.walk(root): + names[:] = [name for name in names if name not in EXCLUDED_DIRS] + paths.extend( + (Path(directory) / name).relative_to(root) for name in files + ) + if len(paths) > MAX_TRACKED_PATHS: + raise ImageSurfaceError( + f"tracked path count exceeds {MAX_TRACKED_PATHS}: {len(paths)}" ) - for name in file_names: - relative = (directory_path / name).relative_to(root) - if is_excluded(relative) or not is_maintained_text_surface(relative): - continue - discovered.add(relative) - return tuple(sorted(discovered)) - - -def normalize_reference(reference: str) -> str: - stripped = reference.strip().strip(",;").strip("\"'") - if stripped.startswith("docker://"): - stripped = stripped.removeprefix("docker://") - return stripped - - -def repository_and_tag(reference: str) -> tuple[str, str | None]: - unpinned = normalize_reference(reference).split("@", 1)[0] - slash = unpinned.rfind("/") - colon = unpinned.rfind(":") - if colon > slash: - return unpinned[:colon], unpinned[colon + 1:] - return unpinned, None - - -def is_debian_derived(reference: str) -> bool: - repository, tag = repository_and_tag(reference) - if tag is None: - return False - image_name = repository.rsplit("/", 1)[-1].casefold() - lowered_tag = tag.casefold() - generation_tags = ("trixie", "book" + "worm", "bullseye", "buster") - return ( - image_name == "debian" - or "debian" in image_name - or any(marker in lowered_tag for marker in generation_tags) - or re.search(r"(?:^|[-_.])debian-?1[0-9](?:$|[-_.])", lowered_tag) - is not None + return tuple( + sorted(path for path in paths if not is_excluded(path) and (root / path).is_file() and not (root / path).is_symlink()) ) +def read_text(root: Path, path: Path) -> tuple[str | None, int]: + target = root / path + try: + size = target.stat().st_size + except FileNotFoundError: + return None, 0 + if size > MAX_TEXT_FILE_BYTES: + with target.open("rb") as stream: + sample = stream.read(8192) + try: + sample.decode() + except UnicodeDecodeError: + return None, 0 + if b"\0" in sample: + return None, 0 + raise ImageSurfaceError( + f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes" + ) + data = target.read_bytes() + if b"\0" in data: + return None, 0 + try: + return data.decode(), size + except UnicodeDecodeError: + return None, 0 -def retired_generation_marker(reference: str) -> str | None: - lowered = reference.casefold() - for marker in ("book" + "worm", "debian" + "12"): - if marker in lowered: - return marker - return None +def references(text: str) -> list[str]: + return [ + match.group("ref").removeprefix("docker://").strip("\"',;") + for match in OCI_RE.finditer(text) + ] +def repository_and_tag(reference: str) -> tuple[str, str | None]: + value = reference.split("@", 1)[0] + slash, colon = value.rfind("/"), value.rfind(":") + return (value[:colon], value[colon + 1 :]) if colon > slash else (value, None) -def is_debian_default_version_only(reference: str) -> bool: +def is_debian_family(reference: str) -> bool: repository, tag = repository_and_tag(reference) - if tag is None: - return False - image_name = repository.rsplit("/", 1)[-1].casefold() + name, tag = repository.rsplit("/", 1)[-1].casefold(), (tag or "").casefold() return ( - image_name in DEBIAN_DEFAULT_IMAGE_NAMES - and VERSION_ONLY_TAG_RE.fullmatch(tag) is not None + "debian" in name + or name == "buildpack-deps" + or "trixie" in tag + or re.search(r"debian-?13", tag) is not None + or name in DEFAULT_DEBIAN_FAMILIES + and (tag in {"latest", "slim"} or re.fullmatch(r"[0-9]+(?:[._][0-9]+)*", tag) is not None) ) +def assignment(path: Path, line: str) -> tuple[str, str] | None: + match = (YAML_KEY_RE if path.suffix in {".yaml", ".yml"} else ASSIGN_RE).match(line) + if match is None: + return None + name = match.group("name") + normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() + return (name, match.group("value")) if normalized.endswith("image") or normalized == "container" else None + +def is_container_consumer(line: str) -> bool: + for command in CLI_RE.finditer(line): + tail = command.group("tail") + action = ACTION_RE.search(tail) + if action is None: + continue + prefix = set(re.findall(r"[A-Za-z][A-Za-z0-9-]*", tail[: action.start()])) + if not prefix & NON_IMAGE_NAMESPACES: + return True + return False -def is_untagged_debian_derived(reference: str) -> bool: - if UNTAGGED_IMAGE_REFERENCE_RE.fullmatch(reference) is None: - return False - repository, tag = repository_and_tag(reference) - if tag is not None or "@" in reference: - return False - image_name = repository.rsplit("/", 1)[-1].casefold() - return "debian" in image_name or image_name in DEBIAN_DEFAULT_IMAGE_NAMES - - -def is_image_assignment(name: str) -> bool: - lowered = name.casefold().replace("$", "") - return lowered in {"container", "image"} or lowered.endswith("image") - - -def numbered_logical_lines( - numbered_lines: list[tuple[int, str]], -) -> list[tuple[int, str]]: - lines: list[tuple[int, str]] = [] - start = 0 - parts: list[str] = [] - for line_number, raw_line in numbered_lines: +def policy_image_name(name: str) -> bool: + value, markers = name.casefold().replace("$", ""), {"base", "builder", "debian", "runtime"} + return re.sub("[^a-z0-9]", "", value) == "image" or value.startswith(tuple(markers)) or bool(set(re.split(r"[_-]+", value)) & markers) + +def markdown_code_flags(path: Path, lines: list[str]) -> list[bool]: + if path.suffix not in MARKDOWN_SUFFIXES: + return [False] * len(lines) + active, marker, result = False, "", [] + for line in lines: + fence = re.match(r"(```+|~~~+)\s*([A-Za-z0-9_-]*)", line.lstrip()) + if fence: + token, language = fence.groups() + if marker and token.startswith(marker): + active, marker = False, "" + elif not marker: + active, marker = language.casefold() in FENCE_LANGS, token[:3] + result.append(False) + else: + result.append(active) + return result + +def logical_lines( + lines: list[str], flags: list[bool] +) -> list[tuple[int, str, bool]]: + result, parts, start, active = [], [], 1, False + for number, line in enumerate(lines, 1): if not parts: - start = line_number - stripped = raw_line.rstrip() + start, active = number, flags[number - 1] + stripped = line.rstrip() continued = stripped.endswith("\\") parts.append(stripped[:-1] if continued else stripped) if not continued: - lines.append((start, " ".join(parts))) + result.append((start, " ".join(parts), active)) parts = [] if parts: - lines.append((start, " ".join(parts))) - return lines - - -def logical_lines(text: str) -> list[tuple[int, str]]: - return numbered_logical_lines(list(enumerate(text.splitlines(), 1))) - - -def shell_tokens(command: str) -> list[str]: - try: - lexer = shlex.shlex(command, posix=True, punctuation_chars=True) - lexer.whitespace_split = True - lexer.commenters = "#" - return list(lexer) - except ValueError: - return [] - - -def shell_command_segments(tokens: list[str]) -> list[list[str]]: - segments: list[list[str]] = [] - segment: list[str] = [] - for token in tokens: - if token in SHELL_COMMAND_SEPARATORS: - if segment: - segments.append(segment) - segment = [] - continue - segment.append(token) - if segment: - segments.append(segment) - return segments - - -def skip_explicit_options( - tokens: list[str], - index: int, - *, - boolean_options: set[str], - value_options: set[str], - groupable_short_options: set[str], - attached_value_options: set[str], - optional_value_options: set[str] | None = None, - split_string_options: set[str] | None = None, -) -> tuple[int | None, str | None]: - optional_value_options = optional_value_options or set() - split_string_options = split_string_options or set() - while index < len(tokens): - candidate = tokens[index] - if candidate == "--": - return index + 1, None - if not candidate.startswith("-") or candidate == "-": - return index, None - split_option = candidate.split("=", 1)[0] - attached_split = ( - "-S" - if "-S" in split_string_options - and candidate.startswith("-S") - and candidate != "-S" - else None - ) - if split_option in split_string_options or attached_split is not None: - if "=" in candidate: - payload = candidate.split("=", 1)[1] - suffix = tokens[index + 1 :] - elif candidate in split_string_options: - if index + 1 >= len(tokens): - return None, None - payload = tokens[index + 1] - suffix = tokens[index + 2 :] - else: - payload = candidate[len(attached_split) :] - suffix = tokens[index + 1 :] - return None, ( - payload if not suffix else f"{payload} {shlex.join(suffix)}" + result.append((start, " ".join(parts), active)) + return result + +def scan_surface(path: Path, text: str) -> list[str]: + if len(text.encode()) > MAX_TEXT_FILE_BYTES: + raise ImageSurfaceError(f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes") + lines, failures = text.splitlines(), [] + markdown_flags = markdown_code_flags(path, text.splitlines()) + code_file = is_dockerfile(path) or path.suffix in CODE_SUFFIXES + records: list[tuple[int, str, str, set[str], bool, bool]] = [] + resolved, declared = set(), set() + for number, line in enumerate(lines, 1): + if len(line) > MAX_LINE_CHARS: + raise ImageSurfaceError( + f"{path}:{number}: line exceeds {MAX_LINE_CHARS} characters" ) - if candidate in boolean_options: - index += 1 - continue - if candidate in value_options: - if index + 1 >= len(tokens): - return None, None - index += 2 - continue - if candidate.startswith("--") and "=" in candidate: - option = candidate.split("=", 1)[0] - if option in value_options | optional_value_options: - index += 1 - continue - return None, None - if any( - candidate.startswith(option) and len(candidate) > len(option) - for option in attached_value_options - ): - index += 1 - continue - if len(candidate) > 2 and all( - character in groupable_short_options for character in candidate[1:] - ): - index += 1 - continue - return None, None - return index, None - - -def skip_time_prefix(tokens: list[str], index: int) -> int: - index += 1 - if index < len(tokens) and tokens[index] == "-p": - index += 1 - if index < len(tokens) and tokens[index] == "--": - index += 1 - return index - - -def simple_command(tokens: list[str]) -> tuple[int | None, str | None]: - """Locate a container CLI only through the supported shell prefix grammar.""" - - index = 0 - if index < len(tokens) and tokens[index] == "$": - index += 1 - - while index < len(tokens) and tokens[index] in SHELL_OPEN_GROUP_PREFIXES: - index += 1 - if index < len(tokens) and tokens[index] in SHELL_COMMAND_CONTROL_PREFIXES: - index += 1 - while index < len(tokens) and tokens[index] in SHELL_OPEN_GROUP_PREFIXES: - index += 1 - - seen_negation = False - seen_time = False - while index < len(tokens): - token = tokens[index] - if token == "!" and not seen_negation: - seen_negation = True - index += 1 - elif token == "time" and not seen_time: - seen_time = True - index = skip_time_prefix(tokens, index) - elif token in SHELL_OPEN_GROUP_PREFIXES: - index += 1 - else: - break - - while index < len(tokens): - while index < len(tokens) and SHELL_ASSIGNMENT_RE.fullmatch(tokens[index]): - index += 1 - if index >= len(tokens): - return None, None - - wrapper = tokens[index] - if wrapper == "time" and not seen_time: - seen_time = True - index = skip_time_prefix(tokens, index) + markdown_code = markdown_flags[number - 1] + comment = line.lstrip().startswith(("#", "//")) and not markdown_code + exemption = ( + path.suffix in MARKDOWN_SUFFIXES + and not markdown_code + and EXEMPTION_RE.search(line) is not None + ) + if "debian13-policy:" in line and not exemption: + failures.append(f"{path}:{number}: invalid Debian image prose exemption") + if exemption: continue - if wrapper == "command": - index, _ = skip_explicit_options( - tokens, - index + 1, - boolean_options=COMMAND_BOOLEAN_OPTIONS, - value_options=set(), - groupable_short_options={"p"}, - attached_value_options=set(), + lowered = line.casefold() + for marker in RETIRED_MARKERS: + if marker in lowered: + failures.append( + f"{path}:{number}: retired Debian image generation marker remains: {marker}" + ) + item = assignment(path, line) if path.suffix in CODE_SUFFIXES else None + consumer = is_container_consumer(line) and (code_file or markdown_code) and not comment + reference_context = ( + not comment and (is_dockerfile(path) + or path.suffix in {".yaml", ".yml"} + or markdown_code + or path.suffix in {".sh", ".bash"} or item is not None + or consumer) + ) + if reference_context: + for reference in references(line): + if not is_debian_family(reference): + continue + prefix = f"{path}:{number}: Debian-derived image reference" + if not DIGEST_RE.search(reference): + failures.append( + f"{prefix} is not pinned by immutable digest: {reference}" + ) + if "trixie" not in reference.casefold() and re.search( + r"debian-?13", reference, re.IGNORECASE + ) is None: + failures.append( + f"{prefix} does not declare Trixie/Debian 13: {reference}" + ) + if item: + name, value = item + canonical = name.casefold() + dependencies = { + found.group("name").casefold() + for found in IMAGE_VAR_RE.finditer(value) + } + has_literal = bool(references(value)) + positional = canonical == "image" and re.fullmatch( + r"""["']?\$(?:\{)?[1-9](?:\})?["']?;?""", value.strip() ) - elif wrapper == "env": - index, payload = skip_explicit_options( - tokens, - index + 1, - boolean_options=ENV_BOOLEAN_OPTIONS, - value_options=ENV_VALUE_OPTIONS, - groupable_short_options={"i", "v"}, - attached_value_options={"-C", "-S", "-u"}, - split_string_options={"-S", "--split-string"}, + computed = not positional and ( + not has_literal + and not dependencies + or re.search(r"\s[+%]\s|`|\$\(|\.format\(|\bf[\"']", value) + is not None ) - if payload is not None: - return None, payload - elif wrapper == "sudo": - index, _ = skip_explicit_options( - tokens, - index + 1, - boolean_options=SUDO_BOOLEAN_OPTIONS, - value_options=SUDO_VALUE_OPTIONS, - groupable_short_options={"A", "E", "H", "P", "S", "b", "i", "k", "n"}, - attached_value_options={ - "-C", - "-D", - "-g", - "-h", - "-p", - "-r", - "-R", - "-t", - "-T", - "-u", - }, - optional_value_options={"--preserve-env"}, + strict = ( + path.suffix in STRICT_ASSIGNMENT_SUFFIXES + and policy_image_name(name) ) - else: - return index, None - if index is None: - return None, None - return index, None - - -def skip_options( - tokens: list[str], - index: int, - *, - boolean_options: set[str], - value_options: set[str], -) -> int: - while index < len(tokens): - candidate = tokens[index] - if candidate == "--": - return index + 1 - if not candidate.startswith("-"): - return index - if ( - "=" in candidate - or candidate in boolean_options - or any( - candidate.startswith(f"{option}=") - for option in value_options - if option.startswith("--") + records.append( + (number, canonical, value, dependencies, computed, strict) ) + declared.add(canonical) + if has_literal and not computed or positional: + resolved.add(canonical) + if not comment and BARE_DEBIAN_RE.search(line) and ( + item is not None or (code_file or markdown_code) and (is_dockerfile(path) or consumer) ): - index += 1 - elif candidate in value_options: - index += 2 - else: - index += 2 - return index - - -def container_command_image_reference( - tokens: list[str], - container_index: int, -) -> str | None: - if ( - container_index >= len(tokens) - or tokens[container_index] not in {"docker", "podman"} - ): - return None - action_index = skip_options( - tokens, - container_index + 1, - boolean_options=DOCKER_GLOBAL_BOOLEAN_OPTIONS, - value_options=DOCKER_GLOBAL_VALUE_OPTIONS, - ) - if action_index >= len(tokens): - return None - action = tokens[action_index] - if action in {"container", "image"} and action_index + 1 < len(tokens): - action_index += 1 - action = tokens[action_index] - if action not in CONTAINER_COMMANDS: - return None - index = action_index + 1 - index = skip_options( - tokens, - index, - boolean_options=CONTAINER_BOOLEAN_OPTIONS, - value_options=set(), - ) - if index < len(tokens): - return normalize_reference(tokens[index]) - return None - - -def shell_command_payload(tokens: list[str], command_index: int) -> str | None: - command = tokens[command_index].rsplit("/", 1)[-1] - if command not in SUPPORTED_SHELL_COMMANDS: - return None - index = command_index + 1 - while index < len(tokens): - option = tokens[index] - if option == "-c": - return tokens[index + 1] if index + 1 < len(tokens) else None - if ( - option.startswith("-") - and not option.startswith("--") - and len(option) > 2 - and all(character in SHELL_COMMAND_SHORT_OPTIONS for character in option[1:]) - ): - if "c" in option[1:]: - return tokens[index + 1] if index + 1 < len(tokens) else None - index += 1 - continue - if ( - len(option) == 2 - and option.startswith("-") - and option[1] in SHELL_COMMAND_SHORT_OPTIONS - ): - index += 1 - continue - return None - return None - - -def command_segment_image_references( - tokens: list[str], - depth: int, -) -> list[str]: - command_index, split_payload = simple_command(tokens) - if split_payload is not None: - return command_image_references_in_command(split_payload, depth=depth + 1) - if command_index is None or command_index >= len(tokens): - return [] - reference = container_command_image_reference(tokens, command_index) - if reference is not None: - return [reference] - payload = shell_command_payload(tokens, command_index) - if payload is not None: - return command_image_references_in_command(payload, depth=depth + 1) - return [] - - -def command_image_references_in_command( - command: str, - *, - depth: int = 0, -) -> list[str]: - if depth > MAX_SHELL_RECURSION_DEPTH: - raise ImageSurfaceError( - f"shell command nesting exceeds {MAX_SHELL_RECURSION_DEPTH} levels" - ) - if len(command) > MAX_SHELL_COMMAND_CHARS: - raise ImageSurfaceError( - f"shell command exceeds {MAX_SHELL_COMMAND_CHARS} characters" - ) - tokens = shell_tokens(command) - if len(tokens) > MAX_SHELL_COMMAND_TOKENS: - raise ImageSurfaceError( - f"shell command exceeds {MAX_SHELL_COMMAND_TOKENS} tokens" - ) - references: list[str] = [] - for segment in shell_command_segments(tokens): - references.extend(command_segment_image_references(segment, depth)) - return references - - -def decode_string_literal(value: str) -> str: - return value.replace(r"\/", "/").replace(r"\"", '"').replace(r"\'", "'") - - -def string_literals(line: str) -> list[str]: - values: list[str] = [] - for match in STRING_LITERAL_RE.finditer(line): - raw = match.group("double") if match.group("double") is not None else match.group("single") - values.append(decode_string_literal(raw)) - return values - - -def reference_candidates(value: str) -> list[str]: - stripped = normalize_reference(value) - candidates = [ - normalize_reference(match.group("reference")) - for match in CONTAINER_REFERENCE_RE.finditer(value) - ] - if stripped and is_untagged_debian_derived(stripped): - candidates.append(stripped) - return candidates - - -def assignment_match(relative: Path, line: str) -> tuple[str, str] | None: - if relative.suffix in PYTHON_SUFFIXES: - match = PY_IMAGE_ASSIGNMENT_RE.match(line) - if match is None: - return None - return match.group("name"), match.group("value") - if relative.suffix in JS_TS_SUFFIXES: - match = JS_TS_IMAGE_ASSIGNMENT_RE.match(line) - if match is None: - return None - return match.group("name"), match.group("value") - match = IMAGE_ASSIGNMENT_RE.match(line) - if match is None: - return None - return match.group("name"), match.group("reference") - - -def assignment_continues(relative: Path, value: str) -> bool: - stripped = value.strip() - if relative.suffix not in PYTHON_SUFFIXES | JS_TS_SUFFIXES: - return False - if stripped.endswith("\\") or stripped in {"", "(", "["}: - return True - if stripped.count("(") > stripped.count(")"): - return True - if relative.suffix in JS_TS_SUFFIXES and not stripped.endswith(";"): - return not string_literals(stripped) - return False - - -def image_assignment_references(relative: Path, text: str) -> list[tuple[int, str]]: - if relative.suffix in YAML_SUFFIXES: - references = set(yaml_declarative_image_references(relative, text)) - for line_number, command in yaml_executable_commands(relative, text): - matched = assignment_match(relative, command) - if matched is None: - continue - name, value = matched - if not is_image_assignment(name): - continue - for reference in reference_candidates(value): - references.add((line_number, reference)) - return sorted(references) - - references: set[tuple[int, str]] = set() - active = False - for line_number, line in enumerate(text.splitlines(), 1): - if active: - for literal in string_literals(line): - for reference in reference_candidates(literal): - references.add((line_number, reference)) - stripped = line.strip() - if stripped.endswith((")", "];", ";")) or stripped == ")": - active = False - continue - - matched = assignment_match(relative, line) - if matched is None: - continue - name, value = matched - if not is_image_assignment(name): + failures.append( + f"{path}:{number}: bare Debian image reference is not pinned " + "and does not declare Trixie/Debian 13: debian" + ) + changed = True + while changed: + changed = False + for _, name, _, dependencies, computed, strict in records: + if name not in resolved and not computed and dependencies & resolved: + resolved.add(name) + changed = True + for number, name, value, _, computed, strict in records: + if strict and (computed or name not in resolved): + failures.append( + f"{path}:{number}: computed or unresolved image assignment " + f"is not allowed: {value.strip()}" + ) + for number, line, markdown_code in logical_lines(lines, markdown_flags): + if line.lstrip().startswith(("#", "//")) or not (code_file or markdown_code) or not is_container_consumer(line): continue - literals = string_literals(value) - if literals: - for literal in literals: - for reference in reference_candidates(literal): - references.add((line_number, reference)) - else: - for reference in reference_candidates(value): - references.add((line_number, reference)) - active = assignment_continues(relative, value) - return sorted(references) - - -def require_yaml() -> None: - if yaml is None: - detail = f": {YAML_IMPORT_ERROR}" if YAML_IMPORT_ERROR is not None else "" - raise ImageSurfaceError( - "PyYAML is required to scan maintained YAML image surfaces" + detail - ) - - -@lru_cache(maxsize=4) -def compose_yaml_documents(text: str) -> tuple[Node, ...]: - require_yaml() - if len(text) > MAX_YAML_TEXT_CHARS: - raise ImageSurfaceError( - f"YAML input exceeds {MAX_YAML_TEXT_CHARS} characters" - ) - try: - documents: list[Node] = [] - document_count = 0 - for root in yaml.compose_all(text, Loader=yaml.SafeLoader): - document_count += 1 - if root is not None: - documents.append(root) - if document_count > MAX_YAML_DOCUMENTS: - raise ImageSurfaceError( - f"YAML input exceeds {MAX_YAML_DOCUMENTS} documents" - ) - return tuple(documents) - except RecursionError as error: - raise ImageSurfaceError("YAML parser recursion limit exceeded") from error - except yaml.YAMLError as error: - detail = str(error).splitlines()[0] if str(error) else type(error).__name__ - raise ImageSurfaceError(f"invalid YAML: {detail}") from error - - -def yaml_mapping_entries( - root: Node, -) -> list[tuple[tuple[str | int, ...], Node, Node]]: - entries: list[tuple[tuple[str | int, ...], Node, Node]] = [] - stack: list[tuple[Node, tuple[str | int, ...], int, tuple[int, ...]]] = [ - (root, (), 0, ()) - ] - visited_nodes = 0 - while stack: - node, path, depth, ancestors = stack.pop() - if depth > MAX_YAML_DEPTH: - raise ImageSurfaceError(f"YAML nesting exceeds {MAX_YAML_DEPTH} levels") - if id(node) in ancestors: - raise ImageSurfaceError("recursive YAML aliases are not supported") - visited_nodes += 1 + (len(node.value) if isinstance(node, MappingNode) else 0) - if visited_nodes > MAX_YAML_NODES: - raise ImageSurfaceError(f"YAML input exceeds {MAX_YAML_NODES} nodes") - child_ancestors = ancestors + (id(node),) - if isinstance(node, MappingNode): - for key_node, value_node in reversed(node.value): - key = ( - key_node.value - if isinstance(key_node, ScalarNode) - and key_node.tag == "tag:yaml.org,2002:str" - else "" - ) - child_path = path + (key,) - entries.append((child_path, key_node, value_node)) - stack.append((value_node, child_path, depth + 1, child_ancestors)) - elif isinstance(node, SequenceNode): - for index, value_node in reversed(list(enumerate(node.value))): - stack.append( - (value_node, path + (index,), depth + 1, child_ancestors) - ) - return entries - - -def yaml_document_context(relative: Path, root: Node) -> tuple[bool, bool, bool]: - fields = ( - { - key.value: value - for key, value in root.value - if isinstance(key, ScalarNode) - and key.tag == "tag:yaml.org,2002:str" + variables = { + match.group("name").casefold() for match in IMAGE_VAR_RE.finditer(line) } - if isinstance(root, MappingNode) - else {} - ) - name = relative.name.casefold() - compose = ( - name in {"compose.yaml", "compose.yml"} - or name.startswith("docker-compose") - or isinstance(fields.get("services"), MappingNode) - and not {"registry", "starter", "integrations", "entities"} & set(fields) - ) - return ( - is_workflow(relative) or isinstance(fields.get("jobs"), MappingNode), - compose, - {"apiVersion", "kind"} <= set(fields), - ) - - -def yaml_field_context( - path: tuple[str | int, ...], - context: tuple[bool, bool, bool], -) -> str: - """Classify only GitHub Actions, Compose, Kubernetes, and image variables.""" - - workflow, compose, kubernetes = context - if ( - workflow - and len(path) == 5 - and path[:1] == ("jobs",) - and path[2] == "steps" - and path[4] in {"run", "uses"} - ): - return f"workflow_{path[4]}" - if workflow and path[:1] == ("jobs",) and ( - len(path) == 3 and path[2] == "container" - or len(path) == 4 and path[2:] == ("container", "image") - or len(path) == 5 and path[2] == "services" and path[4] == "image" - ): - return "image" - if compose and len(path) == 3 and path[0] == "services": - return f"compose_{path[2]}" - if kubernetes and len(path) >= 4 and ( - path[-3] in KUBERNETES_CONTAINER_COLLECTIONS - and isinstance(path[-2], int) - and "spec" in path[:-3] - ): - return f"kubernetes_{path[-1]}" - key = str(path[-1]) if path else "" - lowered = key.casefold().replace("-", "_") - return ( - "image" - if key == "IMAGE" - or lowered.endswith("_image") - or key != lowered and lowered.endswith("image") - else "" - ) - - -def yaml_scalar_commands( - node: ScalarNode, - lines: list[str], -) -> list[tuple[int, str]]: - if node.tag != "tag:yaml.org,2002:str" or not node.value: - return [] - if node.style == ">": - source_line = node.start_mark.line + 1 - return [ - (source_line, command) - for _, command in logical_lines(node.value) - ] - if node.style != "|": - source_line = node.start_mark.line + 1 - return [ - (source_line + line_number - 1, command) - for line_number, command in logical_lines(node.value) - ] - - content = [ - (index + 1, lines[index]) - for index in range( - node.start_mark.line + 1, - min(node.end_mark.line, len(lines)), - ) - ] - indents = [ - len(line) - len(line.lstrip(" ")) - for _, line in content - if line.strip() - ] - if not indents: - return [] - content_indent = min(indents) - return numbered_logical_lines( - [ - (line_number, line[content_indent:] if line.strip() else "") - for line_number, line in content - ] - ) - - -def yaml_sequence_argv(node: Node) -> list[str] | None: - if not isinstance(node, SequenceNode): - return None - values = [ - item.value - for item in node.value - if isinstance(item, ScalarNode) - and item.tag == "tag:yaml.org,2002:str" - ] - return values if len(values) == len(node.value) else None - - -@lru_cache(maxsize=4) -def yaml_surface_values( - relative: Path, - text: str, -) -> tuple[tuple[tuple[int, str], ...], tuple[tuple[int, str], ...]]: - """Return commands and image references from the finite YAML support matrix.""" - - roots = compose_yaml_documents(text) - lines = text.splitlines() - commands: list[tuple[int, str]] = [] - references: set[tuple[int, str]] = set() - for root in roots: - document_context = yaml_document_context(relative, root) - kubernetes_argv: dict[ - tuple[str | int, ...], dict[str, tuple[int, list[str]]] - ] = {} - for path, key_node, value_node in yaml_mapping_entries(root): - context = yaml_field_context(path, document_context) - if context == "workflow_run" and isinstance(value_node, ScalarNode): - commands.extend(yaml_scalar_commands(value_node, lines)) - continue - if context in {"compose_command", "compose_entrypoint"}: - if isinstance(value_node, ScalarNode): - commands.extend(yaml_scalar_commands(value_node, lines)) - else: - argv = yaml_sequence_argv(value_node) - if argv: - commands.append((key_node.start_mark.line + 1, shlex.join(argv))) - continue - if context in {"kubernetes_command", "kubernetes_args"}: - argv = yaml_sequence_argv(value_node) - if argv is not None: - kubernetes_argv.setdefault(path[:-1], {})[context] = ( - key_node.start_mark.line + 1, - argv, - ) - continue - if ( - context - not in { - "compose_image", - "image", - "kubernetes_image", - "workflow_uses", - } - or not isinstance(key_node, ScalarNode) - or not isinstance(value_node, ScalarNode) - or value_node.tag != "tag:yaml.org,2002:str" - ): - continue - value = value_node.value - if context == "workflow_uses" and not value.casefold().startswith("docker://"): - continue - for reference in reference_candidates(normalize_reference(value)): - references.add((key_node.start_mark.line + 1, reference)) - for fields in kubernetes_argv.values(): - command = fields.get("kubernetes_command") - args = fields.get("kubernetes_args", (0, []))[1] - if command is not None and command[1] + args: - commands.append((command[0], shlex.join(command[1] + args))) - return tuple(commands), tuple(sorted(references)) - - -def yaml_executable_commands(relative: Path, text: str) -> list[tuple[int, str]]: - return list(yaml_surface_values(relative, text)[0]) - - -def yaml_declarative_image_references( - relative: Path, - text: str, -) -> list[tuple[int, str]]: - return list(yaml_surface_values(relative, text)[1]) - - -def command_image_references(relative: Path, text: str) -> list[tuple[int, str]]: - if relative.suffix not in SHELL_SUFFIXES | YAML_SUFFIXES and relative.suffix: - return [] - references: set[tuple[int, str]] = set() - commands = ( - yaml_executable_commands(relative, text) - if relative.suffix in YAML_SUFFIXES - else logical_lines(text) - ) - for line_number, command in commands: - for reference in command_image_references_in_command(command): - references.add((line_number, reference)) - return sorted(references) - - -def markdown_code_blocks(text: str) -> list[tuple[int, str, str]]: - blocks: list[tuple[int, str, str]] = [] - block_lines: list[str] = [] - block_start = 0 - language = "" - in_block = False - for line_number, line in enumerate(text.splitlines(), 1): - stripped = line.strip() - if stripped.startswith("```"): - if in_block: - blocks.append((block_start, language, "\n".join(block_lines))) - block_lines = [] - in_block = False - else: - block_start = line_number + 1 - info = stripped.removeprefix("```").strip().split(maxsplit=1) - language = info[0].casefold() if info else "" - in_block = True - continue - if in_block: - block_lines.append(line) - return blocks - - -def dockerfile_reference_lines(text: str) -> list[tuple[int, str]]: - references: list[tuple[int, str]] = [] - stage_names: set[str] = set() - for match in FROM_RE.finditer(text): - base = normalize_reference(match.group(1)) - line = text.count("\n", 0, match.start()) + 1 - if base.casefold() != "scratch" and base.casefold() not in stage_names: - references.append((line, base)) - stage_name = match.group(2) - if stage_name: - stage_names.add(stage_name.casefold()) - references.extend(copy_from_references(text, stage_names)) - return references - - -def markdown_image_references(text: str) -> list[tuple[int, str]]: - references: set[tuple[int, str]] = set() - for start_line, language, block in markdown_code_blocks(text): - if language in MARKDOWN_SHELL_FENCE_LANGS: - block_relative = Path("example.sh") - block_references = command_image_references(block_relative, block) - elif language in MARKDOWN_YAML_FENCE_LANGS: - block_relative = Path("example.yaml") - block_references = image_references(block_relative, block) - elif language in MARKDOWN_DOCKERFILE_FENCE_LANGS: - block_references = dockerfile_reference_lines(block) - else: - continue - for line_number, reference in block_references: - references.add((start_line + line_number - 1, reference)) - return sorted(references) - - -def image_references(relative: Path, text: str) -> list[tuple[int, str]]: - if relative.suffix in MARKDOWN_SUFFIXES: - return markdown_image_references(text) - return sorted( - set(image_assignment_references(relative, text)) - | set(command_image_references(relative, text)) - ) - - -def reference_requires_digest(reference: str) -> bool: - if DIGEST_PIN_RE.search(reference): - return False - _, tag = repository_and_tag(reference) - if tag is not None: - return is_debian_derived(reference) or is_debian_default_version_only(reference) - return is_untagged_debian_derived(reference) - - -def read(root: Path, relative: Path, failures: list[str]) -> str: - path = root / relative - try: - return path.read_text(encoding="utf-8") - except FileNotFoundError: - failures.append(f"missing maintained image surface: {relative}") - return "" - except UnicodeDecodeError: - failures.append(f"maintained image surface is not UTF-8 text: {relative}") - return "" - + if not any(re.search("[A-Za-z]", repository_and_tag(item)[0]) for item in references(line)) and not BARE_DEBIAN_RE.search(line) and not variables & (resolved | declared): + failures.append( + f"{path}:{number}: Docker/Podman image consumer must use a " + "literal or a resolved *_IMAGE assignment" + ) + return failures def require( - text: str, - needle: str, - relative: Path, - detail: str, - failures: list[str], + text: str, needle: str, path: Path, detail: str, failures: list[str] ) -> None: if needle not in text: - failures.append(f"{relative}: missing {detail}: {needle!r}") - - -def copy_from_references(text: str, stage_names: set[str]) -> list[tuple[int, str]]: - references: list[tuple[int, str]] = [] - for line_number, line in logical_lines(text): - match = COPY_RE.match(line) - if match is None: - continue - try: - tokens = shlex.split(match.group("args"), comments=True, posix=True) - except ValueError: - continue - index = 0 - while index < len(tokens): - token = tokens[index] - source: str | None = None - if token == "--from" and index + 1 < len(tokens): - source = tokens[index + 1] - index += 2 - elif token.startswith("--from="): - source = token.split("=", 1)[1] - index += 1 - elif token.startswith("--"): - index += 1 - else: - break - if source is None: - continue - normalized = normalize_reference(source) - if normalized.casefold() in stage_names or normalized.isdigit(): - continue - references.append((line_number, normalized)) - return references - - -def append_reference_failures( - failures: list[str], - relative: Path, - line: int, - reference: str, -) -> None: - marker = retired_generation_marker(reference) - if marker is not None: - failures.append( - f"{relative}:{line}: retired Debian image generation marker " - f"remains in image reference: {marker}: {reference}" - ) - if reference_requires_digest(reference): - failures.append( - f"{relative}:{line}: Debian-derived image reference is not pinned " - f"by immutable digest: {reference}" - ) - + failures.append(f"{path}: missing {detail}: {needle!r}") def runtime_stage(text: str) -> str: marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" offset = text.find(marker) return text[offset:] if offset >= 0 else "" - -def registryctl_tutorial_cache_step(text: str) -> str: - start = text.find("- name: Cache source-under-test Cargo build") - if start < 0: - return "" - end = text.find("\n - name: Execute registryctl tutorials from source", start) - return text[start:] if end < 0 else text[start:end] - +def product_contracts(texts: dict[Path, str], failures: list[str]) -> None: + for path in PRODUCT_DOCKERFILES: + text, runtime = texts.get(path, ""), runtime_stage(texts.get(path, "")) + require(text, f"FROM {DISTROLESS_RUNTIME} AS runtime", path, "Distroless Debian 13 non-root final runtime", failures) + require(runtime, "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3", path, "binary healthcheck", failures) + for forbidden in ("\nRUN ", "apt-get", "/bin/sh", "curl ", "wget "): + if forbidden in runtime: + failures.append(f"{path}: final Distroless runtime contains {forbidden.strip()!r}") + for path in PRODUCT_DOCKERFILES[:3]: + require(texts.get(path, ""), f"FROM {RUST_BUILDER} AS builder", path, "pinned Debian 13 Rust builder", failures) + for path in PRODUCT_DOCKERFILES[3:]: + text = texts.get(path, "") + if not text.startswith(f"# syntax={DOCKERFILE_FRONTEND}\n"): + failures.append(f"{path}: pinned Dockerfile frontend must be the first line") + for needle, detail in ( + (f"FROM {DEBIAN_PREPARATION} AS runtime-root", "runtime preparation base"), + ("ARG SOURCE_DATE_EPOCH=0", "fixed release filesystem timestamp"), + ("RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin", "ephemeral release input mount"), + ('find /workspace/runtime-root -exec touch -h --date="@${SOURCE_DATE_EPOCH}" {} +', "normalized release filesystem metadata"), + ): + require(text, needle, path, detail, failures) + for path in RELAY_DOCKERFILES: + text = texts.get(path, "") + require(text, "/usr/local/bin/registry-relay-rhai-worker", path, "Relay worker binary", failures) + require(runtime_stage(text), 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', path, "absolute Relay entrypoint", failures) + require(texts.get(PRODUCT_DOCKERFILES[2], ""), 'ARG REGISTRY_NOTARY_FEATURES="registry-notary-cel,pkcs11"', PRODUCT_DOCKERFILES[2], "PKCS#11-enabled product build", failures) + for path in NOTARY_DOCKERFILES: + text = texts.get(path, "") + for source, needle, detail in ( + (text, "registry-notary-cel-worker", "Notary CEL worker binary"), + (runtime_stage(text), 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', "absolute Notary entrypoint"), + (text, "chown -R 65532:65532", "numeric nonroot-owned Notary runtime directories"), + (runtime_stage(text), "WORKDIR /var/lib/registry-notary", "Notary working directory"), + ): + require(source, needle, path, detail, failures) + if re.search(r"^\s*(?:COPY|ADD)\b[^\n]*(?:\.so\b|pkcs11[^/\s]*module)", text, re.IGNORECASE | re.MULTILINE): + failures.append(f"{path}: vendor PKCS#11 modules must remain external read-only mounts") + workflow = texts.get(Path(".github/workflows/release.yml"), "") + require(workflow, f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", Path(".github/workflows/release.yml"), "pinned Debian 13 release builder", failures) + require(texts.get(Path("release/scripts/build-release-binaries.sh"), ""), "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", Path("release/scripts/build-release-binaries.sh"), "PKCS#11-enabled release build", failures) + require(texts.get(Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), ""), RUST_BUILDER, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", failures) + tutorial = texts.get(TUTORIAL_SCRIPT, "") + ci = texts.get(CI_WORKFLOW, "") + start = ci.find("- name: Cache source-under-test Cargo build") + end = ci.find("\n - name: Execute registryctl tutorials from source", start) + cache = "" if start < 0 else ci[start:] if end < 0 else ci[start:end] + for source, needle, path, detail in ( + (tutorial, 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', TUTORIAL_SCRIPT, "registryctl tutorial linux target path"), + (tutorial, 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', TUTORIAL_SCRIPT, "registryctl tutorial Cargo home path"), + (cache, "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", CI_WORKFLOW, "registryctl tutorial cache builder identity"), + (cache, "target/registryctl-tutorial-linux-amd64", CI_WORKFLOW, "registryctl tutorial linux target cache path"), + (cache, "target/registryctl-tutorial-cargo-home", CI_WORKFLOW, "registryctl tutorial Cargo home cache path"), + ): + require(source, needle, path, detail, failures) + if "restore-keys:" in cache: + failures.append(f"{CI_WORKFLOW}: registryctl tutorial cache must not restore from pre-builder-identity keys") def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] - maintained_paths = discover_maintained_surfaces(root) - all_paths = tuple(sorted(set(maintained_paths) | set(REQUIRED_PRODUCT_SURFACES))) - texts = { - relative: read(root, relative, failures) - for relative in all_paths - } - - dockerfiles = tuple( - relative for relative in maintained_paths if is_dockerfile(relative) - ) - if not dockerfiles: - failures.append("no maintained Dockerfiles discovered") - - for relative in dockerfiles: - text = texts[relative] - bases = FROM_RE.findall(text) - if not bases: - failures.append(f"{relative}: no FROM instruction found") + try: + discovered = discover_maintained_surfaces(root) + except ImageSurfaceError as error: + return [str(error)] + paths = tuple(sorted(set(discovered) | set(REQUIRED_PRODUCT_SURFACES))) + texts: dict[Path, str] = {} + total = 0 + for path in paths: + try: + text, size = read_text(root, path) + except ImageSurfaceError as error: + failures.append(str(error)) continue - stage_names: set[str] = set() - for base, stage_name in bases: - internal_stage = base.casefold() in stage_names - lowered_base = base.casefold() - for marker in ("book" + "worm", "debian" + "12"): - if not internal_stage and marker in lowered_base: - failures.append( - f"{relative}: retired Debian image generation marker remains: {marker}" - ) - if ( - base.casefold() != "scratch" - and not internal_stage - and not DIGEST_PIN_RE.search(base) - ): - failures.append( - f"{relative}: upstream base is not pinned by immutable digest: {base}" - ) - if stage_name: - stage_names.add(stage_name.casefold()) - for line, reference in copy_from_references(text, stage_names): - append_reference_failures(failures, relative, line, reference) - - for relative in maintained_paths: - if not is_image_reference_surface(relative): + if text is None: + if path in REQUIRED_PRODUCT_SURFACES: + failures.append(f"missing maintained image surface: {path}") continue - text = texts[relative] + total += size + if total > MAX_TOTAL_TEXT_BYTES: + return [f"maintained text exceeds {MAX_TOTAL_TEXT_BYTES} total bytes"] + texts[path] = text try: - references = image_references(relative, text) + failures.extend(scan_surface(path, text)) except ImageSurfaceError as error: - failures.append(f"{relative}: image scan failed: {error}") - continue - for line, reference in references: - append_reference_failures(failures, relative, line, reference) - - for relative in PRODUCT_DOCKERFILES: - text = texts[relative] - require( - text, - f"FROM {DISTROLESS_RUNTIME} AS runtime", - relative, - "Distroless Debian 13 non-root final runtime", - failures, - ) - runtime = runtime_stage(text) - for forbidden in ("\nRUN ", "apt-get", "/bin/sh", "curl ", "wget "): - if forbidden in runtime: - failures.append( - f"{relative}: final Distroless runtime contains {forbidden.strip()!r}" - ) - require( - runtime, - "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3", - relative, - "binary healthcheck", - failures, - ) - - for relative in RUST_BUILDER_DOCKERFILES: - require( - texts[relative], - f"FROM {RUST_BUILDER} AS builder", - relative, - "pinned Debian 13 Rust builder", - failures, - ) - - for relative in PREPARATION_DOCKERFILES: - text = texts[relative] - if not text.startswith(f"# syntax={DOCKERFILE_FRONTEND}\n"): - failures.append( - f"{relative}: pinned Dockerfile frontend must be the first line" - ) - require( - text, - f"FROM {DEBIAN_PREPARATION} AS runtime-root", - relative, - "pinned Debian 13 runtime preparation base", - failures, - ) - require( - text, - "ARG SOURCE_DATE_EPOCH=0", - relative, - "fixed release filesystem timestamp", - failures, - ) - require( - text, - "RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin", - relative, - "ephemeral release input mount", - failures, - ) - require( - text, - 'find /workspace/runtime-root -exec touch -h --date="@${SOURCE_DATE_EPOCH}" {} +', - relative, - "normalized release filesystem metadata", - failures, - ) - - for relative in RELAY_DOCKERFILES: - text = texts[relative] - require( - text, - "/usr/local/bin/registry-relay-rhai-worker", - relative, - "Relay worker binary", - failures, - ) - require( - runtime_stage(text), - 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', - relative, - "absolute Relay entrypoint", - failures, - ) - - product_notary = texts[Path("products/notary/Dockerfile")] - require( - product_notary, - 'ARG REGISTRY_NOTARY_FEATURES="registry-notary-cel,pkcs11"', - Path("products/notary/Dockerfile"), - "PKCS#11-enabled product build", - failures, - ) - for relative in NOTARY_DOCKERFILES: - text = texts[relative] - require( - text, - "registry-notary-cel-worker", - relative, - "Notary CEL worker binary", - failures, - ) - require( - runtime_stage(text), - 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', - relative, - "absolute Notary entrypoint", - failures, - ) - require( - text, - "chown -R 65532:65532", - relative, - "numeric nonroot-owned Notary runtime directories", - failures, - ) - require( - runtime_stage(text), - "WORKDIR /var/lib/registry-notary", - relative, - "Notary working directory", - failures, - ) - if re.search( - r"^\s*(?:COPY|ADD)\b[^\n]*(?:\.so\b|pkcs11[^/\s]*module)", - text, - re.IGNORECASE | re.MULTILINE, - ): - failures.append( - f"{relative}: vendor PKCS#11 modules must remain external read-only mounts" - ) - - ci_workflow = texts[CI_WORKFLOW] - workflow = texts[Path(".github/workflows/release.yml")] - binary_recipe = texts[Path("release/scripts/build-release-binaries.sh")] - require( - workflow, - f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", - Path(".github/workflows/release.yml"), - "pinned Debian 13 release builder", - failures, - ) - require( - binary_recipe, - "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", - Path("release/scripts/build-release-binaries.sh"), - "PKCS#11-enabled release build", - failures, - ) - - live_journey = texts[ - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") - ] - require( - live_journey, - RUST_BUILDER, - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), - "pinned Debian 13 live-journey builder", - failures, - ) - tutorial_checker = texts[REGISTRYCTL_TUTORIAL_SCRIPT] - tutorial_cache = registryctl_tutorial_cache_step(ci_workflow) - require( - tutorial_checker, - 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', - REGISTRYCTL_TUTORIAL_SCRIPT, - "registryctl tutorial linux target path matching container target", - failures, - ) - require( - tutorial_checker, - 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', - REGISTRYCTL_TUTORIAL_SCRIPT, - "registryctl tutorial Cargo home path matching container Cargo home", - failures, - ) - require( - tutorial_cache, - "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", - CI_WORKFLOW, - "registryctl tutorial cache key including builder-bearing script", - failures, - ) - require( - tutorial_cache, - "target/registryctl-tutorial-linux-amd64", - CI_WORKFLOW, - "registryctl tutorial linux target cache path", - failures, - ) - require( - tutorial_cache, - "target/registryctl-tutorial-cargo-home", - CI_WORKFLOW, - "registryctl tutorial Cargo home cache path", - failures, - ) - if "restore-keys:" in tutorial_cache: - failures.append( - f"{CI_WORKFLOW}: registryctl tutorial cache must not restore from " - "pre-builder-identity keys" - ) - + failures.append(str(error)) + dockerfiles = [path for path in discovered if is_dockerfile(path)] + if not dockerfiles: + failures.append("no maintained Dockerfiles discovered") + for path in dockerfiles: + bases, stages = FROM_RE.findall(texts.get(path, "")), set() + if not bases: + failures.append(f"{path}: no FROM instruction found") + for base, stage in bases: + if base.casefold() not in stages | {"scratch"} and not DIGEST_RE.search(base): + failures.append(f"{path}: upstream base is not pinned by immutable digest: {base}") + if stage: + stages.add(stage.casefold()) + product_contracts(texts, failures) return failures - def main() -> int: failures = check_repository() if failures: @@ -1548,6 +495,5 @@ def main() -> int: print("Debian 13 image contract check passed.") return 0 - if __name__ == "__main__": raise SystemExit(main()) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index a022c7ab..3b560ddd 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -3,6 +3,7 @@ import importlib.util import shutil +import subprocess import tempfile import unittest from pathlib import Path @@ -11,6 +12,8 @@ ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "release/scripts/check-debian13-images.py" +DIGEST = "a" * 64 +PINNED_RUST = f"rust:1.95-trixie@sha256:{DIGEST}" def load_module(): @@ -26,7 +29,17 @@ class Debian13ImageCheckTest(unittest.TestCase): def setUp(self) -> None: self.module = load_module() - def copy_required_surfaces(self, root: Path) -> None: + def scan(self, relative: str, text: str) -> list[str]: + return self.module.scan_surface(Path(relative), text) + + def assert_failure(self, relative: str, text: str, fragment: str) -> None: + failures = self.scan(relative, text) + self.assertTrue(any(fragment in item for item in failures), failures) + + def assert_clean(self, relative: str, text: str) -> None: + self.assertEqual([], self.scan(relative, text)) + + def copy_required(self, root: Path) -> None: for relative in self.module.REQUIRED_PRODUCT_SURFACES: destination = root / relative destination.parent.mkdir(parents=True, exist_ok=True) @@ -35,1012 +48,395 @@ def copy_required_surfaces(self, root: Path) -> None: def test_real_repository_follows_debian13_contract(self) -> None: self.assertEqual([], self.module.check_repository(ROOT)) - def test_discovery_covers_dockerfiles_and_active_scripts(self) -> None: + def test_discovery_uses_tracked_files_and_documented_exclusions(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - dockerfile = Path("Dockerfile.worker") - script = Path("docs/site/scripts/build-example.sh") - note = Path("release/notes/v0.1.0.md") - research = Path("docs/site/.research/container-notes.md") - for relative in (dockerfile, script, note, research): - destination = root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text("fixture\n", encoding="utf-8") + tracked = ( + Path("Dockerfile.worker"), + Path("scripts/check.sh"), + Path("release/notes/old.md"), + Path("docs/.research/notes.md"), + Path("external/vendor.md"), + Path("crates/example/resources/scalar/generated.js"), + Path("release/scripts/test_check_debian13_images.py"), + ) + for relative in tracked: + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("fixture\n") + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run( + ["git", "-C", str(root), "add", "."], + check=True, + ) + (root / "untracked.sh").write_text("fixture\n") discovered = self.module.discover_maintained_surfaces(root) - self.assertIn(dockerfile, discovered) - self.assertIn(script, discovered) - self.assertNotIn(note, discovered) - self.assertNotIn(research, discovered) - - def test_discovered_dockerfile_rejects_retired_unpinned_base(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - dockerfile = root / "products/example/Dockerfile" - dockerfile.parent.mkdir(parents=True, exist_ok=True) - dockerfile.write_text( - "FROM rust:1.95-" + "book" + "worm\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "retired Debian image generation marker" in failure - for failure in failures - ), - failures, - ) - self.assertTrue( - any( - "upstream base is not pinned by immutable digest" in failure - for failure in failures - ), - failures, - ) - - def test_discovered_dockerfile_rejects_external_copy_from_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - dockerfile = root / "products/example/Dockerfile" - dockerfile.parent.mkdir(parents=True, exist_ok=True) - dockerfile.write_text( - "FROM scratch\n" - "COPY --from=debian:book" + "worm@sha256:" + "a" * 64 - + " /etc/os-release /os-release\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "products/example/Dockerfile:2" in failure - and "retired Debian image generation marker" in failure - for failure in failures - ), - failures, - ) - - def test_discovered_script_rejects_unpinned_debian_derived_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker run --rm rust:1.95-" + "trixie cargo build --locked\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "Debian-derived image reference is not pinned by immutable digest" - in failure - and "docs/site/scripts/build-example.sh:2" in failure - for failure in failures - ), - failures, - ) - - def test_discovered_script_rejects_version_only_debian_default_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker run --rm node:22 npm test\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "Debian-derived image reference is not pinned by immutable digest" - in failure - and "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": node:22") - for failure in failures - ), - failures, - ) - - def test_discovered_script_parses_docker_container_run(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker container run --rm rust:1.95-trixie cargo build --locked\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "Debian-derived image reference is not pinned by immutable digest" - in failure - and "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": rust:1.95-trixie") - for failure in failures - ), - failures, - ) - - def test_discovered_script_treats_docker_boolean_run_flags_as_valueless(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker run --rm --init --no-healthcheck " - "--oom-kill-disable --sig-proxy debian true\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - - def test_discovered_script_rejects_private_registry_untagged_debian_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker run --rm localhost:5000/debian true\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": localhost:5000/debian") - for failure in failures - ), - failures, - ) - - def test_discovered_script_strips_same_line_shell_separator_after_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker run --rm debian; echo ok\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) + self.assertIn(Path("Dockerfile.worker"), discovered) + self.assertIn(Path("scripts/check.sh"), discovered) + for relative in tracked[2:] + (Path("untracked.sh"),): + self.assertNotIn(relative, discovered) - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) + def test_path_file_line_and_total_bounds_fail_explicitly(self) -> None: + listed = b"\0".join( + f"file-{index}".encode() + for index in range(self.module.MAX_TRACKED_PATHS + 1) + ) + completed = subprocess.CompletedProcess([], 0, stdout=listed) + with ( + mock.patch.object( + self.module.subprocess, + "run", + return_value=completed, + ), + self.assertRaisesRegex( + self.module.ImageSurfaceError, + "tracked path count exceeds", + ), + ): + self.module.discover_maintained_surfaces(Path("/unused")) - def test_discovered_script_parses_docker_global_options(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker --context ci run --rm debian true\n", - encoding="utf-8", + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "text file exceeds", + ): + self.scan( + "large.yaml", + "x" * (self.module.MAX_TEXT_FILE_BYTES + 1), ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, + with self.assertRaisesRegex( + self.module.ImageSurfaceError, + "line exceeds", + ): + self.scan( + "line.yaml", + "x" * (self.module.MAX_LINE_CHARS + 1), ) - def test_discovered_script_scans_after_shell_list_operators(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - 'cd "$PWD" && docker run --rm debian true\n' - "echo ok; podman run --rm node:22 true\n" - "echo ok & docker run --rm python:3.12 true\n" - "echo ok |& podman run --rm rust:1.95-trixie true\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:3" in failure - and failure.endswith(": node:22") - for failure in failures - ), - failures, - ) - self.assertTrue(any("build-example.sh:4" in item for item in failures)) - self.assertTrue(any("build-example.sh:5" in item for item in failures)) - - def test_shell_control_prefixes_find_image_references(self) -> None: - cases = { - "if docker run --rm debian true; then": ["debian"], - "while podman run --rm node:22 true; do": ["node:22"], - "until docker run --rm python:3.12 true; do": ["python:3.12"], - "! docker run --rm rust:1.95-trixie true": ["rust:1.95-trixie"], - "time docker run --rm debian true": ["debian"], - "time -p docker run --rm node:22 true": ["node:22"], - "time -- podman run --rm python:3.12 true": ["python:3.12"], - "{ docker run --rm debian true; }": ["debian"], - "(podman run --rm node:22 true)": ["node:22"], - "if ! time -p docker run --rm rust:1.95-trixie true; then": [ - "rust:1.95-trixie" - ], - "if true; then docker run --rm debian true; fi": ["debian"], - "while true; do podman run --rm node:22 true; done": ["node:22"], - } - - for command, expected in cases.items(): - with self.subTest(command=command): - self.assertEqual( - expected, - self.module.command_image_references_in_command(command), + self.copy_required(root) + with mock.patch.object(self.module, "MAX_TOTAL_TEXT_BYTES", 1): + failures = self.module.check_repository(root) + self.assertEqual( + ["maintained text exceeds 1 total bytes"], + failures, + ) + + def test_debian_family_literal_policy_is_table_driven(self) -> None: + cases = ( + ("Dockerfile", f"FROM {PINNED_RUST}\n", None), + ("compose.yaml", f"services:\n app:\n image: {PINNED_RUST}\n", None), + ("script.sh", "docker run --rm rust:1.95-trixie true\n", "not pinned by immutable digest"), + ("workflow.yml", f"env:\n BUILDER_IMAGE: rust:1.95@sha256:{DIGEST}\n", "does not declare Trixie/Debian 13"), + ("script.sh", "docker run --rm node:22 npm test\n", "not pinned by immutable digest"), + ("script.sh", "docker run --rm node:latest npm test\n", "not pinned by immutable digest"), + ("script.sh", "docker run --rm node:22-alpine npm test\n", None), + ("script.sh", f"docker run --rm node:22@sha256:{DIGEST} npm test\n", "does not declare Trixie/Debian 13"), + ("images.py", "BUILDER_IMAGE: str = 'python:3.13-slim-trixie'\n", "not pinned by immutable digest"), + ("images.ts", "const builderImage = 'golang:1.25-trixie';\n", "not pinned by immutable digest"), + ("module.js", "import path from 'node:path';\n", None), + ("data.yaml", "identifier: did:web\nport: 65532:65532\n", None), + ) + for relative, text, expected in cases: + with self.subTest(relative=relative, text=text): + failures = self.scan(relative, text) + if expected is None: + self.assertEqual([], failures) + else: + self.assertTrue( + any(expected in failure for failure in failures), + failures, + ) + + def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None: + for text in ( + "Historical book" + "worm base\n", + "# image used bullseye during testing\n", + "unused_image: debian" + "12\n", + ): + with self.subTest(text=text): + self.assert_failure( + "notes.txt", + text, + "retired Debian image generation marker", ) - def test_shell_wrapper_options_find_image_references(self) -> None: - cases = { - "sudo -E docker run --rm debian true": ["debian"], - "env -i podman run --rm node:22 true": ["node:22"], - "command -- docker run --rm python:3.12 true": ["python:3.12"], - "sudo -u root env -u EXAMPLE command -p " - "docker run --rm rust:1.95-trixie true": ["rust:1.95-trixie"], - "sudo --preserve-env=CI time -p " - "podman run --rm debian true": ["debian"], - "sudo -A docker run --rm debian true": ["debian"], - "sudo -i podman run --rm node:22 true": ["node:22"], - "sudo -P docker run --rm python:3.12 true": ["python:3.12"], - } + self.assert_clean( + "design.md", + "The book" + 'worm comparison remains historical. ' + '\n', + ) + invalid = ( + "```sh\n" + "# book" + 'worm \n' + "```\n" + ) + failures = self.scan("design.md", invalid) + self.assertTrue( + any("invalid Debian image prose exemption" in item for item in failures), + failures, + ) + self.assertTrue( + any("retired Debian image" in item for item in failures), + failures, + ) - for command, expected in cases.items(): + def test_wrappers_options_operators_and_malformed_shell_still_scan_literals(self) -> None: + commands = ( + "docker --tlsverify pull -a rust:1.95-trixie", + "podman --remote pull --all-tags rust:1.95-trixie", + "docker image pull --disable-content-trust rust:1.95-trixie", + "/usr/bin/docker run --rm rust:1.95-trixie", + "env -S '-i /usr/bin/docker run --rm rust:1.95-trixie'", + "sudo -s /usr/bin/docker run --rm rust:1.95-trixie", + "bash --noprofile -c 'docker run --rm rust:1.95-trixie'", + "if ! time -p docker run --rm rust:1.95-trixie; then true; fi", + "echo ok; podman run --unknown value rust:1.95-trixie", + "docker unknown rust:1.95-trixie", + "docker run 'rust:1.95-trixie", + ) + for command in commands: with self.subTest(command=command): - self.assertEqual( - expected, - self.module.command_image_references_in_command(command), + self.assert_failure( + "script.sh", + command + "\n", + "not pinned by immutable digest", ) - def test_shell_launchers_are_unwrapped_with_a_depth_bound(self) -> None: - cases = { - 'sh -c "docker run --rm debian true"': ["debian"], - 'sudo -A bash -lc "podman run --rm node:22 true"': ["node:22"], - "env -S 'bash -c \"docker run --rm python:3.12 true\"'": [ - "python:3.12" - ], - "env --split-string='sh -c \"podman run --rm rust:1.95-trixie\"'": [ - "rust:1.95-trixie" - ], - "env -Ssh -c 'docker run --rm debian true'": ["debian"], - } - for command, expected in cases.items(): + for command in ( + f"docker run --unknown value {PINNED_RUST}", + f"sudo --unknown docker unknown {PINNED_RUST}", + f"bash --unknown -c 'docker run {PINNED_RUST}'", + ): with self.subTest(command=command): - self.assertEqual( - expected, - self.module.command_image_references_in_command(command), + self.assert_clean("script.sh", command + "\n") + + def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: + cases = ( + ("Dockerfile", "FROM debian\n"), + ("Dockerfile", "COPY --from=localhost:5000/debian /x /x\n"), + ("Dockerfile", "RUN --mount=from=debian,target=/x true\n"), + ("compose.yaml", "services:\n app:\n image: debian\n"), + ("workflow.yml", "jobs:\n test:\n container: debian\n"), + ("images.py", "BUILDER_IMAGE = 'debian'\n"), + ("script.sh", "docker run --rm debian true\n"), + ("guide.md", "```console\n$ podman pull debian\n```\n"), + ) + for relative, text in cases: + with self.subTest(relative=relative, text=text): + self.assert_failure( + relative, + text, + "bare Debian image reference", ) - nested = "docker run --rm debian true" - for _ in range(self.module.MAX_SHELL_RECURSION_DEPTH + 1): - nested = f"sh -c {self.module.shlex.quote(nested)}" - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "shell command nesting exceeds", + for relative, text in ( + ("guide.md", "Debian is the supported distribution.\n"), + ("guide.md", "```text\ndocker run debian\n```\n"), + ("script.sh", "# docker run --rm debian\n"), + ("module.js", '// docker run "$OTHER_IMAGE"\n'), + ("module.py", "distribution = 'debian'\n"), + ("module.py", "distribution = 'Debian'\n"), ): - self.module.command_image_references_in_command(nested) - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "shell command exceeds", - ): - self.module.command_image_references_in_command( - "x" * (self.module.MAX_SHELL_COMMAND_CHARS + 1) - ) - - def test_shell_parser_rejects_invalid_control_and_wrapper_prefixes(self) -> None: - commands = ( - "if then docker run --rm debian true", - "while until docker run --rm debian true", - "! if docker run --rm debian true", - "time while docker run --rm debian true", - "time -- -p docker run --rm debian true", - "sudo if docker run --rm debian true", - "sudo --unknown docker run --rm debian true", - "sudo -v docker run --rm debian true", - "env --unknown docker run --rm debian true", - "command -v docker run --rm debian true", + with self.subTest(relative=relative, text=text): + self.assert_clean(relative, text) + + def test_image_assignments_resolve_literals_and_reject_computation(self) -> None: + clean = ( + f'DEFAULT_BUILDER_IMAGE="{PINNED_RUST}"\n' + 'BUILDER_IMAGE="${DEFAULT_BUILDER_IMAGE}"\n' + 'docker run --rm "$BUILDER_IMAGE" true\n' ) - - for command in commands: - with self.subTest(command=command): - self.assertEqual( - [], - self.module.command_image_references_in_command(command), - ) - - def test_command_scan_ignores_prose_prefixes(self) -> None: - commands = ( - "echo if docker run --rm debian true", - "printf '%s\\n' time docker run --rm node:22 true", - "description: if docker run --rm debian true", - "notes: time -p docker run --rm node:22 true", - "a sentence about { docker run --rm debian true; }", + self.assert_clean("build.sh", clean) + self.assert_clean( + "function.sh", + 'start() {\n local image="$1"\n docker run "$image"\n}\n', + ) + self.assert_clean( + "local.sh", + 'RELAY_IMAGE="registryctl-relay:$RUN_ID"\n' + 'docker run "$RELAY_IMAGE"\n', ) - for command in commands: - with self.subTest(command=command): - self.assertEqual( - [], - self.module.command_image_references_in_command(command), - ) - - def test_yaml_supported_executable_contexts_find_image_references(self) -> None: - cases = { - "workflow literal": ( - Path(".github/workflows/example.yml"), - "jobs:\n" - " test:\n" - " steps:\n" - " - run: |\n" - " command -- docker run --rm python:3.12 true\n", - [(5, "python:3.12")], + cases = ( + ( + "images.py", + "BUILDER_IMAGE = make_image()\n", + "computed or unresolved image assignment", ), - "workflow folded reports scalar start": ( - Path(".github/workflows/example.yml"), - "jobs:\n" - " test:\n" - " steps:\n" - " - run: >\n" - " sudo -E docker run --rm\n" - " rust:1.95-trixie true\n", - [(4, "rust:1.95-trixie")], + ( + "images.ts", + "const builderImage = 'rust:1.95-' + 'trixie';\n", + "computed or unresolved image assignment", ), - "compose argv": ( - Path("compose.yaml"), - "services:\n" - " helper:\n" - " image: debian\n" - " command: [sh, -c, \"podman run --rm node:22 true\"]\n", - [(3, "debian"), (4, "node:22")], + ( + "cycle.sh", + 'BASE_IMAGE="$BUILDER_IMAGE"\n' + 'BUILDER_IMAGE="$BASE_IMAGE"\n', + "computed or unresolved image assignment", ), - "Kubernetes command and args": ( - Path("pod.yaml"), - "apiVersion: v1\n" - "kind: Pod\n" - "spec:\n" - " containers:\n" - " - name: helper\n" - " image: debian\n" - " command: [sh, -c]\n" - " args: [\"docker run --rm python:3.12 true\"]\n", - [(6, "debian"), (7, "python:3.12")], + ( + "build.sh", + 'docker run --rm "$OTHER_IMAGE"\n', + "must use a literal or a resolved *_IMAGE assignment", ), - "image variable": ( - Path("images.yaml"), - "BUILDER_IMAGE: debian\n", - [(1, "debian")], + ( + "build.sh", + 'docker run --publish 127.0.0.1:8080 "$OTHER_IMAGE"\n', + "must use a literal or a resolved *_IMAGE assignment", + ), + ( + "build.sh", + 'OTHER_IMAGE="$(compute)"\nBUILDER_IMAGE="$OTHER_IMAGE"\n', + "computed or unresolved image assignment", + ), + ( + "build.sh", + 'docker pull "$container"\n', + "must use a literal or a resolved *_IMAGE assignment", ), - } - - for name, (relative, text, expected) in cases.items(): - with self.subTest(name=name): - self.assertEqual( - expected, - self.module.image_references(relative, text), - ) - - def test_yaml_domain_data_is_not_executable(self) -> None: - text = ( - "experiment:\n" - " run: docker run --rm debian true\n" - "capability:\n" - " script: docker run --rm node:22 true\n" - "metadata:\n" - " command: [docker, run, --rm, python:3.12]\n" - ) - - self.assertEqual([], self.module.image_references(Path("data.yaml"), text)) - - def test_yaml_all_documents_are_scanned(self) -> None: - text = ( - "experiment:\n" - " run: docker run --rm debian true\n" - "---\n" - "services:\n" - " helper:\n" - " image: node:22\n" - "---\n" - "jobs:\n" - " test:\n" - " steps:\n" - " - run: docker run --rm python:3.12 true\n" ) - self.assertEqual( - [(6, "node:22"), (11, "python:3.12")], - self.module.image_references(Path("example.yaml"), text), + for relative, text, expected in cases: + with self.subTest(relative=relative): + self.assert_failure(relative, text, expected) + + def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> None: + cases = ( + ( + "compose.yaml", + "x-service: &defaults\n image: rust:1.95-trixie\n" + "services:\n app:\n <<: *defaults\n", + ), + ( + ".github/workflows/example.yml", + "jobs:\n test:\n strategy:\n matrix:\n" + " image: [rust:1.95-trixie]\n include:\n" + " - image: rust:1.95-trixie\n" + " unused_image: [rust:1.95-trixie]\n" + " container: ${{ matrix.image }}\n", + ), + ("pod.yaml", "apiVersion: v1\nkind: Pod\nspec:\n containers:\n - image: rust:1.95-trixie\n"), + ("malformed.yaml", "jobs: [\nimage: rust:1.95-trixie\n"), + ("documents.yaml", "description: first\n---\nimage: rust:1.95-trixie\n"), ) - - def test_malformed_yaml_fails_the_repository_check(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - manifest = root / "products/example/broken.yaml" - manifest.parent.mkdir(parents=True, exist_ok=True) - manifest.write_text("jobs:\n broken: [\n", encoding="utf-8") - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "products/example/broken.yaml: image scan failed: invalid YAML" - in failure - for failure in failures - ), - failures, - ) - - def test_yaml_parser_recursion_and_depth_fail_explicitly(self) -> None: - self.module.compose_yaml_documents.cache_clear() - with mock.patch.object( - self.module.yaml, - "compose_all", - side_effect=RecursionError, - ): - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "YAML parser recursion limit exceeded", - ): - self.module.image_references(Path("recursive.yaml"), "jobs: {}\n") - - deep = "".join( - f"{' ' * depth}level_{depth}:\n" for depth in range(6) - ) + " value: true\n" - with mock.patch.object(self.module, "MAX_YAML_DEPTH", 3): - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "YAML nesting exceeds", - ): - self.module.image_references(Path("deep.yaml"), deep) - with mock.patch.object(self.module, "MAX_YAML_DOCUMENTS", 2): - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "YAML input exceeds 2 documents", - ): - self.module.image_references( - Path("documents.yaml"), - "---\n---\n---\n", + for relative, text in cases: + with self.subTest(relative=relative): + self.assert_failure( + relative, + text, + "not pinned by immutable digest", ) - def test_missing_pyyaml_fails_with_dependency_name(self) -> None: - self.module.compose_yaml_documents.cache_clear() - with ( - mock.patch.object(self.module, "yaml", None), - mock.patch.object( - self.module, - "YAML_IMPORT_ERROR", - ImportError("No module named yaml"), - ), - ): - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "PyYAML is required", - ): - self.module.image_references(Path("missing.yaml"), "jobs: {}\n") - - def test_discovered_script_rejects_untagged_debian_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "docker run --rm debian apt-get update\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "Debian-derived image reference is not pinned by immutable digest" - in failure - and "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - - def test_discovered_yaml_rejects_untagged_debian_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - compose = root / "products/example/compose.yaml" - compose.parent.mkdir(parents=True, exist_ok=True) - compose.write_text( - "services:\n" - " helper:\n" - " image: debian\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "Debian-derived image reference is not pinned by immutable digest" - in failure - and "products/example/compose.yaml:3" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - - def test_discovered_kubernetes_yaml_rejects_container_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - manifest = root / "products/example/pod.yaml" - manifest.parent.mkdir(parents=True, exist_ok=True) - manifest.write_text( - "apiVersion: v1\n" - "kind: Pod\n" - "spec:\n" - " containers:\n" - " - name: helper\n" - " image: debian\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "products/example/pod.yaml:6" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - - def test_untagged_detection_ignores_comments_and_yaml_prose(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/explain-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "# docker run --rm debian apt-get update\n", - encoding="utf-8", - ) - metadata = root / "products/example/metadata.yaml" - metadata.parent.mkdir(parents=True, exist_ok=True) - retired_bookworm = f"rust:1.95-{'book' + 'worm'}@sha256:{'a' * 64}" - metadata.write_text( - 'description: "docker run --rm debian is an unsafe example"\n' - f"# docker run --rm {retired_bookworm} cargo build\n" - f"notes: {retired_bookworm}\n" - "base: debian\n" - "container_note: debian\n", - encoding="utf-8", - ) - - self.assertEqual([], self.module.check_repository(root)) + def test_markdown_scans_only_executable_fences(self) -> None: + self.assert_failure( + "guide.md", + "```sh\n" + "docker run --rm rust:1.95-trixie\n" + "```\n", + "not pinned by immutable digest", + ) + self.assert_clean( + "guide.md", + "Example prose mentions rust:1.95-trixie.\n" + "```text\n" + "docker run --rm rust:1.95-trixie\n" + "```\n", + ) - def test_discovered_workflow_rejects_untagged_container_image(self) -> None: + def test_repository_dockerfile_discovery_checks_external_and_internal_stages(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.copy_required_surfaces(root) - workflow = root / ".github/workflows/example.yml" - workflow.parent.mkdir(parents=True, exist_ok=True) - workflow.write_text( - "jobs:\n" - " test:\n" - " runs-on: ubuntu-latest\n" - " container: debian\n", - encoding="utf-8", + self.copy_required(root) + dockerfile = root / "products/example/Dockerfile" + dockerfile.parent.mkdir(parents=True) + dockerfile.write_text( + "FROM scratch AS assets\n" + "FROM assets AS final\n" + "COPY --from=debian /etc/os-release /os-release\n" ) - failures = self.module.check_repository(root) - self.assertTrue( - any( - ".github/workflows/example.yml:4" in failure - and failure.endswith(": debian") - for failure in failures - ), + any("bare Debian image reference" in item for item in failures), failures, ) - - def test_discovered_workflow_rejects_docker_image_action(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - workflow = root / ".github/workflows/example.yml" - workflow.parent.mkdir(parents=True, exist_ok=True) - workflow.write_text( - "jobs:\n" - " test:\n" - " steps:\n" - " - uses: docker://debian:book" + "worm@sha256:" + "a" * 64 + "\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( + self.assertFalse( any( - ".github/workflows/example.yml:4" in failure - and "retired Debian image generation marker" in failure - for failure in failures + "upstream base is not pinned" in item and "assets" in item + for item in failures ), failures, ) - def test_discovered_script_rejects_retired_digest_pinned_image(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - retired_bookworm = f"rust:1.95-{'book' + 'worm'}@sha256:{'a' * 64}" - script.write_text( - "#!/usr/bin/env bash\n" - f"docker run --rm {retired_bookworm} cargo build --locked\n", - encoding="utf-8", - ) - + self.copy_required(root) + dockerfile = root / "Dockerfile.worker" + dockerfile.write_text("FROM alpine:3.22\n") failures = self.module.check_repository(root) - - self.assertTrue( - any( - "retired Debian image generation marker remains in image reference" - in failure - and "docs/site/scripts/build-example.sh:2" in failure - and "bookworm" in failure - for failure in failures - ), - failures, - ) - - def test_discovered_js_ts_rejects_bare_image_declarations(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/images.ts" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - 'const image = "debian";\n' - "let builderImage: string = 'node:22';\n" - 'var container = "python:3.12";\n', - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue(any("images.ts:1" in failure for failure in failures), failures) - self.assertTrue( - any( - "images.ts:2" in failure and failure.endswith(": node:22") - for failure in failures - ), - failures, - ) self.assertTrue( any( - "images.ts:3" in failure and failure.endswith(": python:3.12") - for failure in failures + "Dockerfile.worker: upstream base is not pinned" in item + for item in failures ), failures, ) - def test_discovered_shell_rejects_prefixed_image_declarations(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "docs/site/scripts/build-example.sh" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - "#!/usr/bin/env bash\n" - "local BUILDER_IMAGE=debian\n" - "readonly RUNTIME_IMAGE=rust:1.95-book" + "worm\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:2" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - self.assertTrue( - any( - "docs/site/scripts/build-example.sh:3" in failure - and "retired Debian image generation marker" in failure - for failure in failures - ), - failures, - ) + def test_product_contract_mutations_are_reported(self) -> None: + mutations = ( + (Path("crates/registry-relay/Dockerfile"), "/usr/local/bin/registry-relay-rhai-worker", "/usr/local/bin/removed-worker", "Relay worker binary"), + (Path("products/notary/Dockerfile"), "registry-notary-cel-worker", "removed-cel-worker", "Notary CEL worker binary"), + (Path("release/docker/Dockerfile.registry-relay"), "# syntax=", "# moved-syntax=", "frontend must be the first line"), + (Path("release/scripts/build-release-binaries.sh"), "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", "--features registry-notary/registry-notary-cel", "PKCS#11-enabled release build"), + (Path("docs/site/scripts/check-registryctl-tutorials.sh"), 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', 'LINUX_TARGET="$REPO_ROOT/target/other"', "registryctl tutorial linux target path"), + (Path(".github/workflows/ci.yml"), "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", "hashFiles('Cargo.lock')", "registryctl tutorial cache builder identity"), + ) + for path, old, new, expected in mutations: + with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.copy_required(root) + target = root / path + text = target.read_text() + self.assertIn(old, text) + target.write_text(text.replace(old, new)) + failures = self.module.check_repository(root) + self.assertTrue( + any(expected in item for item in failures), + failures, + ) - def test_wrapped_python_and_ts_image_assignments_keep_context(self) -> None: + def test_missing_required_surface_and_cache_restore_key_are_reported(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.copy_required_surfaces(root) - python_script = root / "products/example/scripts/build_image.py" - python_script.parent.mkdir(parents=True, exist_ok=True) - python_script.write_text( - "BUILDER_IMAGE = (\n" - " 'rust:1.95-trixie'\n" - ")\n", - encoding="utf-8", - ) - ts_script = root / "docs/site/scripts/images.ts" - ts_script.parent.mkdir(parents=True, exist_ok=True) - ts_script.write_text( - "const builderImage =\n" - " 'node:22';\n", - encoding="utf-8", - ) - + self.copy_required(root) + missing = root / "products/notary/Dockerfile" + missing.unlink() failures = self.module.check_repository(root) - - self.assertTrue( - any( - "products/example/scripts/build_image.py:2" in failure - and failure.endswith(": rust:1.95-trixie") - for failure in failures - ), - failures, - ) self.assertTrue( - any( - "docs/site/scripts/images.ts:2" in failure - and failure.endswith(": node:22") - for failure in failures - ), + any("missing maintained image surface" in item for item in failures), failures, ) - def test_registryctl_tutorial_cache_key_must_include_builder_identity(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.copy_required_surfaces(root) - workflow = root / self.module.CI_WORKFLOW - text = workflow.read_text(encoding="utf-8") + self.copy_required(root) + workflow = root / ".github/workflows/ci.yml" + text = workflow.read_text() + marker = "- name: Execute registryctl tutorials from source" workflow.write_text( - text.replace( - "key: registryctl-tutorial-${{ runner.os }}-${{ " - "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh') " - "}}-${{ hashFiles('Cargo.lock') }}", - "key: registryctl-tutorial-${{ runner.os }}-rust-1.95.0-" - "${{ hashFiles('Cargo.lock') }}\n" - " restore-keys: |\n" - " registryctl-tutorial-${{ runner.os }}-rust-1.95.0-", - ), - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "cache key including builder-bearing script" in failure - for failure in failures - ), - failures, - ) - self.assertTrue( - any( - "must not restore from pre-builder-identity keys" in failure - for failure in failures - ), - failures, - ) - - def test_registryctl_tutorial_host_paths_must_match_container_paths(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - tutorial = root / self.module.REGISTRYCTL_TUTORIAL_SCRIPT - text = tutorial.read_text(encoding="utf-8") - linux_target = ( - 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"' - ) - cargo_home = ( - 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"' - ) - tutorial.write_text( - text.replace( - linux_target, - linux_target.removesuffix('"') + '-$BUILDER_CACHE_KEY"', - ).replace( - cargo_home, - cargo_home.removesuffix('"') + '-$BUILDER_CACHE_KEY"', - ), - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "linux target path matching container target" in failure - for failure in failures - ), - failures, + text.replace(marker, "restore-keys: old-key\n " + marker, 1) ) - self.assertTrue( - any( - "Cargo home path matching container Cargo home" in failure - for failure in failures - ), - failures, - ) - - def test_dockerfile_internal_stage_reference_needs_no_digest(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - dockerfile = root / "products/example/Dockerfile" - dockerfile.parent.mkdir(parents=True, exist_ok=True) - dockerfile.write_text( - f"FROM {self.module.RUST_BUILDER} AS builder\n" - "FROM builder AS runtime\n" - "COPY --from=builder /usr/local/bin/tool /usr/local/bin/tool\n", - encoding="utf-8", - ) - - self.assertEqual([], self.module.check_repository(root)) - - def test_discovered_python_script_rejects_unpinned_builder_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - script = root / "products/example/scripts/build_image.py" - script.parent.mkdir(parents=True, exist_ok=True) - script.write_text( - 'BUILDER_IMAGE = "rust:1.95-' + 'trixie"\n', - encoding="utf-8", - ) - failures = self.module.check_repository(root) - self.assertTrue( - any( - "Debian-derived image reference is not pinned by immutable digest" - in failure - and "products/example/scripts/build_image.py:1" in failure - for failure in failures - ), + any("must not restore" in item for item in failures), failures, ) - def test_discovered_markdown_rejects_executable_code_block_image(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - doc = root / "crates/registry-relay/docs/example.md" - doc.parent.mkdir(parents=True, exist_ok=True) - doc.write_text( - "Run the example:\n" - "```bash\n" - "docker run --rm debian true\n" - "```\n" - "```console\n" - "$ docker run --rm node:22 true\n" - "```\n", - encoding="utf-8", - ) - - failures = self.module.check_repository(root) - - self.assertTrue( - any( - "crates/registry-relay/docs/example.md:3" in failure - and failure.endswith(": debian") - for failure in failures - ), - failures, - ) - self.assertTrue( - any( - "crates/registry-relay/docs/example.md:6" in failure - and failure.endswith(": node:22") - for failure in failures - ), - failures, - ) - - def test_markdown_scan_ignores_prose_and_nonexecutable_fences(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - doc = root / "crates/registry-relay/docs/example.md" - doc.parent.mkdir(parents=True, exist_ok=True) - doc.write_text( - "A sentence can discuss docker run --rm debian true.\n" - "```text\n" - "docker run --rm debian true\n" - "```\n", - encoding="utf-8", - ) - - self.assertEqual([], self.module.check_repository(root)) - - def test_history_and_research_are_outside_the_maintained_boundary(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required_surfaces(root) - retired_reference = "Debian " + "12 and Book" + "worm\n" - for relative in ( - Path("release/notes/v0.1.0.md"), - Path("docs/site/.research/container-notes.md"), - ): - destination = root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(retired_reference, encoding="utf-8") - - self.assertEqual([], self.module.check_repository(root)) - if __name__ == "__main__": unittest.main() From ef498401545d1221ef5e6ebde7c80034397f331e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 01:24:26 +0700 Subject: [PATCH 11/34] fix(release): cover default Debian image families Signed-off-by: Jeremi Joslin --- .../scripts/run-live-consultation-journey.sh | 2 +- release/scripts/check-debian13-images.py | 49 ++++++++++--------- release/scripts/test_check_debian13_images.py | 28 +++++++---- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/crates/registry-relay/scripts/run-live-consultation-journey.sh b/crates/registry-relay/scripts/run-live-consultation-journey.sh index 72ec2bce..3a31e5da 100755 --- a/crates/registry-relay/scripts/run-live-consultation-journey.sh +++ b/crates/registry-relay/scripts/run-live-consultation-journey.sh @@ -6,7 +6,7 @@ set +v set -euo pipefail umask 077 -readonly POSTGRES_IMAGE="postgres:16" +readonly POSTGRES_IMAGE="postgres:16-trixie@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" [[ "$#" -le 1 ]] || { printf '%s\n' "usage: $0 [dhis2|rhai|synthetic]" >&2 diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 8ee2e7a2..60daedc9 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -43,18 +43,12 @@ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), TUTORIAL_SCRIPT, ) -RELAY_DOCKERFILES = ( - PRODUCT_DOCKERFILES[0], - PRODUCT_DOCKERFILES[1], - PRODUCT_DOCKERFILES[4], -) +RELAY_DOCKERFILES = (PRODUCT_DOCKERFILES[0], PRODUCT_DOCKERFILES[1], PRODUCT_DOCKERFILES[4]) NOTARY_DOCKERFILES = (PRODUCT_DOCKERFILES[2], PRODUCT_DOCKERFILES[3]) # Scan tracked UTF-8 text, excluding only historical evidence, third-party or # generated material, build output, and this checker's negative fixtures. -EXCLUDED_DIRS = set( - ".git .repo-docs-cache .research .venv __pycache__ dist node_modules target".split() -) +EXCLUDED_DIRS = set(".git .repo-docs-cache .research .venv __pycache__ dist node_modules target".split()) EXCLUDED_PREFIXES = ("external/", "release/notes/") EXCLUDED_EXACT = { "release/scripts/check-debian13-images.py", @@ -69,15 +63,17 @@ "book" + "worm", "bullseye", "buster", "debian" + "12", "debian-12", "debian" + "11", "debian-11", "debian" + "10", "debian-10", ) -DEFAULT_DEBIAN_FAMILIES = {"golang", "node", "python", "rust"} +DEFAULT_DEBIAN_FAMILIES = {"golang", "node", "postgres", "python", "rust"} OCI_RE = re.compile( r"(?(?:docker://)?" r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+" r"(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-fA-F]{64})" r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" ) -BARE_DEBIAN_RE = re.compile( - r"(?(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*" + rf"(?Pdebian|{'|'.join(sorted(DEFAULT_DEBIAN_FAMILIES))}))" + r"(?![A-Za-z0-9._/@+:-])" ) DIGEST_RE = re.compile(r"@sha256:[0-9a-f]{64}$") FROM_RE = re.compile( @@ -105,10 +101,8 @@ MARKDOWN_SUFFIXES = {".md", ".mdx"} CODE_SUFFIXES = set(".bash .js .mjs .py .sh .ts .yaml .yml".split()) STRICT_ASSIGNMENT_SUFFIXES = CODE_SUFFIXES - {".yaml", ".yml"} -FENCE_LANGS = set( - "bash console dockerfile javascript js python sh shell terminal ts " - "typescript yaml yml zsh".split() -) +FENCE_LANGS = set("bash console dockerfile javascript js python sh shell terminal " + "ts typescript yaml yml zsh".split()) class ImageSurfaceError(RuntimeError): @@ -195,13 +189,15 @@ def repository_and_tag(reference: str) -> tuple[str, str | None]: def is_debian_family(reference: str) -> bool: repository, tag = repository_and_tag(reference) name, tag = repository.rsplit("/", 1)[-1].casefold(), (tag or "").casefold() - return ( + default = name in DEFAULT_DEBIAN_FAMILIES + excluded = default and ("alpine" in tag or "windows" in tag) + return not excluded and ( "debian" in name or name == "buildpack-deps" or "trixie" in tag or re.search(r"debian-?13", tag) is not None - or name in DEFAULT_DEBIAN_FAMILIES - and (tag in {"latest", "slim"} or re.fullmatch(r"[0-9]+(?:[._][0-9]+)*", tag) is not None) + or default and (not tag or tag in {"latest", "slim"} or + re.fullmatch(r"[0-9]+(?:[._][0-9]+)*(?:-slim)?", tag) is not None) ) def assignment(path: Path, line: str) -> tuple[str, str] | None: @@ -342,12 +338,16 @@ def scan_surface(path: Path, text: str) -> list[str]: declared.add(canonical) if has_literal and not computed or positional: resolved.add(canonical) - if not comment and BARE_DEBIAN_RE.search(line) and ( - item is not None or (code_file or markdown_code) and (is_dockerfile(path) or consumer) - ): + bare = BARE_DEFAULT_FAMILY_RE.search(line) + bare_assignment = item is not None and ( + path.suffix in {".yaml", ".yml"} and re.sub("[^A-Za-z0-9]", "", item[0]).casefold() in {"image", "container"} + or path.suffix in STRICT_ASSIGNMENT_SUFFIXES and policy_image_name(item[0])) + if not comment and bare and (is_dockerfile(path) or consumer or bare_assignment): + family = bare.group("name") + label = "Debian" if family == "debian" else f"Debian-default {family}" failures.append( - f"{path}:{number}: bare Debian image reference is not pinned " - "and does not declare Trixie/Debian 13: debian" + f"{path}:{number}: bare {label} image reference is not pinned " + f"and does not declare Trixie/Debian 13: {bare.group('ref')}" ) changed = True while changed: @@ -368,7 +368,7 @@ def scan_surface(path: Path, text: str) -> list[str]: variables = { match.group("name").casefold() for match in IMAGE_VAR_RE.finditer(line) } - if not any(re.search("[A-Za-z]", repository_and_tag(item)[0]) for item in references(line)) and not BARE_DEBIAN_RE.search(line) and not variables & (resolved | declared): + if not any(re.search("[A-Za-z]", repository_and_tag(item)[0]) for item in references(line)) and not BARE_DEFAULT_FAMILY_RE.search(line) and not variables & (resolved | declared): failures.append( f"{path}:{number}: Docker/Podman image consumer must use a " "literal or a resolved *_IMAGE assignment" @@ -427,6 +427,7 @@ def product_contracts(texts: dict[Path, str], failures: list[str]) -> None: require(workflow, f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", Path(".github/workflows/release.yml"), "pinned Debian 13 release builder", failures) require(texts.get(Path("release/scripts/build-release-binaries.sh"), ""), "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", Path("release/scripts/build-release-binaries.sh"), "PKCS#11-enabled release build", failures) require(texts.get(Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), ""), RUST_BUILDER, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", failures) + require(texts.get(Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), ""), "postgres:16-trixie@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey PostgreSQL", failures) tutorial = texts.get(TUTORIAL_SCRIPT, "") ci = texts.get(CI_WORKFLOW, "") start = ci.find("- name: Cache source-under-test Cargo build") diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 3b560ddd..3fc6931d 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -125,20 +125,31 @@ def test_path_file_line_and_total_bounds_fail_explicitly(self) -> None: ) def test_debian_family_literal_policy_is_table_driven(self) -> None: - cases = ( + cases = [ ("Dockerfile", f"FROM {PINNED_RUST}\n", None), ("compose.yaml", f"services:\n app:\n image: {PINNED_RUST}\n", None), ("script.sh", "docker run --rm rust:1.95-trixie true\n", "not pinned by immutable digest"), ("workflow.yml", f"env:\n BUILDER_IMAGE: rust:1.95@sha256:{DIGEST}\n", "does not declare Trixie/Debian 13"), - ("script.sh", "docker run --rm node:22 npm test\n", "not pinned by immutable digest"), - ("script.sh", "docker run --rm node:latest npm test\n", "not pinned by immutable digest"), - ("script.sh", "docker run --rm node:22-alpine npm test\n", None), - ("script.sh", f"docker run --rm node:22@sha256:{DIGEST} npm test\n", "does not declare Trixie/Debian 13"), ("images.py", "BUILDER_IMAGE: str = 'python:3.13-slim-trixie'\n", "not pinned by immutable digest"), ("images.ts", "const builderImage = 'golang:1.25-trixie';\n", "not pinned by immutable digest"), ("module.js", "import path from 'node:path';\n", None), ("data.yaml", "identifier: did:web\nport: 65532:65532\n", None), - ) + ] + for family, version in (("rust", "1.95"), ("node", "22"), ("python", "3.13"), + ("golang", "1.25"), ("postgres", "16")): + cases.extend(( + ("compose.yaml", f"{'container' if family == 'node' else 'image'}: {family}\n", "bare Debian-default"), + ("build.sh", f"BUILDER_IMAGE='{family}'\n", "bare Debian-default"), + ("script.sh", f"docker run --rm {family} true\n", "bare Debian-default"), + ("script.sh", f"docker run --rm {family}:{version}\n", "not pinned"), + ("script.sh", f"docker run --rm {family}:{version}-slim\n", "not pinned"), + ("workflow.yml", f"container: {family}@sha256:{DIGEST}\n", "does not declare"), + ("compose.yaml", f"image: registry.test:5000/team/{family}\n", "bare Debian-default"), + ("workflow.yml", f"container: registry.test/team/{family}@sha256:{DIGEST}\n", "does not declare"), + ("script.sh", f"docker run {family}:{version}-alpine\n", None), + ("script.sh", f"docker run {family}:{version}-windows\n", None), + ("guide.md", f"The {family} image is available.\n", None), + )) for relative, text, expected in cases: with self.subTest(relative=relative, text=text): failures = self.scan(relative, text) @@ -149,7 +160,6 @@ def test_debian_family_literal_policy_is_table_driven(self) -> None: any(expected in failure for failure in failures), failures, ) - def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None: for text in ( "Historical book" + "worm base\n", @@ -186,7 +196,6 @@ def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None any("retired Debian image" in item for item in failures), failures, ) - def test_wrappers_options_operators_and_malformed_shell_still_scan_literals(self) -> None: commands = ( "docker --tlsverify pull -a rust:1.95-trixie", @@ -216,7 +225,6 @@ def test_wrappers_options_operators_and_malformed_shell_still_scan_literals(self ): with self.subTest(command=command): self.assert_clean("script.sh", command + "\n") - def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: cases = ( ("Dockerfile", "FROM debian\n"), @@ -246,7 +254,6 @@ def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: ): with self.subTest(relative=relative, text=text): self.assert_clean(relative, text) - def test_image_assignments_resolve_literals_and_reject_computation(self) -> None: clean = ( f'DEFAULT_BUILDER_IMAGE="{PINNED_RUST}"\n' @@ -395,6 +402,7 @@ def test_product_contract_mutations_are_reported(self) -> None: (Path("release/scripts/build-release-binaries.sh"), "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", "--features registry-notary/registry-notary-cel", "PKCS#11-enabled release build"), (Path("docs/site/scripts/check-registryctl-tutorials.sh"), 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', 'LINUX_TARGET="$REPO_ROOT/target/other"', "registryctl tutorial linux target path"), (Path(".github/workflows/ci.yml"), "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", "hashFiles('Cargo.lock')", "registryctl tutorial cache builder identity"), + (Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "postgres:16-trixie@sha256:", "postgres:16@", "pinned Debian 13 live-journey PostgreSQL"), ) for path, old, new, expected in mutations: with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: From d73c9e494dfc64fb2972c76f0545c25ff8cc5e15 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 01:26:05 +0700 Subject: [PATCH 12/34] style(release): format image boundary scanner Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 331 ++++++++++++++---- release/scripts/test_check_debian13_images.py | 171 ++++++--- 2 files changed, 394 insertions(+), 108 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 60daedc9..f660ee02 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -43,12 +43,18 @@ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), TUTORIAL_SCRIPT, ) -RELAY_DOCKERFILES = (PRODUCT_DOCKERFILES[0], PRODUCT_DOCKERFILES[1], PRODUCT_DOCKERFILES[4]) +RELAY_DOCKERFILES = ( + PRODUCT_DOCKERFILES[0], + PRODUCT_DOCKERFILES[1], + PRODUCT_DOCKERFILES[4], +) NOTARY_DOCKERFILES = (PRODUCT_DOCKERFILES[2], PRODUCT_DOCKERFILES[3]) # Scan tracked UTF-8 text, excluding only historical evidence, third-party or # generated material, build output, and this checker's negative fixtures. -EXCLUDED_DIRS = set(".git .repo-docs-cache .research .venv __pycache__ dist node_modules target".split()) +EXCLUDED_DIRS = set( + ".git .repo-docs-cache .research .venv __pycache__ dist node_modules target".split() +) EXCLUDED_PREFIXES = ("external/", "release/notes/") EXCLUDED_EXACT = { "release/scripts/check-debian13-images.py", @@ -60,8 +66,15 @@ MAX_LINE_CHARS = 131_072 RETIRED_MARKERS = ( - "book" + "worm", "bullseye", "buster", "debian" + "12", "debian-12", - "debian" + "11", "debian-11", "debian" + "10", "debian-10", + "book" + "worm", + "bullseye", + "buster", + "debian" + "12", + "debian-12", + "debian" + "11", + "debian-11", + "debian" + "10", + "debian-10", ) DEFAULT_DEBIAN_FAMILIES = {"golang", "node", "postgres", "python", "rust"} OCI_RE = re.compile( @@ -101,13 +114,16 @@ MARKDOWN_SUFFIXES = {".md", ".mdx"} CODE_SUFFIXES = set(".bash .js .mjs .py .sh .ts .yaml .yml".split()) STRICT_ASSIGNMENT_SUFFIXES = CODE_SUFFIXES - {".yaml", ".yml"} -FENCE_LANGS = set("bash console dockerfile javascript js python sh shell terminal " - "ts typescript yaml yml zsh".split()) +FENCE_LANGS = set( + "bash console dockerfile javascript js python sh shell terminal " + "ts typescript yaml yml zsh".split() +) class ImageSurfaceError(RuntimeError): """A maintained text surface exceeded a scanner boundary.""" + def is_dockerfile(path: Path) -> bool: return ( path.name == "Dockerfile" @@ -115,6 +131,7 @@ def is_dockerfile(path: Path) -> bool: or path.name.endswith(".Dockerfile") ) + def is_excluded(path: Path) -> bool: value = path.as_posix() return ( @@ -126,6 +143,7 @@ def is_excluded(path: Path) -> bool: or "/resources/scalar/" in f"/{value}/" ) + def discover_maintained_surfaces(root: Path) -> tuple[Path, ...]: command = subprocess.run( ["git", "-C", str(root), "ls-files", "-z"], @@ -138,17 +156,22 @@ def discover_maintained_surfaces(root: Path) -> tuple[Path, ...]: paths = [] for directory, names, files in os.walk(root): names[:] = [name for name in names if name not in EXCLUDED_DIRS] - paths.extend( - (Path(directory) / name).relative_to(root) for name in files - ) + paths.extend((Path(directory) / name).relative_to(root) for name in files) if len(paths) > MAX_TRACKED_PATHS: raise ImageSurfaceError( f"tracked path count exceeds {MAX_TRACKED_PATHS}: {len(paths)}" ) return tuple( - sorted(path for path in paths if not is_excluded(path) and (root / path).is_file() and not (root / path).is_symlink()) + sorted( + path + for path in paths + if not is_excluded(path) + and (root / path).is_file() + and not (root / path).is_symlink() + ) ) + def read_text(root: Path, path: Path) -> tuple[str | None, int]: target = root / path try: @@ -175,17 +198,20 @@ def read_text(root: Path, path: Path) -> tuple[str | None, int]: except UnicodeDecodeError: return None, 0 + def references(text: str) -> list[str]: return [ match.group("ref").removeprefix("docker://").strip("\"',;") for match in OCI_RE.finditer(text) ] + def repository_and_tag(reference: str) -> tuple[str, str | None]: value = reference.split("@", 1)[0] slash, colon = value.rfind("/"), value.rfind(":") return (value[:colon], value[colon + 1 :]) if colon > slash else (value, None) + def is_debian_family(reference: str) -> bool: repository, tag = repository_and_tag(reference) name, tag = repository.rsplit("/", 1)[-1].casefold(), (tag or "").casefold() @@ -196,17 +222,27 @@ def is_debian_family(reference: str) -> bool: or name == "buildpack-deps" or "trixie" in tag or re.search(r"debian-?13", tag) is not None - or default and (not tag or tag in {"latest", "slim"} or - re.fullmatch(r"[0-9]+(?:[._][0-9]+)*(?:-slim)?", tag) is not None) + or default + and ( + not tag + or tag in {"latest", "slim"} + or re.fullmatch(r"[0-9]+(?:[._][0-9]+)*(?:-slim)?", tag) is not None + ) ) + def assignment(path: Path, line: str) -> tuple[str, str] | None: match = (YAML_KEY_RE if path.suffix in {".yaml", ".yml"} else ASSIGN_RE).match(line) if match is None: return None name = match.group("name") normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() - return (name, match.group("value")) if normalized.endswith("image") or normalized == "container" else None + return ( + (name, match.group("value")) + if normalized.endswith("image") or normalized == "container" + else None + ) + def is_container_consumer(line: str) -> bool: for command in CLI_RE.finditer(line): @@ -219,9 +255,18 @@ def is_container_consumer(line: str) -> bool: return True return False + def policy_image_name(name: str) -> bool: - value, markers = name.casefold().replace("$", ""), {"base", "builder", "debian", "runtime"} - return re.sub("[^a-z0-9]", "", value) == "image" or value.startswith(tuple(markers)) or bool(set(re.split(r"[_-]+", value)) & markers) + value, markers = ( + name.casefold().replace("$", ""), + {"base", "builder", "debian", "runtime"}, + ) + return ( + re.sub("[^a-z0-9]", "", value) == "image" + or value.startswith(tuple(markers)) + or bool(set(re.split(r"[_-]+", value)) & markers) + ) + def markdown_code_flags(path: Path, lines: list[str]) -> list[bool]: if path.suffix not in MARKDOWN_SUFFIXES: @@ -240,9 +285,8 @@ def markdown_code_flags(path: Path, lines: list[str]) -> list[bool]: result.append(active) return result -def logical_lines( - lines: list[str], flags: list[bool] -) -> list[tuple[int, str, bool]]: + +def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, bool]]: result, parts, start, active = [], [], 1, False for number, line in enumerate(lines, 1): if not parts: @@ -257,9 +301,12 @@ def logical_lines( result.append((start, " ".join(parts), active)) return result + def scan_surface(path: Path, text: str) -> list[str]: if len(text.encode()) > MAX_TEXT_FILE_BYTES: - raise ImageSurfaceError(f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes") + raise ImageSurfaceError( + f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes" + ) lines, failures = text.splitlines(), [] markdown_flags = markdown_code_flags(path, text.splitlines()) code_file = is_dockerfile(path) or path.suffix in CODE_SUFFIXES @@ -288,13 +335,16 @@ def scan_surface(path: Path, text: str) -> list[str]: f"{path}:{number}: retired Debian image generation marker remains: {marker}" ) item = assignment(path, line) if path.suffix in CODE_SUFFIXES else None - consumer = is_container_consumer(line) and (code_file or markdown_code) and not comment - reference_context = ( - not comment and (is_dockerfile(path) + consumer = ( + is_container_consumer(line) and (code_file or markdown_code) and not comment + ) + reference_context = not comment and ( + is_dockerfile(path) or path.suffix in {".yaml", ".yml"} or markdown_code - or path.suffix in {".sh", ".bash"} or item is not None - or consumer) + or path.suffix in {".sh", ".bash"} + or item is not None + or consumer ) if reference_context: for reference in references(line): @@ -305,9 +355,10 @@ def scan_surface(path: Path, text: str) -> list[str]: failures.append( f"{prefix} is not pinned by immutable digest: {reference}" ) - if "trixie" not in reference.casefold() and re.search( - r"debian-?13", reference, re.IGNORECASE - ) is None: + if ( + "trixie" not in reference.casefold() + and re.search(r"debian-?13", reference, re.IGNORECASE) is None + ): failures.append( f"{prefix} does not declare Trixie/Debian 13: {reference}" ) @@ -315,8 +366,7 @@ def scan_surface(path: Path, text: str) -> list[str]: name, value = item canonical = name.casefold() dependencies = { - found.group("name").casefold() - for found in IMAGE_VAR_RE.finditer(value) + found.group("name").casefold() for found in IMAGE_VAR_RE.finditer(value) } has_literal = bool(references(value)) positional = canonical == "image" and re.fullmatch( @@ -325,24 +375,27 @@ def scan_surface(path: Path, text: str) -> list[str]: computed = not positional and ( not has_literal and not dependencies - or re.search(r"\s[+%]\s|`|\$\(|\.format\(|\bf[\"']", value) - is not None - ) - strict = ( - path.suffix in STRICT_ASSIGNMENT_SUFFIXES - and policy_image_name(name) + or re.search(r"\s[+%]\s|`|\$\(|\.format\(|\bf[\"']", value) is not None ) - records.append( - (number, canonical, value, dependencies, computed, strict) + strict = path.suffix in STRICT_ASSIGNMENT_SUFFIXES and policy_image_name( + name ) + records.append((number, canonical, value, dependencies, computed, strict)) declared.add(canonical) if has_literal and not computed or positional: resolved.add(canonical) bare = BARE_DEFAULT_FAMILY_RE.search(line) bare_assignment = item is not None and ( - path.suffix in {".yaml", ".yml"} and re.sub("[^A-Za-z0-9]", "", item[0]).casefold() in {"image", "container"} - or path.suffix in STRICT_ASSIGNMENT_SUFFIXES and policy_image_name(item[0])) - if not comment and bare and (is_dockerfile(path) or consumer or bare_assignment): + path.suffix in {".yaml", ".yml"} + and re.sub("[^A-Za-z0-9]", "", item[0]).casefold() in {"image", "container"} + or path.suffix in STRICT_ASSIGNMENT_SUFFIXES + and policy_image_name(item[0]) + ) + if ( + not comment + and bare + and (is_dockerfile(path) or consumer or bare_assignment) + ): family = bare.group("name") label = "Debian" if family == "debian" else f"Debian-default {family}" failures.append( @@ -363,86 +416,220 @@ def scan_surface(path: Path, text: str) -> list[str]: f"is not allowed: {value.strip()}" ) for number, line, markdown_code in logical_lines(lines, markdown_flags): - if line.lstrip().startswith(("#", "//")) or not (code_file or markdown_code) or not is_container_consumer(line): + if ( + line.lstrip().startswith(("#", "//")) + or not (code_file or markdown_code) + or not is_container_consumer(line) + ): continue variables = { match.group("name").casefold() for match in IMAGE_VAR_RE.finditer(line) } - if not any(re.search("[A-Za-z]", repository_and_tag(item)[0]) for item in references(line)) and not BARE_DEFAULT_FAMILY_RE.search(line) and not variables & (resolved | declared): + if ( + not any( + re.search("[A-Za-z]", repository_and_tag(item)[0]) + for item in references(line) + ) + and not BARE_DEFAULT_FAMILY_RE.search(line) + and not variables & (resolved | declared) + ): failures.append( f"{path}:{number}: Docker/Podman image consumer must use a " "literal or a resolved *_IMAGE assignment" ) return failures + def require( text: str, needle: str, path: Path, detail: str, failures: list[str] ) -> None: if needle not in text: failures.append(f"{path}: missing {detail}: {needle!r}") + def runtime_stage(text: str) -> str: marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" offset = text.find(marker) return text[offset:] if offset >= 0 else "" + def product_contracts(texts: dict[Path, str], failures: list[str]) -> None: for path in PRODUCT_DOCKERFILES: text, runtime = texts.get(path, ""), runtime_stage(texts.get(path, "")) - require(text, f"FROM {DISTROLESS_RUNTIME} AS runtime", path, "Distroless Debian 13 non-root final runtime", failures) - require(runtime, "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3", path, "binary healthcheck", failures) + require( + text, + f"FROM {DISTROLESS_RUNTIME} AS runtime", + path, + "Distroless Debian 13 non-root final runtime", + failures, + ) + require( + runtime, + "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3", + path, + "binary healthcheck", + failures, + ) for forbidden in ("\nRUN ", "apt-get", "/bin/sh", "curl ", "wget "): if forbidden in runtime: - failures.append(f"{path}: final Distroless runtime contains {forbidden.strip()!r}") + failures.append( + f"{path}: final Distroless runtime contains {forbidden.strip()!r}" + ) for path in PRODUCT_DOCKERFILES[:3]: - require(texts.get(path, ""), f"FROM {RUST_BUILDER} AS builder", path, "pinned Debian 13 Rust builder", failures) + require( + texts.get(path, ""), + f"FROM {RUST_BUILDER} AS builder", + path, + "pinned Debian 13 Rust builder", + failures, + ) for path in PRODUCT_DOCKERFILES[3:]: text = texts.get(path, "") if not text.startswith(f"# syntax={DOCKERFILE_FRONTEND}\n"): - failures.append(f"{path}: pinned Dockerfile frontend must be the first line") + failures.append( + f"{path}: pinned Dockerfile frontend must be the first line" + ) for needle, detail in ( (f"FROM {DEBIAN_PREPARATION} AS runtime-root", "runtime preparation base"), ("ARG SOURCE_DATE_EPOCH=0", "fixed release filesystem timestamp"), - ("RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin", "ephemeral release input mount"), - ('find /workspace/runtime-root -exec touch -h --date="@${SOURCE_DATE_EPOCH}" {} +', "normalized release filesystem metadata"), + ( + "RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin", + "ephemeral release input mount", + ), + ( + 'find /workspace/runtime-root -exec touch -h --date="@${SOURCE_DATE_EPOCH}" {} +', + "normalized release filesystem metadata", + ), ): require(text, needle, path, detail, failures) for path in RELAY_DOCKERFILES: text = texts.get(path, "") - require(text, "/usr/local/bin/registry-relay-rhai-worker", path, "Relay worker binary", failures) - require(runtime_stage(text), 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', path, "absolute Relay entrypoint", failures) - require(texts.get(PRODUCT_DOCKERFILES[2], ""), 'ARG REGISTRY_NOTARY_FEATURES="registry-notary-cel,pkcs11"', PRODUCT_DOCKERFILES[2], "PKCS#11-enabled product build", failures) + require( + text, + "/usr/local/bin/registry-relay-rhai-worker", + path, + "Relay worker binary", + failures, + ) + require( + runtime_stage(text), + 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', + path, + "absolute Relay entrypoint", + failures, + ) + require( + texts.get(PRODUCT_DOCKERFILES[2], ""), + 'ARG REGISTRY_NOTARY_FEATURES="registry-notary-cel,pkcs11"', + PRODUCT_DOCKERFILES[2], + "PKCS#11-enabled product build", + failures, + ) for path in NOTARY_DOCKERFILES: text = texts.get(path, "") for source, needle, detail in ( (text, "registry-notary-cel-worker", "Notary CEL worker binary"), - (runtime_stage(text), 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', "absolute Notary entrypoint"), - (text, "chown -R 65532:65532", "numeric nonroot-owned Notary runtime directories"), - (runtime_stage(text), "WORKDIR /var/lib/registry-notary", "Notary working directory"), + ( + runtime_stage(text), + 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', + "absolute Notary entrypoint", + ), + ( + text, + "chown -R 65532:65532", + "numeric nonroot-owned Notary runtime directories", + ), + ( + runtime_stage(text), + "WORKDIR /var/lib/registry-notary", + "Notary working directory", + ), ): require(source, needle, path, detail, failures) - if re.search(r"^\s*(?:COPY|ADD)\b[^\n]*(?:\.so\b|pkcs11[^/\s]*module)", text, re.IGNORECASE | re.MULTILINE): - failures.append(f"{path}: vendor PKCS#11 modules must remain external read-only mounts") + if re.search( + r"^\s*(?:COPY|ADD)\b[^\n]*(?:\.so\b|pkcs11[^/\s]*module)", + text, + re.IGNORECASE | re.MULTILINE, + ): + failures.append( + f"{path}: vendor PKCS#11 modules must remain external read-only mounts" + ) workflow = texts.get(Path(".github/workflows/release.yml"), "") - require(workflow, f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", Path(".github/workflows/release.yml"), "pinned Debian 13 release builder", failures) - require(texts.get(Path("release/scripts/build-release-binaries.sh"), ""), "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", Path("release/scripts/build-release-binaries.sh"), "PKCS#11-enabled release build", failures) - require(texts.get(Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), ""), RUST_BUILDER, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", failures) - require(texts.get(Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), ""), "postgres:16-trixie@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey PostgreSQL", failures) + require( + workflow, + f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", + Path(".github/workflows/release.yml"), + "pinned Debian 13 release builder", + failures, + ) + require( + texts.get(Path("release/scripts/build-release-binaries.sh"), ""), + "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", + Path("release/scripts/build-release-binaries.sh"), + "PKCS#11-enabled release build", + failures, + ) + require( + texts.get( + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "" + ), + RUST_BUILDER, + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + "pinned Debian 13 live-journey builder", + failures, + ) + require( + texts.get( + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "" + ), + "postgres:16-trixie@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + "pinned Debian 13 live-journey PostgreSQL", + failures, + ) tutorial = texts.get(TUTORIAL_SCRIPT, "") ci = texts.get(CI_WORKFLOW, "") start = ci.find("- name: Cache source-under-test Cargo build") end = ci.find("\n - name: Execute registryctl tutorials from source", start) cache = "" if start < 0 else ci[start:] if end < 0 else ci[start:end] for source, needle, path, detail in ( - (tutorial, 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', TUTORIAL_SCRIPT, "registryctl tutorial linux target path"), - (tutorial, 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', TUTORIAL_SCRIPT, "registryctl tutorial Cargo home path"), - (cache, "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", CI_WORKFLOW, "registryctl tutorial cache builder identity"), - (cache, "target/registryctl-tutorial-linux-amd64", CI_WORKFLOW, "registryctl tutorial linux target cache path"), - (cache, "target/registryctl-tutorial-cargo-home", CI_WORKFLOW, "registryctl tutorial Cargo home cache path"), + ( + tutorial, + 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', + TUTORIAL_SCRIPT, + "registryctl tutorial linux target path", + ), + ( + tutorial, + 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', + TUTORIAL_SCRIPT, + "registryctl tutorial Cargo home path", + ), + ( + cache, + "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", + CI_WORKFLOW, + "registryctl tutorial cache builder identity", + ), + ( + cache, + "target/registryctl-tutorial-linux-amd64", + CI_WORKFLOW, + "registryctl tutorial linux target cache path", + ), + ( + cache, + "target/registryctl-tutorial-cargo-home", + CI_WORKFLOW, + "registryctl tutorial Cargo home cache path", + ), ): require(source, needle, path, detail, failures) if "restore-keys:" in cache: - failures.append(f"{CI_WORKFLOW}: registryctl tutorial cache must not restore from pre-builder-identity keys") + failures.append( + f"{CI_WORKFLOW}: registryctl tutorial cache must not restore from pre-builder-identity keys" + ) + def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] @@ -479,13 +666,18 @@ def check_repository(root: Path = ROOT) -> list[str]: if not bases: failures.append(f"{path}: no FROM instruction found") for base, stage in bases: - if base.casefold() not in stages | {"scratch"} and not DIGEST_RE.search(base): - failures.append(f"{path}: upstream base is not pinned by immutable digest: {base}") + if base.casefold() not in stages | {"scratch"} and not DIGEST_RE.search( + base + ): + failures.append( + f"{path}: upstream base is not pinned by immutable digest: {base}" + ) if stage: stages.add(stage.casefold()) product_contracts(texts, failures) return failures + def main() -> int: failures = check_repository() if failures: @@ -496,5 +688,6 @@ def main() -> int: print("Debian 13 image contract check passed.") return 0 + if __name__ == "__main__": raise SystemExit(main()) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 3fc6931d..74e9480c 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -128,28 +128,79 @@ def test_debian_family_literal_policy_is_table_driven(self) -> None: cases = [ ("Dockerfile", f"FROM {PINNED_RUST}\n", None), ("compose.yaml", f"services:\n app:\n image: {PINNED_RUST}\n", None), - ("script.sh", "docker run --rm rust:1.95-trixie true\n", "not pinned by immutable digest"), - ("workflow.yml", f"env:\n BUILDER_IMAGE: rust:1.95@sha256:{DIGEST}\n", "does not declare Trixie/Debian 13"), - ("images.py", "BUILDER_IMAGE: str = 'python:3.13-slim-trixie'\n", "not pinned by immutable digest"), - ("images.ts", "const builderImage = 'golang:1.25-trixie';\n", "not pinned by immutable digest"), + ( + "script.sh", + "docker run --rm rust:1.95-trixie true\n", + "not pinned by immutable digest", + ), + ( + "workflow.yml", + f"env:\n BUILDER_IMAGE: rust:1.95@sha256:{DIGEST}\n", + "does not declare Trixie/Debian 13", + ), + ( + "images.py", + "BUILDER_IMAGE: str = 'python:3.13-slim-trixie'\n", + "not pinned by immutable digest", + ), + ( + "images.ts", + "const builderImage = 'golang:1.25-trixie';\n", + "not pinned by immutable digest", + ), ("module.js", "import path from 'node:path';\n", None), ("data.yaml", "identifier: did:web\nport: 65532:65532\n", None), ] - for family, version in (("rust", "1.95"), ("node", "22"), ("python", "3.13"), - ("golang", "1.25"), ("postgres", "16")): - cases.extend(( - ("compose.yaml", f"{'container' if family == 'node' else 'image'}: {family}\n", "bare Debian-default"), - ("build.sh", f"BUILDER_IMAGE='{family}'\n", "bare Debian-default"), - ("script.sh", f"docker run --rm {family} true\n", "bare Debian-default"), - ("script.sh", f"docker run --rm {family}:{version}\n", "not pinned"), - ("script.sh", f"docker run --rm {family}:{version}-slim\n", "not pinned"), - ("workflow.yml", f"container: {family}@sha256:{DIGEST}\n", "does not declare"), - ("compose.yaml", f"image: registry.test:5000/team/{family}\n", "bare Debian-default"), - ("workflow.yml", f"container: registry.test/team/{family}@sha256:{DIGEST}\n", "does not declare"), - ("script.sh", f"docker run {family}:{version}-alpine\n", None), - ("script.sh", f"docker run {family}:{version}-windows\n", None), - ("guide.md", f"The {family} image is available.\n", None), - )) + for family, version in ( + ("rust", "1.95"), + ("node", "22"), + ("python", "3.13"), + ("golang", "1.25"), + ("postgres", "16"), + ): + cases.extend( + ( + ( + "compose.yaml", + f"{'container' if family == 'node' else 'image'}: {family}\n", + "bare Debian-default", + ), + ("build.sh", f"BUILDER_IMAGE='{family}'\n", "bare Debian-default"), + ( + "script.sh", + f"docker run --rm {family} true\n", + "bare Debian-default", + ), + ( + "script.sh", + f"docker run --rm {family}:{version}\n", + "not pinned", + ), + ( + "script.sh", + f"docker run --rm {family}:{version}-slim\n", + "not pinned", + ), + ( + "workflow.yml", + f"container: {family}@sha256:{DIGEST}\n", + "does not declare", + ), + ( + "compose.yaml", + f"image: registry.test:5000/team/{family}\n", + "bare Debian-default", + ), + ( + "workflow.yml", + f"container: registry.test/team/{family}@sha256:{DIGEST}\n", + "does not declare", + ), + ("script.sh", f"docker run {family}:{version}-alpine\n", None), + ("script.sh", f"docker run {family}:{version}-windows\n", None), + ("guide.md", f"The {family} image is available.\n", None), + ) + ) for relative, text, expected in cases: with self.subTest(relative=relative, text=text): failures = self.scan(relative, text) @@ -160,6 +211,7 @@ def test_debian_family_literal_policy_is_table_driven(self) -> None: any(expected in failure for failure in failures), failures, ) + def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None: for text in ( "Historical book" + "worm base\n", @@ -176,14 +228,14 @@ def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None self.assert_clean( "design.md", "The book" - 'worm comparison remains historical. ' - '\n', ) invalid = ( "```sh\n" "# book" - 'worm \n' "```\n" ) @@ -196,7 +248,10 @@ def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None any("retired Debian image" in item for item in failures), failures, ) - def test_wrappers_options_operators_and_malformed_shell_still_scan_literals(self) -> None: + + def test_wrappers_options_operators_and_malformed_shell_still_scan_literals( + self, + ) -> None: commands = ( "docker --tlsverify pull -a rust:1.95-trixie", "podman --remote pull --all-tags rust:1.95-trixie", @@ -225,6 +280,7 @@ def test_wrappers_options_operators_and_malformed_shell_still_scan_literals(self ): with self.subTest(command=command): self.assert_clean("script.sh", command + "\n") + def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: cases = ( ("Dockerfile", "FROM debian\n"), @@ -254,6 +310,7 @@ def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: ): with self.subTest(relative=relative, text=text): self.assert_clean(relative, text) + def test_image_assignments_resolve_literals_and_reject_computation(self) -> None: clean = ( f'DEFAULT_BUILDER_IMAGE="{PINNED_RUST}"\n' @@ -267,8 +324,7 @@ def test_image_assignments_resolve_literals_and_reject_computation(self) -> None ) self.assert_clean( "local.sh", - 'RELAY_IMAGE="registryctl-relay:$RUN_ID"\n' - 'docker run "$RELAY_IMAGE"\n', + 'RELAY_IMAGE="registryctl-relay:$RUN_ID"\ndocker run "$RELAY_IMAGE"\n', ) cases = ( @@ -284,8 +340,7 @@ def test_image_assignments_resolve_literals_and_reject_computation(self) -> None ), ( "cycle.sh", - 'BASE_IMAGE="$BUILDER_IMAGE"\n' - 'BUILDER_IMAGE="$BASE_IMAGE"\n', + 'BASE_IMAGE="$BUILDER_IMAGE"\nBUILDER_IMAGE="$BASE_IMAGE"\n', "computed or unresolved image assignment", ), ( @@ -328,7 +383,10 @@ def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> Non " unused_image: [rust:1.95-trixie]\n" " container: ${{ matrix.image }}\n", ), - ("pod.yaml", "apiVersion: v1\nkind: Pod\nspec:\n containers:\n - image: rust:1.95-trixie\n"), + ( + "pod.yaml", + "apiVersion: v1\nkind: Pod\nspec:\n containers:\n - image: rust:1.95-trixie\n", + ), ("malformed.yaml", "jobs: [\nimage: rust:1.95-trixie\n"), ("documents.yaml", "description: first\n---\nimage: rust:1.95-trixie\n"), ) @@ -343,9 +401,7 @@ def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> Non def test_markdown_scans_only_executable_fences(self) -> None: self.assert_failure( "guide.md", - "```sh\n" - "docker run --rm rust:1.95-trixie\n" - "```\n", + "```sh\ndocker run --rm rust:1.95-trixie\n```\n", "not pinned by immutable digest", ) self.assert_clean( @@ -356,7 +412,9 @@ def test_markdown_scans_only_executable_fences(self) -> None: "```\n", ) - def test_repository_dockerfile_discovery_checks_external_and_internal_stages(self) -> None: + def test_repository_dockerfile_discovery_checks_external_and_internal_stages( + self, + ) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.copy_required(root) @@ -396,13 +454,48 @@ def test_repository_dockerfile_discovery_checks_external_and_internal_stages(sel def test_product_contract_mutations_are_reported(self) -> None: mutations = ( - (Path("crates/registry-relay/Dockerfile"), "/usr/local/bin/registry-relay-rhai-worker", "/usr/local/bin/removed-worker", "Relay worker binary"), - (Path("products/notary/Dockerfile"), "registry-notary-cel-worker", "removed-cel-worker", "Notary CEL worker binary"), - (Path("release/docker/Dockerfile.registry-relay"), "# syntax=", "# moved-syntax=", "frontend must be the first line"), - (Path("release/scripts/build-release-binaries.sh"), "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", "--features registry-notary/registry-notary-cel", "PKCS#11-enabled release build"), - (Path("docs/site/scripts/check-registryctl-tutorials.sh"), 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', 'LINUX_TARGET="$REPO_ROOT/target/other"', "registryctl tutorial linux target path"), - (Path(".github/workflows/ci.yml"), "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", "hashFiles('Cargo.lock')", "registryctl tutorial cache builder identity"), - (Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "postgres:16-trixie@sha256:", "postgres:16@", "pinned Debian 13 live-journey PostgreSQL"), + ( + Path("crates/registry-relay/Dockerfile"), + "/usr/local/bin/registry-relay-rhai-worker", + "/usr/local/bin/removed-worker", + "Relay worker binary", + ), + ( + Path("products/notary/Dockerfile"), + "registry-notary-cel-worker", + "removed-cel-worker", + "Notary CEL worker binary", + ), + ( + Path("release/docker/Dockerfile.registry-relay"), + "# syntax=", + "# moved-syntax=", + "frontend must be the first line", + ), + ( + Path("release/scripts/build-release-binaries.sh"), + "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", + "--features registry-notary/registry-notary-cel", + "PKCS#11-enabled release build", + ), + ( + Path("docs/site/scripts/check-registryctl-tutorials.sh"), + 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', + 'LINUX_TARGET="$REPO_ROOT/target/other"', + "registryctl tutorial linux target path", + ), + ( + Path(".github/workflows/ci.yml"), + "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", + "hashFiles('Cargo.lock')", + "registryctl tutorial cache builder identity", + ), + ( + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + "postgres:16-trixie@sha256:", + "postgres:16@", + "pinned Debian 13 live-journey PostgreSQL", + ), ) for path, old, new, expected in mutations: with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: From 21363b59be16a88c13b0dc6fca627ddafdd5dcb7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 01:36:21 +0700 Subject: [PATCH 13/34] fix(release): scan OCI namespace variants Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 6 ++-- release/scripts/test_check_debian13_images.py | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index f660ee02..dc61026d 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -79,16 +79,16 @@ DEFAULT_DEBIAN_FAMILIES = {"golang", "node", "postgres", "python", "rust"} OCI_RE = re.compile( r"(?(?:docker://)?" - r"(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+" + r"(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+" r"(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-fA-F]{64})" r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" ) BARE_DEFAULT_FAMILY_RE = re.compile( - rf"(?(?:[A-Za-z0-9.-]+(?::[0-9]+)?/)*" + rf"(?(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*" rf"(?Pdebian|{'|'.join(sorted(DEFAULT_DEBIAN_FAMILIES))}))" r"(?![A-Za-z0-9._/@+:-])" ) -DIGEST_RE = re.compile(r"@sha256:[0-9a-f]{64}$") +DIGEST_RE = re.compile(r"@sha256:[0-9a-f]{64}$", re.IGNORECASE) FROM_RE = re.compile( r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", re.IGNORECASE | re.MULTILINE, diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 74e9480c..75403167 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -138,6 +138,41 @@ def test_debian_family_literal_policy_is_table_driven(self) -> None: f"env:\n BUILDER_IMAGE: rust:1.95@sha256:{DIGEST}\n", "does not declare Trixie/Debian 13", ), + ( + "compose.yaml", + "image: registry.local/team_name/debian:trixie\n", + "not pinned by immutable digest", + ), + ( + "script.sh", + "docker run registry.local/first_team/second_team/rust:1.95-trixie\n", + "not pinned by immutable digest", + ), + ( + "compose.yaml", + "image: registry.local/team_name/rust\n", + "bare Debian-default", + ), + ( + "compose.yaml", + "image: registry.local/first_team/second_team/postgres\n", + "bare Debian-default", + ), + ( + "Dockerfile", + f"FROM rust:1.95-trixie@sha256:{DIGEST.upper()}\n", + None, + ), + ( + "guide.md", + "Use registry.local/team_name/debian:trixie in prose.\n", + None, + ), + ( + "guide.md", + "Mirror registry.local/first_team/second_team/rust in prose.\n", + None, + ), ( "images.py", "BUILDER_IMAGE: str = 'python:3.13-slim-trixie'\n", From 6f6c37e712b3715b44903d760f82cd7120fa27c5 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 01:52:12 +0700 Subject: [PATCH 14/34] fix(release): close image scanner bypasses Signed-off-by: Jeremi Joslin --- .../workflows/notary-postgres-conformance.yml | 8 -- .../notary/scripts/postgresql-conformance.sh | 6 +- release/scripts/check-debian13-images.py | 96 +++++++++++++--- release/scripts/test_check_debian13_images.py | 107 +++++++++++++++++- 4 files changed, 189 insertions(+), 28 deletions(-) diff --git a/.github/workflows/notary-postgres-conformance.yml b/.github/workflows/notary-postgres-conformance.yml index 2013c4dc..21101aa4 100644 --- a/.github/workflows/notary-postgres-conformance.yml +++ b/.github/workflows/notary-postgres-conformance.yml @@ -59,14 +59,8 @@ jobs: matrix: include: - postgresql: "16" - source_image: postgres:16.13-alpine - target_image: postgres:16.14-alpine - postgresql: "17" - source_image: postgres:17.9-alpine - target_image: postgres:17.10-alpine - postgresql: "18" - source_image: postgres:18.3-alpine - target_image: postgres:18.4-alpine steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -88,8 +82,6 @@ jobs: - name: Run Notary PostgreSQL conformance env: NOTARY_BIN: target/notary-postgres-conformance/debug/registry-notary - NOTARY_POSTGRES_SOURCE_IMAGE: ${{ matrix.source_image }} - NOTARY_POSTGRES_TARGET_IMAGE: ${{ matrix.target_image }} run: | set -euo pipefail diagnostics="${RUNNER_TEMP}/notary-postgresql-${{ matrix.postgresql }}-docker-events.log" diff --git a/products/notary/scripts/postgresql-conformance.sh b/products/notary/scripts/postgresql-conformance.sh index 54a0b440..88ce826a 100755 --- a/products/notary/scripts/postgresql-conformance.sh +++ b/products/notary/scripts/postgresql-conformance.sh @@ -28,9 +28,9 @@ case "${postgresql_major}" in ;; esac -source_image="${NOTARY_POSTGRES_SOURCE_IMAGE:-${default_source_image}}" -target_image="${NOTARY_POSTGRES_TARGET_IMAGE:-${default_target_image}}" -restore_image="${NOTARY_POSTGRES_RESTORE_IMAGE:-${default_restore_image}}" +source_image="${default_source_image}" +target_image="${default_target_image}" +restore_image="${default_restore_image}" notary_bin="${NOTARY_BIN:-target/debug/registry-notary}" run_id="${GITHUB_RUN_ID:-local}-$$" postgres_container="notary-postgres-conformance-${run_id}" diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index dc61026d..b4dcd4e9 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -108,6 +108,10 @@ CLI_RE = re.compile(r"(?:^|[\s;&|({])(?:\S*/)?(?:docker|podman)\b(?P[^\n]*)") ACTION_RE = re.compile(r"\b(?:create|pull|run)\b") NON_IMAGE_NAMESPACES = set("build buildx compose exec network volume".split()) +SHELL_VAR_RE = re.compile( + r"\$(?:\{[A-Za-z_][A-Za-z0-9_]*(?::-[^{}]*)?\}|[A-Za-z_][A-Za-z0-9_]*)" +) +COMPUTED_VALUE_RE = re.compile(r"[+%]|`|\$\(|\.format\(|\bf[\"']") EXEMPTION_RE = re.compile( r'' ) @@ -125,10 +129,11 @@ class ImageSurfaceError(RuntimeError): def is_dockerfile(path: Path) -> bool: + name = path.name.casefold() return ( - path.name == "Dockerfile" - or path.name.startswith("Dockerfile.") - or path.name.endswith(".Dockerfile") + name == "dockerfile" + or name.startswith("dockerfile.") + or name.endswith(".dockerfile") ) @@ -268,6 +273,14 @@ def policy_image_name(name: str) -> bool: ) +def has_safe_image_template(value: str) -> bool: + if COMPUTED_VALUE_RE.search(value): + return False + expanded = SHELL_VAR_RE.sub("dynamic", value) + found = references(expanded) + return bool(found) and all(not is_debian_family(reference) for reference in found) + + def markdown_code_flags(path: Path, lines: list[str]) -> list[bool]: if path.suffix not in MARKDOWN_SUFFIXES: return [False] * len(lines) @@ -302,6 +315,39 @@ def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, b return result +def yaml_container_consumers(lines: list[str]) -> dict[int, str]: + fields: dict[int, dict[str, tuple[int, str]]] = {} + consumers = {} + for number, line in enumerate(lines, 1): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + for depth in tuple(fields): + if depth > indent: + del fields[depth] + match = YAML_KEY_RE.match(line) + if match is None or match.group("name").casefold() not in { + "entrypoint", + "command", + }: + continue + name, value = match.group("name").casefold(), match.group("value") + fields.setdefault(indent, {})[name] = (number, value) + if {"entrypoint", "command"} <= fields[indent].keys(): + command_number, command = fields[indent]["command"] + entrypoint = fields[indent]["entrypoint"][1] + engine = re.match( + r"^\s*\[?\s*[\"']?(?P(?:[^\s,\"'\]]*/)?(?:docker|podman))" + r"[\"']?(?=$|[\s,\]])", + entrypoint, + ) + if engine: + combined = f"{engine.group('engine')} {command}" + if is_container_consumer(combined): + consumers[command_number] = combined + return consumers + + def scan_surface(path: Path, text: str) -> list[str]: if len(text.encode()) > MAX_TEXT_FILE_BYTES: raise ImageSurfaceError( @@ -310,8 +356,11 @@ def scan_surface(path: Path, text: str) -> list[str]: lines, failures = text.splitlines(), [] markdown_flags = markdown_code_flags(path, text.splitlines()) code_file = is_dockerfile(path) or path.suffix in CODE_SUFFIXES + yaml_consumers = ( + yaml_container_consumers(lines) if path.suffix in {".yaml", ".yml"} else {} + ) records: list[tuple[int, str, str, set[str], bool, bool]] = [] - resolved, declared = set(), set() + resolved = set() for number, line in enumerate(lines, 1): if len(line) > MAX_LINE_CHARS: raise ImageSurfaceError( @@ -335,8 +384,11 @@ def scan_surface(path: Path, text: str) -> list[str]: f"{path}:{number}: retired Debian image generation marker remains: {marker}" ) item = assignment(path, line) if path.suffix in CODE_SUFFIXES else None + consumer_line = yaml_consumers.get(number, line) consumer = ( - is_container_consumer(line) and (code_file or markdown_code) and not comment + is_container_consumer(consumer_line) + and (code_file or markdown_code) + and not comment ) reference_context = not comment and ( is_dockerfile(path) @@ -369,20 +421,21 @@ def scan_surface(path: Path, text: str) -> list[str]: found.group("name").casefold() for found in IMAGE_VAR_RE.finditer(value) } has_literal = bool(references(value)) + has_template = ( + not has_literal and not dependencies and has_safe_image_template(value) + ) positional = canonical == "image" and re.fullmatch( r"""["']?\$(?:\{)?[1-9](?:\})?["']?;?""", value.strip() ) computed = not positional and ( - not has_literal - and not dependencies - or re.search(r"\s[+%]\s|`|\$\(|\.format\(|\bf[\"']", value) is not None + (not has_literal and not has_template and not dependencies) + or COMPUTED_VALUE_RE.search(value) is not None ) strict = path.suffix in STRICT_ASSIGNMENT_SUFFIXES and policy_image_name( name ) records.append((number, canonical, value, dependencies, computed, strict)) - declared.add(canonical) - if has_literal and not computed or positional: + if ((has_literal or has_template) and not computed) or positional: resolved.add(canonical) bare = BARE_DEFAULT_FAMILY_RE.search(line) bare_assignment = item is not None and ( @@ -406,7 +459,12 @@ def scan_surface(path: Path, text: str) -> list[str]: while changed: changed = False for _, name, _, dependencies, computed, strict in records: - if name not in resolved and not computed and dependencies & resolved: + if ( + name not in resolved + and not computed + and dependencies + and dependencies <= resolved + ): resolved.add(name) changed = True for number, name, value, _, computed, strict in records: @@ -415,7 +473,11 @@ def scan_surface(path: Path, text: str) -> list[str]: f"{path}:{number}: computed or unresolved image assignment " f"is not allowed: {value.strip()}" ) - for number, line, markdown_code in logical_lines(lines, markdown_flags): + consumer_records = logical_lines(lines, markdown_flags) + consumer_records.extend( + (number, line, False) for number, line in yaml_consumers.items() + ) + for number, line, markdown_code in consumer_records: if ( line.lstrip().startswith(("#", "//")) or not (code_file or markdown_code) @@ -425,17 +487,23 @@ def scan_surface(path: Path, text: str) -> list[str]: variables = { match.group("name").casefold() for match in IMAGE_VAR_RE.finditer(line) } + unresolved = sorted(variables - resolved) if ( not any( re.search("[A-Za-z]", repository_and_tag(item)[0]) for item in references(line) ) and not BARE_DEFAULT_FAMILY_RE.search(line) - and not variables & (resolved | declared) + and (not variables or unresolved) ): + detail = ( + f"; unresolved image variables: {', '.join(unresolved)}" + if unresolved + else "" + ) failures.append( f"{path}:{number}: Docker/Podman image consumer must use a " - "literal or a resolved *_IMAGE assignment" + f"literal or a statically resolved *_IMAGE assignment{detail}" ) return failures diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 75403167..01bbeb07 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -319,6 +319,7 @@ def test_wrappers_options_operators_and_malformed_shell_still_scan_literals( def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: cases = ( ("Dockerfile", "FROM debian\n"), + ("images/relay.dockerfile", "FROM debian\n"), ("Dockerfile", "COPY --from=localhost:5000/debian /x /x\n"), ("Dockerfile", "RUN --mount=from=debian,target=/x true\n"), ("compose.yaml", "services:\n app:\n image: debian\n"), @@ -336,6 +337,7 @@ def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: ) for relative, text in ( + ("images/relay.dockerfile", f"FROM {PINNED_RUST}\n"), ("guide.md", "Debian is the supported distribution.\n"), ("guide.md", "```text\ndocker run debian\n```\n"), ("script.sh", "# docker run --rm debian\n"), @@ -361,8 +363,36 @@ def test_image_assignments_resolve_literals_and_reject_computation(self) -> None "local.sh", 'RELAY_IMAGE="registryctl-relay:$RUN_ID"\ndocker run "$RELAY_IMAGE"\n', ) + self.assert_clean( + "static.sh", + 'APP_IMAGE="alpine:3.22"\ndocker run --rm "$APP_IMAGE"\n', + ) + self.assert_clean( + "fallback.sh", + 'FIRST_IMAGE="alpine:3.22"\nSECOND_IMAGE="busybox:1.37"\n' + 'APP_IMAGE="${FIRST_IMAGE:-$SECOND_IMAGE}"\n' + 'docker run --rm "$APP_IMAGE"\n', + ) cases = ( + ( + "computed.sh", + 'APP_IMAGE="$(select-image)"\ndocker run --rm "$APP_IMAGE"\n', + "unresolved image variables: app_image", + ), + ( + "alias.sh", + 'OTHER_IMAGE="$(select-image)"\nAPP_IMAGE="$OTHER_IMAGE"\n' + 'docker run --rm "$APP_IMAGE"\n', + "unresolved image variables: app_image", + ), + ( + "multi.sh", + 'GOOD_IMAGE="alpine:3.22"\nBAD_IMAGE="$(select-image)"\n' + 'APP_IMAGE="${GOOD_IMAGE:-$BAD_IMAGE}"\n' + 'docker run --rm "$APP_IMAGE"\n', + "unresolved image variables: app_image", + ), ( "images.py", "BUILDER_IMAGE = make_image()\n", @@ -381,12 +411,12 @@ def test_image_assignments_resolve_literals_and_reject_computation(self) -> None ( "build.sh", 'docker run --rm "$OTHER_IMAGE"\n', - "must use a literal or a resolved *_IMAGE assignment", + "must use a literal or a statically resolved *_IMAGE assignment", ), ( "build.sh", 'docker run --publish 127.0.0.1:8080 "$OTHER_IMAGE"\n', - "must use a literal or a resolved *_IMAGE assignment", + "must use a literal or a statically resolved *_IMAGE assignment", ), ( "build.sh", @@ -396,7 +426,7 @@ def test_image_assignments_resolve_literals_and_reject_computation(self) -> None ( "build.sh", 'docker pull "$container"\n', - "must use a literal or a resolved *_IMAGE assignment", + "must use a literal or a statically resolved *_IMAGE assignment", ), ) for relative, text, expected in cases: @@ -433,6 +463,77 @@ def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> Non "not pinned by immutable digest", ) + def test_compose_entrypoint_and_command_form_one_image_consumer(self) -> None: + self.assert_failure( + "compose.yaml", + "services:\n app:\n entrypoint: [docker]\n" + " command: [run, --rm, debian]\n", + "bare Debian image reference", + ) + self.assert_clean( + "compose.yaml", + "services:\n app:\n entrypoint: [docker]\n" + f" command: [run, --rm, {PINNED_RUST}]\n", + ) + self.assert_clean( + "compose.yaml", + "services:\n first:\n entrypoint: [docker]\n" + " second:\n command: [run, --rm, debian]\n", + ) + self.assert_clean( + "compose.yaml", + "services:\n app:\n entrypoint: [echo, docker]\n" + " command: [run, --rm, debian]\n", + ) + + def test_postgresql_conformance_workflow_selects_static_images(self) -> None: + script_path = Path("products/notary/scripts/postgresql-conformance.sh") + script = (ROOT / script_path).read_text() + workflow = ( + ROOT / ".github/workflows/notary-postgres-conformance.yml" + ).read_text() + selections = { + "16": ( + "postgres:16.13-alpine", + "postgres:16.14-alpine", + "postgres:17.10-alpine", + ), + "17": ( + "postgres:17.9-alpine", + "postgres:17.10-alpine", + "postgres:18.4-alpine", + ), + "18": ( + "postgres:18.3-alpine", + "postgres:18.4-alpine", + "postgres:18.4-alpine", + ), + } + for major, (source, target, restore) in selections.items(): + with self.subTest(major=major): + self.assertIn(f'- postgresql: "{major}"', workflow) + self.assertIn( + f' {major})\n default_source_image="{source}"\n' + f' default_target_image="{target}"\n' + f' default_restore_image="{restore}"\n', + script, + ) + self.assertNotIn("NOTARY_POSTGRES_SOURCE_IMAGE", script + workflow) + self.assertNotIn("NOTARY_POSTGRES_TARGET_IMAGE", script + workflow) + self.assertNotIn("NOTARY_POSTGRES_RESTORE_IMAGE", script + workflow) + self.assert_clean(script_path.as_posix(), script) + + override = ( + 'source_image="${NOTARY_POSTGRES_SOURCE_IMAGE:-${default_source_image}}"' + ) + mutated = script.replace('source_image="${default_source_image}"', override) + self.assertNotEqual(script, mutated) + self.assert_failure( + script_path.as_posix(), + mutated, + "unresolved image variables: source_image", + ) + def test_markdown_scans_only_executable_fences(self) -> None: self.assert_failure( "guide.md", From 34ab475feaff65be9006f2cd85e81f48a8e55de8 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 02:17:45 +0700 Subject: [PATCH 15/34] fix(release): harden image scanner parsing Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 374 ++++++++++++++---- release/scripts/test_check_debian13_images.py | 131 ++++++ 2 files changed, 433 insertions(+), 72 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index b4dcd4e9..86b4a21f 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -101,23 +101,118 @@ YAML_KEY_RE = re.compile( r"^\s*(?:-\s*)?(?P[A-Za-z_][A-Za-z0-9_$-]*)\s*:\s*(?P.+)$" ) +YAML_FIELD_RE = re.compile( + r"^(?P\s*)(?P-\s*)?" + r"(?P[A-Za-z_][A-Za-z0-9_$-]*)\s*:\s*(?P.*)$" +) IMAGE_VAR_RE = re.compile( r"\b(?P[A-Za-z_][A-Za-z0-9_$-]*(?:_image|Image)|IMAGE)\b", re.IGNORECASE, ) CLI_RE = re.compile(r"(?:^|[\s;&|({])(?:\S*/)?(?:docker|podman)\b(?P[^\n]*)") -ACTION_RE = re.compile(r"\b(?:create|pull|run)\b") NON_IMAGE_NAMESPACES = set("build buildx compose exec network volume".split()) -SHELL_VAR_RE = re.compile( - r"\$(?:\{[A-Za-z_][A-Za-z0-9_]*(?::-[^{}]*)?\}|[A-Za-z_][A-Za-z0-9_]*)" +IMAGE_ALIAS_NAME = r"(?:[A-Za-z_][A-Za-z0-9_]*(?:_IMAGE|Image)|IMAGE)" +DIRECT_IMAGE_ALIAS_RE = re.compile( + rf"""^["']?\$(?:{IMAGE_ALIAS_NAME}|\{{{IMAGE_ALIAS_NAME}\}})["']?;?$""", + re.IGNORECASE, +) +IMAGE_FALLBACK_RE = re.compile( + rf"""^["']?\$\{{{IMAGE_ALIAS_NAME}:-(?:\${IMAGE_ALIAS_NAME}|""" + rf"""\$\{{{IMAGE_ALIAS_NAME}\}})\}}["']?;?$""", + re.IGNORECASE, +) +SIMPLE_SHELL_VAR = r"\$(?:[A-Za-z_][A-Za-z0-9_]*|\{[A-Za-z_][A-Za-z0-9_]*\})" +SAFE_TAG_TEMPLATE_RE = re.compile( + rf"""^["']?(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*""" + rf"""[A-Za-z0-9._-]+):(?:[A-Za-z0-9_.-]|{SIMPLE_SHELL_VAR})+["']?;?$""" ) COMPUTED_VALUE_RE = re.compile(r"[+%]|`|\$\(|\.format\(|\bf[\"']") +CLI_TOKEN_RE = re.compile(r""""[^"\n]*"|'[^'\n]*'|[^\s,;\[\]]+""") +CONTAINER_VALUE_OPTIONS = { + "--add-host", + "--annotation", + "--attach", + "--cap-add", + "--cap-drop", + "--cgroupns", + "--cidfile", + "--cpus", + "--device", + "--dns", + "--dns-search", + "--entrypoint", + "--env", + "--env-file", + "--expose", + "--gpus", + "--group-add", + "--health-cmd", + "--health-interval", + "--health-retries", + "--health-timeout", + "--hostname", + "--ipc", + "--label", + "--link", + "--log-driver", + "--log-opt", + "--mac-address", + "--memory", + "--mount", + "--name", + "--network", + "--pid", + "--platform", + "--publish", + "--pull", + "--restart", + "--runtime", + "--security-opt", + "--shm-size", + "--stop-signal", + "--stop-timeout", + "--tmpfs", + "--ulimit", + "--user", + "--volume", + "--workdir", + "-a", + "-e", + "-h", + "-l", + "-m", + "-p", + "-u", + "-v", + "-w", +} +CONTAINER_FLAG_OPTIONS = { + "--detach", + "--init", + "--interactive", + "--privileged", + "--quiet", + "--read-only", + "--rm", + "--tty", + "-d", + "-i", + "-it", + "-q", + "-t", +} +PULL_FLAG_OPTIONS = {"--all-tags", "--disable-content-trust", "-a", "-q"} +SHORT_VALUE_OPTIONS = ("-e", "-h", "-l", "-m", "-p", "-u", "-v", "-w") +DOCKER_CONTEXT_RE = re.compile( + r"docker-image://(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*" + r"[A-Za-z0-9._-]+(?:\:[A-Za-z0-9_][A-Za-z0-9._-]*)?" + r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" +) EXEMPTION_RE = re.compile( r'' ) MARKDOWN_SUFFIXES = {".md", ".mdx"} CODE_SUFFIXES = set(".bash .js .mjs .py .sh .ts .yaml .yml".split()) -STRICT_ASSIGNMENT_SUFFIXES = CODE_SUFFIXES - {".yaml", ".yml"} FENCE_LANGS = set( "bash console dockerfile javascript js python sh shell terminal " "ts typescript yaml yml zsh".split() @@ -211,6 +306,15 @@ def references(text: str) -> list[str]: ] +def exact_reference(value: str) -> str | None: + candidate = value.strip().removesuffix(";").strip() + if len(candidate) >= 2 and candidate[0] == candidate[-1] and candidate[0] in "\"'": + candidate = candidate[1:-1] + found = references(candidate) + normalized = candidate.removeprefix("docker://") + return found[0] if len(found) == 1 and found[0] == normalized else None + + def repository_and_tag(reference: str) -> tuple[str, str | None]: value = reference.split("@", 1)[0] slash, colon = value.rfind("/"), value.rfind(":") @@ -241,6 +345,9 @@ def assignment(path: Path, line: str) -> tuple[str, str] | None: if match is None: return None name = match.group("name") + value = match.group("value").strip() + if value.rstrip(",") in {"(", "[", "{"} or value.endswith(","): + return None normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() return ( (name, match.group("value")) @@ -249,16 +356,78 @@ def assignment(path: Path, line: str) -> tuple[str, str] | None: ) -def is_container_consumer(line: str) -> bool: +def command_tokens(value: str) -> list[str]: + command = re.split(r"\s*(?:&&|\|\||[;|])\s*", value, maxsplit=1)[0] + return [ + token[1:-1] + if len(token) >= 2 and token[0] == token[-1] and token[0] in "\"'" + else token.strip("\"'()") + for token in CLI_TOKEN_RE.findall(command) + ] + + +def container_image_values(line: str) -> list[str]: + values = [] for command in CLI_RE.finditer(line): - tail = command.group("tail") - action = ACTION_RE.search(tail) - if action is None: + tokens = command_tokens(command.group("tail")) + action_index = next( + ( + index + for index, token in enumerate(tokens) + if token.casefold() in {"create", "pull", "run"} + ), + None, + ) + if action_index is None: continue - prefix = set(re.findall(r"[A-Za-z][A-Za-z0-9-]*", tail[: action.start()])) + prefix = {token.casefold().lstrip("-") for token in tokens[:action_index]} if not prefix & NON_IMAGE_NAMESPACES: - return True - return False + action = tokens[action_index].casefold() + value_options = CONTAINER_VALUE_OPTIONS - ( + {"-a", "--attach"} if action == "pull" else set() + ) + flag_options = CONTAINER_FLAG_OPTIONS | ( + PULL_FLAG_OPTIONS if action == "pull" else set() + ) + ambiguous, index = False, action_index + 1 + while index < len(tokens): + token = tokens[index] + if token == "\\": + index += 1 + continue + if token == "--": + index += 1 + break + if not token.startswith("-"): + break + option = token.split("=", 1)[0] + attached = ( + option in SHORT_VALUE_OPTIONS + and token != option + and not token.startswith("--") + ) + if option in value_options: + index += 1 if "=" in token or attached else 2 + elif option in flag_options: + index += 1 + else: + ambiguous = True + index += 1 + if index >= len(tokens): + values.append("") + elif ambiguous: + values.extend( + token + for token in tokens[index:] + if token != "\\" and not token.startswith("-") + ) + else: + values.append(tokens[index]) + return values + + +def is_container_consumer(line: str) -> bool: + return bool(container_image_values(line)) def policy_image_name(name: str) -> bool: @@ -273,12 +442,53 @@ def policy_image_name(name: str) -> bool: ) +def is_image_alias(value: str) -> bool: + return ( + DIRECT_IMAGE_ALIAS_RE.fullmatch(value.strip()) is not None + or IMAGE_FALLBACK_RE.fullmatch(value.strip()) is not None + ) + + def has_safe_image_template(value: str) -> bool: - if COMPUTED_VALUE_RE.search(value): + match = SAFE_TAG_TEMPLATE_RE.fullmatch(value) + if COMPUTED_VALUE_RE.search(value) or match is None: return False - expanded = SHELL_VAR_RE.sub("dynamic", value) - found = references(expanded) - return bool(found) and all(not is_debian_family(reference) for reference in found) + return not is_debian_family(match.group("repository")) + + +def build_context_references(text: str) -> list[str]: + return [match.group("ref") for match in DOCKER_CONTEXT_RE.finditer(text)] + + +def append_reference_failures( + path: Path, + number: int, + reference: str, + failures: list[str], + kind: str = "Debian-derived image reference", +) -> None: + if not is_debian_family(reference): + return + prefix = f"{path}:{number}: {kind}" + if not DIGEST_RE.search(reference): + failures.append(f"{prefix} is not pinned by immutable digest: {reference}") + if ( + "trixie" not in reference.casefold() + and re.search(r"debian-?13", reference, re.IGNORECASE) is None + ): + failures.append(f"{prefix} does not declare Trixie/Debian 13: {reference}") + + +def append_bare_failures( + path: Path, number: int, value: str, failures: list[str] +) -> None: + for bare in BARE_DEFAULT_FAMILY_RE.finditer(value): + family = bare.group("name") + label = "Debian" if family == "debian" else f"Debian-default {family}" + failures.append( + f"{path}:{number}: bare {label} image reference is not pinned " + f"and does not declare Trixie/Debian 13: {bare.group('ref')}" + ) def markdown_code_flags(path: Path, lines: list[str]) -> list[bool]: @@ -316,26 +526,40 @@ def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, b def yaml_container_consumers(lines: list[str]) -> dict[int, str]: - fields: dict[int, dict[str, tuple[int, str]]] = {} + parents: list[tuple[int, int]] = [] + fields: dict[tuple[int, str], tuple[int, str]] = {} consumers = {} - for number, line in enumerate(lines, 1): + for offset, line in enumerate(lines): + number = offset + 1 if not line.strip() or line.lstrip().startswith("#"): continue indent = len(line) - len(line.lstrip()) - for depth in tuple(fields): - if depth > indent: - del fields[depth] - match = YAML_KEY_RE.match(line) - if match is None or match.group("name").casefold() not in { - "entrypoint", - "command", - }: + while parents and parents[-1][0] >= indent: + parents.pop() + match = YAML_FIELD_RE.match(line) + if match is None: + parents.append((indent, number)) continue - name, value = match.group("name").casefold(), match.group("value") - fields.setdefault(indent, {})[name] = (number, value) - if {"entrypoint", "command"} <= fields[indent].keys(): - command_number, command = fields[indent]["command"] - entrypoint = fields[indent]["entrypoint"][1] + name = match.group("name").casefold() + scope = number if match.group("item") else parents[-1][1] if parents else 0 + if name in {"entrypoint", "command"}: + value = match.group("value").strip() + if not value: + parts = [] + for following in lines[offset + 1 :]: + if not following.strip() or following.lstrip().startswith("#"): + continue + following_indent = len(following) - len(following.lstrip()) + if following_indent <= indent: + break + item = re.match(r"^\s*-\s*(?P.+)$", following) + if item: + parts.append(item.group("value")) + value = " ".join(parts) + fields[(scope, name)] = (number, value) + if (scope, "entrypoint") in fields and (scope, "command") in fields: + command_number, command = fields[(scope, "command")] + entrypoint = fields[(scope, "entrypoint")][1] engine = re.match( r"^\s*\[?\s*[\"']?(?P(?:[^\s,\"'\]]*/)?(?:docker|podman))" r"[\"']?(?=$|[\s,\]])", @@ -345,17 +569,24 @@ def yaml_container_consumers(lines: list[str]) -> dict[int, str]: combined = f"{engine.group('engine')} {command}" if is_container_consumer(combined): consumers[command_number] = combined + parents.append((indent, number)) return consumers -def scan_surface(path: Path, text: str) -> list[str]: +def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: if len(text.encode()) > MAX_TEXT_FILE_BYTES: raise ImageSurfaceError( f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes" ) lines, failures = text.splitlines(), [] markdown_flags = markdown_code_flags(path, text.splitlines()) - code_file = is_dockerfile(path) or path.suffix in CODE_SUFFIXES + shebang = bool(lines and lines[0].startswith("#!")) + code_file = ( + is_dockerfile(path) or path.suffix in CODE_SUFFIXES or executable or shebang + ) + strict_assignments = ( + code_file and not is_dockerfile(path) and path.suffix not in {".yaml", ".yml"} + ) yaml_consumers = ( yaml_container_consumers(lines) if path.suffix in {".yaml", ".yml"} else {} ) @@ -383,7 +614,7 @@ def scan_surface(path: Path, text: str) -> list[str]: failures.append( f"{path}:{number}: retired Debian image generation marker remains: {marker}" ) - item = assignment(path, line) if path.suffix in CODE_SUFFIXES else None + item = assignment(path, line) if code_file else None consumer_line = yaml_consumers.get(number, line) consumer = ( is_container_consumer(consumer_line) @@ -392,69 +623,61 @@ def scan_surface(path: Path, text: str) -> list[str]: ) reference_context = not comment and ( is_dockerfile(path) - or path.suffix in {".yaml", ".yml"} + or path.suffix in {".yaml", ".yml", ".sh", ".bash"} or markdown_code - or path.suffix in {".sh", ".bash"} or item is not None or consumer ) - if reference_context: + context_references = build_context_references(line) + if reference_context and not consumer: for reference in references(line): - if not is_debian_family(reference): - continue - prefix = f"{path}:{number}: Debian-derived image reference" - if not DIGEST_RE.search(reference): - failures.append( - f"{prefix} is not pinned by immutable digest: {reference}" - ) - if ( - "trixie" not in reference.casefold() - and re.search(r"debian-?13", reference, re.IGNORECASE) is None - ): - failures.append( - f"{prefix} does not declare Trixie/Debian 13: {reference}" - ) + if reference not in context_references: + append_reference_failures(path, number, reference, failures) + if reference_context: + for reference in context_references: + append_reference_failures( + path, + number, + reference, + failures, + "Docker build context", + ) if item: name, value = item canonical = name.casefold() dependencies = { found.group("name").casefold() for found in IMAGE_VAR_RE.finditer(value) } - has_literal = bool(references(value)) + has_literal = exact_reference(value) is not None + alias = is_image_alias(value) has_template = ( not has_literal and not dependencies and has_safe_image_template(value) ) positional = canonical == "image" and re.fullmatch( - r"""["']?\$(?:\{)?[1-9](?:\})?["']?;?""", value.strip() + r"""["']?\$(?:[1-9]|\{[1-9](?::-[^{}]+)?\})["']?;?""", + value.strip(), ) computed = not positional and ( - (not has_literal and not has_template and not dependencies) + (not has_literal and not alias and not has_template) or COMPUTED_VALUE_RE.search(value) is not None ) - strict = path.suffix in STRICT_ASSIGNMENT_SUFFIXES and policy_image_name( - name - ) + strict = strict_assignments and policy_image_name(name) records.append((number, canonical, value, dependencies, computed, strict)) if ((has_literal or has_template) and not computed) or positional: resolved.add(canonical) - bare = BARE_DEFAULT_FAMILY_RE.search(line) bare_assignment = item is not None and ( path.suffix in {".yaml", ".yml"} and re.sub("[^A-Za-z0-9]", "", item[0]).casefold() in {"image", "container"} - or path.suffix in STRICT_ASSIGNMENT_SUFFIXES + or strict_assignments and policy_image_name(item[0]) ) if ( not comment - and bare - and (is_dockerfile(path) or consumer or bare_assignment) + and not consumer + and BARE_DEFAULT_FAMILY_RE.search(line) + and (is_dockerfile(path) or bare_assignment) ): - family = bare.group("name") - label = "Debian" if family == "debian" else f"Debian-default {family}" - failures.append( - f"{path}:{number}: bare {label} image reference is not pinned " - f"and does not declare Trixie/Debian 13: {bare.group('ref')}" - ) + append_bare_failures(path, number, line, failures) changed = True while changed: changed = False @@ -478,22 +701,28 @@ def scan_surface(path: Path, text: str) -> list[str]: (number, line, False) for number, line in yaml_consumers.items() ) for number, line, markdown_code in consumer_records: + image_values = container_image_values(line) if ( line.lstrip().startswith(("#", "//")) or not (code_file or markdown_code) - or not is_container_consumer(line) + or not image_values ): continue + image_text = " ".join(image_values) + for reference in references(image_text): + append_reference_failures(path, number, reference, failures) + append_bare_failures(path, number, image_text, failures) variables = { - match.group("name").casefold() for match in IMAGE_VAR_RE.finditer(line) + match.group("name").casefold() + for match in IMAGE_VAR_RE.finditer(image_text) } unresolved = sorted(variables - resolved) if ( not any( re.search("[A-Za-z]", repository_and_tag(item)[0]) - for item in references(line) + for item in references(image_text) ) - and not BARE_DEFAULT_FAMILY_RE.search(line) + and not BARE_DEFAULT_FAMILY_RE.search(image_text) and (not variables or unresolved) ): detail = ( @@ -505,7 +734,7 @@ def scan_surface(path: Path, text: str) -> list[str]: f"{path}:{number}: Docker/Podman image consumer must use a " f"literal or a statically resolved *_IMAGE assignment{detail}" ) - return failures + return list(dict.fromkeys(failures)) def require( @@ -723,7 +952,8 @@ def check_repository(root: Path = ROOT) -> list[str]: return [f"maintained text exceeds {MAX_TOTAL_TEXT_BYTES} total bytes"] texts[path] = text try: - failures.extend(scan_surface(path, text)) + executable = bool((root / path).stat().st_mode & 0o111) + failures.extend(scan_surface(path, text, executable=executable)) except ImageSurfaceError as error: failures.append(str(error)) dockerfiles = [path for path in discovered if is_dockerfile(path)] diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 01bbeb07..ebe340fe 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -433,6 +433,32 @@ def test_image_assignments_resolve_literals_and_reject_computation(self) -> None with self.subTest(relative=relative): self.assert_failure(relative, text, expected) + def test_image_templates_use_only_bounded_static_forms(self) -> None: + cases = ( + 'APP_IMAGE="${UNSAFE_IMAGE:-alpine:3.22}"\n', + 'APP_IMAGE="${BASE_IMAGE/alpine/$TARGET}"\n', + 'APP_IMAGE="${REPOSITORY}:${TAG}"\n', + 'APP_IMAGE="rust:$TAG"\n', + ) + for assignment in cases: + with self.subTest(assignment=assignment): + self.assert_failure( + "template.sh", + assignment + 'docker run --rm "$APP_IMAGE"\n', + "unresolved image variables: app_image", + ) + + self.assert_clean( + "aliases.sh", + 'FIRST_IMAGE="alpine:3.22"\nSECOND_IMAGE="busybox:1.37"\n' + 'APP_IMAGE="${FIRST_IMAGE:-$SECOND_IMAGE}"\n' + 'docker run --rm "$APP_IMAGE"\n', + ) + self.assert_clean( + "tag.sh", + 'APP_IMAGE="registryctl-relay:$RUN_ID"\ndocker run --rm "$APP_IMAGE"\n', + ) + def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> None: cases = ( ( @@ -486,6 +512,111 @@ def test_compose_entrypoint_and_command_form_one_image_consumer(self) -> None: " command: [run, --rm, debian]\n", ) + def test_compose_block_sequences_preserve_mapping_and_list_scope(self) -> None: + self.assert_failure( + "compose.yaml", + "services:\n app:\n entrypoint:\n - docker\n" + " command:\n - run\n - --rm\n - debian\n", + "bare Debian image reference", + ) + self.assert_clean( + "compose.yaml", + "services:\n app:\n entrypoint:\n - docker\n" + f" command:\n - run\n - --rm\n - {PINNED_RUST}\n", + ) + self.assert_clean( + "compose.yaml", + "items:\n - entrypoint: [docker]\n - command: [run, --rm, debian]\n", + ) + self.assert_failure( + "compose.yaml", + "items:\n - name: app\n entrypoint: [docker]\n" + " command: [run, --rm, debian]\n", + "bare Debian image reference", + ) + + def test_container_cli_scans_only_the_bounded_image_operand(self) -> None: + for command in ( + "docker run --name postgres alpine:3.22 true\n", + "docker run alpine:3.22 python -V\n", + "docker create --env NAME=postgres busybox:1.37 python\n", + ): + with self.subTest(command=command): + self.assert_clean("helper.sh", command) + + for command, family in ( + ("docker run --name app postgres true\n", "postgres"), + ("docker run python -V\n", "python"), + ("docker pull --platform linux/amd64 rust:1.95\n", "rust:1.95"), + ): + with self.subTest(command=command): + self.assert_failure("helper.sh", command, family) + + def test_multiline_container_commands_scan_the_joined_operand(self) -> None: + self.assert_failure( + "helper.sh", + "docker run --rm \\\n debian true\n", + "bare Debian image reference", + ) + self.assert_clean( + "helper.sh", + f"docker run --rm \\\n {PINNED_RUST} true\n", + ) + + def test_extensionless_executable_and_shebang_helpers_are_code(self) -> None: + text = "#!/bin/sh\ndocker run --rm debian\n" + self.assert_failure( + "release/scripts/registry-release", + text, + "bare Debian image reference", + ) + failures = self.module.scan_surface( + Path("tools/runner"), + "docker run --rm debian\n", + executable=True, + ) + self.assertTrue( + any("bare Debian image reference" in failure for failure in failures), + failures, + ) + self.assert_clean( + "notes/helper", + "This prose says docker run --rm debian without a shebang.\n", + ) + + def test_docker_image_build_contexts_follow_the_image_policy(self) -> None: + self.assert_failure( + "build.sh", + "docker buildx build --build-context base=docker-image://debian .\n", + "Docker build context", + ) + self.assert_failure( + "build.sh", + "docker build --build-context base=docker-image://rust:1.95-trixie .\n", + "Docker build context", + ) + failures = self.scan( + "build.sh", + "docker build --build-context base=docker-image://rust:1.95-trixie .\n", + ) + self.assertFalse( + any("Debian-derived image reference" in failure for failure in failures), + failures, + ) + self.assert_clean( + "build.sh", + "docker buildx build --build-context " + f"base=docker-image://{PINNED_RUST} .\n", + ) + self.assert_clean( + "build.sh", + "docker buildx build --build-context base=docker-image://alpine:3.22 .\n", + ) + self.assert_clean( + "guide.md", + "Prose mentions --build-context base=docker-image://debian.\n", + ) + def test_postgresql_conformance_workflow_selects_static_images(self) -> None: script_path = Path("products/notary/scripts/postgresql-conformance.sh") script = (ROOT / script_path).read_text() From ba1e3b0059515cef372c629b1430e62d4ec1ca77 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 02:44:16 +0700 Subject: [PATCH 16/34] fix(release): fail closed on dynamic image surfaces Signed-off-by: Jeremi Joslin --- crates/registryctl/src/templates/compose.yaml | 1 + .../conformance/relay-oidc/docker-compose.yml | 1 + release/scripts/check-debian13-images.py | 300 ++++++++++++++++-- release/scripts/test_check_debian13_images.py | 184 +++++++++++ 4 files changed, 454 insertions(+), 32 deletions(-) diff --git a/crates/registryctl/src/templates/compose.yaml b/crates/registryctl/src/templates/compose.yaml index 2da0228d..8a80895d 100644 --- a/crates/registryctl/src/templates/compose.yaml +++ b/crates/registryctl/src/templates/compose.yaml @@ -1,6 +1,7 @@ # Generated by registryctl. services: registry-relay: + # debian13-policy: allow-validated-image validator="registryctl-image-lock" reason="registryctl renders the validated digest image lock into this whole value" image: {{relay_image}} user: "${REGISTRY_STACK_RUNTIME_UID:-65532}:${REGISTRY_STACK_RUNTIME_GID:-65532}" env_file: diff --git a/release/conformance/relay-oidc/docker-compose.yml b/release/conformance/relay-oidc/docker-compose.yml index afed49ab..9926521b 100644 --- a/release/conformance/relay-oidc/docker-compose.yml +++ b/release/conformance/relay-oidc/docker-compose.yml @@ -87,6 +87,7 @@ services: - zitadel-seed:/seed:ro relay: + # debian13-policy: allow-validated-image validator="relay-oidc-smoke" reason="the smoke runner validates the fixed Relay repository and lowercase digest" image: ${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:?runner must provide digest-pinned Registry Relay image} platform: linux/amd64 network_mode: service:zitadel diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 86b4a21f..a31e69c5 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -89,6 +89,11 @@ r"(?![A-Za-z0-9._/@+:-])" ) DIGEST_RE = re.compile(r"@sha256:[0-9a-f]{64}$", re.IGNORECASE) +NON_DEBIAN_TAG_RE = re.compile( + r"(?:^|[._-])(?:alpine(?:[0-9]+(?:\.[0-9]+)*)?" + r"|windows(?:servercore|nanoserver)?)(?:$|[._-])", + re.IGNORECASE, +) FROM_RE = re.compile( r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", re.IGNORECASE | re.MULTILINE, @@ -122,12 +127,19 @@ re.IGNORECASE, ) SIMPLE_SHELL_VAR = r"\$(?:[A-Za-z_][A-Za-z0-9_]*|\{[A-Za-z_][A-Za-z0-9_]*\})" +SIMPLE_GITHUB_EXPRESSION = ( + r"\$\{\{\s*[A-Za-z_][A-Za-z0-9_-]*" + r"(?:\.[A-Za-z_][A-Za-z0-9_-]*)*\s*\}\}" +) +SIMPLE_DYNAMIC_VALUE = rf"(?:{SIMPLE_SHELL_VAR}|{SIMPLE_GITHUB_EXPRESSION})" +SIMPLE_DYNAMIC_VALUE_RE = re.compile(SIMPLE_DYNAMIC_VALUE) SAFE_TAG_TEMPLATE_RE = re.compile( rf"""^["']?(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*""" - rf"""[A-Za-z0-9._-]+):(?:[A-Za-z0-9_.-]|{SIMPLE_SHELL_VAR})+["']?;?$""" + rf"""[A-Za-z0-9._-]+):(?P(?:[A-Za-z0-9_.-]|""" + rf"""{SIMPLE_DYNAMIC_VALUE})+)["']?;?$""" ) COMPUTED_VALUE_RE = re.compile(r"[+%]|`|\$\(|\.format\(|\bf[\"']") -CLI_TOKEN_RE = re.compile(r""""[^"\n]*"|'[^'\n]*'|[^\s,;\[\]]+""") +CLI_TOKEN_RE = re.compile(r""""[^"\n]*"|'[^'\n]*'|[^\s;\[\]]+""") CONTAINER_VALUE_OPTIONS = { "--add-host", "--annotation", @@ -137,6 +149,7 @@ "--cgroupns", "--cidfile", "--cpus", + "--cpuset-cpus", "--device", "--dns", "--dns-search", @@ -161,6 +174,7 @@ "--mount", "--name", "--network", + "--network-alias", "--pid", "--platform", "--publish", @@ -208,9 +222,55 @@ r"[A-Za-z0-9._-]+(?:\:[A-Za-z0-9_][A-Za-z0-9._-]*)?" r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" ) +DOCKER_CONTEXT_TOKEN_RE = re.compile( + r"""docker-image://(?P[^\s,;"']+)""", +) EXEMPTION_RE = re.compile( r'' ) +VALIDATED_IMAGE_EXEMPTION_RE = re.compile( + r"^\s*#\s*debian13-policy:\s*allow-validated-image\s+" + r'validator="(?P[a-z0-9-]+)"\s+reason="[^"]{12,}"\s*$' +) +VALIDATED_IMAGE_CONTRACTS = { + "registryctl-image-lock": ( + Path("crates/registryctl/src/templates/compose.yaml"), + "{{relay_image}}", + ( + ( + Path("crates/registryctl/src/lib.rs"), + 'validate_locked_image_ref(\n "images.registry-relay"', + ), + ( + Path("crates/registryctl/src/lib.rs"), + "fn image_lock_rejects_mutable_or_noncanonical_image_references()", + ), + ( + Path("crates/registryctl/tests/image_lock.rs"), + "assert!(compose.contains(RELAY_IMAGE));", + ), + ), + ), + "relay-oidc-smoke": ( + Path("release/conformance/relay-oidc/docker-compose.yml"), + "${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:" + "?runner must provide digest-pinned Registry Relay image}", + ( + ( + Path("release/scripts/relay-oidc-smoke.py"), + "def validate_relay_image(value: str) -> str:", + ), + ( + Path("release/scripts/relay-oidc-smoke.py"), + "relay_image = validate_relay_image(args.relay_image)", + ), + ( + Path("release/scripts/test_relay_oidc_smoke.py"), + "def test_relay_image_requires_exact_repository_and_lowercase_digest", + ), + ), + ), +} MARKDOWN_SUFFIXES = {".md", ".mdx"} CODE_SUFFIXES = set(".bash .js .mjs .py .sh .ts .yaml .yml".split()) FENCE_LANGS = set( @@ -325,7 +385,7 @@ def is_debian_family(reference: str) -> bool: repository, tag = repository_and_tag(reference) name, tag = repository.rsplit("/", 1)[-1].casefold(), (tag or "").casefold() default = name in DEFAULT_DEBIAN_FAMILIES - excluded = default and ("alpine" in tag or "windows" in tag) + excluded = default and NON_DEBIAN_TAG_RE.search(tag) is not None return not excluded and ( "debian" in name or name == "buildpack-deps" @@ -356,12 +416,41 @@ def assignment(path: Path, line: str) -> tuple[str, str] | None: ) +def split_flow_commas(value: str, nested: bool) -> list[str]: + parts, current, depth, quote = [], [], 0, "" + for character in value: + if quote: + current.append(character) + if character == quote: + quote = "" + elif character in "\"'": + quote = character + current.append(character) + elif character == "[": + depth += 1 + current.append(character) + elif character == "]": + depth = max(0, depth - 1) + current.append(character) + elif character == "," and bool(depth) == nested: + parts.append("".join(current)) + current = [] + else: + current.append(character) + parts.append("".join(current)) + return parts + + def command_tokens(value: str) -> list[str]: - command = re.split(r"\s*(?:&&|\|\||[;|])\s*", value, maxsplit=1)[0] + command = re.split( + r"\s*(?:&&|\|\||[;|])\s*", + " ".join(split_flow_commas(value, nested=True)), + maxsplit=1, + )[0] return [ token[1:-1] if len(token) >= 2 and token[0] == token[-1] and token[0] in "\"'" - else token.strip("\"'()") + else token.strip("\"'(),") for token in CLI_TOKEN_RE.findall(command) ] @@ -408,7 +497,7 @@ def container_image_values(line: str) -> list[str]: ) if option in value_options: index += 1 if "=" in token or attached else 2 - elif option in flag_options: + elif option in flag_options or re.fullmatch(r"-[diqt]+", option): index += 1 else: ambiguous = True @@ -442,6 +531,11 @@ def policy_image_name(name: str) -> bool: ) +def yaml_policy_image_name(name: str) -> bool: + normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() + return normalized in {"image", "container"} or policy_image_name(name) + + def is_image_alias(value: str) -> bool: return ( DIRECT_IMAGE_ALIAS_RE.fullmatch(value.strip()) is not None @@ -453,13 +547,26 @@ def has_safe_image_template(value: str) -> bool: match = SAFE_TAG_TEMPLATE_RE.fullmatch(value) if COMPUTED_VALUE_RE.search(value) or match is None: return False - return not is_debian_family(match.group("repository")) + repository = match.group("repository") + name = repository.rsplit("/", 1)[-1].casefold() + if name not in DEFAULT_DEBIAN_FAMILIES: + return not is_debian_family(repository) + static_tag = SIMPLE_DYNAMIC_VALUE_RE.sub("dynamic", match.group("tag")).casefold() + return NON_DEBIAN_TAG_RE.search(static_tag) is not None def build_context_references(text: str) -> list[str]: return [match.group("ref") for match in DOCKER_CONTEXT_RE.finditer(text)] +def computed_build_contexts(text: str) -> list[str]: + return [ + match.group("value") + for match in DOCKER_CONTEXT_TOKEN_RE.finditer(text) + if "$" in match.group("value") or "{" in match.group("value") + ] + + def append_reference_failures( path: Path, number: int, @@ -525,14 +632,88 @@ def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, b return result -def yaml_container_consumers(lines: list[str]) -> dict[int, str]: +def flow_mapping_fields(line: str) -> list[dict[str, str]]: + mappings = [] + for mapping in re.finditer(r"\{(?P[^{}]*)\}", line): + fields = {} + for part in split_flow_commas(mapping.group("body"), nested=False): + name, separator, value = part.partition(":") + if separator and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", name.strip()): + fields[name.strip().casefold()] = value.strip() + if fields: + mappings.append(fields) + return mappings + + +def yaml_container_consumers( + lines: list[str], +) -> tuple[dict[int, str], list[tuple[int, str]]]: parents: list[tuple[int, int]] = [] fields: dict[tuple[int, str], tuple[int, str]] = {} - consumers = {} + anchors: dict[str, str] = {} + consumers, unresolved = {}, [] + + def combine(number: int, entrypoint: str, command: str) -> None: + tokens = command_tokens(command) + action = tokens[0].casefold() if tokens else "" + alias = re.fullmatch(r"\*(?P[A-Za-z0-9_-]+)", entrypoint.strip()) + if alias: + resolved = anchors.get(alias.group("name")) + if resolved is None: + if action in {"create", "pull", "run"}: + unresolved.append( + ( + number, + "cannot statically resolve container entrypoint; " + "use a literal docker/podman entrypoint", + ) + ) + return + entrypoint = resolved + elif entrypoint.strip().startswith(("$", "{{")) and action in { + "create", + "pull", + "run", + }: + unresolved.append( + ( + number, + "cannot statically resolve container entrypoint; " + "use a literal docker/podman entrypoint", + ) + ) + return + engine = re.match( + r"^\s*\[?\s*[\"']?(?P(?:[^\s,\"'\]]*/)?(?:docker|podman))" + r"[\"']?(?=$|[\s,\]])", + entrypoint, + ) + if engine: + combined = f"{engine.group('engine')} {command}" + if is_container_consumer(combined): + consumers[number] = combined + elif command.strip().startswith(("*", "$", "{{")): + unresolved.append( + ( + number, + "cannot statically resolve container command; " + "use a literal create/pull/run command", + ) + ) + for offset, line in enumerate(lines): number = offset + 1 if not line.strip() or line.lstrip().startswith("#"): continue + for anchor in re.finditer( + r"&(?P[A-Za-z0-9_-]+)\s+" + r"(?P\[[^\]\n]+\]|[\"']?(?:docker|podman)[\"']?)", + line, + ): + anchors[anchor.group("name")] = anchor.group("value") + for mapping in flow_mapping_fields(line): + if {"entrypoint", "command"} <= mapping.keys(): + combine(number, mapping["entrypoint"], mapping["command"]) indent = len(line) - len(line.lstrip()) while parents and parents[-1][0] >= indent: parents.pop() @@ -544,7 +725,8 @@ def yaml_container_consumers(lines: list[str]) -> dict[int, str]: scope = number if match.group("item") else parents[-1][1] if parents else 0 if name in {"entrypoint", "command"}: value = match.group("value").strip() - if not value: + block_scalar = re.fullmatch(r"[>|][+-]?", value) is not None + if not value or block_scalar: parts = [] for following in lines[offset + 1 :]: if not following.strip() or following.lstrip().startswith("#"): @@ -552,25 +734,42 @@ def yaml_container_consumers(lines: list[str]) -> dict[int, str]: following_indent = len(following) - len(following.lstrip()) if following_indent <= indent: break - item = re.match(r"^\s*-\s*(?P.+)$", following) - if item: - parts.append(item.group("value")) + if block_scalar: + parts.append(following.strip()) + else: + item = re.match(r"^\s*-\s*(?P.+)$", following) + if item: + parts.append(item.group("value")) value = " ".join(parts) fields[(scope, name)] = (number, value) if (scope, "entrypoint") in fields and (scope, "command") in fields: command_number, command = fields[(scope, "command")] entrypoint = fields[(scope, "entrypoint")][1] - engine = re.match( - r"^\s*\[?\s*[\"']?(?P(?:[^\s,\"'\]]*/)?(?:docker|podman))" - r"[\"']?(?=$|[\s,\]])", - entrypoint, - ) - if engine: - combined = f"{engine.group('engine')} {command}" - if is_container_consumer(combined): - consumers[command_number] = combined + combine(command_number, entrypoint, command) parents.append((indent, number)) - return consumers + return consumers, unresolved + + +def yaml_image_exemptions(path: Path, lines: list[str]) -> tuple[set[int], set[int]]: + if path.suffix not in {".yaml", ".yml"}: + return set(), set() + comments, assignments = set(), set() + for offset, line in enumerate(lines[:-1]): + exemption = VALIDATED_IMAGE_EXEMPTION_RE.fullmatch(line) + if exemption is None: + continue + contract = VALIDATED_IMAGE_CONTRACTS.get(exemption.group("validator")) + if contract is None or contract[0] != path: + continue + item = assignment(path, lines[offset + 1]) + if ( + item is not None + and yaml_policy_image_name(item[0]) + and item[1].strip() == contract[1] + ): + comments.add(offset + 1) + assignments.add(offset + 2) + return comments, assignments def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: @@ -584,12 +783,16 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: code_file = ( is_dockerfile(path) or path.suffix in CODE_SUFFIXES or executable or shebang ) - strict_assignments = ( + strict_code_assignments = ( code_file and not is_dockerfile(path) and path.suffix not in {".yaml", ".yml"} ) - yaml_consumers = ( - yaml_container_consumers(lines) if path.suffix in {".yaml", ".yml"} else {} + yaml_consumers, yaml_errors = ( + yaml_container_consumers(lines) + if path.suffix in {".yaml", ".yml"} + else ({}, []) ) + failures.extend(f"{path}:{number}: {message}" for number, message in yaml_errors) + exemption_comments, exempt_assignments = yaml_image_exemptions(path, lines) records: list[tuple[int, str, str, set[str], bool, bool]] = [] resolved = set() for number, line in enumerate(lines, 1): @@ -599,13 +802,22 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: ) markdown_code = markdown_flags[number - 1] comment = line.lstrip().startswith(("#", "//")) and not markdown_code - exemption = ( + exemption = number in exemption_comments or ( path.suffix in MARKDOWN_SUFFIXES and not markdown_code and EXEMPTION_RE.search(line) is not None ) if "debian13-policy:" in line and not exemption: - failures.append(f"{path}:{number}: invalid Debian image prose exemption") + kind = ( + "prose exemption" + if path.suffix in MARKDOWN_SUFFIXES + else "policy annotation" + ) + failures.append( + f"{path}:{number}: invalid Debian image {kind}; use a literal " + "static repository and digest unless an exact reviewed validator " + "contract is allowlisted" + ) if exemption: continue lowered = line.casefold() @@ -642,6 +854,12 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: failures, "Docker build context", ) + for context in computed_build_contexts(line): + failures.append( + f"{path}:{number}: computed Docker build context is not allowed: " + f"docker-image://{context}; use a literal static " + "docker-image:// reference" + ) if item: name, value = item canonical = name.casefold() @@ -661,14 +879,18 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: (not has_literal and not alias and not has_template) or COMPUTED_VALUE_RE.search(value) is not None ) - strict = strict_assignments and policy_image_name(name) + strict = (strict_code_assignments and policy_image_name(name)) or ( + path.suffix in {".yaml", ".yml"} + and yaml_policy_image_name(name) + and number not in exempt_assignments + ) records.append((number, canonical, value, dependencies, computed, strict)) if ((has_literal or has_template) and not computed) or positional: resolved.add(canonical) bare_assignment = item is not None and ( path.suffix in {".yaml", ".yml"} - and re.sub("[^A-Za-z0-9]", "", item[0]).casefold() in {"image", "container"} - or strict_assignments + and yaml_policy_image_name(item[0]) + or strict_code_assignments and policy_image_name(item[0]) ) if ( @@ -694,7 +916,8 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: if strict and (computed or name not in resolved): failures.append( f"{path}:{number}: computed or unresolved image assignment " - f"is not allowed: {value.strip()}" + f"is not allowed: {value.strip()}; use a literal static repository " + "and digest, or a reviewed allow-validated-image validator contract" ) consumer_records = logical_lines(lines, markdown_flags) consumer_records.extend( @@ -928,6 +1151,18 @@ def product_contracts(texts: dict[Path, str], failures: list[str]) -> None: ) +def validated_image_contracts(texts: dict[Path, str], failures: list[str]) -> None: + for validator, (_, _, requirements) in VALIDATED_IMAGE_CONTRACTS.items(): + for path, needle in requirements: + require( + texts.get(path, ""), + needle, + path, + f"{validator} validated dynamic image contract", + failures, + ) + + def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] try: @@ -973,6 +1208,7 @@ def check_repository(root: Path = ROOT) -> list[str]: if stage: stages.add(stage.casefold()) product_contracts(texts, failures) + validated_image_contracts(texts, failures) return failures diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index ebe340fe..56d61f3b 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -459,6 +459,128 @@ def test_image_templates_use_only_bounded_static_forms(self) -> None: 'APP_IMAGE="registryctl-relay:$RUN_ID"\ndocker run --rm "$APP_IMAGE"\n', ) + def test_yaml_policy_image_keys_fail_closed_for_dynamic_default_families( + self, + ) -> None: + cases = ( + ("compose.yaml", "services:\n app:\n image: rust:$TAG\n"), + ("compose.yaml", "services:\n app:\n image: rust:${TAG}\n"), + ("compose.yaml", "services:\n app:\n image: debian:${TAG}\n"), + ("compose.yaml", "services:\n app:\n image: postgres:$PG_TAG\n"), + ( + "compose.yaml", + "services:\n app:\n image: postgres:$PG_TAG-notalpine\n", + ), + ( + ".github/workflows/example.yml", + "jobs:\n test:\n container: rust:$TAG\n", + ), + ("images.yaml", "builder_image: rust:$TAG\n"), + ) + for relative, text in cases: + with self.subTest(relative=relative, text=text): + self.assert_failure( + relative, + text, + "computed or unresolved image assignment", + ) + + self.assert_clean( + "compose.yaml", + f"services:\n app:\n image: {PINNED_RUST}\n", + ) + self.assert_clean( + "compose.yaml", + "services:\n app:\n image: registryctl-relay:$TAG\n", + ) + self.assert_clean( + ".github/workflows/example.yml", + "jobs:\n test:\n services:\n postgres:\n" + " image: postgres:${{ matrix.postgresql }}-alpine\n", + ) + self.assert_clean( + "metadata.yaml", + "metadata_image: rust:$TAG\nartifact_image: postgres:$PG_TAG\n", + ) + + def test_validated_image_annotations_are_path_and_contract_scoped(self) -> None: + registryctl_path = "crates/registryctl/src/templates/compose.yaml" + registryctl_annotation = ( + "# debian13-policy: allow-validated-image " + 'validator="registryctl-image-lock" ' + 'reason="registryctl renders a validated digest image lock here"\n' + ) + relay_path = "release/conformance/relay-oidc/docker-compose.yml" + relay_annotation = ( + "# debian13-policy: allow-validated-image " + 'validator="relay-oidc-smoke" ' + 'reason="the runner validates the fixed repository and digest"\n' + ) + self.assert_clean( + registryctl_path, + registryctl_annotation + "image: {{relay_image}}\n", + ) + self.assert_clean( + relay_path, + relay_annotation + "image: ${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:" + "?runner must provide digest-pinned Registry Relay image}\n", + ) + self.assert_failure( + "other/compose.yaml", + registryctl_annotation + "image: {{relay_image}}\n", + "invalid Debian image policy annotation", + ) + for value in ("rust:$TAG", "rust:{{relay_image}}"): + with self.subTest(value=value): + self.assert_failure( + registryctl_path, + registryctl_annotation + f"image: {value}\n", + "computed or unresolved image assignment", + ) + relay_value = ( + "${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:" + "?runner must provide digest-pinned Registry Relay image}" + ) + for relative, annotation, value in ( + (registryctl_path, registryctl_annotation, relay_value), + (relay_path, relay_annotation, "{{relay_image}}"), + ( + registryctl_path, + registryctl_annotation, + "${EVIL_IMAGE:?unreviewed image input}", + ), + ): + with self.subTest(relative=relative, value=value): + self.assert_failure( + relative, + annotation + f"image: {value}\n", + "invalid Debian image policy annotation", + ) + + texts = { + path: (ROOT / path).read_text() + for _, _, requirements in self.module.VALIDATED_IMAGE_CONTRACTS.values() + for path, _ in requirements + } + failures: list[str] = [] + self.module.validated_image_contracts(texts, failures) + self.assertEqual([], failures) + for validator, ( + _, + _, + requirements, + ) in self.module.VALIDATED_IMAGE_CONTRACTS.items(): + for path, needle in requirements: + with self.subTest(validator=validator, path=path): + mutated = dict(texts) + mutated[path] = mutated[path].replace(needle, "", 1) + failures = [] + self.module.validated_image_contracts(mutated, failures) + self.assertTrue( + any(validator in failure for failure in failures), + failures, + ) + def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> None: cases = ( ( @@ -535,17 +657,71 @@ def test_compose_block_sequences_preserve_mapping_and_list_scope(self) -> None: "bare Debian image reference", ) + def test_compose_bounded_flow_folded_and_anchored_consumers(self) -> None: + self.assert_failure( + "compose.yaml", + "services:\n app: {entrypoint: [docker], command: [run, --rm, debian]}\n", + "bare Debian image reference", + ) + self.assert_failure( + "compose.yaml", + "services:\n app:\n entrypoint: [docker]\n" + " command: >-\n run\n --rm\n debian\n", + "bare Debian image reference", + ) + self.assert_failure( + "compose.yaml", + "x-engine: &container-engine [docker]\nservices:\n app:\n" + " entrypoint: *container-engine\n" + " command: [run, --rm, debian]\n", + "bare Debian image reference", + ) + self.assert_failure( + "compose.yaml", + "services:\n app:\n entrypoint: *unknown-engine\n" + " command: [run, --rm, debian]\n", + "cannot statically resolve container entrypoint", + ) + self.assert_failure( + "compose.yaml", + "services:\n app:\n entrypoint: ${CONTAINER_ENGINE}\n" + " command: [run, --rm, debian]\n", + "cannot statically resolve container entrypoint", + ) + self.assert_failure( + "compose.yaml", + "services:\n app:\n entrypoint: [docker]\n" + " command: *container-command\n", + "cannot statically resolve container command", + ) + self.assert_clean( + "compose.yaml", + "services:\n" + f" app: {{entrypoint: [docker], command: [run, --rm, {PINNED_RUST}]}}\n", + ) + def test_container_cli_scans_only_the_bounded_image_operand(self) -> None: for command in ( "docker run --name postgres alpine:3.22 true\n", "docker run alpine:3.22 python -V\n", "docker create --env NAME=postgres busybox:1.37 python\n", + "docker run --network-alias postgres alpine:3.22 true\n", + "docker run --mount type=bind,source=/tmp,target=/mnt alpine:3.22\n", + "docker run --cpuset-cpus 0 alpine:3.22\n", + "docker run -dit alpine:3.22\n", ): with self.subTest(command=command): self.assert_clean("helper.sh", command) for command, family in ( ("docker run --name app postgres true\n", "postgres"), + ("docker run --network-alias app postgres true\n", "postgres"), + ( + "docker run --mount type=bind,source=/tmp,target=/mnt postgres\n", + "postgres", + ), + ("docker run --cpuset-cpus 0 postgres\n", "postgres"), + ("docker run -dit postgres\n", "postgres"), ("docker run python -V\n", "python"), ("docker pull --platform linux/amd64 rust:1.95\n", "rust:1.95"), ): @@ -595,6 +771,14 @@ def test_docker_image_build_contexts_follow_the_image_policy(self) -> None: "docker build --build-context base=docker-image://rust:1.95-trixie .\n", "Docker build context", ) + for value in ("$RUST_IMAGE", "${RUST_IMAGE}"): + with self.subTest(value=value): + self.assert_failure( + "build.sh", + "docker buildx build --build-context " + f"base=docker-image://{value} .\n", + "computed Docker build context is not allowed", + ) failures = self.scan( "build.sh", "docker build --build-context base=docker-image://rust:1.95-trixie .\n", From d97b33adcf5ec11f9408cecbf5e809221dd93ccb Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 02:46:01 +0700 Subject: [PATCH 17/34] fix(release): reject computed image contexts Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 4 ++-- release/scripts/test_check_debian13_images.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index a31e69c5..8c1db9d9 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -223,7 +223,7 @@ r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" ) DOCKER_CONTEXT_TOKEN_RE = re.compile( - r"""docker-image://(?P[^\s,;"']+)""", + r"""docker-image://(?P[^\s,;"'\]\}]+)""", ) EXEMPTION_RE = re.compile( r'' @@ -563,7 +563,7 @@ def computed_build_contexts(text: str) -> list[str]: return [ match.group("value") for match in DOCKER_CONTEXT_TOKEN_RE.finditer(text) - if "$" in match.group("value") or "{" in match.group("value") + if DOCKER_CONTEXT_RE.fullmatch(f"docker-image://{match.group('value')}") is None ] diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 56d61f3b..f55ac8e6 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -771,7 +771,7 @@ def test_docker_image_build_contexts_follow_the_image_policy(self) -> None: "docker build --build-context base=docker-image://rust:1.95-trixie .\n", "Docker build context", ) - for value in ("$RUST_IMAGE", "${RUST_IMAGE}"): + for value in ("$RUST_IMAGE", "${RUST_IMAGE}", "`resolve-image`"): with self.subTest(value=value): self.assert_failure( "build.sh", From da12db3dd9f44378e4e7cfd0bab1143fe2ab87ac Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 03:09:11 +0700 Subject: [PATCH 18/34] fix(release): close image scanner bypasses Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 193 ++++++++++++++---- release/scripts/test_check_debian13_images.py | 132 +++++++++++- release/scripts/test_relay_oidc_smoke.py | 44 ++++ 3 files changed, 333 insertions(+), 36 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 8c1db9d9..513e930d 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -144,15 +144,34 @@ "--add-host", "--annotation", "--attach", + "--blkio-weight", + "--blkio-weight-device", "--cap-add", "--cap-drop", + "--cgroup-parent", "--cgroupns", "--cidfile", + "--cpu-count", + "--cpu-percent", + "--cpu-period", + "--cpu-quota", + "--cpu-rt-period", + "--cpu-rt-runtime", + "--cpu-shares", "--cpus", "--cpuset-cpus", + "--cpuset-mems", + "--detach-keys", "--device", + "--device-cgroup-rule", + "--device-read-bps", + "--device-read-iops", + "--device-write-bps", + "--device-write-iops", "--dns", + "--dns-option", "--dns-search", + "--domainname", "--entrypoint", "--env", "--env-file", @@ -162,20 +181,34 @@ "--health-cmd", "--health-interval", "--health-retries", + "--health-start-interval", + "--health-start-period", "--health-timeout", "--hostname", + "--io-maxbandwidth", + "--io-maxiops", + "--ip", + "--ip6", "--ipc", + "--isolation", "--label", + "--label-file", "--link", + "--link-local-ip", "--log-driver", "--log-opt", "--mac-address", "--memory", + "--memory-reservation", + "--memory-swap", + "--memory-swappiness", "--mount", "--name", "--network", "--network-alias", + "--oom-score-adj", "--pid", + "--pids-limit", "--platform", "--publish", "--pull", @@ -185,12 +218,19 @@ "--shm-size", "--stop-signal", "--stop-timeout", + "--storage-opt", + "--sysctl", "--tmpfs", "--ulimit", "--user", + "--userns", + "--uts", "--volume", + "--volume-driver", + "--volumes-from", "--workdir", "-a", + "-c", "-e", "-h", "-l", @@ -204,11 +244,17 @@ "--detach", "--init", "--interactive", + "--no-healthcheck", + "--oom-kill-disable", "--privileged", + "--publish-all", "--quiet", "--read-only", "--rm", + "--sig-proxy", "--tty", + "--use-api-socket", + "-P", "-d", "-i", "-it", @@ -216,7 +262,7 @@ "-t", } PULL_FLAG_OPTIONS = {"--all-tags", "--disable-content-trust", "-a", "-q"} -SHORT_VALUE_OPTIONS = ("-e", "-h", "-l", "-m", "-p", "-u", "-v", "-w") +SHORT_VALUE_OPTIONS = ("-c", "-e", "-h", "-l", "-m", "-p", "-u", "-v", "-w") DOCKER_CONTEXT_RE = re.compile( r"docker-image://(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*" r"[A-Za-z0-9._-]+(?:\:[A-Za-z0-9_][A-Za-z0-9._-]*)?" @@ -241,6 +287,11 @@ Path("crates/registryctl/src/lib.rs"), 'validate_locked_image_ref(\n "images.registry-relay"', ), + ( + Path("crates/registryctl/src/lib.rs"), + 'include_str!("templates/compose.yaml")' + '.replace("{{relay_image}}", image_lock.relay_image())', + ), ( Path("crates/registryctl/src/lib.rs"), "fn image_lock_rejects_mutable_or_noncanonical_image_references()", @@ -264,10 +315,18 @@ Path("release/scripts/relay-oidc-smoke.py"), "relay_image = validate_relay_image(args.relay_image)", ), + ( + Path("release/scripts/relay-oidc-smoke.py"), + '"REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE": relay_image,', + ), ( Path("release/scripts/test_relay_oidc_smoke.py"), "def test_relay_image_requires_exact_repository_and_lowercase_digest", ), + ( + Path("release/scripts/test_relay_oidc_smoke.py"), + "def test_execute_live_binds_validated_argument_over_ambient_image", + ), ), ), } @@ -400,22 +459,27 @@ def is_debian_family(reference: str) -> bool: ) -def assignment(path: Path, line: str) -> tuple[str, str] | None: - match = (YAML_KEY_RE if path.suffix in {".yaml", ".yml"} else ASSIGN_RE).match(line) - if match is None: - return None - name = match.group("name") - value = match.group("value").strip() - if value.rstrip(",") in {"(", "[", "{"} or value.endswith(","): +def image_assignment(name: str, value: str) -> tuple[str, str] | None: + stripped = value.strip() + if stripped.rstrip(",") in {"(", "[", "{"} or stripped.endswith(","): return None normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() return ( - (name, match.group("value")) + (name, value) if normalized.endswith("image") or normalized == "container" else None ) +def assignment(path: Path, line: str) -> tuple[str, str] | None: + match = (YAML_KEY_RE if path.suffix in {".yaml", ".yml"} else ASSIGN_RE).match(line) + return ( + image_assignment(match.group("name"), match.group("value")) + if match is not None + else None + ) + + def split_flow_commas(value: str, nested: bool) -> list[str]: parts, current, depth, quote = [], [], 0, "" for character in value: @@ -426,10 +490,10 @@ def split_flow_commas(value: str, nested: bool) -> list[str]: elif character in "\"'": quote = character current.append(character) - elif character == "[": + elif character in "([{": depth += 1 current.append(character) - elif character == "]": + elif character in ")]}": depth = max(0, depth - 1) current.append(character) elif character == "," and bool(depth) == nested: @@ -455,8 +519,8 @@ def command_tokens(value: str) -> list[str]: ] -def container_image_values(line: str) -> list[str]: - values = [] +def container_image_operands(line: str) -> tuple[list[str], list[str]]: + values, errors = [], [] for command in CLI_RE.finditer(line): tokens = command_tokens(command.group("tail")) action_index = next( @@ -478,7 +542,7 @@ def container_image_values(line: str) -> list[str]: flag_options = CONTAINER_FLAG_OPTIONS | ( PULL_FLAG_OPTIONS if action == "pull" else set() ) - ambiguous, index = False, action_index + 1 + unknown_option, index = False, action_index + 1 while index < len(tokens): token = tokens[index] if token == "\\": @@ -500,23 +564,23 @@ def container_image_values(line: str) -> list[str]: elif option in flag_options or re.fullmatch(r"-[diqt]+", option): index += 1 else: - ambiguous = True - index += 1 + errors.append( + f"unsupported Docker/Podman option has unknown arity: {option}" + ) + unknown_option = True + break + if unknown_option: + continue if index >= len(tokens): values.append("") - elif ambiguous: - values.extend( - token - for token in tokens[index:] - if token != "\\" and not token.startswith("-") - ) else: values.append(tokens[index]) - return values + return values, errors def is_container_consumer(line: str) -> bool: - return bool(container_image_values(line)) + values, errors = container_image_operands(line) + return bool(values or errors) def policy_image_name(name: str) -> bool: @@ -632,11 +696,29 @@ def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, b return result +def flow_mapping_bodies(line: str) -> list[str]: + bodies, starts, quote, escaped = [], [], "", False + for index, character in enumerate(line): + if quote: + if character == quote and not escaped: + quote = "" + escaped = character == "\\" and not escaped + if character != "\\": + escaped = False + elif character in "\"'": + quote = character + elif character == "{": + starts.append(index + 1) + elif character == "}" and starts: + bodies.append(line[starts.pop() : index]) + return bodies + + def flow_mapping_fields(line: str) -> list[dict[str, str]]: mappings = [] - for mapping in re.finditer(r"\{(?P[^{}]*)\}", line): + for body in flow_mapping_bodies(line): fields = {} - for part in split_flow_commas(mapping.group("body"), nested=False): + for part in split_flow_commas(body, nested=False): name, separator, value = part.partition(":") if separator and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", name.strip()): fields[name.strip().casefold()] = value.strip() @@ -645,6 +727,33 @@ def flow_mapping_fields(line: str) -> list[dict[str, str]]: return mappings +def flow_mapping_image_assignments(line: str) -> list[tuple[str, str]]: + return [ + item + for fields in flow_mapping_fields(line) + for name, value in fields.items() + if (item := image_assignment(name, value)) is not None + ] + + +def yaml_block_scalar_flags(lines: list[str]) -> list[bool]: + flags, scalar_indent = [], None + scalar = re.compile( + r"^\s*(?:-\s*)?[A-Za-z_][A-Za-z0-9_-]*\s*:\s*[>|][+-]?(?:\s+#.*)?$" + ) + for line in lines: + stripped = line.strip() + indent = len(line) - len(line.lstrip()) + inside = scalar_indent is not None and (not stripped or indent > scalar_indent) + if scalar_indent is not None and stripped and indent <= scalar_indent: + scalar_indent = None + inside = False + flags.append(inside) + if not inside and scalar.fullmatch(line): + scalar_indent = indent + return flags + + def yaml_container_consumers( lines: list[str], ) -> tuple[dict[int, str], list[tuple[int, str]]]: @@ -791,6 +900,11 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: if path.suffix in {".yaml", ".yml"} else ({}, []) ) + yaml_scalar_flags = ( + yaml_block_scalar_flags(lines) + if path.suffix in {".yaml", ".yml"} + else [False] * len(lines) + ) failures.extend(f"{path}:{number}: {message}" for number, message in yaml_errors) exemption_comments, exempt_assignments = yaml_image_exemptions(path, lines) records: list[tuple[int, str, str, set[str], bool, bool]] = [] @@ -826,7 +940,13 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: failures.append( f"{path}:{number}: retired Debian image generation marker remains: {marker}" ) - item = assignment(path, line) if code_file else None + items = [] + if code_file: + item = assignment(path, line) + if item is not None: + items.append(item) + if path.suffix in {".yaml", ".yml"} and not yaml_scalar_flags[number - 1]: + items.extend(flow_mapping_image_assignments(line)) consumer_line = yaml_consumers.get(number, line) consumer = ( is_container_consumer(consumer_line) @@ -837,7 +957,7 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: is_dockerfile(path) or path.suffix in {".yaml", ".yml", ".sh", ".bash"} or markdown_code - or item is not None + or bool(items) or consumer ) context_references = build_context_references(line) @@ -860,8 +980,7 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: f"docker-image://{context}; use a literal static " "docker-image:// reference" ) - if item: - name, value = item + for name, value in items: canonical = name.casefold() dependencies = { found.group("name").casefold() for found in IMAGE_VAR_RE.finditer(value) @@ -887,11 +1006,12 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: records.append((number, canonical, value, dependencies, computed, strict)) if ((has_literal or has_template) and not computed) or positional: resolved.add(canonical) - bare_assignment = item is not None and ( + bare_assignment = any( path.suffix in {".yaml", ".yml"} - and yaml_policy_image_name(item[0]) + and yaml_policy_image_name(name) or strict_code_assignments - and policy_image_name(item[0]) + and policy_image_name(name) + for name, _ in items ) if ( not comment @@ -924,13 +1044,16 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: (number, line, False) for number, line in yaml_consumers.items() ) for number, line, markdown_code in consumer_records: - image_values = container_image_values(line) + image_values, option_errors = container_image_operands(line) if ( line.lstrip().startswith(("#", "//")) or not (code_file or markdown_code) - or not image_values + or not (image_values or option_errors) ): continue + failures.extend(f"{path}:{number}: {message}" for message in option_errors) + if not image_values: + continue image_text = " ".join(image_values) for reference in references(image_text): append_reference_failures(path, number, reference, failures) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index f55ac8e6..b89da644 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -296,7 +296,6 @@ def test_wrappers_options_operators_and_malformed_shell_still_scan_literals( "sudo -s /usr/bin/docker run --rm rust:1.95-trixie", "bash --noprofile -c 'docker run --rm rust:1.95-trixie'", "if ! time -p docker run --rm rust:1.95-trixie; then true; fi", - "echo ok; podman run --unknown value rust:1.95-trixie", "docker unknown rust:1.95-trixie", "docker run 'rust:1.95-trixie", ) @@ -309,7 +308,17 @@ def test_wrappers_options_operators_and_malformed_shell_still_scan_literals( ) for command in ( + "echo ok; podman run --unknown value rust:1.95-trixie", f"docker run --unknown value {PINNED_RUST}", + ): + with self.subTest(command=command): + self.assert_failure( + "script.sh", + command + "\n", + "unsupported Docker/Podman option has unknown arity: --unknown", + ) + + for command in ( f"sudo --unknown docker unknown {PINNED_RUST}", f"bash --unknown -c 'docker run {PINNED_RUST}'", ): @@ -503,6 +512,45 @@ def test_yaml_policy_image_keys_fail_closed_for_dynamic_default_families( "metadata_image: rust:$TAG\nartifact_image: postgres:$PG_TAG\n", ) + def test_yaml_flow_mapping_image_keys_use_assignment_policy(self) -> None: + cases = ( + "services:\n app: {image: rust:$TAG}\n", + 'services:\n app: {image: "rust:${TAG}"}\n', + "jobs:\n test: {container: postgres:$PG_TAG}\n", + "images: {builder_image: rust:$TAG}\n", + "x-service: &defaults {image: rust:$TAG}\n" + "services:\n app:\n <<: *defaults\n", + ) + for text in cases: + with self.subTest(text=text): + self.assert_failure( + "compose.yaml", + text, + "computed or unresolved image assignment", + ) + + self.assert_clean( + "compose.yaml", + f"services:\n app: {{image: {PINNED_RUST}}}\n", + ) + self.assert_clean( + "compose.yaml", + "services:\n app: {image: registryctl-relay:$TAG}\n", + ) + self.assert_clean( + "metadata.yaml", + "metadata: {metadata_image: rust:$TAG}\n", + ) + self.assert_clean( + ".github/workflows/example.yml", + "steps:\n - run: |\n jq '{image: $builder_image}' input.json\n", + ) + self.assert_clean( + "compose.yaml", + f"x-service: &defaults {{image: {PINNED_RUST}}}\n" + "services:\n app:\n <<: *defaults\n", + ) + def test_validated_image_annotations_are_path_and_contract_scoped(self) -> None: registryctl_path = "crates/registryctl/src/templates/compose.yaml" registryctl_annotation = ( @@ -728,6 +776,88 @@ def test_container_cli_scans_only_the_bounded_image_operand(self) -> None: with self.subTest(command=command): self.assert_failure("helper.sh", command, family) + def test_documented_container_options_preserve_the_image_operand(self) -> None: + value_options = ( + ("--blkio-weight", "500"), + ("--blkio-weight-device", "/dev/sda:500"), + ("--cgroup-parent", "registry.slice"), + ("--cpu-count", "2"), + ("--cpu-percent", "50"), + ("--cpu-period", "100000"), + ("--cpu-quota", "50000"), + ("--cpu-rt-period", "1000000"), + ("--cpu-rt-runtime", "950000"), + ("--cpu-shares", "512"), + ("-c", "512"), + ("--cpuset-mems", "0"), + ("--detach-keys", "ctrl-x"), + ("--device-cgroup-rule", "c 42:* rmw"), + ("--device-read-bps", "/dev/sda:1mb"), + ("--device-read-iops", "/dev/sda:1000"), + ("--device-write-bps", "/dev/sda:1mb"), + ("--device-write-iops", "/dev/sda:1000"), + ("--dns-option", "ndots:2"), + ("--domainname", "example.test"), + ("--health-start-interval", "1s"), + ("--health-start-period", "5s"), + ("--io-maxbandwidth", "10mb"), + ("--io-maxiops", "1000"), + ("--ip", "172.30.100.104"), + ("--ip6", "2001:db8::33"), + ("--isolation", "process"), + ("--label-file", "labels.txt"), + ("--link-local-ip", "169.254.1.2"), + ("--memory-reservation", "256m"), + ("--memory-swap", "1g"), + ("--memory-swappiness", "60"), + ("--oom-score-adj", "100"), + ("--pids-limit", "100"), + ("--storage-opt", "size=10G"), + ("--sysctl", "net.ipv4.ip_forward=1"), + ("--userns", "host"), + ("--uts", "host"), + ("--volume-driver", "local"), + ("--volumes-from", "data"), + ) + for option, value in value_options: + with self.subTest(option=option, image="alpine"): + self.assert_clean( + "helper.sh", + f"docker run {option} {value!r} alpine:3.22 true\n", + ) + with self.subTest(option=option, image="postgres"): + self.assert_failure( + "helper.sh", + f"docker create {option} {value!r} postgres true\n", + "postgres", + ) + + for option in ( + "--no-healthcheck", + "--oom-kill-disable", + "--publish-all", + "-P", + "--sig-proxy", + "--use-api-socket", + ): + with self.subTest(option=option, image="alpine"): + self.assert_clean( + "helper.sh", + f"docker run {option} alpine:3.22 true\n", + ) + with self.subTest(option=option, image="postgres"): + self.assert_failure( + "helper.sh", + f"docker run {option} postgres true\n", + "postgres", + ) + + self.assert_failure( + "helper.sh", + "docker run --future-option value alpine:3.22 true\n", + "unsupported Docker/Podman option has unknown arity: --future-option", + ) + def test_multiline_container_commands_scan_the_joined_operand(self) -> None: self.assert_failure( "helper.sh", diff --git a/release/scripts/test_relay_oidc_smoke.py b/release/scripts/test_relay_oidc_smoke.py index 34dd3427..43a092fc 100644 --- a/release/scripts/test_relay_oidc_smoke.py +++ b/release/scripts/test_relay_oidc_smoke.py @@ -175,6 +175,50 @@ def test_relay_image_requires_exact_repository_and_lowercase_digest(self) -> Non with self.assertRaises(self.runner.SmokeError): self.runner.validate_relay_image(image) + def test_execute_live_binds_validated_argument_over_ambient_image(self) -> None: + requested = "ghcr.io/registrystack/registry-relay@sha256:" + "a" * 64 + ambient = "ghcr.io/registrystack/registry-relay@sha256:" + "b" * 64 + image_variable = "REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE" + captured_environments: list[dict[str, str]] = [] + + def run_checked( + command: list[str], *, env: dict[str, str], **_kwargs: object + ) -> None: + captured_environments.append(dict(env)) + if "up" in command: + raise self.runner.SmokeError("stop after environment capture") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + args = self.runner.argparse.Namespace( + relay_image=requested, + candidate_source_ref="c" * 40, + release_id="1.0.0-rc.1", + output_dir=str(root / "output"), + host_port=19191, + ) + with ( + patch.object(self.runner, "DEFAULT_WORK_ROOT", root / "work"), + patch.object(self.runner, "validate_assets", return_value={}), + patch.object( + self.runner.shutil, "which", return_value="/usr/bin/docker" + ), + patch.object(self.runner, "run_checked", side_effect=run_checked), + patch.dict( + self.runner.os.environ, + {image_variable: ambient}, + clear=False, + ), + ): + with self.assertRaisesRegex( + self.runner.SmokeError, "stop after environment capture" + ): + self.runner.execute_live(args) + + self.assertEqual(2, len(captured_environments)) + for environment in captured_environments: + self.assertEqual(requested, environment[image_variable]) + def test_plan_is_offline_and_does_not_claim_live_evidence(self) -> None: plan = self.runner.plan_document( "ghcr.io/registrystack/registry-relay@sha256:" + "a" * 64, From 98da966fabb70efb6e8eb6ae73875da2832d1853 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 03:28:33 +0700 Subject: [PATCH 19/34] fix(release): bound image policy grammar Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 313 ++++++++++++++---- release/scripts/test_check_debian13_images.py | 134 ++++++++ 2 files changed, 387 insertions(+), 60 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 513e930d..5c6caeae 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -103,12 +103,22 @@ r"(?P[A-Za-z_][A-Za-z0-9_$-]*)(?:\s*:[^=]+)?" r"\s*=\s*(?P.+)$" ) +YAML_PLAIN_KEY = r"[A-Za-z_][A-Za-z0-9_$-]*" +YAML_SIMPLE_KEY = rf"""(?:{YAML_PLAIN_KEY}|"{YAML_PLAIN_KEY}"|'{YAML_PLAIN_KEY}')""" YAML_KEY_RE = re.compile( - r"^\s*(?:-\s*)?(?P[A-Za-z_][A-Za-z0-9_$-]*)\s*:\s*(?P.+)$" + rf"^\s*(?:-\s*)?(?P{YAML_SIMPLE_KEY})\s*:\s*(?P.+)$" ) YAML_FIELD_RE = re.compile( r"^(?P\s*)(?P-\s*)?" - r"(?P[A-Za-z_][A-Za-z0-9_$-]*)\s*:\s*(?P.*)$" + rf"(?P{YAML_SIMPLE_KEY})\s*:\s*(?P.*)$" +) +YAML_EMPTY_KEY_RE = re.compile(rf"^\s*(?:-\s*)?(?P{YAML_SIMPLE_KEY})\s*:\s*$") +YAML_EXPLICIT_KEY_RE = re.compile( + rf"^\s*(?:-\s*)?\?\s*(?P{YAML_SIMPLE_KEY})(?:\s*:\s*.*)?$" +) +YAML_SCALAR_ANCHOR_RE = re.compile( + rf"^\s*(?:-\s*)?{YAML_SIMPLE_KEY}\s*:\s*" + r"&(?P[A-Za-z0-9_-]+)\s+(?P.+)$" ) IMAGE_VAR_RE = re.compile( r"\b(?P[A-Za-z_][A-Za-z0-9_$-]*(?:_image|Image)|IMAGE)\b", @@ -257,12 +267,11 @@ "-P", "-d", "-i", - "-it", "-q", "-t", } PULL_FLAG_OPTIONS = {"--all-tags", "--disable-content-trust", "-a", "-q"} -SHORT_VALUE_OPTIONS = ("-c", "-e", "-h", "-l", "-m", "-p", "-u", "-v", "-w") +CONTAINER_TERMINAL_OPTIONS = {"--help"} DOCKER_CONTEXT_RE = re.compile( r"docker-image://(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*" r"[A-Za-z0-9._-]+(?:\:[A-Za-z0-9_][A-Za-z0-9._-]*)?" @@ -459,22 +468,31 @@ def is_debian_family(reference: str) -> bool: ) +def image_assignment_name(name: str) -> bool: + normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() + return normalized.endswith("image") or normalized == "container" + + def image_assignment(name: str, value: str) -> tuple[str, str] | None: stripped = value.strip() if stripped.rstrip(",") in {"(", "[", "{"} or stripped.endswith(","): return None - normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() - return ( - (name, value) - if normalized.endswith("image") or normalized == "container" - else None - ) + return (name, value) if image_assignment_name(name) else None + + +def yaml_key_name(value: str) -> str: + return value[1:-1] if value[:1] in "\"'" and value[-1:] == value[:1] else value def assignment(path: Path, line: str) -> tuple[str, str] | None: match = (YAML_KEY_RE if path.suffix in {".yaml", ".yml"} else ASSIGN_RE).match(line) return ( - image_assignment(match.group("name"), match.group("value")) + image_assignment( + yaml_key_name(match.group("name")) + if path.suffix in {".yaml", ".yml"} + else match.group("name"), + match.group("value"), + ) if match is not None else None ) @@ -519,8 +537,27 @@ def command_tokens(value: str) -> list[str]: ] -def container_image_operands(line: str) -> tuple[list[str], list[str]]: - values, errors = [], [] +def short_option_consumption( + token: str, value_options: set[str], flag_options: set[str] +) -> tuple[int, str | None]: + if len(token) == 1: + return 0, "unsupported Docker/Podman short-option cluster: -" + for index, character in enumerate(token[1:]): + option = f"-{character}" + if option in flag_options: + continue + if option in value_options: + return (1 if index + 2 < len(token) else 2), None + return ( + 0, + "unsupported Docker/Podman short-option cluster: " + f"{token}; unknown option: {option}", + ) + return 1, None + + +def container_image_operands(line: str) -> tuple[list[str], list[str], bool]: + values, errors, recognized = [], [], False for command in CLI_RE.finditer(line): tokens = command_tokens(command.group("tail")) action_index = next( @@ -535,6 +572,7 @@ def container_image_operands(line: str) -> tuple[list[str], list[str]]: continue prefix = {token.casefold().lstrip("-") for token in tokens[:action_index]} if not prefix & NON_IMAGE_NAMESPACES: + recognized = True action = tokens[action_index].casefold() value_options = CONTAINER_VALUE_OPTIONS - ( {"-a", "--attach"} if action == "pull" else set() @@ -542,7 +580,7 @@ def container_image_operands(line: str) -> tuple[list[str], list[str]]: flag_options = CONTAINER_FLAG_OPTIONS | ( PULL_FLAG_OPTIONS if action == "pull" else set() ) - unknown_option, index = False, action_index + 1 + terminal_option, unknown_option, index = False, False, action_index + 1 while index < len(tokens): token = tokens[index] if token == "\\": @@ -553,34 +591,46 @@ def container_image_operands(line: str) -> tuple[list[str], list[str]]: break if not token.startswith("-"): break - option = token.split("=", 1)[0] - attached = ( - option in SHORT_VALUE_OPTIONS - and token != option - and not token.startswith("--") - ) - if option in value_options: - index += 1 if "=" in token or attached else 2 - elif option in flag_options or re.fullmatch(r"-[diqt]+", option): - index += 1 + if token.startswith("--"): + option = token.split("=", 1)[0] + if token in CONTAINER_TERMINAL_OPTIONS: + terminal_option = True + index = len(tokens) + break + if option in value_options: + index += 1 if "=" in token else 2 + elif option in flag_options: + index += 1 + else: + errors.append( + "unsupported Docker/Podman option has unknown arity: " + f"{option}" + ) + unknown_option = True + break else: - errors.append( - f"unsupported Docker/Podman option has unknown arity: {option}" + consumed, error = short_option_consumption( + token, value_options, flag_options ) - unknown_option = True - break + if error is not None: + errors.append(error) + unknown_option = True + break + index += consumed if unknown_option: continue + if terminal_option: + continue if index >= len(tokens): values.append("") else: values.append(tokens[index]) - return values, errors + return values, errors, recognized def is_container_consumer(line: str) -> bool: - values, errors = container_image_operands(line) - return bool(values or errors) + _, _, recognized = container_image_operands(line) + return recognized def policy_image_name(name: str) -> bool: @@ -696,7 +746,23 @@ def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, b return result -def flow_mapping_bodies(line: str) -> list[str]: +def yaml_code(line: str) -> str: + quote, escaped = "", False + for index, character in enumerate(line): + if quote: + if character == quote and not escaped: + quote = "" + escaped = quote == '"' and character == "\\" and not escaped + if character != "\\": + escaped = False + elif character in "\"'": + quote = character + elif character == "#" and (index == 0 or line[index - 1].isspace()): + return line[:index].rstrip() + return line + + +def flow_mapping_parts(line: str) -> tuple[list[str], list[str]]: bodies, starts, quote, escaped = [], [], "", False for index, character in enumerate(line): if quote: @@ -711,35 +777,127 @@ def flow_mapping_bodies(line: str) -> list[str]: starts.append(index + 1) elif character == "}" and starts: bodies.append(line[starts.pop() : index]) - return bodies + return bodies, [line[start:] for start in starts] -def flow_mapping_fields(line: str) -> list[dict[str, str]]: +def flow_mapping_pairs(line: str) -> list[list[tuple[str, str]]]: mappings = [] - for body in flow_mapping_bodies(line): - fields = {} + for body in flow_mapping_parts(line)[0]: + fields = [] for part in split_flow_commas(body, nested=False): name, separator, value = part.partition(":") - if separator and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", name.strip()): - fields[name.strip().casefold()] = value.strip() + raw_name = name.strip() + if separator and re.fullmatch(YAML_SIMPLE_KEY, raw_name): + fields.append((yaml_key_name(raw_name), value.strip())) if fields: mappings.append(fields) return mappings +def flow_mapping_fields(line: str) -> list[dict[str, str]]: + return [ + {name.casefold(): value for name, value in pairs} + for pairs in flow_mapping_pairs(line) + ] + + def flow_mapping_image_assignments(line: str) -> list[tuple[str, str]]: return [ item - for fields in flow_mapping_fields(line) - for name, value in fields.items() + for pairs in flow_mapping_pairs(line) + for name, value in pairs + if value if (item := image_assignment(name, value)) is not None ] +def flow_explicit_policy_key(body: str) -> str | None: + for part in split_flow_commas(body, nested=False): + match = re.match(rf"^\s*\?\s*(?P{YAML_SIMPLE_KEY})(?:\s|:|$)", part) + if match is not None: + name = yaml_key_name(match.group("name")) + if image_assignment_name(name): + return name + return None + + +def unsupported_yaml_policy_keys(line: str) -> list[str]: + names = [] + for pattern in (YAML_EMPTY_KEY_RE, YAML_EXPLICIT_KEY_RE): + match = pattern.match(line) + if match is not None: + name = yaml_key_name(match.group("name")) + if image_assignment_name(name): + names.append(name) + closed, opened = flow_mapping_parts(line) + for body in closed: + explicit = flow_explicit_policy_key(body) + if explicit is not None: + names.append(explicit) + for pairs in flow_mapping_pairs(f"{{{body}}}"): + names.extend( + name + for name, value in pairs + if not value and image_assignment_name(name) + ) + for body in opened: + explicit = flow_explicit_policy_key(body) + if explicit is not None: + names.append(explicit) + for part in split_flow_commas(body, nested=False): + raw_name, separator, _ = part.partition(":") + raw_name = raw_name.strip() + if separator and re.fullmatch(YAML_SIMPLE_KEY, raw_name): + name = yaml_key_name(raw_name) + if image_assignment_name(name): + names.append(name) + return list(dict.fromkeys(names)) + + +def compliant_scalar_anchor(line: str) -> tuple[str, str] | None: + match = YAML_SCALAR_ANCHOR_RE.match(line) + if match is None: + return None + reference = exact_reference(match.group("value")) + if reference is None: + return None + if is_debian_family(reference) and ( + DIGEST_RE.search(reference) is None + or "trixie" not in reference.casefold() + and re.search(r"debian-?13", reference, re.IGNORECASE) is None + ): + return None + return match.group("anchor"), reference + + +def yaml_anchor_declarations(line: str) -> list[str]: + names, quote, escaped, index = [], "", False, 0 + while index < len(line): + character = line[index] + if quote: + if character == quote and not escaped: + quote = "" + escaped = quote == '"' and character == "\\" and not escaped + if character != "\\": + escaped = False + elif character in "\"'": + quote = character + elif character == "&" and ( + index == 0 or line[index - 1].isspace() or line[index - 1] in "[{,:-" + ): + match = re.match(r"[A-Za-z0-9_-]+", line[index + 1 :]) + if match is not None: + names.append(match.group()) + index += len(match.group()) + index += 1 + return names + + def yaml_block_scalar_flags(lines: list[str]) -> list[bool]: flags, scalar_indent = [], None scalar = re.compile( - r"^\s*(?:-\s*)?[A-Za-z_][A-Za-z0-9_-]*\s*:\s*[>|][+-]?(?:\s+#.*)?$" + rf"^\s*(?:-\s*)?{YAML_SIMPLE_KEY}\s*:\s*" + r"[>|](?:[1-9][+-]?|[+-][1-9]?)?(?:\s+#.*)?$" ) for line in lines: stripped = line.strip() @@ -812,25 +970,26 @@ def combine(number: int, entrypoint: str, command: str) -> None: for offset, line in enumerate(lines): number = offset + 1 - if not line.strip() or line.lstrip().startswith("#"): + source = yaml_code(line) + if not source.strip(): continue for anchor in re.finditer( r"&(?P[A-Za-z0-9_-]+)\s+" r"(?P\[[^\]\n]+\]|[\"']?(?:docker|podman)[\"']?)", - line, + source, ): anchors[anchor.group("name")] = anchor.group("value") - for mapping in flow_mapping_fields(line): + for mapping in flow_mapping_fields(source): if {"entrypoint", "command"} <= mapping.keys(): combine(number, mapping["entrypoint"], mapping["command"]) - indent = len(line) - len(line.lstrip()) + indent = len(source) - len(source.lstrip()) while parents and parents[-1][0] >= indent: parents.pop() - match = YAML_FIELD_RE.match(line) + match = YAML_FIELD_RE.match(source) if match is None: parents.append((indent, number)) continue - name = match.group("name").casefold() + name = yaml_key_name(match.group("name")).casefold() scope = number if match.group("item") else parents[-1][1] if parents else 0 if name in {"entrypoint", "command"}: value = match.group("value").strip() @@ -908,7 +1067,7 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: failures.extend(f"{path}:{number}: {message}" for number, message in yaml_errors) exemption_comments, exempt_assignments = yaml_image_exemptions(path, lines) records: list[tuple[int, str, str, set[str], bool, bool]] = [] - resolved = set() + resolved, yaml_anchors = set(), {} for number, line in enumerate(lines, 1): if len(line) > MAX_LINE_CHARS: raise ImageSurfaceError( @@ -934,6 +1093,7 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: ) if exemption: continue + yaml_source = yaml_code(line) if path.suffix in {".yaml", ".yml"} else line lowered = line.casefold() for marker in RETIRED_MARKERS: if marker in lowered: @@ -942,12 +1102,32 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: ) items = [] if code_file: - item = assignment(path, line) + scalar_yaml = ( + path.suffix in {".yaml", ".yml"} and yaml_scalar_flags[number - 1] + ) + if ( + path.suffix in {".yaml", ".yml"} + and not scalar_yaml + and re.fullmatch(r"\s*(?:---|\.\.\.)\s*", yaml_source) + ): + yaml_anchors.clear() + item = assignment(path, yaml_source) if not scalar_yaml else None if item is not None: items.append(item) - if path.suffix in {".yaml", ".yml"} and not yaml_scalar_flags[number - 1]: - items.extend(flow_mapping_image_assignments(line)) - consumer_line = yaml_consumers.get(number, line) + if path.suffix in {".yaml", ".yml"} and not scalar_yaml: + items.extend(flow_mapping_image_assignments(yaml_source)) + for name in unsupported_yaml_policy_keys(yaml_source): + failures.append( + f"{path}:{number}: unsupported YAML image policy-key " + f"syntax: {name}; use a simple block key or a single-line " + "flow mapping" + ) + for anchor_name in yaml_anchor_declarations(yaml_source): + yaml_anchors.pop(anchor_name, None) + anchor = compliant_scalar_anchor(yaml_source) + if anchor is not None: + yaml_anchors[anchor[0]] = anchor[1] + consumer_line = yaml_consumers.get(number, yaml_source) consumer = ( is_container_consumer(consumer_line) and (code_file or markdown_code) @@ -960,9 +1140,9 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: or bool(items) or consumer ) - context_references = build_context_references(line) + context_references = build_context_references(yaml_source) if reference_context and not consumer: - for reference in references(line): + for reference in references(yaml_source): if reference not in context_references: append_reference_failures(path, number, reference, failures) if reference_context: @@ -974,7 +1154,7 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: failures, "Docker build context", ) - for context in computed_build_contexts(line): + for context in computed_build_contexts(yaml_source): failures.append( f"{path}:{number}: computed Docker build context is not allowed: " f"docker-image://{context}; use a literal static " @@ -982,10 +1162,18 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: ) for name, value in items: canonical = name.casefold() + anchor = re.fullmatch(r"\*(?P[A-Za-z0-9_-]+)", value.strip()) + anchored_reference = ( + yaml_anchors.get(anchor.group("name")) + if anchor is not None and path.suffix in {".yaml", ".yml"} + else None + ) dependencies = { found.group("name").casefold() for found in IMAGE_VAR_RE.finditer(value) } - has_literal = exact_reference(value) is not None + has_literal = ( + exact_reference(value) is not None or anchored_reference is not None + ) alias = is_image_alias(value) has_template = ( not has_literal and not dependencies and has_safe_image_template(value) @@ -1016,10 +1204,10 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: if ( not comment and not consumer - and BARE_DEFAULT_FAMILY_RE.search(line) + and BARE_DEFAULT_FAMILY_RE.search(yaml_source) and (is_dockerfile(path) or bare_assignment) ): - append_bare_failures(path, number, line, failures) + append_bare_failures(path, number, yaml_source, failures) changed = True while changed: changed = False @@ -1039,12 +1227,17 @@ def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: f"is not allowed: {value.strip()}; use a literal static repository " "and digest, or a reviewed allow-validated-image validator contract" ) - consumer_records = logical_lines(lines, markdown_flags) + source_lines = ( + [yaml_code(line) for line in lines] + if path.suffix in {".yaml", ".yml"} + else lines + ) + consumer_records = logical_lines(source_lines, markdown_flags) consumer_records.extend( (number, line, False) for number, line in yaml_consumers.items() ) for number, line, markdown_code in consumer_records: - image_values, option_errors = container_image_operands(line) + image_values, option_errors, _ = container_image_operands(line) if ( line.lstrip().startswith(("#", "//")) or not (code_file or markdown_code) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index b89da644..ef4c7691 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -516,10 +516,13 @@ def test_yaml_flow_mapping_image_keys_use_assignment_policy(self) -> None: cases = ( "services:\n app: {image: rust:$TAG}\n", 'services:\n app: {image: "rust:${TAG}"}\n', + 'services:\n app: {"image": rust:$TAG}\n', "jobs:\n test: {container: postgres:$PG_TAG}\n", "images: {builder_image: rust:$TAG}\n", "x-service: &defaults {image: rust:$TAG}\n" "services:\n app:\n <<: *defaults\n", + 'x-service: &defaults {"image": rust:$TAG}\n' + "services:\n app:\n <<: *defaults\n", ) for text in cases: with self.subTest(text=text): @@ -533,6 +536,10 @@ def test_yaml_flow_mapping_image_keys_use_assignment_policy(self) -> None: "compose.yaml", f"services:\n app: {{image: {PINNED_RUST}}}\n", ) + self.assert_clean( + "compose.yaml", + f'services:\n app: {{"image": {PINNED_RUST}}}\n', + ) self.assert_clean( "compose.yaml", "services:\n app: {image: registryctl-relay:$TAG}\n", @@ -550,6 +557,101 @@ def test_yaml_flow_mapping_image_keys_use_assignment_policy(self) -> None: f"x-service: &defaults {{image: {PINNED_RUST}}}\n" "services:\n app:\n <<: *defaults\n", ) + self.assert_clean( + "compose.yaml", + f'x-service: &defaults {{"image": {PINNED_RUST}}}\n' + "services:\n app:\n <<: *defaults\n", + ) + + def test_yaml_quoted_and_unsupported_policy_key_syntax_fails_closed( + self, + ) -> None: + for text in ( + 'services:\n app:\n "image": rust:$TAG\n', + "services:\n app:\n 'container': postgres:$PG_TAG\n", + ): + with self.subTest(text=text): + self.assert_failure( + "compose.yaml", + text, + "computed or unresolved image assignment", + ) + + for text in ( + "services:\n app: {image:\n rust:$TAG}\n", + "services:\n app:\n ? image\n : rust:$TAG\n", + "services:\n app:\n image:\n rust:$TAG\n", + ): + with self.subTest(text=text): + self.assert_failure( + "compose.yaml", + text, + "unsupported YAML image policy-key syntax", + ) + + self.assert_clean( + "compose.yaml", + f'services:\n app:\n "image": {PINNED_RUST}\n', + ) + self.assert_clean( + "compose.yaml", + f"jobs:\n test:\n 'container': {PINNED_RUST}\n", + ) + + def test_yaml_policy_flow_text_inside_comments_is_ignored(self) -> None: + self.assert_clean( + "compose.yaml", + "# app: {image: rust:$TAG}\n" + "services:\n" + f" app: {{image: {PINNED_RUST}}} # ignored: {{image: rust:$TAG}}\n", + ) + + def test_yaml_scalar_image_aliases_require_local_literal_anchors(self) -> None: + self.assert_clean( + "compose.yaml", + f"x-image: &runtime {PINNED_RUST}\n" + "services:\n app:\n image: *runtime\n", + ) + self.assert_clean( + "compose.yaml", + f'"x-image": &runtime {PINNED_RUST}\n' + 'services:\n app: {"image": *runtime}\n', + ) + for text, expected in ( + ( + "x-image: &runtime rust:$TAG\nservices:\n app:\n image: *runtime\n", + "computed or unresolved image assignment", + ), + ( + "x-image: &runtime rust:1.95-trixie\n" + "services:\n app:\n image: *runtime\n", + "not pinned by immutable digest", + ), + ( + f"services:\n app:\n image: *runtime\n" + f"x-image: &runtime {PINNED_RUST}\n", + "computed or unresolved image assignment", + ), + ( + f"x-image: &runtime {PINNED_RUST}\n---\n" + "services:\n app:\n image: *runtime\n", + "computed or unresolved image assignment", + ), + ( + f"x-image: &runtime {PINNED_RUST}\n" + "replacement: &runtime rust:$TAG\n" + "services:\n app:\n image: *runtime\n", + "computed or unresolved image assignment", + ), + ( + f"x-image: &runtime {PINNED_RUST}\n" + "replacement: {value: &runtime rust:$TAG}\n" + "services:\n app:\n image: *runtime\n", + "computed or unresolved image assignment", + ), + ): + with self.subTest(text=text): + self.assert_failure("compose.yaml", text, expected) def test_validated_image_annotations_are_path_and_contract_scoped(self) -> None: registryctl_path = "crates/registryctl/src/templates/compose.yaml" @@ -776,6 +878,38 @@ def test_container_cli_scans_only_the_bounded_image_operand(self) -> None: with self.subTest(command=command): self.assert_failure("helper.sh", command, family) + def test_container_short_option_grammar_and_help_are_bounded(self) -> None: + for command in ( + "docker run --help\n", + "docker run --help rust:1.95-trixie\n", + "docker run -c512 alpine:3.22 true\n", + "docker run -itv/tmp:/tmp alpine:3.22 true\n", + "docker run -itp8080:80 alpine:3.22 true\n", + "docker run -itc512 alpine:3.22 true\n", + ): + with self.subTest(command=command): + self.assert_clean("helper.sh", command) + + for command in ( + "docker run -c512 postgres true\n", + "docker run -itv/tmp:/tmp postgres true\n", + "docker run -itp8080:80 postgres true\n", + "docker run -itc512 postgres true\n", + ): + with self.subTest(command=command): + self.assert_failure("helper.sh", command, "postgres") + + for command in ( + "docker run -itxvalue alpine:3.22 true\n", + "docker run -Z alpine:3.22 true\n", + ): + with self.subTest(command=command): + self.assert_failure( + "helper.sh", + command, + "unsupported Docker/Podman short-option cluster", + ) + def test_documented_container_options_preserve_the_image_operand(self) -> None: value_options = ( ("--blkio-weight", "500"), From 4399a04dfb1d4a2753ac67fbc9da8224bcb03ef6 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 03:53:02 +0700 Subject: [PATCH 20/34] fix(release): narrow Debian image boundary check Signed-off-by: Jeremi Joslin --- .../workflows/notary-postgres-conformance.yml | 8 + .../scripts/run-live-consultation-journey.sh | 2 +- crates/registryctl/src/templates/compose.yaml | 1 - .../notary/scripts/postgresql-conformance.sh | 6 +- .../conformance/relay-oidc/docker-compose.yml | 1 - release/scripts/check-debian13-images.py | 1564 ++--------------- release/scripts/test_check_debian13_images.py | 1340 ++------------ release/scripts/test_registry_release.py | 42 + release/scripts/test_relay_oidc_smoke.py | 44 - 9 files changed, 353 insertions(+), 2655 deletions(-) diff --git a/.github/workflows/notary-postgres-conformance.yml b/.github/workflows/notary-postgres-conformance.yml index 21101aa4..2013c4dc 100644 --- a/.github/workflows/notary-postgres-conformance.yml +++ b/.github/workflows/notary-postgres-conformance.yml @@ -59,8 +59,14 @@ jobs: matrix: include: - postgresql: "16" + source_image: postgres:16.13-alpine + target_image: postgres:16.14-alpine - postgresql: "17" + source_image: postgres:17.9-alpine + target_image: postgres:17.10-alpine - postgresql: "18" + source_image: postgres:18.3-alpine + target_image: postgres:18.4-alpine steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -82,6 +88,8 @@ jobs: - name: Run Notary PostgreSQL conformance env: NOTARY_BIN: target/notary-postgres-conformance/debug/registry-notary + NOTARY_POSTGRES_SOURCE_IMAGE: ${{ matrix.source_image }} + NOTARY_POSTGRES_TARGET_IMAGE: ${{ matrix.target_image }} run: | set -euo pipefail diagnostics="${RUNNER_TEMP}/notary-postgresql-${{ matrix.postgresql }}-docker-events.log" diff --git a/crates/registry-relay/scripts/run-live-consultation-journey.sh b/crates/registry-relay/scripts/run-live-consultation-journey.sh index 3a31e5da..72ec2bce 100755 --- a/crates/registry-relay/scripts/run-live-consultation-journey.sh +++ b/crates/registry-relay/scripts/run-live-consultation-journey.sh @@ -6,7 +6,7 @@ set +v set -euo pipefail umask 077 -readonly POSTGRES_IMAGE="postgres:16-trixie@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" +readonly POSTGRES_IMAGE="postgres:16" [[ "$#" -le 1 ]] || { printf '%s\n' "usage: $0 [dhis2|rhai|synthetic]" >&2 diff --git a/crates/registryctl/src/templates/compose.yaml b/crates/registryctl/src/templates/compose.yaml index 8a80895d..2da0228d 100644 --- a/crates/registryctl/src/templates/compose.yaml +++ b/crates/registryctl/src/templates/compose.yaml @@ -1,7 +1,6 @@ # Generated by registryctl. services: registry-relay: - # debian13-policy: allow-validated-image validator="registryctl-image-lock" reason="registryctl renders the validated digest image lock into this whole value" image: {{relay_image}} user: "${REGISTRY_STACK_RUNTIME_UID:-65532}:${REGISTRY_STACK_RUNTIME_GID:-65532}" env_file: diff --git a/products/notary/scripts/postgresql-conformance.sh b/products/notary/scripts/postgresql-conformance.sh index 88ce826a..54a0b440 100755 --- a/products/notary/scripts/postgresql-conformance.sh +++ b/products/notary/scripts/postgresql-conformance.sh @@ -28,9 +28,9 @@ case "${postgresql_major}" in ;; esac -source_image="${default_source_image}" -target_image="${default_target_image}" -restore_image="${default_restore_image}" +source_image="${NOTARY_POSTGRES_SOURCE_IMAGE:-${default_source_image}}" +target_image="${NOTARY_POSTGRES_TARGET_IMAGE:-${default_target_image}}" +restore_image="${NOTARY_POSTGRES_RESTORE_IMAGE:-${default_restore_image}}" notary_bin="${NOTARY_BIN:-target/debug/registry-notary}" run_id="${GITHUB_RUN_ID:-local}-$$" postgres_container="notary-postgres-conformance-${run_id}" diff --git a/release/conformance/relay-oidc/docker-compose.yml b/release/conformance/relay-oidc/docker-compose.yml index 9926521b..afed49ab 100644 --- a/release/conformance/relay-oidc/docker-compose.yml +++ b/release/conformance/relay-oidc/docker-compose.yml @@ -87,7 +87,6 @@ services: - zitadel-seed:/seed:ro relay: - # debian13-policy: allow-validated-image validator="relay-oidc-smoke" reason="the smoke runner validates the fixed Relay repository and lowercase digest" image: ${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:?runner must provide digest-pinned Registry Relay image} platform: linux/amd64 network_mode: service:zitadel diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 5c6caeae..5befea9a 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -3,13 +3,13 @@ from __future__ import annotations -import os import re -import subprocess import sys from pathlib import Path + ROOT = Path(__file__).resolve().parents[2] + RUST_BUILDER = ( "rust:1.95-trixie@sha256:" "f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3" @@ -27,1370 +27,229 @@ "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) -CI_WORKFLOW = Path(".github/workflows/ci.yml") -TUTORIAL_SCRIPT = Path("docs/site/scripts/check-registryctl-tutorials.sh") -PRODUCT_DOCKERFILES = ( +DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), Path("crates/registry-relay/Dockerfile.demo"), Path("products/notary/Dockerfile"), Path("release/docker/Dockerfile.registry-notary"), Path("release/docker/Dockerfile.registry-relay"), ) -REQUIRED_PRODUCT_SURFACES = PRODUCT_DOCKERFILES + ( - CI_WORKFLOW, + +# These are the maintained image and image-policy surfaces. Historical release +# notes are immutable evidence and intentionally are not rewritten by this gate. +MAINTAINED_TEXT_PATHS = DOCKERFILES + ( Path(".github/workflows/release.yml"), Path("release/scripts/build-release-binaries.sh"), + Path("docs/site/scripts/check-registryctl-tutorials.sh"), + Path("crates/registry-relay/docs/ops.md"), + Path("crates/registry-relay/docs/security-assurance.md"), + Path("crates/registry-relay/scripts/check_docker_build_contract.py"), Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), - TUTORIAL_SCRIPT, -) -RELAY_DOCKERFILES = ( - PRODUCT_DOCKERFILES[0], - PRODUCT_DOCKERFILES[1], - PRODUCT_DOCKERFILES[4], + Path("products/notary/docs/security-assurance.md"), ) -NOTARY_DOCKERFILES = (PRODUCT_DOCKERFILES[2], PRODUCT_DOCKERFILES[3]) -# Scan tracked UTF-8 text, excluding only historical evidence, third-party or -# generated material, build output, and this checker's negative fixtures. -EXCLUDED_DIRS = set( - ".git .repo-docs-cache .research .venv __pycache__ dist node_modules target".split() -) -EXCLUDED_PREFIXES = ("external/", "release/notes/") -EXCLUDED_EXACT = { - "release/scripts/check-debian13-images.py", - "release/scripts/test_check_debian13_images.py", -} -MAX_TRACKED_PATHS = 10_000 -MAX_TEXT_FILE_BYTES = 2_000_000 -MAX_TOTAL_TEXT_BYTES = 32_000_000 -MAX_LINE_CHARS = 131_072 - -RETIRED_MARKERS = ( - "book" + "worm", - "bullseye", - "buster", - "debian" + "12", - "debian-12", - "debian" + "11", - "debian-11", - "debian" + "10", - "debian-10", -) -DEFAULT_DEBIAN_FAMILIES = {"golang", "node", "postgres", "python", "rust"} -OCI_RE = re.compile( - r"(?(?:docker://)?" - r"(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*[A-Za-z0-9._-]+" - r"(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-fA-F]{64})" - r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" -) -BARE_DEFAULT_FAMILY_RE = re.compile( - rf"(?(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*" - rf"(?Pdebian|{'|'.join(sorted(DEFAULT_DEBIAN_FAMILIES))}))" - r"(?![A-Za-z0-9._/@+:-])" -) -DIGEST_RE = re.compile(r"@sha256:[0-9a-f]{64}$", re.IGNORECASE) -NON_DEBIAN_TAG_RE = re.compile( - r"(?:^|[._-])(?:alpine(?:[0-9]+(?:\.[0-9]+)*)?" - r"|windows(?:servercore|nanoserver)?)(?:$|[._-])", - re.IGNORECASE, -) -FROM_RE = re.compile( - r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", - re.IGNORECASE | re.MULTILINE, -) -ASSIGN_RE = re.compile( - r"^\s*(?:-\s*)?(?:(?:export|local|readonly|const|let|var)\b[^A-Za-z_]*)?" - r"(?P[A-Za-z_][A-Za-z0-9_$-]*)(?:\s*:[^=]+)?" - r"\s*=\s*(?P.+)$" -) -YAML_PLAIN_KEY = r"[A-Za-z_][A-Za-z0-9_$-]*" -YAML_SIMPLE_KEY = rf"""(?:{YAML_PLAIN_KEY}|"{YAML_PLAIN_KEY}"|'{YAML_PLAIN_KEY}')""" -YAML_KEY_RE = re.compile( - rf"^\s*(?:-\s*)?(?P{YAML_SIMPLE_KEY})\s*:\s*(?P.+)$" -) -YAML_FIELD_RE = re.compile( - r"^(?P\s*)(?P-\s*)?" - rf"(?P{YAML_SIMPLE_KEY})\s*:\s*(?P.*)$" -) -YAML_EMPTY_KEY_RE = re.compile(rf"^\s*(?:-\s*)?(?P{YAML_SIMPLE_KEY})\s*:\s*$") -YAML_EXPLICIT_KEY_RE = re.compile( - rf"^\s*(?:-\s*)?\?\s*(?P{YAML_SIMPLE_KEY})(?:\s*:\s*.*)?$" -) -YAML_SCALAR_ANCHOR_RE = re.compile( - rf"^\s*(?:-\s*)?{YAML_SIMPLE_KEY}\s*:\s*" - r"&(?P[A-Za-z0-9_-]+)\s+(?P.+)$" -) -IMAGE_VAR_RE = re.compile( - r"\b(?P[A-Za-z_][A-Za-z0-9_$-]*(?:_image|Image)|IMAGE)\b", - re.IGNORECASE, -) -CLI_RE = re.compile(r"(?:^|[\s;&|({])(?:\S*/)?(?:docker|podman)\b(?P[^\n]*)") -NON_IMAGE_NAMESPACES = set("build buildx compose exec network volume".split()) -IMAGE_ALIAS_NAME = r"(?:[A-Za-z_][A-Za-z0-9_]*(?:_IMAGE|Image)|IMAGE)" -DIRECT_IMAGE_ALIAS_RE = re.compile( - rf"""^["']?\$(?:{IMAGE_ALIAS_NAME}|\{{{IMAGE_ALIAS_NAME}\}})["']?;?$""", - re.IGNORECASE, -) -IMAGE_FALLBACK_RE = re.compile( - rf"""^["']?\$\{{{IMAGE_ALIAS_NAME}:-(?:\${IMAGE_ALIAS_NAME}|""" - rf"""\$\{{{IMAGE_ALIAS_NAME}\}})\}}["']?;?$""", - re.IGNORECASE, -) -SIMPLE_SHELL_VAR = r"\$(?:[A-Za-z_][A-Za-z0-9_]*|\{[A-Za-z_][A-Za-z0-9_]*\})" -SIMPLE_GITHUB_EXPRESSION = ( - r"\$\{\{\s*[A-Za-z_][A-Za-z0-9_-]*" - r"(?:\.[A-Za-z_][A-Za-z0-9_-]*)*\s*\}\}" -) -SIMPLE_DYNAMIC_VALUE = rf"(?:{SIMPLE_SHELL_VAR}|{SIMPLE_GITHUB_EXPRESSION})" -SIMPLE_DYNAMIC_VALUE_RE = re.compile(SIMPLE_DYNAMIC_VALUE) -SAFE_TAG_TEMPLATE_RE = re.compile( - rf"""^["']?(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*""" - rf"""[A-Za-z0-9._-]+):(?P(?:[A-Za-z0-9_.-]|""" - rf"""{SIMPLE_DYNAMIC_VALUE})+)["']?;?$""" -) -COMPUTED_VALUE_RE = re.compile(r"[+%]|`|\$\(|\.format\(|\bf[\"']") -CLI_TOKEN_RE = re.compile(r""""[^"\n]*"|'[^'\n]*'|[^\s;\[\]]+""") -CONTAINER_VALUE_OPTIONS = { - "--add-host", - "--annotation", - "--attach", - "--blkio-weight", - "--blkio-weight-device", - "--cap-add", - "--cap-drop", - "--cgroup-parent", - "--cgroupns", - "--cidfile", - "--cpu-count", - "--cpu-percent", - "--cpu-period", - "--cpu-quota", - "--cpu-rt-period", - "--cpu-rt-runtime", - "--cpu-shares", - "--cpus", - "--cpuset-cpus", - "--cpuset-mems", - "--detach-keys", - "--device", - "--device-cgroup-rule", - "--device-read-bps", - "--device-read-iops", - "--device-write-bps", - "--device-write-iops", - "--dns", - "--dns-option", - "--dns-search", - "--domainname", - "--entrypoint", - "--env", - "--env-file", - "--expose", - "--gpus", - "--group-add", - "--health-cmd", - "--health-interval", - "--health-retries", - "--health-start-interval", - "--health-start-period", - "--health-timeout", - "--hostname", - "--io-maxbandwidth", - "--io-maxiops", - "--ip", - "--ip6", - "--ipc", - "--isolation", - "--label", - "--label-file", - "--link", - "--link-local-ip", - "--log-driver", - "--log-opt", - "--mac-address", - "--memory", - "--memory-reservation", - "--memory-swap", - "--memory-swappiness", - "--mount", - "--name", - "--network", - "--network-alias", - "--oom-score-adj", - "--pid", - "--pids-limit", - "--platform", - "--publish", - "--pull", - "--restart", - "--runtime", - "--security-opt", - "--shm-size", - "--stop-signal", - "--stop-timeout", - "--storage-opt", - "--sysctl", - "--tmpfs", - "--ulimit", - "--user", - "--userns", - "--uts", - "--volume", - "--volume-driver", - "--volumes-from", - "--workdir", - "-a", - "-c", - "-e", - "-h", - "-l", - "-m", - "-p", - "-u", - "-v", - "-w", -} -CONTAINER_FLAG_OPTIONS = { - "--detach", - "--init", - "--interactive", - "--no-healthcheck", - "--oom-kill-disable", - "--privileged", - "--publish-all", - "--quiet", - "--read-only", - "--rm", - "--sig-proxy", - "--tty", - "--use-api-socket", - "-P", - "-d", - "-i", - "-q", - "-t", -} -PULL_FLAG_OPTIONS = {"--all-tags", "--disable-content-trust", "-a", "-q"} -CONTAINER_TERMINAL_OPTIONS = {"--help"} -DOCKER_CONTEXT_RE = re.compile( - r"docker-image://(?P(?:[A-Za-z0-9._-]+(?::[0-9]+)?/)*" - r"[A-Za-z0-9._-]+(?:\:[A-Za-z0-9_][A-Za-z0-9._-]*)?" - r"(?:@sha256:[0-9a-fA-F]{64})?)(?![A-Za-z0-9._/@+-])" -) -DOCKER_CONTEXT_TOKEN_RE = re.compile( - r"""docker-image://(?P[^\s,;"'\]\}]+)""", -) -EXEMPTION_RE = re.compile( - r'' -) -VALIDATED_IMAGE_EXEMPTION_RE = re.compile( - r"^\s*#\s*debian13-policy:\s*allow-validated-image\s+" - r'validator="(?P[a-z0-9-]+)"\s+reason="[^"]{12,}"\s*$' +RUST_BUILDER_DOCKERFILES = DOCKERFILES[:3] +PREPARATION_DOCKERFILES = DOCKERFILES[3:] +RELAY_DOCKERFILES = ( + Path("crates/registry-relay/Dockerfile"), + Path("crates/registry-relay/Dockerfile.demo"), + Path("release/docker/Dockerfile.registry-relay"), ) -VALIDATED_IMAGE_CONTRACTS = { - "registryctl-image-lock": ( - Path("crates/registryctl/src/templates/compose.yaml"), - "{{relay_image}}", - ( - ( - Path("crates/registryctl/src/lib.rs"), - 'validate_locked_image_ref(\n "images.registry-relay"', - ), - ( - Path("crates/registryctl/src/lib.rs"), - 'include_str!("templates/compose.yaml")' - '.replace("{{relay_image}}", image_lock.relay_image())', - ), - ( - Path("crates/registryctl/src/lib.rs"), - "fn image_lock_rejects_mutable_or_noncanonical_image_references()", - ), - ( - Path("crates/registryctl/tests/image_lock.rs"), - "assert!(compose.contains(RELAY_IMAGE));", - ), - ), - ), - "relay-oidc-smoke": ( - Path("release/conformance/relay-oidc/docker-compose.yml"), - "${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:" - "?runner must provide digest-pinned Registry Relay image}", - ( - ( - Path("release/scripts/relay-oidc-smoke.py"), - "def validate_relay_image(value: str) -> str:", - ), - ( - Path("release/scripts/relay-oidc-smoke.py"), - "relay_image = validate_relay_image(args.relay_image)", - ), - ( - Path("release/scripts/relay-oidc-smoke.py"), - '"REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE": relay_image,', - ), - ( - Path("release/scripts/test_relay_oidc_smoke.py"), - "def test_relay_image_requires_exact_repository_and_lowercase_digest", - ), - ( - Path("release/scripts/test_relay_oidc_smoke.py"), - "def test_execute_live_binds_validated_argument_over_ambient_image", - ), - ), - ), -} -MARKDOWN_SUFFIXES = {".md", ".mdx"} -CODE_SUFFIXES = set(".bash .js .mjs .py .sh .ts .yaml .yml".split()) -FENCE_LANGS = set( - "bash console dockerfile javascript js python sh shell terminal " - "ts typescript yaml yml zsh".split() +NOTARY_DOCKERFILES = ( + Path("products/notary/Dockerfile"), + Path("release/docker/Dockerfile.registry-notary"), ) +FROM_RE = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)", re.MULTILINE) +DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") -class ImageSurfaceError(RuntimeError): - """A maintained text surface exceeded a scanner boundary.""" - - -def is_dockerfile(path: Path) -> bool: - name = path.name.casefold() - return ( - name == "dockerfile" - or name.startswith("dockerfile.") - or name.endswith(".dockerfile") - ) - - -def is_excluded(path: Path) -> bool: - value = path.as_posix() - return ( - value in EXCLUDED_EXACT - or any(part in EXCLUDED_DIRS for part in path.parts) - or any(value.startswith(prefix) for prefix in EXCLUDED_PREFIXES) - or path.name in {"CHANGELOG.md", "release-notes.md"} - or value == "docs/site/src/content/docs/changelog.mdx" - or "/resources/scalar/" in f"/{value}/" - ) - - -def discover_maintained_surfaces(root: Path) -> tuple[Path, ...]: - command = subprocess.run( - ["git", "-C", str(root), "ls-files", "-z"], - capture_output=True, - check=False, - ) - if command.returncode == 0: - paths = [Path(item.decode()) for item in command.stdout.split(b"\0") if item] - else: - paths = [] - for directory, names, files in os.walk(root): - names[:] = [name for name in names if name not in EXCLUDED_DIRS] - paths.extend((Path(directory) / name).relative_to(root) for name in files) - if len(paths) > MAX_TRACKED_PATHS: - raise ImageSurfaceError( - f"tracked path count exceeds {MAX_TRACKED_PATHS}: {len(paths)}" - ) - return tuple( - sorted( - path - for path in paths - if not is_excluded(path) - and (root / path).is_file() - and not (root / path).is_symlink() - ) - ) - -def read_text(root: Path, path: Path) -> tuple[str | None, int]: - target = root / path +def read(root: Path, relative: Path, failures: list[str]) -> str: + path = root / relative try: - size = target.stat().st_size + return path.read_text(encoding="utf-8") except FileNotFoundError: - return None, 0 - if size > MAX_TEXT_FILE_BYTES: - with target.open("rb") as stream: - sample = stream.read(8192) - try: - sample.decode() - except UnicodeDecodeError: - return None, 0 - if b"\0" in sample: - return None, 0 - raise ImageSurfaceError( - f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes" - ) - data = target.read_bytes() - if b"\0" in data: - return None, 0 - try: - return data.decode(), size - except UnicodeDecodeError: - return None, 0 - - -def references(text: str) -> list[str]: - return [ - match.group("ref").removeprefix("docker://").strip("\"',;") - for match in OCI_RE.finditer(text) - ] - - -def exact_reference(value: str) -> str | None: - candidate = value.strip().removesuffix(";").strip() - if len(candidate) >= 2 and candidate[0] == candidate[-1] and candidate[0] in "\"'": - candidate = candidate[1:-1] - found = references(candidate) - normalized = candidate.removeprefix("docker://") - return found[0] if len(found) == 1 and found[0] == normalized else None - - -def repository_and_tag(reference: str) -> tuple[str, str | None]: - value = reference.split("@", 1)[0] - slash, colon = value.rfind("/"), value.rfind(":") - return (value[:colon], value[colon + 1 :]) if colon > slash else (value, None) - - -def is_debian_family(reference: str) -> bool: - repository, tag = repository_and_tag(reference) - name, tag = repository.rsplit("/", 1)[-1].casefold(), (tag or "").casefold() - default = name in DEFAULT_DEBIAN_FAMILIES - excluded = default and NON_DEBIAN_TAG_RE.search(tag) is not None - return not excluded and ( - "debian" in name - or name == "buildpack-deps" - or "trixie" in tag - or re.search(r"debian-?13", tag) is not None - or default - and ( - not tag - or tag in {"latest", "slim"} - or re.fullmatch(r"[0-9]+(?:[._][0-9]+)*(?:-slim)?", tag) is not None - ) - ) - - -def image_assignment_name(name: str) -> bool: - normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() - return normalized.endswith("image") or normalized == "container" - - -def image_assignment(name: str, value: str) -> tuple[str, str] | None: - stripped = value.strip() - if stripped.rstrip(",") in {"(", "[", "{"} or stripped.endswith(","): - return None - return (name, value) if image_assignment_name(name) else None - - -def yaml_key_name(value: str) -> str: - return value[1:-1] if value[:1] in "\"'" and value[-1:] == value[:1] else value - - -def assignment(path: Path, line: str) -> tuple[str, str] | None: - match = (YAML_KEY_RE if path.suffix in {".yaml", ".yml"} else ASSIGN_RE).match(line) - return ( - image_assignment( - yaml_key_name(match.group("name")) - if path.suffix in {".yaml", ".yml"} - else match.group("name"), - match.group("value"), - ) - if match is not None - else None - ) - + failures.append(f"missing maintained image surface: {relative}") + return "" -def split_flow_commas(value: str, nested: bool) -> list[str]: - parts, current, depth, quote = [], [], 0, "" - for character in value: - if quote: - current.append(character) - if character == quote: - quote = "" - elif character in "\"'": - quote = character - current.append(character) - elif character in "([{": - depth += 1 - current.append(character) - elif character in ")]}": - depth = max(0, depth - 1) - current.append(character) - elif character == "," and bool(depth) == nested: - parts.append("".join(current)) - current = [] - else: - current.append(character) - parts.append("".join(current)) - return parts - -def command_tokens(value: str) -> list[str]: - command = re.split( - r"\s*(?:&&|\|\||[;|])\s*", - " ".join(split_flow_commas(value, nested=True)), - maxsplit=1, - )[0] - return [ - token[1:-1] - if len(token) >= 2 and token[0] == token[-1] and token[0] in "\"'" - else token.strip("\"'(),") - for token in CLI_TOKEN_RE.findall(command) - ] - - -def short_option_consumption( - token: str, value_options: set[str], flag_options: set[str] -) -> tuple[int, str | None]: - if len(token) == 1: - return 0, "unsupported Docker/Podman short-option cluster: -" - for index, character in enumerate(token[1:]): - option = f"-{character}" - if option in flag_options: - continue - if option in value_options: - return (1 if index + 2 < len(token) else 2), None - return ( - 0, - "unsupported Docker/Podman short-option cluster: " - f"{token}; unknown option: {option}", - ) - return 1, None - - -def container_image_operands(line: str) -> tuple[list[str], list[str], bool]: - values, errors, recognized = [], [], False - for command in CLI_RE.finditer(line): - tokens = command_tokens(command.group("tail")) - action_index = next( - ( - index - for index, token in enumerate(tokens) - if token.casefold() in {"create", "pull", "run"} - ), - None, - ) - if action_index is None: - continue - prefix = {token.casefold().lstrip("-") for token in tokens[:action_index]} - if not prefix & NON_IMAGE_NAMESPACES: - recognized = True - action = tokens[action_index].casefold() - value_options = CONTAINER_VALUE_OPTIONS - ( - {"-a", "--attach"} if action == "pull" else set() - ) - flag_options = CONTAINER_FLAG_OPTIONS | ( - PULL_FLAG_OPTIONS if action == "pull" else set() - ) - terminal_option, unknown_option, index = False, False, action_index + 1 - while index < len(tokens): - token = tokens[index] - if token == "\\": - index += 1 - continue - if token == "--": - index += 1 - break - if not token.startswith("-"): - break - if token.startswith("--"): - option = token.split("=", 1)[0] - if token in CONTAINER_TERMINAL_OPTIONS: - terminal_option = True - index = len(tokens) - break - if option in value_options: - index += 1 if "=" in token else 2 - elif option in flag_options: - index += 1 - else: - errors.append( - "unsupported Docker/Podman option has unknown arity: " - f"{option}" - ) - unknown_option = True - break - else: - consumed, error = short_option_consumption( - token, value_options, flag_options - ) - if error is not None: - errors.append(error) - unknown_option = True - break - index += consumed - if unknown_option: - continue - if terminal_option: - continue - if index >= len(tokens): - values.append("") - else: - values.append(tokens[index]) - return values, errors, recognized - - -def is_container_consumer(line: str) -> bool: - _, _, recognized = container_image_operands(line) - return recognized - - -def policy_image_name(name: str) -> bool: - value, markers = ( - name.casefold().replace("$", ""), - {"base", "builder", "debian", "runtime"}, - ) - return ( - re.sub("[^a-z0-9]", "", value) == "image" - or value.startswith(tuple(markers)) - or bool(set(re.split(r"[_-]+", value)) & markers) - ) - - -def yaml_policy_image_name(name: str) -> bool: - normalized = re.sub("[^A-Za-z0-9]", "", name).casefold() - return normalized in {"image", "container"} or policy_image_name(name) - - -def is_image_alias(value: str) -> bool: - return ( - DIRECT_IMAGE_ALIAS_RE.fullmatch(value.strip()) is not None - or IMAGE_FALLBACK_RE.fullmatch(value.strip()) is not None - ) - - -def has_safe_image_template(value: str) -> bool: - match = SAFE_TAG_TEMPLATE_RE.fullmatch(value) - if COMPUTED_VALUE_RE.search(value) or match is None: - return False - repository = match.group("repository") - name = repository.rsplit("/", 1)[-1].casefold() - if name not in DEFAULT_DEBIAN_FAMILIES: - return not is_debian_family(repository) - static_tag = SIMPLE_DYNAMIC_VALUE_RE.sub("dynamic", match.group("tag")).casefold() - return NON_DEBIAN_TAG_RE.search(static_tag) is not None - - -def build_context_references(text: str) -> list[str]: - return [match.group("ref") for match in DOCKER_CONTEXT_RE.finditer(text)] - - -def computed_build_contexts(text: str) -> list[str]: - return [ - match.group("value") - for match in DOCKER_CONTEXT_TOKEN_RE.finditer(text) - if DOCKER_CONTEXT_RE.fullmatch(f"docker-image://{match.group('value')}") is None - ] - - -def append_reference_failures( - path: Path, - number: int, - reference: str, +def require( + text: str, + needle: str, + relative: Path, + detail: str, failures: list[str], - kind: str = "Debian-derived image reference", -) -> None: - if not is_debian_family(reference): - return - prefix = f"{path}:{number}: {kind}" - if not DIGEST_RE.search(reference): - failures.append(f"{prefix} is not pinned by immutable digest: {reference}") - if ( - "trixie" not in reference.casefold() - and re.search(r"debian-?13", reference, re.IGNORECASE) is None - ): - failures.append(f"{prefix} does not declare Trixie/Debian 13: {reference}") - - -def append_bare_failures( - path: Path, number: int, value: str, failures: list[str] ) -> None: - for bare in BARE_DEFAULT_FAMILY_RE.finditer(value): - family = bare.group("name") - label = "Debian" if family == "debian" else f"Debian-default {family}" - failures.append( - f"{path}:{number}: bare {label} image reference is not pinned " - f"and does not declare Trixie/Debian 13: {bare.group('ref')}" - ) - - -def markdown_code_flags(path: Path, lines: list[str]) -> list[bool]: - if path.suffix not in MARKDOWN_SUFFIXES: - return [False] * len(lines) - active, marker, result = False, "", [] - for line in lines: - fence = re.match(r"(```+|~~~+)\s*([A-Za-z0-9_-]*)", line.lstrip()) - if fence: - token, language = fence.groups() - if marker and token.startswith(marker): - active, marker = False, "" - elif not marker: - active, marker = language.casefold() in FENCE_LANGS, token[:3] - result.append(False) - else: - result.append(active) - return result - - -def logical_lines(lines: list[str], flags: list[bool]) -> list[tuple[int, str, bool]]: - result, parts, start, active = [], [], 1, False - for number, line in enumerate(lines, 1): - if not parts: - start, active = number, flags[number - 1] - stripped = line.rstrip() - continued = stripped.endswith("\\") - parts.append(stripped[:-1] if continued else stripped) - if not continued: - result.append((start, " ".join(parts), active)) - parts = [] - if parts: - result.append((start, " ".join(parts), active)) - return result - - -def yaml_code(line: str) -> str: - quote, escaped = "", False - for index, character in enumerate(line): - if quote: - if character == quote and not escaped: - quote = "" - escaped = quote == '"' and character == "\\" and not escaped - if character != "\\": - escaped = False - elif character in "\"'": - quote = character - elif character == "#" and (index == 0 or line[index - 1].isspace()): - return line[:index].rstrip() - return line - - -def flow_mapping_parts(line: str) -> tuple[list[str], list[str]]: - bodies, starts, quote, escaped = [], [], "", False - for index, character in enumerate(line): - if quote: - if character == quote and not escaped: - quote = "" - escaped = character == "\\" and not escaped - if character != "\\": - escaped = False - elif character in "\"'": - quote = character - elif character == "{": - starts.append(index + 1) - elif character == "}" and starts: - bodies.append(line[starts.pop() : index]) - return bodies, [line[start:] for start in starts] - - -def flow_mapping_pairs(line: str) -> list[list[tuple[str, str]]]: - mappings = [] - for body in flow_mapping_parts(line)[0]: - fields = [] - for part in split_flow_commas(body, nested=False): - name, separator, value = part.partition(":") - raw_name = name.strip() - if separator and re.fullmatch(YAML_SIMPLE_KEY, raw_name): - fields.append((yaml_key_name(raw_name), value.strip())) - if fields: - mappings.append(fields) - return mappings - - -def flow_mapping_fields(line: str) -> list[dict[str, str]]: - return [ - {name.casefold(): value for name, value in pairs} - for pairs in flow_mapping_pairs(line) - ] - - -def flow_mapping_image_assignments(line: str) -> list[tuple[str, str]]: - return [ - item - for pairs in flow_mapping_pairs(line) - for name, value in pairs - if value - if (item := image_assignment(name, value)) is not None - ] - - -def flow_explicit_policy_key(body: str) -> str | None: - for part in split_flow_commas(body, nested=False): - match = re.match(rf"^\s*\?\s*(?P{YAML_SIMPLE_KEY})(?:\s|:|$)", part) - if match is not None: - name = yaml_key_name(match.group("name")) - if image_assignment_name(name): - return name - return None - - -def unsupported_yaml_policy_keys(line: str) -> list[str]: - names = [] - for pattern in (YAML_EMPTY_KEY_RE, YAML_EXPLICIT_KEY_RE): - match = pattern.match(line) - if match is not None: - name = yaml_key_name(match.group("name")) - if image_assignment_name(name): - names.append(name) - closed, opened = flow_mapping_parts(line) - for body in closed: - explicit = flow_explicit_policy_key(body) - if explicit is not None: - names.append(explicit) - for pairs in flow_mapping_pairs(f"{{{body}}}"): - names.extend( - name - for name, value in pairs - if not value and image_assignment_name(name) - ) - for body in opened: - explicit = flow_explicit_policy_key(body) - if explicit is not None: - names.append(explicit) - for part in split_flow_commas(body, nested=False): - raw_name, separator, _ = part.partition(":") - raw_name = raw_name.strip() - if separator and re.fullmatch(YAML_SIMPLE_KEY, raw_name): - name = yaml_key_name(raw_name) - if image_assignment_name(name): - names.append(name) - return list(dict.fromkeys(names)) - - -def compliant_scalar_anchor(line: str) -> tuple[str, str] | None: - match = YAML_SCALAR_ANCHOR_RE.match(line) - if match is None: - return None - reference = exact_reference(match.group("value")) - if reference is None: - return None - if is_debian_family(reference) and ( - DIGEST_RE.search(reference) is None - or "trixie" not in reference.casefold() - and re.search(r"debian-?13", reference, re.IGNORECASE) is None - ): - return None - return match.group("anchor"), reference - - -def yaml_anchor_declarations(line: str) -> list[str]: - names, quote, escaped, index = [], "", False, 0 - while index < len(line): - character = line[index] - if quote: - if character == quote and not escaped: - quote = "" - escaped = quote == '"' and character == "\\" and not escaped - if character != "\\": - escaped = False - elif character in "\"'": - quote = character - elif character == "&" and ( - index == 0 or line[index - 1].isspace() or line[index - 1] in "[{,:-" - ): - match = re.match(r"[A-Za-z0-9_-]+", line[index + 1 :]) - if match is not None: - names.append(match.group()) - index += len(match.group()) - index += 1 - return names - - -def yaml_block_scalar_flags(lines: list[str]) -> list[bool]: - flags, scalar_indent = [], None - scalar = re.compile( - rf"^\s*(?:-\s*)?{YAML_SIMPLE_KEY}\s*:\s*" - r"[>|](?:[1-9][+-]?|[+-][1-9]?)?(?:\s+#.*)?$" - ) - for line in lines: - stripped = line.strip() - indent = len(line) - len(line.lstrip()) - inside = scalar_indent is not None and (not stripped or indent > scalar_indent) - if scalar_indent is not None and stripped and indent <= scalar_indent: - scalar_indent = None - inside = False - flags.append(inside) - if not inside and scalar.fullmatch(line): - scalar_indent = indent - return flags - - -def yaml_container_consumers( - lines: list[str], -) -> tuple[dict[int, str], list[tuple[int, str]]]: - parents: list[tuple[int, int]] = [] - fields: dict[tuple[int, str], tuple[int, str]] = {} - anchors: dict[str, str] = {} - consumers, unresolved = {}, [] - - def combine(number: int, entrypoint: str, command: str) -> None: - tokens = command_tokens(command) - action = tokens[0].casefold() if tokens else "" - alias = re.fullmatch(r"\*(?P[A-Za-z0-9_-]+)", entrypoint.strip()) - if alias: - resolved = anchors.get(alias.group("name")) - if resolved is None: - if action in {"create", "pull", "run"}: - unresolved.append( - ( - number, - "cannot statically resolve container entrypoint; " - "use a literal docker/podman entrypoint", - ) - ) - return - entrypoint = resolved - elif entrypoint.strip().startswith(("$", "{{")) and action in { - "create", - "pull", - "run", - }: - unresolved.append( - ( - number, - "cannot statically resolve container entrypoint; " - "use a literal docker/podman entrypoint", - ) - ) - return - engine = re.match( - r"^\s*\[?\s*[\"']?(?P(?:[^\s,\"'\]]*/)?(?:docker|podman))" - r"[\"']?(?=$|[\s,\]])", - entrypoint, - ) - if engine: - combined = f"{engine.group('engine')} {command}" - if is_container_consumer(combined): - consumers[number] = combined - elif command.strip().startswith(("*", "$", "{{")): - unresolved.append( - ( - number, - "cannot statically resolve container command; " - "use a literal create/pull/run command", - ) - ) - - for offset, line in enumerate(lines): - number = offset + 1 - source = yaml_code(line) - if not source.strip(): - continue - for anchor in re.finditer( - r"&(?P[A-Za-z0-9_-]+)\s+" - r"(?P\[[^\]\n]+\]|[\"']?(?:docker|podman)[\"']?)", - source, - ): - anchors[anchor.group("name")] = anchor.group("value") - for mapping in flow_mapping_fields(source): - if {"entrypoint", "command"} <= mapping.keys(): - combine(number, mapping["entrypoint"], mapping["command"]) - indent = len(source) - len(source.lstrip()) - while parents and parents[-1][0] >= indent: - parents.pop() - match = YAML_FIELD_RE.match(source) - if match is None: - parents.append((indent, number)) - continue - name = yaml_key_name(match.group("name")).casefold() - scope = number if match.group("item") else parents[-1][1] if parents else 0 - if name in {"entrypoint", "command"}: - value = match.group("value").strip() - block_scalar = re.fullmatch(r"[>|][+-]?", value) is not None - if not value or block_scalar: - parts = [] - for following in lines[offset + 1 :]: - if not following.strip() or following.lstrip().startswith("#"): - continue - following_indent = len(following) - len(following.lstrip()) - if following_indent <= indent: - break - if block_scalar: - parts.append(following.strip()) - else: - item = re.match(r"^\s*-\s*(?P.+)$", following) - if item: - parts.append(item.group("value")) - value = " ".join(parts) - fields[(scope, name)] = (number, value) - if (scope, "entrypoint") in fields and (scope, "command") in fields: - command_number, command = fields[(scope, "command")] - entrypoint = fields[(scope, "entrypoint")][1] - combine(command_number, entrypoint, command) - parents.append((indent, number)) - return consumers, unresolved + if needle not in text: + failures.append(f"{relative}: missing {detail}: {needle!r}") -def yaml_image_exemptions(path: Path, lines: list[str]) -> tuple[set[int], set[int]]: - if path.suffix not in {".yaml", ".yml"}: - return set(), set() - comments, assignments = set(), set() - for offset, line in enumerate(lines[:-1]): - exemption = VALIDATED_IMAGE_EXEMPTION_RE.fullmatch(line) - if exemption is None: - continue - contract = VALIDATED_IMAGE_CONTRACTS.get(exemption.group("validator")) - if contract is None or contract[0] != path: - continue - item = assignment(path, lines[offset + 1]) - if ( - item is not None - and yaml_policy_image_name(item[0]) - and item[1].strip() == contract[1] - ): - comments.add(offset + 1) - assignments.add(offset + 2) - return comments, assignments +def runtime_stage(text: str) -> str: + marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" + offset = text.find(marker) + return text[offset:] if offset >= 0 else "" -def scan_surface(path: Path, text: str, executable: bool = False) -> list[str]: - if len(text.encode()) > MAX_TEXT_FILE_BYTES: - raise ImageSurfaceError( - f"{path}: text file exceeds {MAX_TEXT_FILE_BYTES} bytes" - ) - lines, failures = text.splitlines(), [] - markdown_flags = markdown_code_flags(path, text.splitlines()) - shebang = bool(lines and lines[0].startswith("#!")) - code_file = ( - is_dockerfile(path) or path.suffix in CODE_SUFFIXES or executable or shebang - ) - strict_code_assignments = ( - code_file and not is_dockerfile(path) and path.suffix not in {".yaml", ".yml"} - ) - yaml_consumers, yaml_errors = ( - yaml_container_consumers(lines) - if path.suffix in {".yaml", ".yml"} - else ({}, []) - ) - yaml_scalar_flags = ( - yaml_block_scalar_flags(lines) - if path.suffix in {".yaml", ".yml"} - else [False] * len(lines) - ) - failures.extend(f"{path}:{number}: {message}" for number, message in yaml_errors) - exemption_comments, exempt_assignments = yaml_image_exemptions(path, lines) - records: list[tuple[int, str, str, set[str], bool, bool]] = [] - resolved, yaml_anchors = set(), {} - for number, line in enumerate(lines, 1): - if len(line) > MAX_LINE_CHARS: - raise ImageSurfaceError( - f"{path}:{number}: line exceeds {MAX_LINE_CHARS} characters" - ) - markdown_code = markdown_flags[number - 1] - comment = line.lstrip().startswith(("#", "//")) and not markdown_code - exemption = number in exemption_comments or ( - path.suffix in MARKDOWN_SUFFIXES - and not markdown_code - and EXEMPTION_RE.search(line) is not None - ) - if "debian13-policy:" in line and not exemption: - kind = ( - "prose exemption" - if path.suffix in MARKDOWN_SUFFIXES - else "policy annotation" - ) - failures.append( - f"{path}:{number}: invalid Debian image {kind}; use a literal " - "static repository and digest unless an exact reviewed validator " - "contract is allowlisted" - ) - if exemption: - continue - yaml_source = yaml_code(line) if path.suffix in {".yaml", ".yml"} else line - lowered = line.casefold() - for marker in RETIRED_MARKERS: +def check_repository(root: Path = ROOT) -> list[str]: + failures: list[str] = [] + texts = { + relative: read(root, relative, failures) + for relative in MAINTAINED_TEXT_PATHS + } + + retired_markers = ("book" + "worm", "debian" + "12") + for relative, text in texts.items(): + lowered = text.casefold() + for marker in retired_markers: if marker in lowered: failures.append( - f"{path}:{number}: retired Debian image generation marker remains: {marker}" - ) - items = [] - if code_file: - scalar_yaml = ( - path.suffix in {".yaml", ".yml"} and yaml_scalar_flags[number - 1] - ) - if ( - path.suffix in {".yaml", ".yml"} - and not scalar_yaml - and re.fullmatch(r"\s*(?:---|\.\.\.)\s*", yaml_source) - ): - yaml_anchors.clear() - item = assignment(path, yaml_source) if not scalar_yaml else None - if item is not None: - items.append(item) - if path.suffix in {".yaml", ".yml"} and not scalar_yaml: - items.extend(flow_mapping_image_assignments(yaml_source)) - for name in unsupported_yaml_policy_keys(yaml_source): - failures.append( - f"{path}:{number}: unsupported YAML image policy-key " - f"syntax: {name}; use a simple block key or a single-line " - "flow mapping" - ) - for anchor_name in yaml_anchor_declarations(yaml_source): - yaml_anchors.pop(anchor_name, None) - anchor = compliant_scalar_anchor(yaml_source) - if anchor is not None: - yaml_anchors[anchor[0]] = anchor[1] - consumer_line = yaml_consumers.get(number, yaml_source) - consumer = ( - is_container_consumer(consumer_line) - and (code_file or markdown_code) - and not comment - ) - reference_context = not comment and ( - is_dockerfile(path) - or path.suffix in {".yaml", ".yml", ".sh", ".bash"} - or markdown_code - or bool(items) - or consumer - ) - context_references = build_context_references(yaml_source) - if reference_context and not consumer: - for reference in references(yaml_source): - if reference not in context_references: - append_reference_failures(path, number, reference, failures) - if reference_context: - for reference in context_references: - append_reference_failures( - path, - number, - reference, - failures, - "Docker build context", + f"{relative}: retired Debian image generation marker remains: {marker}" ) - for context in computed_build_contexts(yaml_source): + + for relative in DOCKERFILES: + text = texts[relative] + bases = FROM_RE.findall(text) + if not bases: + failures.append(f"{relative}: no FROM instruction found") + continue + for base in bases: + if not DIGEST_PIN_RE.search(base): failures.append( - f"{path}:{number}: computed Docker build context is not allowed: " - f"docker-image://{context}; use a literal static " - "docker-image:// reference" + f"{relative}: upstream base is not pinned by immutable digest: {base}" ) - for name, value in items: - canonical = name.casefold() - anchor = re.fullmatch(r"\*(?P[A-Za-z0-9_-]+)", value.strip()) - anchored_reference = ( - yaml_anchors.get(anchor.group("name")) - if anchor is not None and path.suffix in {".yaml", ".yml"} - else None - ) - dependencies = { - found.group("name").casefold() for found in IMAGE_VAR_RE.finditer(value) - } - has_literal = ( - exact_reference(value) is not None or anchored_reference is not None - ) - alias = is_image_alias(value) - has_template = ( - not has_literal and not dependencies and has_safe_image_template(value) - ) - positional = canonical == "image" and re.fullmatch( - r"""["']?\$(?:[1-9]|\{[1-9](?::-[^{}]+)?\})["']?;?""", - value.strip(), - ) - computed = not positional and ( - (not has_literal and not alias and not has_template) - or COMPUTED_VALUE_RE.search(value) is not None - ) - strict = (strict_code_assignments and policy_image_name(name)) or ( - path.suffix in {".yaml", ".yml"} - and yaml_policy_image_name(name) - and number not in exempt_assignments - ) - records.append((number, canonical, value, dependencies, computed, strict)) - if ((has_literal or has_template) and not computed) or positional: - resolved.add(canonical) - bare_assignment = any( - path.suffix in {".yaml", ".yml"} - and yaml_policy_image_name(name) - or strict_code_assignments - and policy_image_name(name) - for name, _ in items - ) - if ( - not comment - and not consumer - and BARE_DEFAULT_FAMILY_RE.search(yaml_source) - and (is_dockerfile(path) or bare_assignment) - ): - append_bare_failures(path, number, yaml_source, failures) - changed = True - while changed: - changed = False - for _, name, _, dependencies, computed, strict in records: - if ( - name not in resolved - and not computed - and dependencies - and dependencies <= resolved - ): - resolved.add(name) - changed = True - for number, name, value, _, computed, strict in records: - if strict and (computed or name not in resolved): - failures.append( - f"{path}:{number}: computed or unresolved image assignment " - f"is not allowed: {value.strip()}; use a literal static repository " - "and digest, or a reviewed allow-validated-image validator contract" - ) - source_lines = ( - [yaml_code(line) for line in lines] - if path.suffix in {".yaml", ".yml"} - else lines - ) - consumer_records = logical_lines(source_lines, markdown_flags) - consumer_records.extend( - (number, line, False) for number, line in yaml_consumers.items() - ) - for number, line, markdown_code in consumer_records: - image_values, option_errors, _ = container_image_operands(line) - if ( - line.lstrip().startswith(("#", "//")) - or not (code_file or markdown_code) - or not (image_values or option_errors) - ): - continue - failures.extend(f"{path}:{number}: {message}" for message in option_errors) - if not image_values: - continue - image_text = " ".join(image_values) - for reference in references(image_text): - append_reference_failures(path, number, reference, failures) - append_bare_failures(path, number, image_text, failures) - variables = { - match.group("name").casefold() - for match in IMAGE_VAR_RE.finditer(image_text) - } - unresolved = sorted(variables - resolved) - if ( - not any( - re.search("[A-Za-z]", repository_and_tag(item)[0]) - for item in references(image_text) - ) - and not BARE_DEFAULT_FAMILY_RE.search(image_text) - and (not variables or unresolved) - ): - detail = ( - f"; unresolved image variables: {', '.join(unresolved)}" - if unresolved - else "" - ) - failures.append( - f"{path}:{number}: Docker/Podman image consumer must use a " - f"literal or a statically resolved *_IMAGE assignment{detail}" - ) - return list(dict.fromkeys(failures)) - - -def require( - text: str, needle: str, path: Path, detail: str, failures: list[str] -) -> None: - if needle not in text: - failures.append(f"{path}: missing {detail}: {needle!r}") - - -def runtime_stage(text: str) -> str: - marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" - offset = text.find(marker) - return text[offset:] if offset >= 0 else "" - -def product_contracts(texts: dict[Path, str], failures: list[str]) -> None: - for path in PRODUCT_DOCKERFILES: - text, runtime = texts.get(path, ""), runtime_stage(texts.get(path, "")) require( text, f"FROM {DISTROLESS_RUNTIME} AS runtime", - path, + relative, "Distroless Debian 13 non-root final runtime", failures, ) + runtime = runtime_stage(text) + for forbidden in ("\nRUN ", "apt-get", "/bin/sh", "curl ", "wget "): + if forbidden in runtime: + failures.append( + f"{relative}: final Distroless runtime contains {forbidden.strip()!r}" + ) require( runtime, "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3", - path, + relative, "binary healthcheck", failures, ) - for forbidden in ("\nRUN ", "apt-get", "/bin/sh", "curl ", "wget "): - if forbidden in runtime: - failures.append( - f"{path}: final Distroless runtime contains {forbidden.strip()!r}" - ) - for path in PRODUCT_DOCKERFILES[:3]: + + for relative in RUST_BUILDER_DOCKERFILES: require( - texts.get(path, ""), + texts[relative], f"FROM {RUST_BUILDER} AS builder", - path, + relative, "pinned Debian 13 Rust builder", failures, ) - for path in PRODUCT_DOCKERFILES[3:]: - text = texts.get(path, "") + + for relative in PREPARATION_DOCKERFILES: + text = texts[relative] if not text.startswith(f"# syntax={DOCKERFILE_FRONTEND}\n"): failures.append( - f"{path}: pinned Dockerfile frontend must be the first line" + f"{relative}: pinned Dockerfile frontend must be the first line" ) - for needle, detail in ( - (f"FROM {DEBIAN_PREPARATION} AS runtime-root", "runtime preparation base"), - ("ARG SOURCE_DATE_EPOCH=0", "fixed release filesystem timestamp"), - ( - "RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin", - "ephemeral release input mount", - ), - ( - 'find /workspace/runtime-root -exec touch -h --date="@${SOURCE_DATE_EPOCH}" {} +', - "normalized release filesystem metadata", - ), - ): - require(text, needle, path, detail, failures) - for path in RELAY_DOCKERFILES: - text = texts.get(path, "") + require( + text, + f"FROM {DEBIAN_PREPARATION} AS runtime-root", + relative, + "pinned Debian 13 runtime preparation base", + failures, + ) + require( + text, + "ARG SOURCE_DATE_EPOCH=0", + relative, + "fixed release filesystem timestamp", + failures, + ) + require( + text, + "RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin", + relative, + "ephemeral release input mount", + failures, + ) + require( + text, + 'find /workspace/runtime-root -exec touch -h --date="@${SOURCE_DATE_EPOCH}" {} +', + relative, + "normalized release filesystem metadata", + failures, + ) + + for relative in RELAY_DOCKERFILES: + text = texts[relative] require( text, "/usr/local/bin/registry-relay-rhai-worker", - path, + relative, "Relay worker binary", failures, ) require( runtime_stage(text), 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', - path, + relative, "absolute Relay entrypoint", failures, ) + + product_notary = texts[Path("products/notary/Dockerfile")] require( - texts.get(PRODUCT_DOCKERFILES[2], ""), + product_notary, 'ARG REGISTRY_NOTARY_FEATURES="registry-notary-cel,pkcs11"', - PRODUCT_DOCKERFILES[2], + Path("products/notary/Dockerfile"), "PKCS#11-enabled product build", failures, ) - for path in NOTARY_DOCKERFILES: - text = texts.get(path, "") - for source, needle, detail in ( - (text, "registry-notary-cel-worker", "Notary CEL worker binary"), - ( - runtime_stage(text), - 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', - "absolute Notary entrypoint", - ), - ( - text, - "chown -R 65532:65532", - "numeric nonroot-owned Notary runtime directories", - ), - ( - runtime_stage(text), - "WORKDIR /var/lib/registry-notary", - "Notary working directory", - ), - ): - require(source, needle, path, detail, failures) + for relative in NOTARY_DOCKERFILES: + text = texts[relative] + require( + text, + "registry-notary-cel-worker", + relative, + "Notary CEL worker binary", + failures, + ) + require( + runtime_stage(text), + 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', + relative, + "absolute Notary entrypoint", + failures, + ) + require( + text, + "chown -R 65532:65532", + relative, + "numeric nonroot-owned Notary runtime directories", + failures, + ) + require( + runtime_stage(text), + "WORKDIR /var/lib/registry-notary", + relative, + "Notary working directory", + failures, + ) if re.search( r"^\s*(?:COPY|ADD)\b[^\n]*(?:\.so\b|pkcs11[^/\s]*module)", text, re.IGNORECASE | re.MULTILINE, ): failures.append( - f"{path}: vendor PKCS#11 modules must remain external read-only mounts" + f"{relative}: vendor PKCS#11 modules must remain external read-only mounts" ) - workflow = texts.get(Path(".github/workflows/release.yml"), "") + + workflow = texts[Path(".github/workflows/release.yml")] + binary_recipe = texts[Path("release/scripts/build-release-binaries.sh")] + tutorial_check = texts[Path("docs/site/scripts/check-registryctl-tutorials.sh")] require( workflow, f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", @@ -1399,132 +258,31 @@ def product_contracts(texts: dict[Path, str], failures: list[str]) -> None: failures, ) require( - texts.get(Path("release/scripts/build-release-binaries.sh"), ""), + binary_recipe, "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", Path("release/scripts/build-release-binaries.sh"), "PKCS#11-enabled release build", failures, ) require( - texts.get( - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "" - ), - RUST_BUILDER, - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), - "pinned Debian 13 live-journey builder", + tutorial_check, + f'BUILDER_IMAGE="{RUST_BUILDER}"', + Path("docs/site/scripts/check-registryctl-tutorials.sh"), + "pinned Debian 13 registryctl tutorial builder", failures, ) + + live_journey = texts[ + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") + ] require( - texts.get( - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "" - ), - "postgres:16-trixie@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", + live_journey, + RUST_BUILDER, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), - "pinned Debian 13 live-journey PostgreSQL", + "pinned Debian 13 live-journey builder", failures, ) - tutorial = texts.get(TUTORIAL_SCRIPT, "") - ci = texts.get(CI_WORKFLOW, "") - start = ci.find("- name: Cache source-under-test Cargo build") - end = ci.find("\n - name: Execute registryctl tutorials from source", start) - cache = "" if start < 0 else ci[start:] if end < 0 else ci[start:end] - for source, needle, path, detail in ( - ( - tutorial, - 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', - TUTORIAL_SCRIPT, - "registryctl tutorial linux target path", - ), - ( - tutorial, - 'CARGO_HOME_DIR="$REPO_ROOT/target/registryctl-tutorial-cargo-home"', - TUTORIAL_SCRIPT, - "registryctl tutorial Cargo home path", - ), - ( - cache, - "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", - CI_WORKFLOW, - "registryctl tutorial cache builder identity", - ), - ( - cache, - "target/registryctl-tutorial-linux-amd64", - CI_WORKFLOW, - "registryctl tutorial linux target cache path", - ), - ( - cache, - "target/registryctl-tutorial-cargo-home", - CI_WORKFLOW, - "registryctl tutorial Cargo home cache path", - ), - ): - require(source, needle, path, detail, failures) - if "restore-keys:" in cache: - failures.append( - f"{CI_WORKFLOW}: registryctl tutorial cache must not restore from pre-builder-identity keys" - ) - - -def validated_image_contracts(texts: dict[Path, str], failures: list[str]) -> None: - for validator, (_, _, requirements) in VALIDATED_IMAGE_CONTRACTS.items(): - for path, needle in requirements: - require( - texts.get(path, ""), - needle, - path, - f"{validator} validated dynamic image contract", - failures, - ) - -def check_repository(root: Path = ROOT) -> list[str]: - failures: list[str] = [] - try: - discovered = discover_maintained_surfaces(root) - except ImageSurfaceError as error: - return [str(error)] - paths = tuple(sorted(set(discovered) | set(REQUIRED_PRODUCT_SURFACES))) - texts: dict[Path, str] = {} - total = 0 - for path in paths: - try: - text, size = read_text(root, path) - except ImageSurfaceError as error: - failures.append(str(error)) - continue - if text is None: - if path in REQUIRED_PRODUCT_SURFACES: - failures.append(f"missing maintained image surface: {path}") - continue - total += size - if total > MAX_TOTAL_TEXT_BYTES: - return [f"maintained text exceeds {MAX_TOTAL_TEXT_BYTES} total bytes"] - texts[path] = text - try: - executable = bool((root / path).stat().st_mode & 0o111) - failures.extend(scan_surface(path, text, executable=executable)) - except ImageSurfaceError as error: - failures.append(str(error)) - dockerfiles = [path for path in discovered if is_dockerfile(path)] - if not dockerfiles: - failures.append("no maintained Dockerfiles discovered") - for path in dockerfiles: - bases, stages = FROM_RE.findall(texts.get(path, "")), set() - if not bases: - failures.append(f"{path}: no FROM instruction found") - for base, stage in bases: - if base.casefold() not in stages | {"scratch"} and not DIGEST_RE.search( - base - ): - failures.append( - f"{path}: upstream base is not pinned by immutable digest: {base}" - ) - if stage: - stages.add(stage.casefold()) - product_contracts(texts, failures) - validated_image_contracts(texts, failures) return failures diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index ef4c7691..83d41fc5 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -3,17 +3,30 @@ import importlib.util import shutil -import subprocess import tempfile import unittest from pathlib import Path -from unittest import mock ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "release/scripts/check-debian13-images.py" -DIGEST = "a" * 64 -PINNED_RUST = f"rust:1.95-trixie@sha256:{DIGEST}" +TUTORIAL_CHECK = Path("docs/site/scripts/check-registryctl-tutorials.sh") + +EXPECTED_SURFACES = { + Path(".github/workflows/release.yml"), + Path("crates/registry-relay/Dockerfile"), + Path("crates/registry-relay/Dockerfile.demo"), + Path("crates/registry-relay/docs/ops.md"), + Path("crates/registry-relay/docs/security-assurance.md"), + Path("crates/registry-relay/scripts/check_docker_build_contract.py"), + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + TUTORIAL_CHECK, + Path("products/notary/Dockerfile"), + Path("products/notary/docs/security-assurance.md"), + Path("release/docker/Dockerfile.registry-notary"), + Path("release/docker/Dockerfile.registry-relay"), + Path("release/scripts/build-release-binaries.sh"), +} def load_module(): @@ -26,1233 +39,156 @@ def load_module(): class Debian13ImageCheckTest(unittest.TestCase): - def setUp(self) -> None: - self.module = load_module() - - def scan(self, relative: str, text: str) -> list[str]: - return self.module.scan_surface(Path(relative), text) - - def assert_failure(self, relative: str, text: str, fragment: str) -> None: - failures = self.scan(relative, text) - self.assertTrue(any(fragment in item for item in failures), failures) - - def assert_clean(self, relative: str, text: str) -> None: - self.assertEqual([], self.scan(relative, text)) - - def copy_required(self, root: Path) -> None: - for relative in self.module.REQUIRED_PRODUCT_SURFACES: + @classmethod + def setUpClass(cls) -> None: + cls.module = load_module() + + def fixture(self) -> Path: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + for relative in self.module.MAINTAINED_TEXT_PATHS: destination = root / relative destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ROOT / relative, destination) + return root - def test_real_repository_follows_debian13_contract(self) -> None: - self.assertEqual([], self.module.check_repository(ROOT)) - - def test_discovery_uses_tracked_files_and_documented_exclusions(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - tracked = ( - Path("Dockerfile.worker"), - Path("scripts/check.sh"), - Path("release/notes/old.md"), - Path("docs/.research/notes.md"), - Path("external/vendor.md"), - Path("crates/example/resources/scalar/generated.js"), - Path("release/scripts/test_check_debian13_images.py"), - ) - for relative in tracked: - target = root / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text("fixture\n") - subprocess.run(["git", "init", "-q", str(root)], check=True) - subprocess.run( - ["git", "-C", str(root), "add", "."], - check=True, - ) - (root / "untracked.sh").write_text("fixture\n") - - discovered = self.module.discover_maintained_surfaces(root) - - self.assertIn(Path("Dockerfile.worker"), discovered) - self.assertIn(Path("scripts/check.sh"), discovered) - for relative in tracked[2:] + (Path("untracked.sh"),): - self.assertNotIn(relative, discovered) - - def test_path_file_line_and_total_bounds_fail_explicitly(self) -> None: - listed = b"\0".join( - f"file-{index}".encode() - for index in range(self.module.MAX_TRACKED_PATHS + 1) - ) - completed = subprocess.CompletedProcess([], 0, stdout=listed) - with ( - mock.patch.object( - self.module.subprocess, - "run", - return_value=completed, - ), - self.assertRaisesRegex( - self.module.ImageSurfaceError, - "tracked path count exceeds", - ), - ): - self.module.discover_maintained_surfaces(Path("/unused")) - - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "text file exceeds", - ): - self.scan( - "large.yaml", - "x" * (self.module.MAX_TEXT_FILE_BYTES + 1), - ) - with self.assertRaisesRegex( - self.module.ImageSurfaceError, - "line exceeds", - ): - self.scan( - "line.yaml", - "x" * (self.module.MAX_LINE_CHARS + 1), - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required(root) - with mock.patch.object(self.module, "MAX_TOTAL_TEXT_BYTES", 1): - failures = self.module.check_repository(root) - self.assertEqual( - ["maintained text exceeds 1 total bytes"], - failures, - ) - - def test_debian_family_literal_policy_is_table_driven(self) -> None: - cases = [ - ("Dockerfile", f"FROM {PINNED_RUST}\n", None), - ("compose.yaml", f"services:\n app:\n image: {PINNED_RUST}\n", None), - ( - "script.sh", - "docker run --rm rust:1.95-trixie true\n", - "not pinned by immutable digest", - ), - ( - "workflow.yml", - f"env:\n BUILDER_IMAGE: rust:1.95@sha256:{DIGEST}\n", - "does not declare Trixie/Debian 13", - ), - ( - "compose.yaml", - "image: registry.local/team_name/debian:trixie\n", - "not pinned by immutable digest", - ), - ( - "script.sh", - "docker run registry.local/first_team/second_team/rust:1.95-trixie\n", - "not pinned by immutable digest", - ), - ( - "compose.yaml", - "image: registry.local/team_name/rust\n", - "bare Debian-default", - ), - ( - "compose.yaml", - "image: registry.local/first_team/second_team/postgres\n", - "bare Debian-default", - ), - ( - "Dockerfile", - f"FROM rust:1.95-trixie@sha256:{DIGEST.upper()}\n", - None, - ), - ( - "guide.md", - "Use registry.local/team_name/debian:trixie in prose.\n", - None, - ), - ( - "guide.md", - "Mirror registry.local/first_team/second_team/rust in prose.\n", - None, - ), - ( - "images.py", - "BUILDER_IMAGE: str = 'python:3.13-slim-trixie'\n", - "not pinned by immutable digest", - ), - ( - "images.ts", - "const builderImage = 'golang:1.25-trixie';\n", - "not pinned by immutable digest", - ), - ("module.js", "import path from 'node:path';\n", None), - ("data.yaml", "identifier: did:web\nport: 65532:65532\n", None), - ] - for family, version in ( - ("rust", "1.95"), - ("node", "22"), - ("python", "3.13"), - ("golang", "1.25"), - ("postgres", "16"), - ): - cases.extend( - ( - ( - "compose.yaml", - f"{'container' if family == 'node' else 'image'}: {family}\n", - "bare Debian-default", - ), - ("build.sh", f"BUILDER_IMAGE='{family}'\n", "bare Debian-default"), - ( - "script.sh", - f"docker run --rm {family} true\n", - "bare Debian-default", - ), - ( - "script.sh", - f"docker run --rm {family}:{version}\n", - "not pinned", - ), - ( - "script.sh", - f"docker run --rm {family}:{version}-slim\n", - "not pinned", - ), - ( - "workflow.yml", - f"container: {family}@sha256:{DIGEST}\n", - "does not declare", - ), - ( - "compose.yaml", - f"image: registry.test:5000/team/{family}\n", - "bare Debian-default", - ), - ( - "workflow.yml", - f"container: registry.test/team/{family}@sha256:{DIGEST}\n", - "does not declare", - ), - ("script.sh", f"docker run {family}:{version}-alpine\n", None), - ("script.sh", f"docker run {family}:{version}-windows\n", None), - ("guide.md", f"The {family} image is available.\n", None), - ) - ) - for relative, text, expected in cases: - with self.subTest(relative=relative, text=text): - failures = self.scan(relative, text) - if expected is None: - self.assertEqual([], failures) - else: - self.assertTrue( - any(expected in failure for failure in failures), - failures, - ) - - def test_retired_markers_are_global_with_markdown_prose_exemptions(self) -> None: - for text in ( - "Historical book" + "worm base\n", - "# image used bullseye during testing\n", - "unused_image: debian" + "12\n", - ): - with self.subTest(text=text): - self.assert_failure( - "notes.txt", - text, - "retired Debian image generation marker", - ) - - self.assert_clean( - "design.md", - "The book" - "worm comparison remains historical. " - "\n', - ) - invalid = ( - "```sh\n" - "# book" - "worm \n' - "```\n" - ) - failures = self.scan("design.md", invalid) - self.assertTrue( - any("invalid Debian image prose exemption" in item for item in failures), - failures, - ) - self.assertTrue( - any("retired Debian image" in item for item in failures), - failures, - ) - - def test_wrappers_options_operators_and_malformed_shell_still_scan_literals( + def assert_has_failure( self, + root: Path, + fragment: str, ) -> None: - commands = ( - "docker --tlsverify pull -a rust:1.95-trixie", - "podman --remote pull --all-tags rust:1.95-trixie", - "docker image pull --disable-content-trust rust:1.95-trixie", - "/usr/bin/docker run --rm rust:1.95-trixie", - "env -S '-i /usr/bin/docker run --rm rust:1.95-trixie'", - "sudo -s /usr/bin/docker run --rm rust:1.95-trixie", - "bash --noprofile -c 'docker run --rm rust:1.95-trixie'", - "if ! time -p docker run --rm rust:1.95-trixie; then true; fi", - "docker unknown rust:1.95-trixie", - "docker run 'rust:1.95-trixie", - ) - for command in commands: - with self.subTest(command=command): - self.assert_failure( - "script.sh", - command + "\n", - "not pinned by immutable digest", - ) - - for command in ( - "echo ok; podman run --unknown value rust:1.95-trixie", - f"docker run --unknown value {PINNED_RUST}", - ): - with self.subTest(command=command): - self.assert_failure( - "script.sh", - command + "\n", - "unsupported Docker/Podman option has unknown arity: --unknown", - ) - - for command in ( - f"sudo --unknown docker unknown {PINNED_RUST}", - f"bash --unknown -c 'docker run {PINNED_RUST}'", - ): - with self.subTest(command=command): - self.assert_clean("script.sh", command + "\n") - - def test_bare_debian_is_finite_to_image_code_contexts(self) -> None: - cases = ( - ("Dockerfile", "FROM debian\n"), - ("images/relay.dockerfile", "FROM debian\n"), - ("Dockerfile", "COPY --from=localhost:5000/debian /x /x\n"), - ("Dockerfile", "RUN --mount=from=debian,target=/x true\n"), - ("compose.yaml", "services:\n app:\n image: debian\n"), - ("workflow.yml", "jobs:\n test:\n container: debian\n"), - ("images.py", "BUILDER_IMAGE = 'debian'\n"), - ("script.sh", "docker run --rm debian true\n"), - ("guide.md", "```console\n$ podman pull debian\n```\n"), + failures = self.module.check_repository(root) + self.assertTrue( + any(fragment in failure for failure in failures), + failures, ) - for relative, text in cases: - with self.subTest(relative=relative, text=text): - self.assert_failure( - relative, - text, - "bare Debian image reference", - ) - for relative, text in ( - ("images/relay.dockerfile", f"FROM {PINNED_RUST}\n"), - ("guide.md", "Debian is the supported distribution.\n"), - ("guide.md", "```text\ndocker run debian\n```\n"), - ("script.sh", "# docker run --rm debian\n"), - ("module.js", '// docker run "$OTHER_IMAGE"\n'), - ("module.py", "distribution = 'debian'\n"), - ("module.py", "distribution = 'Debian'\n"), - ): - with self.subTest(relative=relative, text=text): - self.assert_clean(relative, text) - - def test_image_assignments_resolve_literals_and_reject_computation(self) -> None: - clean = ( - f'DEFAULT_BUILDER_IMAGE="{PINNED_RUST}"\n' - 'BUILDER_IMAGE="${DEFAULT_BUILDER_IMAGE}"\n' - 'docker run --rm "$BUILDER_IMAGE" true\n' - ) - self.assert_clean("build.sh", clean) - self.assert_clean( - "function.sh", - 'start() {\n local image="$1"\n docker run "$image"\n}\n', - ) - self.assert_clean( - "local.sh", - 'RELAY_IMAGE="registryctl-relay:$RUN_ID"\ndocker run "$RELAY_IMAGE"\n', - ) - self.assert_clean( - "static.sh", - 'APP_IMAGE="alpine:3.22"\ndocker run --rm "$APP_IMAGE"\n', - ) - self.assert_clean( - "fallback.sh", - 'FIRST_IMAGE="alpine:3.22"\nSECOND_IMAGE="busybox:1.37"\n' - 'APP_IMAGE="${FIRST_IMAGE:-$SECOND_IMAGE}"\n' - 'docker run --rm "$APP_IMAGE"\n', - ) + def test_current_repository_follows_contract(self) -> None: + self.assertEqual([], self.module.check_repository(ROOT)) - cases = ( - ( - "computed.sh", - 'APP_IMAGE="$(select-image)"\ndocker run --rm "$APP_IMAGE"\n', - "unresolved image variables: app_image", - ), - ( - "alias.sh", - 'OTHER_IMAGE="$(select-image)"\nAPP_IMAGE="$OTHER_IMAGE"\n' - 'docker run --rm "$APP_IMAGE"\n', - "unresolved image variables: app_image", - ), - ( - "multi.sh", - 'GOOD_IMAGE="alpine:3.22"\nBAD_IMAGE="$(select-image)"\n' - 'APP_IMAGE="${GOOD_IMAGE:-$BAD_IMAGE}"\n' - 'docker run --rm "$APP_IMAGE"\n', - "unresolved image variables: app_image", - ), - ( - "images.py", - "BUILDER_IMAGE = make_image()\n", - "computed or unresolved image assignment", - ), - ( - "images.ts", - "const builderImage = 'rust:1.95-' + 'trixie';\n", - "computed or unresolved image assignment", - ), - ( - "cycle.sh", - 'BASE_IMAGE="$BUILDER_IMAGE"\nBUILDER_IMAGE="$BASE_IMAGE"\n', - "computed or unresolved image assignment", - ), - ( - "build.sh", - 'docker run --rm "$OTHER_IMAGE"\n', - "must use a literal or a statically resolved *_IMAGE assignment", - ), - ( - "build.sh", - 'docker run --publish 127.0.0.1:8080 "$OTHER_IMAGE"\n', - "must use a literal or a statically resolved *_IMAGE assignment", - ), - ( - "build.sh", - 'OTHER_IMAGE="$(compute)"\nBUILDER_IMAGE="$OTHER_IMAGE"\n', - "computed or unresolved image assignment", - ), - ( - "build.sh", - 'docker pull "$container"\n', - "must use a literal or a statically resolved *_IMAGE assignment", - ), - ) - for relative, text, expected in cases: + def test_inventory_covers_every_maintained_surface(self) -> None: + self.assertEqual(EXPECTED_SURFACES, set(self.module.MAINTAINED_TEXT_PATHS)) + for relative in EXPECTED_SURFACES: with self.subTest(relative=relative): - self.assert_failure(relative, text, expected) + self.assertTrue((ROOT / relative).is_file()) - def test_image_templates_use_only_bounded_static_forms(self) -> None: - cases = ( - 'APP_IMAGE="${UNSAFE_IMAGE:-alpine:3.22}"\n', - 'APP_IMAGE="${BASE_IMAGE/alpine/$TARGET}"\n', - 'APP_IMAGE="${REPOSITORY}:${TAG}"\n', - 'APP_IMAGE="rust:$TAG"\n', - ) - for assignment in cases: - with self.subTest(assignment=assignment): - self.assert_failure( - "template.sh", - assignment + 'docker run --rm "$APP_IMAGE"\n', - "unresolved image variables: app_image", + failures = self.module.check_repository(Path("/does-not-exist")) + for relative in EXPECTED_SURFACES: + with self.subTest(missing=relative): + self.assertIn( + f"missing maintained image surface: {relative}", + failures, ) - self.assert_clean( - "aliases.sh", - 'FIRST_IMAGE="alpine:3.22"\nSECOND_IMAGE="busybox:1.37"\n' - 'APP_IMAGE="${FIRST_IMAGE:-$SECOND_IMAGE}"\n' - 'docker run --rm "$APP_IMAGE"\n', - ) - self.assert_clean( - "tag.sh", - 'APP_IMAGE="registryctl-relay:$RUN_ID"\ndocker run --rm "$APP_IMAGE"\n', - ) - - def test_yaml_policy_image_keys_fail_closed_for_dynamic_default_families( - self, - ) -> None: + def test_retired_markers_are_rejected_regardless_of_text_syntax(self) -> None: cases = ( - ("compose.yaml", "services:\n app:\n image: rust:$TAG\n"), - ("compose.yaml", "services:\n app:\n image: rust:${TAG}\n"), - ("compose.yaml", "services:\n app:\n image: debian:${TAG}\n"), - ("compose.yaml", "services:\n app:\n image: postgres:$PG_TAG\n"), - ( - "compose.yaml", - "services:\n app:\n image: postgres:$PG_TAG-notalpine\n", - ), - ( - ".github/workflows/example.yml", - "jobs:\n test:\n container: rust:$TAG\n", - ), - ("images.yaml", "builder_image: rust:$TAG\n"), + "# historical book" + "worm image\n", + 'IMAGE="debian' + '12"\n', + '{"base": "BOOK' + 'WORM"}\n', ) - for relative, text in cases: - with self.subTest(relative=relative, text=text): - self.assert_failure( - relative, - text, - "computed or unresolved image assignment", + for suffix in cases: + with self.subTest(suffix=suffix): + root = self.fixture() + target = root / TUTORIAL_CHECK + target.write_text( + target.read_text(encoding="utf-8") + suffix, + encoding="utf-8", ) - - self.assert_clean( - "compose.yaml", - f"services:\n app:\n image: {PINNED_RUST}\n", - ) - self.assert_clean( - "compose.yaml", - "services:\n app:\n image: registryctl-relay:$TAG\n", - ) - self.assert_clean( - ".github/workflows/example.yml", - "jobs:\n test:\n services:\n postgres:\n" - " image: postgres:${{ matrix.postgresql }}-alpine\n", - ) - self.assert_clean( - "metadata.yaml", - "metadata_image: rust:$TAG\nartifact_image: postgres:$PG_TAG\n", - ) - - def test_yaml_flow_mapping_image_keys_use_assignment_policy(self) -> None: - cases = ( - "services:\n app: {image: rust:$TAG}\n", - 'services:\n app: {image: "rust:${TAG}"}\n', - 'services:\n app: {"image": rust:$TAG}\n', - "jobs:\n test: {container: postgres:$PG_TAG}\n", - "images: {builder_image: rust:$TAG}\n", - "x-service: &defaults {image: rust:$TAG}\n" - "services:\n app:\n <<: *defaults\n", - 'x-service: &defaults {"image": rust:$TAG}\n' - "services:\n app:\n <<: *defaults\n", - ) - for text in cases: - with self.subTest(text=text): - self.assert_failure( - "compose.yaml", - text, - "computed or unresolved image assignment", + self.assert_has_failure( + root, + f"{TUTORIAL_CHECK}: retired Debian image generation marker remains", ) - self.assert_clean( - "compose.yaml", - f"services:\n app: {{image: {PINNED_RUST}}}\n", - ) - self.assert_clean( - "compose.yaml", - f'services:\n app: {{"image": {PINNED_RUST}}}\n', - ) - self.assert_clean( - "compose.yaml", - "services:\n app: {image: registryctl-relay:$TAG}\n", - ) - self.assert_clean( - "metadata.yaml", - "metadata: {metadata_image: rust:$TAG}\n", - ) - self.assert_clean( - ".github/workflows/example.yml", - "steps:\n - run: |\n jq '{image: $builder_image}' input.json\n", - ) - self.assert_clean( - "compose.yaml", - f"x-service: &defaults {{image: {PINNED_RUST}}}\n" - "services:\n app:\n <<: *defaults\n", - ) - self.assert_clean( - "compose.yaml", - f'x-service: &defaults {{"image": {PINNED_RUST}}}\n' - "services:\n app:\n <<: *defaults\n", - ) - - def test_yaml_quoted_and_unsupported_policy_key_syntax_fails_closed( - self, - ) -> None: - for text in ( - 'services:\n app:\n "image": rust:$TAG\n', - "services:\n app:\n 'container': postgres:$PG_TAG\n", - ): - with self.subTest(text=text): - self.assert_failure( - "compose.yaml", - text, - "computed or unresolved image assignment", - ) - - for text in ( - "services:\n app: {image:\n rust:$TAG}\n", - "services:\n app:\n ? image\n : rust:$TAG\n", - "services:\n app:\n image:\n rust:$TAG\n", - ): - with self.subTest(text=text): - self.assert_failure( - "compose.yaml", - text, - "unsupported YAML image policy-key syntax", - ) - - self.assert_clean( - "compose.yaml", - f'services:\n app:\n "image": {PINNED_RUST}\n', - ) - self.assert_clean( - "compose.yaml", - f"jobs:\n test:\n 'container': {PINNED_RUST}\n", - ) - - def test_yaml_policy_flow_text_inside_comments_is_ignored(self) -> None: - self.assert_clean( - "compose.yaml", - "# app: {image: rust:$TAG}\n" - "services:\n" - f" app: {{image: {PINNED_RUST}}} # ignored: {{image: rust:$TAG}}\n", - ) - - def test_yaml_scalar_image_aliases_require_local_literal_anchors(self) -> None: - self.assert_clean( - "compose.yaml", - f"x-image: &runtime {PINNED_RUST}\n" - "services:\n app:\n image: *runtime\n", - ) - self.assert_clean( - "compose.yaml", - f'"x-image": &runtime {PINNED_RUST}\n' - 'services:\n app: {"image": *runtime}\n', - ) - for text, expected in ( - ( - "x-image: &runtime rust:$TAG\nservices:\n app:\n image: *runtime\n", - "computed or unresolved image assignment", - ), - ( - "x-image: &runtime rust:1.95-trixie\n" - "services:\n app:\n image: *runtime\n", - "not pinned by immutable digest", - ), - ( - f"services:\n app:\n image: *runtime\n" - f"x-image: &runtime {PINNED_RUST}\n", - "computed or unresolved image assignment", - ), - ( - f"x-image: &runtime {PINNED_RUST}\n---\n" - "services:\n app:\n image: *runtime\n", - "computed or unresolved image assignment", - ), - ( - f"x-image: &runtime {PINNED_RUST}\n" - "replacement: &runtime rust:$TAG\n" - "services:\n app:\n image: *runtime\n", - "computed or unresolved image assignment", - ), - ( - f"x-image: &runtime {PINNED_RUST}\n" - "replacement: {value: &runtime rust:$TAG}\n" - "services:\n app:\n image: *runtime\n", - "computed or unresolved image assignment", - ), - ): - with self.subTest(text=text): - self.assert_failure("compose.yaml", text, expected) - - def test_validated_image_annotations_are_path_and_contract_scoped(self) -> None: - registryctl_path = "crates/registryctl/src/templates/compose.yaml" - registryctl_annotation = ( - "# debian13-policy: allow-validated-image " - 'validator="registryctl-image-lock" ' - 'reason="registryctl renders a validated digest image lock here"\n' - ) - relay_path = "release/conformance/relay-oidc/docker-compose.yml" - relay_annotation = ( - "# debian13-policy: allow-validated-image " - 'validator="relay-oidc-smoke" ' - 'reason="the runner validates the fixed repository and digest"\n' - ) - self.assert_clean( - registryctl_path, - registryctl_annotation + "image: {{relay_image}}\n", - ) - self.assert_clean( - relay_path, - relay_annotation + "image: ${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:" - "?runner must provide digest-pinned Registry Relay image}\n", - ) - self.assert_failure( - "other/compose.yaml", - registryctl_annotation + "image: {{relay_image}}\n", - "invalid Debian image policy annotation", - ) - for value in ("rust:$TAG", "rust:{{relay_image}}"): - with self.subTest(value=value): - self.assert_failure( - registryctl_path, - registryctl_annotation + f"image: {value}\n", - "computed or unresolved image assignment", - ) - relay_value = ( - "${REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE:" - "?runner must provide digest-pinned Registry Relay image}" - ) - for relative, annotation, value in ( - (registryctl_path, registryctl_annotation, relay_value), - (relay_path, relay_annotation, "{{relay_image}}"), - ( - registryctl_path, - registryctl_annotation, - "${EVIL_IMAGE:?unreviewed image input}", - ), - ): - with self.subTest(relative=relative, value=value): - self.assert_failure( - relative, - annotation + f"image: {value}\n", - "invalid Debian image policy annotation", - ) - - texts = { - path: (ROOT / path).read_text() - for _, _, requirements in self.module.VALIDATED_IMAGE_CONTRACTS.values() - for path, _ in requirements - } - failures: list[str] = [] - self.module.validated_image_contracts(texts, failures) - self.assertEqual([], failures) - for validator, ( - _, - _, - requirements, - ) in self.module.VALIDATED_IMAGE_CONTRACTS.items(): - for path, needle in requirements: - with self.subTest(validator=validator, path=path): - mutated = dict(texts) - mutated[path] = mutated[path].replace(needle, "", 1) - failures = [] - self.module.validated_image_contracts(mutated, failures) - self.assertTrue( - any(validator in failure for failure in failures), - failures, - ) - - def test_yaml_literals_cover_compose_merges_matrices_and_kubernetes(self) -> None: + def test_tutorial_builder_must_match_the_exact_pinned_image(self) -> None: + exact = f'BUILDER_IMAGE="{self.module.RUST_BUILDER}"' cases = ( - ( - "compose.yaml", - "x-service: &defaults\n image: rust:1.95-trixie\n" - "services:\n app:\n <<: *defaults\n", - ), - ( - ".github/workflows/example.yml", - "jobs:\n test:\n strategy:\n matrix:\n" - " image: [rust:1.95-trixie]\n include:\n" - " - image: rust:1.95-trixie\n" - " unused_image: [rust:1.95-trixie]\n" - " container: ${{ matrix.image }}\n", - ), - ( - "pod.yaml", - "apiVersion: v1\nkind: Pod\nspec:\n containers:\n - image: rust:1.95-trixie\n", - ), - ("malformed.yaml", "jobs: [\nimage: rust:1.95-trixie\n"), - ("documents.yaml", "description: first\n---\nimage: rust:1.95-trixie\n"), - ) - for relative, text in cases: + "", + 'BUILDER_IMAGE="rust:1.94-trixie@sha256:' + "a" * 64 + '"', + 'BUILDER_IMAGE="rust:1.95-trixie"', + ) + for replacement in cases: + with self.subTest(replacement=replacement): + root = self.fixture() + target = root / TUTORIAL_CHECK + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure( + root, + "missing pinned Debian 13 registryctl tutorial builder", + ) + + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: + for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): - self.assert_failure( - relative, - text, - "not pinned by immutable digest", - ) - - def test_compose_entrypoint_and_command_form_one_image_consumer(self) -> None: - self.assert_failure( - "compose.yaml", - "services:\n app:\n entrypoint: [docker]\n" - " command: [run, --rm, debian]\n", - "bare Debian image reference", - ) - self.assert_clean( - "compose.yaml", - "services:\n app:\n entrypoint: [docker]\n" - f" command: [run, --rm, {PINNED_RUST}]\n", - ) - self.assert_clean( - "compose.yaml", - "services:\n first:\n entrypoint: [docker]\n" - " second:\n command: [run, --rm, debian]\n", - ) - self.assert_clean( - "compose.yaml", - "services:\n app:\n entrypoint: [echo, docker]\n" - " command: [run, --rm, debian]\n", - ) - - def test_compose_block_sequences_preserve_mapping_and_list_scope(self) -> None: - self.assert_failure( - "compose.yaml", - "services:\n app:\n entrypoint:\n - docker\n" - " command:\n - run\n - --rm\n - debian\n", - "bare Debian image reference", - ) - self.assert_clean( - "compose.yaml", - "services:\n app:\n entrypoint:\n - docker\n" - f" command:\n - run\n - --rm\n - {PINNED_RUST}\n", - ) - self.assert_clean( - "compose.yaml", - "items:\n - entrypoint: [docker]\n - command: [run, --rm, debian]\n", - ) - self.assert_failure( - "compose.yaml", - "items:\n - name: app\n entrypoint: [docker]\n" - " command: [run, --rm, debian]\n", - "bare Debian image reference", - ) - - def test_compose_bounded_flow_folded_and_anchored_consumers(self) -> None: - self.assert_failure( - "compose.yaml", - "services:\n app: {entrypoint: [docker], command: [run, --rm, debian]}\n", - "bare Debian image reference", - ) - self.assert_failure( - "compose.yaml", - "services:\n app:\n entrypoint: [docker]\n" - " command: >-\n run\n --rm\n debian\n", - "bare Debian image reference", - ) - self.assert_failure( - "compose.yaml", - "x-engine: &container-engine [docker]\nservices:\n app:\n" - " entrypoint: *container-engine\n" - " command: [run, --rm, debian]\n", - "bare Debian image reference", - ) - self.assert_failure( - "compose.yaml", - "services:\n app:\n entrypoint: *unknown-engine\n" - " command: [run, --rm, debian]\n", - "cannot statically resolve container entrypoint", - ) - self.assert_failure( - "compose.yaml", - "services:\n app:\n entrypoint: ${CONTAINER_ENGINE}\n" - " command: [run, --rm, debian]\n", - "cannot statically resolve container entrypoint", - ) - self.assert_failure( - "compose.yaml", - "services:\n app:\n entrypoint: [docker]\n" - " command: *container-command\n", - "cannot statically resolve container command", - ) - self.assert_clean( - "compose.yaml", - "services:\n" - f" app: {{entrypoint: [docker], command: [run, --rm, {PINNED_RUST}]}}\n", - ) - - def test_container_cli_scans_only_the_bounded_image_operand(self) -> None: - for command in ( - "docker run --name postgres alpine:3.22 true\n", - "docker run alpine:3.22 python -V\n", - "docker create --env NAME=postgres busybox:1.37 python\n", - "docker run --network-alias postgres alpine:3.22 true\n", - "docker run --mount type=bind,source=/tmp,target=/mnt alpine:3.22\n", - "docker run --cpuset-cpus 0 alpine:3.22\n", - "docker run -dit alpine:3.22\n", - ): - with self.subTest(command=command): - self.assert_clean("helper.sh", command) - - for command, family in ( - ("docker run --name app postgres true\n", "postgres"), - ("docker run --network-alias app postgres true\n", "postgres"), - ( - "docker run --mount type=bind,source=/tmp,target=/mnt postgres\n", - "postgres", - ), - ("docker run --cpuset-cpus 0 postgres\n", "postgres"), - ("docker run -dit postgres\n", "postgres"), - ("docker run python -V\n", "python"), - ("docker pull --platform linux/amd64 rust:1.95\n", "rust:1.95"), - ): - with self.subTest(command=command): - self.assert_failure("helper.sh", command, family) - - def test_container_short_option_grammar_and_help_are_bounded(self) -> None: - for command in ( - "docker run --help\n", - "docker run --help rust:1.95-trixie\n", - "docker run -c512 alpine:3.22 true\n", - "docker run -itv/tmp:/tmp alpine:3.22 true\n", - "docker run -itp8080:80 alpine:3.22 true\n", - "docker run -itc512 alpine:3.22 true\n", - ): - with self.subTest(command=command): - self.assert_clean("helper.sh", command) - - for command in ( - "docker run -c512 postgres true\n", - "docker run -itv/tmp:/tmp postgres true\n", - "docker run -itp8080:80 postgres true\n", - "docker run -itc512 postgres true\n", - ): - with self.subTest(command=command): - self.assert_failure("helper.sh", command, "postgres") - - for command in ( - "docker run -itxvalue alpine:3.22 true\n", - "docker run -Z alpine:3.22 true\n", - ): - with self.subTest(command=command): - self.assert_failure( - "helper.sh", - command, - "unsupported Docker/Podman short-option cluster", - ) - - def test_documented_container_options_preserve_the_image_operand(self) -> None: - value_options = ( - ("--blkio-weight", "500"), - ("--blkio-weight-device", "/dev/sda:500"), - ("--cgroup-parent", "registry.slice"), - ("--cpu-count", "2"), - ("--cpu-percent", "50"), - ("--cpu-period", "100000"), - ("--cpu-quota", "50000"), - ("--cpu-rt-period", "1000000"), - ("--cpu-rt-runtime", "950000"), - ("--cpu-shares", "512"), - ("-c", "512"), - ("--cpuset-mems", "0"), - ("--detach-keys", "ctrl-x"), - ("--device-cgroup-rule", "c 42:* rmw"), - ("--device-read-bps", "/dev/sda:1mb"), - ("--device-read-iops", "/dev/sda:1000"), - ("--device-write-bps", "/dev/sda:1mb"), - ("--device-write-iops", "/dev/sda:1000"), - ("--dns-option", "ndots:2"), - ("--domainname", "example.test"), - ("--health-start-interval", "1s"), - ("--health-start-period", "5s"), - ("--io-maxbandwidth", "10mb"), - ("--io-maxiops", "1000"), - ("--ip", "172.30.100.104"), - ("--ip6", "2001:db8::33"), - ("--isolation", "process"), - ("--label-file", "labels.txt"), - ("--link-local-ip", "169.254.1.2"), - ("--memory-reservation", "256m"), - ("--memory-swap", "1g"), - ("--memory-swappiness", "60"), - ("--oom-score-adj", "100"), - ("--pids-limit", "100"), - ("--storage-opt", "size=10G"), - ("--sysctl", "net.ipv4.ip_forward=1"), - ("--userns", "host"), - ("--uts", "host"), - ("--volume-driver", "local"), - ("--volumes-from", "data"), - ) - for option, value in value_options: - with self.subTest(option=option, image="alpine"): - self.assert_clean( - "helper.sh", - f"docker run {option} {value!r} alpine:3.22 true\n", + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + base = self.module.FROM_RE.findall(text)[0] + self.assertIn("@sha256:", base) + target.write_text( + text.replace(base, base.split("@sha256:", 1)[0], 1), + encoding="utf-8", + ) + self.assert_has_failure( + root, + f"{relative}: upstream base is not pinned by immutable digest", + ) + + def test_every_runtime_stays_distroless_and_shell_free(self) -> None: + marker = f"FROM {self.module.DISTROLESS_RUNTIME} AS runtime" + mutable_runtime = "FROM debian:trixie-slim AS runtime" + healthcheck = ( + "HEALTHCHECK --interval=30s --timeout=5s " + "--start-period=10s --retries=3" + ) + for relative in self.module.DOCKERFILES: + with self.subTest(relative=relative, invariant="distroless"): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(marker, text) + target.write_text( + text.replace(marker, mutable_runtime, 1), + encoding="utf-8", ) - with self.subTest(option=option, image="postgres"): - self.assert_failure( - "helper.sh", - f"docker create {option} {value!r} postgres true\n", - "postgres", + self.assert_has_failure( + root, + f"{relative}: missing Distroless Debian 13 non-root final runtime", ) - for option in ( - "--no-healthcheck", - "--oom-kill-disable", - "--publish-all", - "-P", - "--sig-proxy", - "--use-api-socket", - ): - with self.subTest(option=option, image="alpine"): - self.assert_clean( - "helper.sh", - f"docker run {option} alpine:3.22 true\n", + with self.subTest(relative=relative, invariant="runtime"): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(marker, text) + target.write_text( + text.replace(marker, marker + "\nRUN true", 1), + encoding="utf-8", ) - with self.subTest(option=option, image="postgres"): - self.assert_failure( - "helper.sh", - f"docker run {option} postgres true\n", - "postgres", + self.assert_has_failure( + root, + f"{relative}: final Distroless runtime contains 'RUN'", ) - self.assert_failure( - "helper.sh", - "docker run --future-option value alpine:3.22 true\n", - "unsupported Docker/Podman option has unknown arity: --future-option", - ) - - def test_multiline_container_commands_scan_the_joined_operand(self) -> None: - self.assert_failure( - "helper.sh", - "docker run --rm \\\n debian true\n", - "bare Debian image reference", - ) - self.assert_clean( - "helper.sh", - f"docker run --rm \\\n {PINNED_RUST} true\n", - ) - - def test_extensionless_executable_and_shebang_helpers_are_code(self) -> None: - text = "#!/bin/sh\ndocker run --rm debian\n" - self.assert_failure( - "release/scripts/registry-release", - text, - "bare Debian image reference", - ) - failures = self.module.scan_surface( - Path("tools/runner"), - "docker run --rm debian\n", - executable=True, - ) - self.assertTrue( - any("bare Debian image reference" in failure for failure in failures), - failures, - ) - self.assert_clean( - "notes/helper", - "This prose says docker run --rm debian without a shebang.\n", - ) - - def test_docker_image_build_contexts_follow_the_image_policy(self) -> None: - self.assert_failure( - "build.sh", - "docker buildx build --build-context base=docker-image://debian .\n", - "Docker build context", - ) - self.assert_failure( - "build.sh", - "docker build --build-context base=docker-image://rust:1.95-trixie .\n", - "Docker build context", - ) - for value in ("$RUST_IMAGE", "${RUST_IMAGE}", "`resolve-image`"): - with self.subTest(value=value): - self.assert_failure( - "build.sh", - "docker buildx build --build-context " - f"base=docker-image://{value} .\n", - "computed Docker build context is not allowed", - ) - failures = self.scan( - "build.sh", - "docker build --build-context base=docker-image://rust:1.95-trixie .\n", - ) - self.assertFalse( - any("Debian-derived image reference" in failure for failure in failures), - failures, - ) - self.assert_clean( - "build.sh", - "docker buildx build --build-context " - f"base=docker-image://{PINNED_RUST} .\n", - ) - self.assert_clean( - "build.sh", - "docker buildx build --build-context base=docker-image://alpine:3.22 .\n", - ) - self.assert_clean( - "guide.md", - "Prose mentions --build-context base=docker-image://debian.\n", - ) - - def test_postgresql_conformance_workflow_selects_static_images(self) -> None: - script_path = Path("products/notary/scripts/postgresql-conformance.sh") - script = (ROOT / script_path).read_text() - workflow = ( - ROOT / ".github/workflows/notary-postgres-conformance.yml" - ).read_text() - selections = { - "16": ( - "postgres:16.13-alpine", - "postgres:16.14-alpine", - "postgres:17.10-alpine", - ), - "17": ( - "postgres:17.9-alpine", - "postgres:17.10-alpine", - "postgres:18.4-alpine", - ), - "18": ( - "postgres:18.3-alpine", - "postgres:18.4-alpine", - "postgres:18.4-alpine", - ), - } - for major, (source, target, restore) in selections.items(): - with self.subTest(major=major): - self.assertIn(f'- postgresql: "{major}"', workflow) - self.assertIn( - f' {major})\n default_source_image="{source}"\n' - f' default_target_image="{target}"\n' - f' default_restore_image="{restore}"\n', - script, + with self.subTest(relative=relative, invariant="healthcheck"): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(healthcheck, text) + target.write_text( + text.replace(healthcheck, "HEALTHCHECK --none", 1), + encoding="utf-8", ) - self.assertNotIn("NOTARY_POSTGRES_SOURCE_IMAGE", script + workflow) - self.assertNotIn("NOTARY_POSTGRES_TARGET_IMAGE", script + workflow) - self.assertNotIn("NOTARY_POSTGRES_RESTORE_IMAGE", script + workflow) - self.assert_clean(script_path.as_posix(), script) - - override = ( - 'source_image="${NOTARY_POSTGRES_SOURCE_IMAGE:-${default_source_image}}"' - ) - mutated = script.replace('source_image="${default_source_image}"', override) - self.assertNotEqual(script, mutated) - self.assert_failure( - script_path.as_posix(), - mutated, - "unresolved image variables: source_image", - ) - - def test_markdown_scans_only_executable_fences(self) -> None: - self.assert_failure( - "guide.md", - "```sh\ndocker run --rm rust:1.95-trixie\n```\n", - "not pinned by immutable digest", - ) - self.assert_clean( - "guide.md", - "Example prose mentions rust:1.95-trixie.\n" - "```text\n" - "docker run --rm rust:1.95-trixie\n" - "```\n", - ) - - def test_repository_dockerfile_discovery_checks_external_and_internal_stages( - self, - ) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required(root) - dockerfile = root / "products/example/Dockerfile" - dockerfile.parent.mkdir(parents=True) - dockerfile.write_text( - "FROM scratch AS assets\n" - "FROM assets AS final\n" - "COPY --from=debian /etc/os-release /os-release\n" - ) - failures = self.module.check_repository(root) - self.assertTrue( - any("bare Debian image reference" in item for item in failures), - failures, - ) - self.assertFalse( - any( - "upstream base is not pinned" in item and "assets" in item - for item in failures - ), - failures, - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required(root) - dockerfile = root / "Dockerfile.worker" - dockerfile.write_text("FROM alpine:3.22\n") - failures = self.module.check_repository(root) - self.assertTrue( - any( - "Dockerfile.worker: upstream base is not pinned" in item - for item in failures - ), - failures, - ) - - def test_product_contract_mutations_are_reported(self) -> None: - mutations = ( - ( - Path("crates/registry-relay/Dockerfile"), - "/usr/local/bin/registry-relay-rhai-worker", - "/usr/local/bin/removed-worker", - "Relay worker binary", - ), - ( - Path("products/notary/Dockerfile"), - "registry-notary-cel-worker", - "removed-cel-worker", - "Notary CEL worker binary", - ), - ( - Path("release/docker/Dockerfile.registry-relay"), - "# syntax=", - "# moved-syntax=", - "frontend must be the first line", - ), - ( - Path("release/scripts/build-release-binaries.sh"), - "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", - "--features registry-notary/registry-notary-cel", - "PKCS#11-enabled release build", - ), - ( - Path("docs/site/scripts/check-registryctl-tutorials.sh"), - 'LINUX_TARGET="$REPO_ROOT/target/registryctl-tutorial-linux-amd64"', - 'LINUX_TARGET="$REPO_ROOT/target/other"', - "registryctl tutorial linux target path", - ), - ( - Path(".github/workflows/ci.yml"), - "hashFiles('docs/site/scripts/check-registryctl-tutorials.sh')", - "hashFiles('Cargo.lock')", - "registryctl tutorial cache builder identity", - ), - ( - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), - "postgres:16-trixie@sha256:", - "postgres:16@", - "pinned Debian 13 live-journey PostgreSQL", - ), - ) - for path, old, new, expected in mutations: - with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required(root) - target = root / path - text = target.read_text() - self.assertIn(old, text) - target.write_text(text.replace(old, new)) - failures = self.module.check_repository(root) - self.assertTrue( - any(expected in item for item in failures), - failures, + self.assert_has_failure( + root, + f"{relative}: missing binary healthcheck", ) - def test_missing_required_surface_and_cache_restore_key_are_reported(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required(root) - missing = root / "products/notary/Dockerfile" - missing.unlink() - failures = self.module.check_repository(root) - self.assertTrue( - any("missing maintained image surface" in item for item in failures), - failures, - ) - - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.copy_required(root) - workflow = root / ".github/workflows/ci.yml" - text = workflow.read_text() - marker = "- name: Execute registryctl tutorials from source" - workflow.write_text( - text.replace(marker, "restore-keys: old-key\n " + marker, 1) - ) - failures = self.module.check_repository(root) - self.assertTrue( - any("must not restore" in item for item in failures), - failures, - ) - if __name__ == "__main__": unittest.main() diff --git a/release/scripts/test_registry_release.py b/release/scripts/test_registry_release.py index 8e12d44d..19e0ffab 100755 --- a/release/scripts/test_registry_release.py +++ b/release/scripts/test_registry_release.py @@ -19,7 +19,49 @@ IMAGE_DIGEST_REF = f"ghcr.io/registrystack/registry-notary@{IMAGE_DIGEST}" +def load_debian13_image_check(): + path = ROOT / "release/scripts/check-debian13-images.py" + spec = importlib.util.spec_from_file_location("check_debian13_images", path) + if spec is None or spec.loader is None: + raise ImportError(f"could not load module spec from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + class RegistryReleaseTest(unittest.TestCase): + def test_maintained_images_follow_debian13_contract(self) -> None: + module = load_debian13_image_check() + self.assertEqual([], module.check_repository(ROOT)) + + def test_debian13_contract_rejects_retired_base_and_unpinned_base(self) -> None: + module = load_debian13_image_check() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for relative in module.MAINTAINED_TEXT_PATHS: + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + (ROOT / relative).read_text(encoding="utf-8"), + encoding="utf-8", + ) + + relay_dockerfile = root / "crates/registry-relay/Dockerfile" + text = relay_dockerfile.read_text(encoding="utf-8") + text = text.replace( + module.RUST_BUILDER, + "rust:1.95-" + "book" + "worm", + 1, + ) + relay_dockerfile.write_text(text, encoding="utf-8") + + failures = module.check_repository(root) + self.assertTrue( + any("retired Debian image generation marker" in failure for failure in failures) + ) + self.assertTrue( + any("not pinned by immutable digest" in failure for failure in failures) + ) def test_contributing_documents_major_functionality_test_policy(self) -> None: text = (ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") diff --git a/release/scripts/test_relay_oidc_smoke.py b/release/scripts/test_relay_oidc_smoke.py index 43a092fc..34dd3427 100644 --- a/release/scripts/test_relay_oidc_smoke.py +++ b/release/scripts/test_relay_oidc_smoke.py @@ -175,50 +175,6 @@ def test_relay_image_requires_exact_repository_and_lowercase_digest(self) -> Non with self.assertRaises(self.runner.SmokeError): self.runner.validate_relay_image(image) - def test_execute_live_binds_validated_argument_over_ambient_image(self) -> None: - requested = "ghcr.io/registrystack/registry-relay@sha256:" + "a" * 64 - ambient = "ghcr.io/registrystack/registry-relay@sha256:" + "b" * 64 - image_variable = "REGISTRY_RELAY_OIDC_SMOKE_RELAY_IMAGE" - captured_environments: list[dict[str, str]] = [] - - def run_checked( - command: list[str], *, env: dict[str, str], **_kwargs: object - ) -> None: - captured_environments.append(dict(env)) - if "up" in command: - raise self.runner.SmokeError("stop after environment capture") - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - args = self.runner.argparse.Namespace( - relay_image=requested, - candidate_source_ref="c" * 40, - release_id="1.0.0-rc.1", - output_dir=str(root / "output"), - host_port=19191, - ) - with ( - patch.object(self.runner, "DEFAULT_WORK_ROOT", root / "work"), - patch.object(self.runner, "validate_assets", return_value={}), - patch.object( - self.runner.shutil, "which", return_value="/usr/bin/docker" - ), - patch.object(self.runner, "run_checked", side_effect=run_checked), - patch.dict( - self.runner.os.environ, - {image_variable: ambient}, - clear=False, - ), - ): - with self.assertRaisesRegex( - self.runner.SmokeError, "stop after environment capture" - ): - self.runner.execute_live(args) - - self.assertEqual(2, len(captured_environments)) - for environment in captured_environments: - self.assertEqual(requested, environment[image_variable]) - def test_plan_is_offline_and_does_not_claim_live_evidence(self) -> None: plan = self.runner.plan_document( "ghcr.io/registrystack/registry-relay@sha256:" + "a" * 64, From 84d5b62259431594d6cade5bc18da3e9b2412dd7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 04:19:00 +0700 Subject: [PATCH 21/34] fix(release): close focused image contract gaps Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 95 +++++++++++++-- release/scripts/test_check_debian13_images.py | 113 +++++++++++++++++- 2 files changed, 193 insertions(+), 15 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 5befea9a..eb870209 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -26,6 +26,12 @@ "docker/dockerfile:1.7@sha256:" "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) +TUTORIAL_CACHE_STEP = "Cache source-under-test Cargo build" +TUTORIAL_CACHE_KEY = ( + " key: registryctl-tutorial-${{ runner.os }}-" + "${{ hashFiles('docs/site/scripts/check-registryctl-tutorials.sh') }}-" + "${{ hashFiles('Cargo.lock') }}" +) DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), @@ -38,6 +44,7 @@ # These are the maintained image and image-policy surfaces. Historical release # notes are immutable evidence and intentionally are not rewritten by this gate. MAINTAINED_TEXT_PATHS = DOCKERFILES + ( + Path(".github/workflows/ci.yml"), Path(".github/workflows/release.yml"), Path("release/scripts/build-release-binaries.sh"), Path("docs/site/scripts/check-registryctl-tutorials.sh"), @@ -62,6 +69,10 @@ FROM_RE = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)", re.MULTILINE) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") +RETIRED_MARKER_RE = re.compile( + r"\b(?:bookworm|debian[ \t_:-]*12)\b", + re.IGNORECASE, +) def read(root: Path, relative: Path, failures: list[str]) -> str: @@ -84,6 +95,35 @@ def require( failures.append(f"{relative}: missing {detail}: {needle!r}") +def require_exact_line( + text: str, + line: str, + relative: Path, + detail: str, + failures: list[str], +) -> None: + if line not in text.splitlines(): + failures.append(f"{relative}: missing {detail}: exact line {line!r}") + + +def workflow_step(text: str, name: str) -> str: + lines = text.splitlines() + header = f" - name: {name}" + matches = [index for index, line in enumerate(lines) if line == header] + if len(matches) != 1: + return "" + start = matches[0] + end = next( + ( + index + for index in range(start + 1, len(lines)) + if lines[index].startswith(" - name: ") + ), + len(lines), + ) + return "\n".join(lines[start:end]) + + def runtime_stage(text: str) -> str: marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" offset = text.find(marker) @@ -97,14 +137,13 @@ def check_repository(root: Path = ROOT) -> list[str]: for relative in MAINTAINED_TEXT_PATHS } - retired_markers = ("book" + "worm", "debian" + "12") for relative, text in texts.items(): - lowered = text.casefold() - for marker in retired_markers: - if marker in lowered: - failures.append( - f"{relative}: retired Debian image generation marker remains: {marker}" - ) + marker = RETIRED_MARKER_RE.search(text) + if marker: + failures.append( + f"{relative}: retired Debian image generation marker remains: " + f"{marker.group(0).casefold()}" + ) for relative in DOCKERFILES: text = texts[relative] @@ -117,6 +156,11 @@ def check_repository(root: Path = ROOT) -> list[str]: failures.append( f"{relative}: upstream base is not pinned by immutable digest: {base}" ) + if bases[-1] != DISTROLESS_RUNTIME: + failures.append( + f"{relative}: final Dockerfile stage must use pinned " + f"Distroless Debian 13 runtime: {bases[-1]}" + ) require( text, @@ -248,15 +292,23 @@ def check_repository(root: Path = ROOT) -> list[str]: ) workflow = texts[Path(".github/workflows/release.yml")] + ci_workflow = texts[Path(".github/workflows/ci.yml")] binary_recipe = texts[Path("release/scripts/build-release-binaries.sh")] tutorial_check = texts[Path("docs/site/scripts/check-registryctl-tutorials.sh")] - require( + require_exact_line( workflow, - f"RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", + f" RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", Path(".github/workflows/release.yml"), "pinned Debian 13 release builder", failures, ) + require_exact_line( + binary_recipe, + f'default_builder_image="{RUST_BUILDER}"', + Path("release/scripts/build-release-binaries.sh"), + "pinned Debian 13 release recipe builder", + failures, + ) require( binary_recipe, "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", @@ -264,7 +316,7 @@ def check_repository(root: Path = ROOT) -> list[str]: "PKCS#11-enabled release build", failures, ) - require( + require_exact_line( tutorial_check, f'BUILDER_IMAGE="{RUST_BUILDER}"', Path("docs/site/scripts/check-registryctl-tutorials.sh"), @@ -275,14 +327,33 @@ def check_repository(root: Path = ROOT) -> list[str]: live_journey = texts[ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") ] - require( + require_exact_line( live_journey, - RUST_BUILDER, + f" {RUST_BUILDER} \\", Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", failures, ) + tutorial_cache = workflow_step(ci_workflow, TUTORIAL_CACHE_STEP) + if not tutorial_cache: + failures.append( + f".github/workflows/ci.yml: missing unique {TUTORIAL_CACHE_STEP!r} step" + ) + else: + require_exact_line( + tutorial_cache, + TUTORIAL_CACHE_KEY, + Path(".github/workflows/ci.yml"), + "registryctl tutorial builder cache key", + failures, + ) + if re.search(r"^\s*restore-keys\s*:", tutorial_cache, re.MULTILINE): + failures.append( + ".github/workflows/ci.yml: registryctl tutorial builder cache " + "must not use restore-keys fallback" + ) + return failures diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 83d41fc5..34ac8d37 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -11,21 +11,28 @@ ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "release/scripts/check-debian13-images.py" TUTORIAL_CHECK = Path("docs/site/scripts/check-registryctl-tutorials.sh") +RELEASE_WORKFLOW = Path(".github/workflows/release.yml") +RELEASE_BINARY_RECIPE = Path("release/scripts/build-release-binaries.sh") +LIVE_JOURNEY = Path( + "crates/registry-relay/scripts/run-live-consultation-journey.sh" +) +CI_WORKFLOW = Path(".github/workflows/ci.yml") EXPECTED_SURFACES = { - Path(".github/workflows/release.yml"), + CI_WORKFLOW, + RELEASE_WORKFLOW, Path("crates/registry-relay/Dockerfile"), Path("crates/registry-relay/Dockerfile.demo"), Path("crates/registry-relay/docs/ops.md"), Path("crates/registry-relay/docs/security-assurance.md"), Path("crates/registry-relay/scripts/check_docker_build_contract.py"), - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + LIVE_JOURNEY, TUTORIAL_CHECK, Path("products/notary/Dockerfile"), Path("products/notary/docs/security-assurance.md"), Path("release/docker/Dockerfile.registry-notary"), Path("release/docker/Dockerfile.registry-relay"), - Path("release/scripts/build-release-binaries.sh"), + RELEASE_BINARY_RECIPE, } @@ -85,6 +92,10 @@ def test_retired_markers_are_rejected_regardless_of_text_syntax(self) -> None: cases = ( "# historical book" + "worm image\n", 'IMAGE="debian' + '12"\n', + "base: Debian 12\n", + "base: debian-12\n", + "base: debian_12\n", + "base: debian:12\n", '{"base": "BOOK' + 'WORM"}\n', ) for suffix in cases: @@ -122,6 +133,48 @@ def test_tutorial_builder_must_match_the_exact_pinned_image(self) -> None: "missing pinned Debian 13 registryctl tutorial builder", ) + def test_builder_contract_lines_cannot_be_shadowed_by_comments(self) -> None: + pin = self.module.RUST_BUILDER + cases = ( + ( + RELEASE_WORKFLOW, + f" RELEASE_BUILDER_IMAGE: {pin}", + f" # RELEASE_BUILDER_IMAGE: {pin}\n" + " RELEASE_BUILDER_IMAGE: rust:1.95-trixie", + "missing pinned Debian 13 release builder", + ), + ( + RELEASE_BINARY_RECIPE, + f'default_builder_image="{pin}"', + f'# default_builder_image="{pin}"\n' + 'default_builder_image="rust:1.95-trixie"', + "missing pinned Debian 13 release recipe builder", + ), + ( + TUTORIAL_CHECK, + f'BUILDER_IMAGE="{pin}"', + f'# BUILDER_IMAGE="{pin}"\nBUILDER_IMAGE="rust:1.95-trixie"', + "missing pinned Debian 13 registryctl tutorial builder", + ), + ( + LIVE_JOURNEY, + f" {pin} \\", + f" # {pin} \\\n rust:1.95-trixie \\", + "missing pinned Debian 13 live-journey builder", + ), + ) + for relative, exact, replacement, failure in cases: + with self.subTest(relative=relative): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): @@ -139,6 +192,60 @@ def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: f"{relative}: upstream base is not pinned by immutable digest", ) + def test_distroless_runtime_is_the_final_dockerfile_stage(self) -> None: + pinned_alpine = "alpine:3.22@sha256:" + "a" * 64 + for relative in self.module.DOCKERFILES: + additions = ( + f"\nFROM {pinned_alpine} AS debug\n", + f"\n# FROM {self.module.DISTROLESS_RUNTIME} AS runtime\n" + f"FROM {self.module.DEBIAN_PREPARATION} AS debug\n", + ) + for addition in additions: + with self.subTest(relative=relative, addition=addition): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text(text + addition, encoding="utf-8") + self.assert_has_failure( + root, + f"{relative}: final Dockerfile stage must use pinned " + "Distroless Debian 13 runtime", + ) + + def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( + self, + ) -> None: + exact = self.module.TUTORIAL_CACHE_KEY + cases = ( + ("", "missing registryctl tutorial builder cache key"), + ( + f" # {exact.strip()}", + "missing registryctl tutorial builder cache key", + ), + ( + " key: registryctl-tutorial-${{ runner.os }}-" + "${{ hashFiles('Cargo.lock') }}", + "missing registryctl tutorial builder cache key", + ), + ( + exact + + "\n restore-keys: |\n" + " registryctl-tutorial-${{ runner.os }}-", + "must not use restore-keys fallback", + ), + ) + for replacement, failure in cases: + with self.subTest(replacement=replacement): + root = self.fixture() + target = root / CI_WORKFLOW + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + def test_every_runtime_stays_distroless_and_shell_free(self) -> None: marker = f"FROM {self.module.DISTROLESS_RUNTIME} AS runtime" mutable_runtime = "FROM debian:trixie-slim AS runtime" From 758619e0686bddca5ffe030404fd154387aed868 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 04:29:56 +0700 Subject: [PATCH 22/34] fix(release): enforce bounded image stages Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 180 ++++++++++-------- release/scripts/test_check_debian13_images.py | 170 ++++++++++++++++- 2 files changed, 269 insertions(+), 81 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index eb870209..a35bd82d 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -67,12 +67,56 @@ Path("release/docker/Dockerfile.registry-notary"), ) -FROM_RE = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)", re.MULTILINE) +RELAY_RUNTIME_DIRECTIVES = ( + ( + "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s " + '--retries=3 CMD ["/usr/local/bin/registry-relay", "healthcheck"]', + "binary healthcheck", + ), + ( + 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', + "absolute Relay entrypoint", + ), + ("WORKDIR /var/lib/registry-relay", "Relay working directory"), +) +NOTARY_RUNTIME_DIRECTIVES = ( + ( + "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s " + '--retries=3 CMD ["/usr/local/bin/registry-notary", "healthcheck"]', + "binary healthcheck", + ), + ( + 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', + "absolute Notary entrypoint", + ), + ("WORKDIR /var/lib/registry-notary", "Notary working directory"), +) + +FROM_RE = re.compile( + r"^[ \t]*FROM[ \t]+(?:--platform=\S+[ \t]+)?" + r"(?P[^\s#]+)" + r"(?:[ \t]+AS[ \t]+(?P[^\s#]+))?" + r"[ \t]*(?:#.*)?$", + re.IGNORECASE | re.MULTILINE, +) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( r"\b(?:bookworm|debian[ \t_:-]*12)\b", re.IGNORECASE, ) +RELEASE_BUILDER_KEY_RE = re.compile( + r"^[ \t]*RELEASE_BUILDER_IMAGE[ \t]*:" +) +DEFAULT_BUILDER_ASSIGNMENT_RE = re.compile( + r"^[ \t]*(?:(?:export|readonly)[ \t]+)?default_builder_image[ \t]*=" +) +TUTORIAL_BUILDER_ASSIGNMENT_RE = re.compile( + r"^[ \t]*(?:(?:export|readonly)[ \t]+)?BUILDER_IMAGE[ \t]*=" +) +LIVE_JOURNEY_BUILDER_RE = re.compile( + r"^[ \t]+rust:[^ \t#]+[ \t]+\\[ \t]*$" +) +CACHE_KEY_RE = re.compile(r"^[ \t]*key[ \t]*:") def read(root: Path, relative: Path, failures: list[str]) -> str: @@ -95,15 +139,24 @@ def require( failures.append(f"{relative}: missing {detail}: {needle!r}") -def require_exact_line( +def require_unique_active_line( text: str, line: str, + active_pattern: re.Pattern[str], relative: Path, detail: str, failures: list[str], ) -> None: - if line not in text.splitlines(): - failures.append(f"{relative}: missing {detail}: exact line {line!r}") + active_lines = [ + candidate + for candidate in text.splitlines() + if active_pattern.match(candidate) + ] + if active_lines != [line]: + failures.append( + f"{relative}: missing {detail}: expected exactly one active " + f"line {line!r}; found {active_lines!r}" + ) def workflow_step(text: str, name: str) -> str: @@ -117,19 +170,13 @@ def workflow_step(text: str, name: str) -> str: ( index for index in range(start + 1, len(lines)) - if lines[index].startswith(" - name: ") + if lines[index].startswith(" - ") ), len(lines), ) return "\n".join(lines[start:end]) -def runtime_stage(text: str) -> str: - marker = f"FROM {DISTROLESS_RUNTIME} AS runtime" - offset = text.find(marker) - return text[offset:] if offset >= 0 else "" - - def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] texts = { @@ -147,50 +194,56 @@ def check_repository(root: Path = ROOT) -> list[str]: for relative in DOCKERFILES: text = texts[relative] - bases = FROM_RE.findall(text) - if not bases: + stage_matches = list(FROM_RE.finditer(text)) + if not stage_matches: failures.append(f"{relative}: no FROM instruction found") continue - for base in bases: + stages = tuple( + (match.group("base"), match.group("alias")) + for match in stage_matches + ) + expected_stages = ( + ( + (RUST_BUILDER, "builder"), + (DISTROLESS_RUNTIME, "runtime"), + ) + if relative in RUST_BUILDER_DOCKERFILES + else ( + (DEBIAN_PREPARATION, "runtime-root"), + (DISTROLESS_RUNTIME, "runtime"), + ) + ) + if stages != expected_stages: + failures.append( + f"{relative}: Dockerfile stage sequence must be exactly " + f"{expected_stages!r}; found {stages!r}" + ) + for base, _alias in stages: if not DIGEST_PIN_RE.search(base): failures.append( f"{relative}: upstream base is not pinned by immutable digest: {base}" ) - if bases[-1] != DISTROLESS_RUNTIME: + runtime = text[stage_matches[-1].start() :] + if re.search(r"^[ \t]*RUN(?:[ \t]|$)", runtime, re.IGNORECASE | re.MULTILINE): failures.append( - f"{relative}: final Dockerfile stage must use pinned " - f"Distroless Debian 13 runtime: {bases[-1]}" + f"{relative}: final Distroless runtime contains 'RUN'" ) - - require( - text, - f"FROM {DISTROLESS_RUNTIME} AS runtime", - relative, - "Distroless Debian 13 non-root final runtime", - failures, - ) - runtime = runtime_stage(text) - for forbidden in ("\nRUN ", "apt-get", "/bin/sh", "curl ", "wget "): + for forbidden in ("apt-get", "/bin/sh", "curl ", "wget "): if forbidden in runtime: failures.append( f"{relative}: final Distroless runtime contains {forbidden.strip()!r}" ) - require( - runtime, - "HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3", - relative, - "binary healthcheck", - failures, - ) - - for relative in RUST_BUILDER_DOCKERFILES: - require( - texts[relative], - f"FROM {RUST_BUILDER} AS builder", - relative, - "pinned Debian 13 Rust builder", - failures, + runtime_directives = ( + RELAY_RUNTIME_DIRECTIVES + if relative in RELAY_DOCKERFILES + else NOTARY_RUNTIME_DIRECTIVES ) + for directive, detail in runtime_directives: + if directive not in runtime.splitlines(): + failures.append( + f"{relative}: missing {detail} in final runtime stage: " + f"{directive!r}" + ) for relative in PREPARATION_DOCKERFILES: text = texts[relative] @@ -198,13 +251,6 @@ def check_repository(root: Path = ROOT) -> list[str]: failures.append( f"{relative}: pinned Dockerfile frontend must be the first line" ) - require( - text, - f"FROM {DEBIAN_PREPARATION} AS runtime-root", - relative, - "pinned Debian 13 runtime preparation base", - failures, - ) require( text, "ARG SOURCE_DATE_EPOCH=0", @@ -236,13 +282,6 @@ def check_repository(root: Path = ROOT) -> list[str]: "Relay worker binary", failures, ) - require( - runtime_stage(text), - 'ENTRYPOINT ["/usr/local/bin/registry-relay"]', - relative, - "absolute Relay entrypoint", - failures, - ) product_notary = texts[Path("products/notary/Dockerfile")] require( @@ -261,13 +300,6 @@ def check_repository(root: Path = ROOT) -> list[str]: "Notary CEL worker binary", failures, ) - require( - runtime_stage(text), - 'ENTRYPOINT ["/usr/local/bin/registry-notary"]', - relative, - "absolute Notary entrypoint", - failures, - ) require( text, "chown -R 65532:65532", @@ -275,13 +307,6 @@ def check_repository(root: Path = ROOT) -> list[str]: "numeric nonroot-owned Notary runtime directories", failures, ) - require( - runtime_stage(text), - "WORKDIR /var/lib/registry-notary", - relative, - "Notary working directory", - failures, - ) if re.search( r"^\s*(?:COPY|ADD)\b[^\n]*(?:\.so\b|pkcs11[^/\s]*module)", text, @@ -295,16 +320,18 @@ def check_repository(root: Path = ROOT) -> list[str]: ci_workflow = texts[Path(".github/workflows/ci.yml")] binary_recipe = texts[Path("release/scripts/build-release-binaries.sh")] tutorial_check = texts[Path("docs/site/scripts/check-registryctl-tutorials.sh")] - require_exact_line( + require_unique_active_line( workflow, f" RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", + RELEASE_BUILDER_KEY_RE, Path(".github/workflows/release.yml"), "pinned Debian 13 release builder", failures, ) - require_exact_line( + require_unique_active_line( binary_recipe, f'default_builder_image="{RUST_BUILDER}"', + DEFAULT_BUILDER_ASSIGNMENT_RE, Path("release/scripts/build-release-binaries.sh"), "pinned Debian 13 release recipe builder", failures, @@ -316,9 +343,10 @@ def check_repository(root: Path = ROOT) -> list[str]: "PKCS#11-enabled release build", failures, ) - require_exact_line( + require_unique_active_line( tutorial_check, f'BUILDER_IMAGE="{RUST_BUILDER}"', + TUTORIAL_BUILDER_ASSIGNMENT_RE, Path("docs/site/scripts/check-registryctl-tutorials.sh"), "pinned Debian 13 registryctl tutorial builder", failures, @@ -327,9 +355,10 @@ def check_repository(root: Path = ROOT) -> list[str]: live_journey = texts[ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") ] - require_exact_line( + require_unique_active_line( live_journey, f" {RUST_BUILDER} \\", + LIVE_JOURNEY_BUILDER_RE, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", failures, @@ -341,9 +370,10 @@ def check_repository(root: Path = ROOT) -> list[str]: f".github/workflows/ci.yml: missing unique {TUTORIAL_CACHE_STEP!r} step" ) else: - require_exact_line( + require_unique_active_line( tutorial_cache, TUTORIAL_CACHE_KEY, + CACHE_KEY_RE, Path(".github/workflows/ci.yml"), "registryctl tutorial builder cache key", failures, diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 34ac8d37..14ffcee5 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -71,6 +71,16 @@ def assert_has_failure( failures, ) + def runtime_directives(self, relative: Path) -> tuple[str, str, str]: + product = "relay" if relative in self.module.RELAY_DOCKERFILES else "notary" + binary = f"/usr/local/bin/registry-{product}" + return ( + "HEALTHCHECK --interval=30s --timeout=5s " + f'--start-period=10s --retries=3 CMD ["{binary}", "healthcheck"]', + f'ENTRYPOINT ["{binary}"]', + f"WORKDIR /var/lib/registry-{product}", + ) + def test_current_repository_follows_contract(self) -> None: self.assertEqual([], self.module.check_repository(ROOT)) @@ -175,13 +185,62 @@ def test_builder_contract_lines_cannot_be_shadowed_by_comments(self) -> None: ) self.assert_has_failure(root, failure) + def test_builder_contracts_reject_earlier_and_later_active_overrides( + self, + ) -> None: + pin = self.module.RUST_BUILDER + cases = ( + ( + RELEASE_WORKFLOW, + f" RELEASE_BUILDER_IMAGE: {pin}", + " RELEASE_BUILDER_IMAGE: rust:1.95-trixie", + "missing pinned Debian 13 release builder", + ), + ( + RELEASE_BINARY_RECIPE, + f'default_builder_image="{pin}"', + 'default_builder_image="rust:1.95-trixie"', + "missing pinned Debian 13 release recipe builder", + ), + ( + TUTORIAL_CHECK, + f'BUILDER_IMAGE="{pin}"', + 'BUILDER_IMAGE="rust:1.95-trixie"', + "missing pinned Debian 13 registryctl tutorial builder", + ), + ) + for relative, exact, override, failure in cases: + for replacement in (f"{override}\n{exact}", f"{exact}\n{override}"): + with self.subTest(relative=relative, replacement=replacement): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + + root = self.fixture() + target = root / LIVE_JOURNEY + exact = f" {pin} \\" + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace(exact, exact + "\n rust:1.95-trixie \\", 1), + encoding="utf-8", + ) + self.assert_has_failure( + root, + "missing pinned Debian 13 live-journey builder", + ) + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): root = self.fixture() target = root / relative text = target.read_text(encoding="utf-8") - base = self.module.FROM_RE.findall(text)[0] + base = next(self.module.FROM_RE.finditer(text)).group("base") self.assertIn("@sha256:", base) target.write_text( text.replace(base, base.split("@sha256:", 1)[0], 1), @@ -196,9 +255,9 @@ def test_distroless_runtime_is_the_final_dockerfile_stage(self) -> None: pinned_alpine = "alpine:3.22@sha256:" + "a" * 64 for relative in self.module.DOCKERFILES: additions = ( - f"\nFROM {pinned_alpine} AS debug\n", + f"\n from {pinned_alpine} as debug\n", f"\n# FROM {self.module.DISTROLESS_RUNTIME} AS runtime\n" - f"FROM {self.module.DEBIAN_PREPARATION} AS debug\n", + f" from {self.module.DEBIAN_PREPARATION} as debug\n", ) for addition in additions: with self.subTest(relative=relative, addition=addition): @@ -208,9 +267,69 @@ def test_distroless_runtime_is_the_final_dockerfile_stage(self) -> None: target.write_text(text + addition, encoding="utf-8") self.assert_has_failure( root, - f"{relative}: final Dockerfile stage must use pinned " - "Distroless Debian 13 runtime", + f"{relative}: Dockerfile stage sequence must be exactly", + ) + + def test_dockerfile_stage_sequence_rejects_aliases_duplicates_and_empty_runtime( + self, + ) -> None: + runtime = f"FROM {self.module.DISTROLESS_RUNTIME} AS runtime" + for relative in self.module.DOCKERFILES: + with self.subTest(relative=relative, mutation="alias"): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace(runtime, runtime.replace("runtime", "final"), 1), + encoding="utf-8", + ) + self.assert_has_failure( + root, + f"{relative}: Dockerfile stage sequence must be exactly", + ) + + with self.subTest(relative=relative, mutation="duplicate"): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text(text + f"\n{runtime}\n", encoding="utf-8") + self.assert_has_failure( + root, + f"{relative}: Dockerfile stage sequence must be exactly", + ) + + with self.subTest(relative=relative, mutation="empty"): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + final_stage = list(self.module.FROM_RE.finditer(text))[-1] + target.write_text( + text[: final_stage.end()] + "\n", + encoding="utf-8", + ) + self.assert_has_failure(root, f"{relative}: missing binary healthcheck") + + def test_runtime_directives_must_be_active_in_the_final_stage(self) -> None: + for relative in self.module.DOCKERFILES: + for directive in self.runtime_directives(relative): + with self.subTest(relative=relative, directive=directive): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + final_stage = list(self.module.FROM_RE.finditer(text))[-1] + prefix = text[: final_stage.start()] + runtime = text[final_stage.start() :] + self.assertIn(f"\n{directive}\n", runtime) + runtime = runtime.replace( + f"\n{directive}\n", + f"\n# {directive}\n", + 1, + ) + target.write_text( + prefix + directive + "\n" + runtime, + encoding="utf-8", ) + self.assert_has_failure(root, f"{relative}: missing") def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( self, @@ -246,6 +365,45 @@ def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( ) self.assert_has_failure(root, failure) + wrong_key = ( + " key: registryctl-tutorial-${{ runner.os }}-" + "${{ hashFiles('Cargo.lock') }}" + ) + root = self.fixture() + target = root / CI_WORKFLOW + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace( + exact, + wrong_key + + "\n - uses: example.invalid/cache@v1\n" + " with:\n" + + exact, + 1, + ), + encoding="utf-8", + ) + self.assert_has_failure( + root, + "missing registryctl tutorial builder cache key", + ) + + root = self.fixture() + target = root / CI_WORKFLOW + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace( + exact, + exact + + "\n - run: true\n" + " env:\n" + " restore-keys: belongs-to-next-step", + 1, + ), + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + def test_every_runtime_stays_distroless_and_shell_free(self) -> None: marker = f"FROM {self.module.DISTROLESS_RUNTIME} AS runtime" mutable_runtime = "FROM debian:trixie-slim AS runtime" @@ -265,7 +423,7 @@ def test_every_runtime_stays_distroless_and_shell_free(self) -> None: ) self.assert_has_failure( root, - f"{relative}: missing Distroless Debian 13 non-root final runtime", + f"{relative}: Dockerfile stage sequence must be exactly", ) with self.subTest(relative=relative, invariant="runtime"): From 62a71b1f2ad3898e8f73724f4b84b03a6ed82330 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 04:35:50 +0700 Subject: [PATCH 23/34] fix(release): allow canonical shell pin forms Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 22 +++++--- release/scripts/test_check_debian13_images.py | 54 +++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index a35bd82d..ffdf927f 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -141,7 +141,7 @@ def require( def require_unique_active_line( text: str, - line: str, + allowed_lines: tuple[str, ...], active_pattern: re.Pattern[str], relative: Path, detail: str, @@ -152,10 +152,10 @@ def require_unique_active_line( for candidate in text.splitlines() if active_pattern.match(candidate) ] - if active_lines != [line]: + if len(active_lines) != 1 or active_lines[0] not in allowed_lines: failures.append( f"{relative}: missing {detail}: expected exactly one active " - f"line {line!r}; found {active_lines!r}" + f"line from {allowed_lines!r}; found {active_lines!r}" ) @@ -322,7 +322,7 @@ def check_repository(root: Path = ROOT) -> list[str]: tutorial_check = texts[Path("docs/site/scripts/check-registryctl-tutorials.sh")] require_unique_active_line( workflow, - f" RELEASE_BUILDER_IMAGE: {RUST_BUILDER}", + (f" RELEASE_BUILDER_IMAGE: {RUST_BUILDER}",), RELEASE_BUILDER_KEY_RE, Path(".github/workflows/release.yml"), "pinned Debian 13 release builder", @@ -330,7 +330,10 @@ def check_repository(root: Path = ROOT) -> list[str]: ) require_unique_active_line( binary_recipe, - f'default_builder_image="{RUST_BUILDER}"', + tuple( + f'{prefix}default_builder_image="{RUST_BUILDER}"' + for prefix in ("", "readonly ", "export ") + ), DEFAULT_BUILDER_ASSIGNMENT_RE, Path("release/scripts/build-release-binaries.sh"), "pinned Debian 13 release recipe builder", @@ -345,7 +348,10 @@ def check_repository(root: Path = ROOT) -> list[str]: ) require_unique_active_line( tutorial_check, - f'BUILDER_IMAGE="{RUST_BUILDER}"', + tuple( + f'{prefix}BUILDER_IMAGE="{RUST_BUILDER}"' + for prefix in ("", "readonly ", "export ") + ), TUTORIAL_BUILDER_ASSIGNMENT_RE, Path("docs/site/scripts/check-registryctl-tutorials.sh"), "pinned Debian 13 registryctl tutorial builder", @@ -357,7 +363,7 @@ def check_repository(root: Path = ROOT) -> list[str]: ] require_unique_active_line( live_journey, - f" {RUST_BUILDER} \\", + (f" {RUST_BUILDER} \\",), LIVE_JOURNEY_BUILDER_RE, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", @@ -372,7 +378,7 @@ def check_repository(root: Path = ROOT) -> list[str]: else: require_unique_active_line( tutorial_cache, - TUTORIAL_CACHE_KEY, + (TUTORIAL_CACHE_KEY,), CACHE_KEY_RE, Path(".github/workflows/ci.yml"), "registryctl tutorial builder cache key", diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 14ffcee5..ef6285e3 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -234,6 +234,60 @@ def test_builder_contracts_reject_earlier_and_later_active_overrides( "missing pinned Debian 13 live-journey builder", ) + def test_shell_builder_contracts_allow_only_one_canonical_assignment( + self, + ) -> None: + pin = self.module.RUST_BUILDER + cases = ( + ( + RELEASE_BINARY_RECIPE, + f'default_builder_image="{pin}"', + "default_builder_image", + "missing pinned Debian 13 release recipe builder", + ), + ( + TUTORIAL_CHECK, + f'BUILDER_IMAGE="{pin}"', + "BUILDER_IMAGE", + "missing pinned Debian 13 registryctl tutorial builder", + ), + ) + for relative, exact, variable, failure in cases: + for prefix in ("readonly ", "export "): + with self.subTest( + relative=relative, + prefix=prefix, + valid=True, + ): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace(exact, prefix + exact, 1), + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + + invalid_replacements = ( + f"readonly {exact}\nexport {exact}", + f'readonly {variable}="rust:1.95-trixie"', + f'export {variable}="rust:1.95-trixie"', + ) + for replacement in invalid_replacements: + with self.subTest( + relative=relative, + replacement=replacement, + valid=False, + ): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): From 9e586721d9ff7c1538102db3ef71e96e7d37f627 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 04:44:22 +0700 Subject: [PATCH 24/34] fix(release): close image contract overrides Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 110 +++++++++++++-- release/scripts/test_check_debian13_images.py | 131 ++++++++++++++++++ 2 files changed, 230 insertions(+), 11 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index ffdf927f..2d16d759 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -32,6 +32,9 @@ "${{ hashFiles('docs/site/scripts/check-registryctl-tutorials.sh') }}-" "${{ hashFiles('Cargo.lock') }}" ) +RELEASE_BUILDER_HANDOFF = 'release_builder_image="${default_builder_image}"' +RELEASE_BUILDER_CONSUMER = ' "${release_builder_image}" \\' +TUTORIAL_BUILDER_CONSUMER = '\t\t"$BUILDER_IMAGE" \\' DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), @@ -93,7 +96,7 @@ ) FROM_RE = re.compile( - r"^[ \t]*FROM[ \t]+(?:--platform=\S+[ \t]+)?" + r"^[ \t]*FROM[ \t]+(?:--platform=(?P\S+)[ \t]+)?" r"(?P[^\s#]+)" r"(?:[ \t]+AS[ \t]+(?P[^\s#]+))?" r"[ \t]*(?:#.*)?$", @@ -101,7 +104,11 @@ ) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( - r"\b(?:bookworm|debian[ \t_:-]*12)\b", + r"\b(?:bookworm|debian[ \t_:-]*v?[ \t_:-]*12)\b", + re.IGNORECASE, +) +RUNTIME_DIRECTIVE_RE = re.compile( + r"^[ \t]*(?PHEALTHCHECK|ENTRYPOINT|WORKDIR|USER)(?:[ \t]+|$)", re.IGNORECASE, ) RELEASE_BUILDER_KEY_RE = re.compile( @@ -113,10 +120,24 @@ TUTORIAL_BUILDER_ASSIGNMENT_RE = re.compile( r"^[ \t]*(?:(?:export|readonly)[ \t]+)?BUILDER_IMAGE[ \t]*=" ) +RELEASE_BUILDER_HANDOFF_RE = re.compile( + r"^[ \t]*(?:(?:export|readonly)[ \t]+)?release_builder_image[ \t]*=" +) +RELEASE_BUILDER_CONSUMER_RE = re.compile( + r'^[ \t]+(?:"?\$\{release_builder_image\}"?|"?rust:[^" \t#]+"?)' + r"[ \t]+\\[ \t]*$" +) +TUTORIAL_BUILDER_CONSUMER_RE = re.compile( + r'^[ \t]+(?:"?\$BUILDER_IMAGE"?|"?rust:[^" \t#]+"?)[ \t]+\\[ \t]*$' +) LIVE_JOURNEY_BUILDER_RE = re.compile( r"^[ \t]+rust:[^ \t#]+[ \t]+\\[ \t]*$" ) CACHE_KEY_RE = re.compile(r"^[ \t]*key[ \t]*:") +RESTORE_KEYS_RE = re.compile( + r"""^[ \t]*(?:restore-keys|'restore-keys'|"restore-keys")[ \t]*:""", + re.MULTILINE, +) def read(root: Path, relative: Path, failures: list[str]) -> str: @@ -177,6 +198,22 @@ def workflow_step(text: str, name: str) -> str: return "\n".join(lines[start:end]) +def shell_continuation_command(text: str, command: str) -> str: + lines = text.splitlines() + matches = [ + index + for index, line in enumerate(lines) + if line.lstrip() == f"{command} \\" + ] + if len(matches) != 1: + return "" + start = matches[0] + end = start + 1 + while end < len(lines) and lines[end - 1].rstrip().endswith("\\"): + end += 1 + return "\n".join(lines[start:end]) + + def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] texts = { @@ -199,18 +236,22 @@ def check_repository(root: Path = ROOT) -> list[str]: failures.append(f"{relative}: no FROM instruction found") continue stages = tuple( - (match.group("base"), match.group("alias")) + ( + match.group("base"), + match.group("alias"), + match.group("platform"), + ) for match in stage_matches ) expected_stages = ( ( - (RUST_BUILDER, "builder"), - (DISTROLESS_RUNTIME, "runtime"), + (RUST_BUILDER, "builder", None), + (DISTROLESS_RUNTIME, "runtime", None), ) if relative in RUST_BUILDER_DOCKERFILES else ( - (DEBIAN_PREPARATION, "runtime-root"), - (DISTROLESS_RUNTIME, "runtime"), + (DEBIAN_PREPARATION, "runtime-root", None), + (DISTROLESS_RUNTIME, "runtime", None), ) ) if stages != expected_stages: @@ -218,7 +259,7 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{relative}: Dockerfile stage sequence must be exactly " f"{expected_stages!r}; found {stages!r}" ) - for base, _alias in stages: + for base, _alias, _platform in stages: if not DIGEST_PIN_RE.search(base): failures.append( f"{relative}: upstream base is not pinned by immutable digest: {base}" @@ -238,12 +279,27 @@ def check_repository(root: Path = ROOT) -> list[str]: if relative in RELAY_DOCKERFILES else NOTARY_RUNTIME_DIRECTIVES ) + active_runtime_directives: dict[str, list[str]] = {} + for line in runtime.splitlines(): + directive_match = RUNTIME_DIRECTIVE_RE.match(line) + if directive_match: + name = directive_match.group("name").casefold() + active_runtime_directives.setdefault(name, []).append(line) for directive, detail in runtime_directives: - if directive not in runtime.splitlines(): + name = directive.partition(" ")[0] + active_lines = active_runtime_directives.get(name.casefold(), []) + if active_lines != [directive]: failures.append( f"{relative}: missing {detail} in final runtime stage: " - f"{directive!r}" + f"expected exactly one active {name} directive " + f"{directive!r}; found {active_lines!r}" ) + active_users = active_runtime_directives.get("user", []) + if active_users: + failures.append( + f"{relative}: final Distroless runtime must inherit the " + f"nonroot base user; found active USER directives {active_users!r}" + ) for relative in PREPARATION_DOCKERFILES: text = texts[relative] @@ -339,6 +395,26 @@ def check_repository(root: Path = ROOT) -> list[str]: "pinned Debian 13 release recipe builder", failures, ) + require_unique_active_line( + binary_recipe, + (RELEASE_BUILDER_HANDOFF,), + RELEASE_BUILDER_HANDOFF_RE, + Path("release/scripts/build-release-binaries.sh"), + "release builder handoff", + failures, + ) + release_builder_command = shell_continuation_command( + binary_recipe, + "docker run --rm", + ) + require_unique_active_line( + release_builder_command, + (RELEASE_BUILDER_CONSUMER,), + RELEASE_BUILDER_CONSUMER_RE, + Path("release/scripts/build-release-binaries.sh"), + "release Docker builder consumer", + failures, + ) require( binary_recipe, "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", @@ -357,6 +433,18 @@ def check_repository(root: Path = ROOT) -> list[str]: "pinned Debian 13 registryctl tutorial builder", failures, ) + tutorial_builder_command = shell_continuation_command( + tutorial_check, + "docker run --rm", + ) + require_unique_active_line( + tutorial_builder_command, + (TUTORIAL_BUILDER_CONSUMER,), + TUTORIAL_BUILDER_CONSUMER_RE, + Path("docs/site/scripts/check-registryctl-tutorials.sh"), + "registryctl tutorial Docker builder consumer", + failures, + ) live_journey = texts[ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") @@ -384,7 +472,7 @@ def check_repository(root: Path = ROOT) -> list[str]: "registryctl tutorial builder cache key", failures, ) - if re.search(r"^\s*restore-keys\s*:", tutorial_cache, re.MULTILINE): + if RESTORE_KEYS_RE.search(tutorial_cache): failures.append( ".github/workflows/ci.yml: registryctl tutorial builder cache " "must not use restore-keys fallback" diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index ef6285e3..4c236b3c 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -106,6 +106,7 @@ def test_retired_markers_are_rejected_regardless_of_text_syntax(self) -> None: "base: debian-12\n", "base: debian_12\n", "base: debian:12\n", + "base: Debian v12\n", '{"base": "BOOK' + 'WORM"}\n', ) for suffix in cases: @@ -121,6 +122,24 @@ def test_retired_markers_are_rejected_regardless_of_text_syntax(self) -> None: f"{TUTORIAL_CHECK}: retired Debian image generation marker remains", ) + def test_retired_marker_does_not_expand_to_slash_or_dot_separators( + self, + ) -> None: + for suffix in ( + "base: Debian/12\n", + "base: Debian.12\n", + "base: Debian/v12\n", + "base: Debian.v12\n", + ): + with self.subTest(suffix=suffix): + root = self.fixture() + target = root / TUTORIAL_CHECK + target.write_text( + target.read_text(encoding="utf-8") + suffix, + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + def test_tutorial_builder_must_match_the_exact_pinned_image(self) -> None: exact = f'BUILDER_IMAGE="{self.module.RUST_BUILDER}"' cases = ( @@ -288,6 +307,43 @@ def test_shell_builder_contracts_allow_only_one_canonical_assignment( ) self.assert_has_failure(root, failure) + def test_builder_handoffs_and_docker_consumers_remain_exact(self) -> None: + cases = ( + ( + RELEASE_BINARY_RECIPE, + self.module.RELEASE_BUILDER_HANDOFF, + 'release_builder_image="rust:1.95-trixie"', + "missing release builder handoff", + ), + ( + RELEASE_BINARY_RECIPE, + self.module.RELEASE_BUILDER_CONSUMER, + ' "rust:1.95-trixie" \\', + "missing release Docker builder consumer", + ), + ( + TUTORIAL_CHECK, + self.module.TUTORIAL_BUILDER_CONSUMER, + '\t\t"rust:1.95-trixie" \\', + "missing registryctl tutorial Docker builder consumer", + ), + ) + for relative, exact, unpinned, failure in cases: + for replacement in (unpinned, f"{exact}\n{exact}"): + with self.subTest( + relative=relative, + replacement=replacement, + ): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): @@ -305,6 +361,32 @@ def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: f"{relative}: upstream base is not pinned by immutable digest", ) + def test_dockerfile_stages_reject_forced_platforms(self) -> None: + for relative in self.module.DOCKERFILES: + for stage_index in (0, 1): + with self.subTest( + relative=relative, + stage_index=stage_index, + ): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + stage = list(self.module.FROM_RE.finditer(text))[stage_index] + original = stage.group(0) + forced = original.replace( + "FROM ", + "FROM --platform=linux/amd64 ", + 1, + ) + target.write_text( + text[: stage.start()] + forced + text[stage.end() :], + encoding="utf-8", + ) + self.assert_has_failure( + root, + f"{relative}: Dockerfile stage sequence must be exactly", + ) + def test_distroless_runtime_is_the_final_dockerfile_stage(self) -> None: pinned_alpine = "alpine:3.22@sha256:" + "a" * 64 for relative in self.module.DOCKERFILES: @@ -385,6 +467,43 @@ def test_runtime_directives_must_be_active_in_the_final_stage(self) -> None: ) self.assert_has_failure(root, f"{relative}: missing") + def test_runtime_directive_overrides_and_users_are_rejected(self) -> None: + overrides = ( + (" healthcheck NONE", "exactly one active HEALTHCHECK"), + (' entrypoint ["/tmp/override"]', "exactly one active ENTRYPOINT"), + (" workdir /tmp", "exactly one active WORKDIR"), + ("USER 0", "must inherit the nonroot base user"), + ) + comments = "\n".join( + ( + "# HEALTHCHECK NONE", + '# ENTRYPOINT ["/tmp/override"]', + "# WORKDIR /tmp", + "# USER 0", + ) + ) + for relative in self.module.DOCKERFILES: + for override, failure in overrides: + with self.subTest(relative=relative, override=override): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + f"\n{override}\n", + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + + with self.subTest(relative=relative, comments=True): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + f"\n{comments}\n", + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( self, ) -> None: @@ -406,6 +525,18 @@ def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( " registryctl-tutorial-${{ runner.os }}-", "must not use restore-keys fallback", ), + ( + exact + + "\n 'restore-keys': |\n" + " registryctl-tutorial-${{ runner.os }}-", + "must not use restore-keys fallback", + ), + ( + exact + + '\n "restore-keys": |\n' + " registryctl-tutorial-${{ runner.os }}-", + "must not use restore-keys fallback", + ), ) for replacement, failure in cases: with self.subTest(replacement=replacement): From 539e2190600925e117bc3cf9311f8afd3016f5ef Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 04:56:49 +0700 Subject: [PATCH 25/34] fix(release): bind image consumers exactly Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 147 +++++++++++++----- release/scripts/test_check_debian13_images.py | 109 ++++++++++++- 2 files changed, 214 insertions(+), 42 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 2d16d759..1c408e06 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -27,14 +27,41 @@ "a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e" ) TUTORIAL_CACHE_STEP = "Cache source-under-test Cargo build" -TUTORIAL_CACHE_KEY = ( - " key: registryctl-tutorial-${{ runner.os }}-" +TUTORIAL_CACHE_VALUE = ( + "registryctl-tutorial-${{ runner.os }}-" "${{ hashFiles('docs/site/scripts/check-registryctl-tutorials.sh') }}-" "${{ hashFiles('Cargo.lock') }}" ) +TUTORIAL_CACHE_KEYS = tuple( + f" {key}: {TUTORIAL_CACHE_VALUE}" + for key in ("key", "'key'", '"key"') +) +TUTORIAL_CACHE_KEY = TUTORIAL_CACHE_KEYS[0] RELEASE_BUILDER_HANDOFF = 'release_builder_image="${default_builder_image}"' RELEASE_BUILDER_CONSUMER = ' "${release_builder_image}" \\' TUTORIAL_BUILDER_CONSUMER = '\t\t"$BUILDER_IMAGE" \\' +LIVE_JOURNEY_BUILDER = f" {RUST_BUILDER} \\" +RELEASE_BUILDER_TAIL = "\n".join( + ( + ' --env RELEASE_RUSTFLAGS="${release_rustflags}" \\', + RELEASE_BUILDER_CONSUMER, + " bash -c 'set -euo pipefail", + ) +) +TUTORIAL_BUILDER_TAIL = "\n".join( + ( + "\t\t--env HOME=/tmp/registryctl-tutorial-home \\", + TUTORIAL_BUILDER_CONSUMER, + "\t\tbash -c 'set -euo pipefail", + ) +) +LIVE_JOURNEY_BUILDER_TAIL = "\n".join( + ( + " --workdir /workspace \\", + LIVE_JOURNEY_BUILDER, + " sh -eu -c \\", + ) +) DOCKERFILES = ( Path("crates/registry-relay/Dockerfile"), @@ -81,6 +108,10 @@ "absolute Relay entrypoint", ), ("WORKDIR /var/lib/registry-relay", "Relay working directory"), + ( + 'CMD ["--config", "/etc/registry-relay/config.yaml"]', + "Relay default command", + ), ) NOTARY_RUNTIME_DIRECTIVES = ( ( @@ -93,6 +124,10 @@ "absolute Notary entrypoint", ), ("WORKDIR /var/lib/registry-notary", "Notary working directory"), + ( + 'CMD ["--config", "/etc/registry-notary/config.yaml"]', + "Notary default command", + ), ) FROM_RE = re.compile( @@ -108,7 +143,8 @@ re.IGNORECASE, ) RUNTIME_DIRECTIVE_RE = re.compile( - r"^[ \t]*(?PHEALTHCHECK|ENTRYPOINT|WORKDIR|USER)(?:[ \t]+|$)", + r"^[ \t]*(?PHEALTHCHECK|ENTRYPOINT|WORKDIR|CMD|USER|VOLUME)" + r"(?:[ \t]+|$)", re.IGNORECASE, ) RELEASE_BUILDER_KEY_RE = re.compile( @@ -123,17 +159,12 @@ RELEASE_BUILDER_HANDOFF_RE = re.compile( r"^[ \t]*(?:(?:export|readonly)[ \t]+)?release_builder_image[ \t]*=" ) -RELEASE_BUILDER_CONSUMER_RE = re.compile( - r'^[ \t]+(?:"?\$\{release_builder_image\}"?|"?rust:[^" \t#]+"?)' - r"[ \t]+\\[ \t]*$" -) -TUTORIAL_BUILDER_CONSUMER_RE = re.compile( - r'^[ \t]+(?:"?\$BUILDER_IMAGE"?|"?rust:[^" \t#]+"?)[ \t]+\\[ \t]*$' -) LIVE_JOURNEY_BUILDER_RE = re.compile( r"^[ \t]+rust:[^ \t#]+[ \t]+\\[ \t]*$" ) -CACHE_KEY_RE = re.compile(r"^[ \t]*key[ \t]*:") +CACHE_KEY_RE = re.compile( + r"""^[ \t]*(?:key|'key'|"key")[ \t]*:""" +) RESTORE_KEYS_RE = re.compile( r"""^[ \t]*(?:restore-keys|'restore-keys'|"restore-keys")[ \t]*:""", re.MULTILINE, @@ -180,6 +211,21 @@ def require_unique_active_line( ) +def require_unique_text( + text: str, + expected: str, + relative: Path, + detail: str, + failures: list[str], +) -> None: + count = text.count(expected) + if count != 1: + failures.append( + f"{relative}: missing {detail}: expected exactly one exact " + f"block {expected!r}; found {count}" + ) + + def workflow_step(text: str, name: str) -> str: lines = text.splitlines() header = f" - name: {name}" @@ -198,20 +244,30 @@ def workflow_step(text: str, name: str) -> str: return "\n".join(lines[start:end]) -def shell_continuation_command(text: str, command: str) -> str: +def shell_continuation_command( + text: str, + command: str, + required_line: str | None = None, +) -> str: lines = text.splitlines() - matches = [ + starts = [ index for index, line in enumerate(lines) if line.lstrip() == f"{command} \\" ] - if len(matches) != 1: - return "" - start = matches[0] - end = start + 1 - while end < len(lines) and lines[end - 1].rstrip().endswith("\\"): - end += 1 - return "\n".join(lines[start:end]) + commands = [] + for start in starts: + end = start + 1 + while end < len(lines) and lines[end - 1].rstrip().endswith("\\"): + end += 1 + commands.append("\n".join(lines[start:end])) + if required_line is not None: + commands = [ + candidate + for candidate in commands + if required_line in candidate.splitlines() + ] + return commands[0] if len(commands) == 1 else "" def check_repository(root: Path = ROOT) -> list[str]: @@ -294,12 +350,16 @@ def check_repository(root: Path = ROOT) -> list[str]: f"expected exactly one active {name} directive " f"{directive!r}; found {active_lines!r}" ) - active_users = active_runtime_directives.get("user", []) - if active_users: - failures.append( - f"{relative}: final Distroless runtime must inherit the " - f"nonroot base user; found active USER directives {active_users!r}" - ) + for name, invariant in ( + ("user", "inherit the nonroot base user"), + ("volume", "declare no writable VOLUME mount surfaces"), + ): + active_lines = active_runtime_directives.get(name, []) + if active_lines: + failures.append( + f"{relative}: final Distroless runtime must {invariant}; " + f"found active {name.upper()} directives {active_lines!r}" + ) for relative in PREPARATION_DOCKERFILES: text = texts[relative] @@ -397,7 +457,10 @@ def check_repository(root: Path = ROOT) -> list[str]: ) require_unique_active_line( binary_recipe, - (RELEASE_BUILDER_HANDOFF,), + tuple( + f"{prefix}{RELEASE_BUILDER_HANDOFF}" + for prefix in ("", "readonly ", "export ") + ), RELEASE_BUILDER_HANDOFF_RE, Path("release/scripts/build-release-binaries.sh"), "release builder handoff", @@ -407,12 +470,11 @@ def check_repository(root: Path = ROOT) -> list[str]: binary_recipe, "docker run --rm", ) - require_unique_active_line( + require_unique_text( release_builder_command, - (RELEASE_BUILDER_CONSUMER,), - RELEASE_BUILDER_CONSUMER_RE, + RELEASE_BUILDER_TAIL, Path("release/scripts/build-release-binaries.sh"), - "release Docker builder consumer", + "release Docker builder command tail", failures, ) require( @@ -437,12 +499,11 @@ def check_repository(root: Path = ROOT) -> list[str]: tutorial_check, "docker run --rm", ) - require_unique_active_line( + require_unique_text( tutorial_builder_command, - (TUTORIAL_BUILDER_CONSUMER,), - TUTORIAL_BUILDER_CONSUMER_RE, + TUTORIAL_BUILDER_TAIL, Path("docs/site/scripts/check-registryctl-tutorials.sh"), - "registryctl tutorial Docker builder consumer", + "registryctl tutorial Docker builder command tail", failures, ) @@ -451,12 +512,24 @@ def check_repository(root: Path = ROOT) -> list[str]: ] require_unique_active_line( live_journey, - (f" {RUST_BUILDER} \\",), + (LIVE_JOURNEY_BUILDER,), LIVE_JOURNEY_BUILDER_RE, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "pinned Debian 13 live-journey builder", failures, ) + live_builder_command = shell_continuation_command( + live_journey, + "docker run --rm", + LIVE_JOURNEY_BUILDER, + ) + require_unique_text( + live_builder_command, + LIVE_JOURNEY_BUILDER_TAIL, + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + "live-journey Docker builder command tail", + failures, + ) tutorial_cache = workflow_step(ci_workflow, TUTORIAL_CACHE_STEP) if not tutorial_cache: @@ -466,7 +539,7 @@ def check_repository(root: Path = ROOT) -> list[str]: else: require_unique_active_line( tutorial_cache, - (TUTORIAL_CACHE_KEY,), + TUTORIAL_CACHE_KEYS, CACHE_KEY_RE, Path(".github/workflows/ci.yml"), "registryctl tutorial builder cache key", diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 4c236b3c..03b84433 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -71,7 +71,7 @@ def assert_has_failure( failures, ) - def runtime_directives(self, relative: Path) -> tuple[str, str, str]: + def runtime_directives(self, relative: Path) -> tuple[str, ...]: product = "relay" if relative in self.module.RELAY_DOCKERFILES else "notary" binary = f"/usr/local/bin/registry-{product}" return ( @@ -79,6 +79,7 @@ def runtime_directives(self, relative: Path) -> tuple[str, str, str]: f'--start-period=10s --retries=3 CMD ["{binary}", "healthcheck"]', f'ENTRYPOINT ["{binary}"]', f"WORKDIR /var/lib/registry-{product}", + f'CMD ["--config", "/etc/registry-{product}/config.yaml"]', ) def test_current_repository_follows_contract(self) -> None: @@ -270,6 +271,12 @@ def test_shell_builder_contracts_allow_only_one_canonical_assignment( "BUILDER_IMAGE", "missing pinned Debian 13 registryctl tutorial builder", ), + ( + RELEASE_BINARY_RECIPE, + self.module.RELEASE_BUILDER_HANDOFF, + "release_builder_image", + "missing release builder handoff", + ), ) for relative, exact, variable, failure in cases: for prefix in ("readonly ", "export "): @@ -319,13 +326,13 @@ def test_builder_handoffs_and_docker_consumers_remain_exact(self) -> None: RELEASE_BINARY_RECIPE, self.module.RELEASE_BUILDER_CONSUMER, ' "rust:1.95-trixie" \\', - "missing release Docker builder consumer", + "missing release Docker builder command tail", ), ( TUTORIAL_CHECK, self.module.TUTORIAL_BUILDER_CONSUMER, '\t\t"rust:1.95-trixie" \\', - "missing registryctl tutorial Docker builder consumer", + "missing registryctl tutorial Docker builder command tail", ), ) for relative, exact, unpinned, failure in cases: @@ -344,6 +351,47 @@ def test_builder_handoffs_and_docker_consumers_remain_exact(self) -> None: ) self.assert_has_failure(root, failure) + def test_docker_builder_tails_reject_preceding_positional_images( + self, + ) -> None: + cases = ( + ( + RELEASE_BINARY_RECIPE, + self.module.RELEASE_BUILDER_CONSUMER, + " ", + "missing release Docker builder command tail", + ), + ( + TUTORIAL_CHECK, + self.module.TUTORIAL_BUILDER_CONSUMER, + "\t\t", + "missing registryctl tutorial Docker builder command tail", + ), + ( + LIVE_JOURNEY, + self.module.LIVE_JOURNEY_BUILDER, + " ", + "missing live-journey Docker builder command tail", + ), + ) + for relative, approved, indentation, failure in cases: + for image in ("alpine:3.22", "debian:trixie-slim"): + with self.subTest(relative=relative, image=image): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(approved, text) + positional = f"{indentation}{image} \\" + target.write_text( + text.replace( + approved, + positional + "\n" + approved, + 1, + ), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): @@ -472,14 +520,24 @@ def test_runtime_directive_overrides_and_users_are_rejected(self) -> None: (" healthcheck NONE", "exactly one active HEALTHCHECK"), (' entrypoint ["/tmp/override"]', "exactly one active ENTRYPOINT"), (" workdir /tmp", "exactly one active WORKDIR"), + ( + ' cmd ["--config", "/tmp/override.yaml"]', + "exactly one active CMD", + ), ("USER 0", "must inherit the nonroot base user"), + ( + " volume /var/lib/registry", + "declare no writable VOLUME mount surfaces", + ), ) comments = "\n".join( ( "# HEALTHCHECK NONE", '# ENTRYPOINT ["/tmp/override"]', "# WORKDIR /tmp", + '# CMD ["--config", "/tmp/override.yaml"]', "# USER 0", + "# VOLUME /var/lib/registry", ) ) for relative in self.module.DOCKERFILES: @@ -494,6 +552,21 @@ def test_runtime_directive_overrides_and_users_are_rejected(self) -> None: ) self.assert_has_failure(root, failure) + with self.subTest(relative=relative, replaced_cmd=True): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + canonical_cmd = self.runtime_directives(relative)[-1] + target.write_text( + text.replace( + canonical_cmd, + 'CMD ["/tmp/override"]', + 1, + ), + encoding="utf-8", + ) + self.assert_has_failure(root, "exactly one active CMD") + with self.subTest(relative=relative, comments=True): root = self.fixture() target = root / relative @@ -508,6 +581,21 @@ def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( self, ) -> None: exact = self.module.TUTORIAL_CACHE_KEY + for allowed in self.module.TUTORIAL_CACHE_KEYS[1:]: + with self.subTest(allowed=allowed): + root = self.fixture() + target = root / CI_WORKFLOW + text = target.read_text(encoding="utf-8") + target.write_text( + text.replace(exact, allowed, 1), + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + + wrong_value = ( + "registryctl-tutorial-${{ runner.os }}-" + "${{ hashFiles('Cargo.lock') }}" + ) cases = ( ("", "missing registryctl tutorial builder cache key"), ( @@ -515,8 +603,19 @@ def test_tutorial_cache_is_bound_to_the_builder_script_without_fallback( "missing registryctl tutorial builder cache key", ), ( - " key: registryctl-tutorial-${{ runner.os }}-" - "${{ hashFiles('Cargo.lock') }}", + f" key: {wrong_value}", + "missing registryctl tutorial builder cache key", + ), + ( + f" 'key': {wrong_value}", + "missing registryctl tutorial builder cache key", + ), + ( + f' "key": {wrong_value}', + "missing registryctl tutorial builder cache key", + ), + ( + exact + "\n" + self.module.TUTORIAL_CACHE_KEYS[1], "missing registryctl tutorial builder cache key", ), ( From ba8558e848291e60b5d270381713617f2532913f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 05:01:25 +0700 Subject: [PATCH 26/34] fix(release): reject positional image bypasses Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 56 +++++++++++++++ release/scripts/test_check_debian13_images.py | 70 ++++++++++++++----- 2 files changed, 107 insertions(+), 19 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 1c408e06..b08982db 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -226,6 +226,41 @@ def require_unique_text( ) +def require_option_lines_before_image( + command: str, + approved_image: str, + relative: Path, + detail: str, + failures: list[str], +) -> None: + lines = command.splitlines() + image_indexes = [ + index for index, line in enumerate(lines) if line == approved_image + ] + if ( + not lines + or lines[0].lstrip() != "docker run --rm \\" + or len(image_indexes) != 1 + ): + failures.append( + f"{relative}: missing {detail}: expected one approved image " + "inside the selected docker run --rm command" + ) + return + invalid_lines = [ + line + for line in lines[1 : image_indexes[0]] + if line.strip() + and not line.lstrip().startswith("#") + and not line.lstrip().startswith("--") + ] + if invalid_lines: + failures.append( + f"{relative}: {detail} contains non-option lines before the " + f"approved image: {invalid_lines!r}" + ) + + def workflow_step(text: str, name: str) -> str: lines = text.splitlines() header = f" - name: {name}" @@ -477,6 +512,13 @@ def check_repository(root: Path = ROOT) -> list[str]: "release Docker builder command tail", failures, ) + require_option_lines_before_image( + release_builder_command, + RELEASE_BUILDER_CONSUMER, + Path("release/scripts/build-release-binaries.sh"), + "release Docker builder command", + failures, + ) require( binary_recipe, "--features registry-notary/registry-notary-cel,registry-notary/pkcs11", @@ -506,6 +548,13 @@ def check_repository(root: Path = ROOT) -> list[str]: "registryctl tutorial Docker builder command tail", failures, ) + require_option_lines_before_image( + tutorial_builder_command, + TUTORIAL_BUILDER_CONSUMER, + Path("docs/site/scripts/check-registryctl-tutorials.sh"), + "registryctl tutorial Docker builder command", + failures, + ) live_journey = texts[ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") @@ -530,6 +579,13 @@ def check_repository(root: Path = ROOT) -> list[str]: "live-journey Docker builder command tail", failures, ) + require_option_lines_before_image( + live_builder_command, + LIVE_JOURNEY_BUILDER, + Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + "live-journey Docker builder command", + failures, + ) tutorial_cache = workflow_step(ci_workflow, TUTORIAL_CACHE_STEP) if not tutorial_cache: diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 03b84433..bd854077 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -357,40 +357,72 @@ def test_docker_builder_tails_reject_preceding_positional_images( cases = ( ( RELEASE_BINARY_RECIPE, + "docker run --rm \\", + self.module.RELEASE_BUILDER_TAIL.splitlines()[0], self.module.RELEASE_BUILDER_CONSUMER, " ", - "missing release Docker builder command tail", + "release Docker builder command contains non-option lines", ), ( TUTORIAL_CHECK, + "\tdocker run --rm \\", + self.module.TUTORIAL_BUILDER_TAIL.splitlines()[0], self.module.TUTORIAL_BUILDER_CONSUMER, "\t\t", - "missing registryctl tutorial Docker builder command tail", + "registryctl tutorial Docker builder command contains " + "non-option lines", ), ( LIVE_JOURNEY, + " docker run --rm \\", + self.module.LIVE_JOURNEY_BUILDER_TAIL.splitlines()[0], self.module.LIVE_JOURNEY_BUILDER, " ", - "missing live-journey Docker builder command tail", + "live-journey Docker builder command contains non-option lines", ), ) - for relative, approved, indentation, failure in cases: + for ( + relative, + command_start, + last_option, + approved, + indentation, + failure, + ) in cases: for image in ("alpine:3.22", "debian:trixie-slim"): - with self.subTest(relative=relative, image=image): - root = self.fixture() - target = root / relative - text = target.read_text(encoding="utf-8") - self.assertIn(approved, text) - positional = f"{indentation}{image} \\" - target.write_text( - text.replace( - approved, - positional + "\n" + approved, - 1, - ), - encoding="utf-8", - ) - self.assert_has_failure(root, failure) + positional = f"{indentation}{image} \\" + positions = ( + ( + "before-first-option", + command_start, + command_start + "\n" + positional, + ), + ( + "before-last-option", + last_option, + positional + "\n" + last_option, + ), + ( + "before-approved-image", + approved, + positional + "\n" + approved, + ), + ) + for position, anchor, replacement in positions: + with self.subTest( + relative=relative, + image=image, + position=position, + ): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(anchor, text) + target.write_text( + text.replace(anchor, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: From 9e2864257708315158e8aa90c0e9333edadc44ea Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 05:07:19 +0700 Subject: [PATCH 27/34] fix(release): pin exact builder command prefixes Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 97 +++++++++++-------- release/scripts/test_check_debian13_images.py | 61 +++++++++--- 2 files changed, 108 insertions(+), 50 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index b08982db..0133ffc7 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -41,24 +41,64 @@ RELEASE_BUILDER_CONSUMER = ' "${release_builder_image}" \\' TUTORIAL_BUILDER_CONSUMER = '\t\t"$BUILDER_IMAGE" \\' LIVE_JOURNEY_BUILDER = f" {RUST_BUILDER} \\" +RELEASE_BUILDER_PREFIX = ( + "docker run --rm \\", + " --platform linux/amd64 \\", + ' --user "$(id -u):$(id -g)" \\', + ' --volume "${repo_root}:/workspace" \\', + ' --volume "${release_cargo_home}:/workspace/.cargo-home" \\', + ' --volume "${release_target_dir}:/workspace/target" \\', + " --workdir /workspace \\", + " --env CARGO_HOME=/workspace/.cargo-home \\", + " --env CARGO_TARGET_DIR=/workspace/target \\", + " --env CARGO_INCREMENTAL=0 \\", + ' --env CARGO_TERM_COLOR="${CARGO_TERM_COLOR:-always}" \\', + " --env HOME=/workspace \\", + ' --env RELEASE_TAG="${tag}" \\', + ' --env RELEASE_RUSTFLAGS="${release_rustflags}" \\', + RELEASE_BUILDER_CONSUMER, +) +TUTORIAL_BUILDER_PREFIX = ( + "\tdocker run --rm \\", + "\t\t--platform linux/amd64 \\", + '\t\t--user "$(id -u):$(id -g)" \\', + '\t\t--volume "$REPO_ROOT:/workspace" \\', + "\t\t--workdir /workspace \\", + "\t\t--env CARGO_HOME=/workspace/target/registryctl-tutorial-cargo-home \\", + "\t\t--env CARGO_TARGET_DIR=/workspace/target/registryctl-tutorial-linux-amd64 \\", + "\t\t--env CARGO_TERM_COLOR=always \\", + "\t\t--env HOME=/tmp/registryctl-tutorial-home \\", + TUTORIAL_BUILDER_CONSUMER, +) +LIVE_JOURNEY_BUILDER_PREFIX = ( + " docker run --rm \\", + " --add-host host.docker.internal:host-gateway \\", + ' --network "$network_name" \\', + " --network-alias rhai-runner \\", + ' --env-file "$rhai_test_env_file" \\', + ' --volume "$repository_root:/workspace" \\', + ' --volume "$certificate_input:/live-postgres-ca:ro" \\', + ' --volume "$HOME/.cargo/registry:/usr/local/cargo/registry" \\', + ' --volume "$HOME/.cargo/git:/usr/local/cargo/git" \\', + " --volume registry-relay-linux-target:/target \\", + " --workdir /workspace \\", + LIVE_JOURNEY_BUILDER, +) RELEASE_BUILDER_TAIL = "\n".join( ( - ' --env RELEASE_RUSTFLAGS="${release_rustflags}" \\', - RELEASE_BUILDER_CONSUMER, + *RELEASE_BUILDER_PREFIX[-2:], " bash -c 'set -euo pipefail", ) ) TUTORIAL_BUILDER_TAIL = "\n".join( ( - "\t\t--env HOME=/tmp/registryctl-tutorial-home \\", - TUTORIAL_BUILDER_CONSUMER, + *TUTORIAL_BUILDER_PREFIX[-2:], "\t\tbash -c 'set -euo pipefail", ) ) LIVE_JOURNEY_BUILDER_TAIL = "\n".join( ( - " --workdir /workspace \\", - LIVE_JOURNEY_BUILDER, + *LIVE_JOURNEY_BUILDER_PREFIX[-2:], " sh -eu -c \\", ) ) @@ -226,38 +266,19 @@ def require_unique_text( ) -def require_option_lines_before_image( +def require_exact_command_prefix( command: str, - approved_image: str, + expected_lines: tuple[str, ...], relative: Path, detail: str, failures: list[str], ) -> None: - lines = command.splitlines() - image_indexes = [ - index for index, line in enumerate(lines) if line == approved_image - ] - if ( - not lines - or lines[0].lstrip() != "docker run --rm \\" - or len(image_indexes) != 1 - ): + actual_lines = tuple(command.splitlines()[: len(expected_lines)]) + if actual_lines != expected_lines: failures.append( - f"{relative}: missing {detail}: expected one approved image " - "inside the selected docker run --rm command" - ) - return - invalid_lines = [ - line - for line in lines[1 : image_indexes[0]] - if line.strip() - and not line.lstrip().startswith("#") - and not line.lstrip().startswith("--") - ] - if invalid_lines: - failures.append( - f"{relative}: {detail} contains non-option lines before the " - f"approved image: {invalid_lines!r}" + f"{relative}: {detail} does not match the exact expected " + f"header/options/image prefix: expected {expected_lines!r}; " + f"found {actual_lines!r}" ) @@ -512,9 +533,9 @@ def check_repository(root: Path = ROOT) -> list[str]: "release Docker builder command tail", failures, ) - require_option_lines_before_image( + require_exact_command_prefix( release_builder_command, - RELEASE_BUILDER_CONSUMER, + RELEASE_BUILDER_PREFIX, Path("release/scripts/build-release-binaries.sh"), "release Docker builder command", failures, @@ -548,9 +569,9 @@ def check_repository(root: Path = ROOT) -> list[str]: "registryctl tutorial Docker builder command tail", failures, ) - require_option_lines_before_image( + require_exact_command_prefix( tutorial_builder_command, - TUTORIAL_BUILDER_CONSUMER, + TUTORIAL_BUILDER_PREFIX, Path("docs/site/scripts/check-registryctl-tutorials.sh"), "registryctl tutorial Docker builder command", failures, @@ -579,9 +600,9 @@ def check_repository(root: Path = ROOT) -> list[str]: "live-journey Docker builder command tail", failures, ) - require_option_lines_before_image( + require_exact_command_prefix( live_builder_command, - LIVE_JOURNEY_BUILDER, + LIVE_JOURNEY_BUILDER_PREFIX, Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), "live-journey Docker builder command", failures, diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index bd854077..99df20ff 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -357,38 +357,35 @@ def test_docker_builder_tails_reject_preceding_positional_images( cases = ( ( RELEASE_BINARY_RECIPE, - "docker run --rm \\", - self.module.RELEASE_BUILDER_TAIL.splitlines()[0], + self.module.RELEASE_BUILDER_PREFIX, self.module.RELEASE_BUILDER_CONSUMER, " ", - "release Docker builder command contains non-option lines", + "does not match the exact expected header/options/image prefix", ), ( TUTORIAL_CHECK, - "\tdocker run --rm \\", - self.module.TUTORIAL_BUILDER_TAIL.splitlines()[0], + self.module.TUTORIAL_BUILDER_PREFIX, self.module.TUTORIAL_BUILDER_CONSUMER, "\t\t", - "registryctl tutorial Docker builder command contains " - "non-option lines", + "does not match the exact expected header/options/image prefix", ), ( LIVE_JOURNEY, - " docker run --rm \\", - self.module.LIVE_JOURNEY_BUILDER_TAIL.splitlines()[0], + self.module.LIVE_JOURNEY_BUILDER_PREFIX, self.module.LIVE_JOURNEY_BUILDER, " ", - "live-journey Docker builder command contains non-option lines", + "does not match the exact expected header/options/image prefix", ), ) for ( relative, - command_start, - last_option, + expected_prefix, approved, indentation, failure, ) in cases: + command_start = expected_prefix[0] + last_option = expected_prefix[-2] for image in ("alpine:3.22", "debian:trixie-slim"): positional = f"{indentation}{image} \\" positions = ( @@ -424,6 +421,46 @@ def test_docker_builder_tails_reject_preceding_positional_images( ) self.assert_has_failure(root, failure) + def test_docker_builder_prefixes_reject_images_on_option_lines( + self, + ) -> None: + cases = ( + ( + RELEASE_BINARY_RECIPE, + self.module.RELEASE_BUILDER_PREFIX, + ), + ( + TUTORIAL_CHECK, + self.module.TUTORIAL_BUILDER_PREFIX, + ), + ( + LIVE_JOURNEY, + self.module.LIVE_JOURNEY_BUILDER_PREFIX, + ), + ) + failure = "does not match the exact expected header/options/image prefix" + for relative, expected_prefix in cases: + for image in ("alpine:3.22", "debian:trixie-slim"): + for position, option in ( + ("early", expected_prefix[1]), + ("late", expected_prefix[-2]), + ): + with self.subTest( + relative=relative, + image=image, + position=position, + ): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(option, text) + mutated = option[:-1] + f"{image} \\" + target.write_text( + text.replace(option, mutated, 1), + encoding="utf-8", + ) + self.assert_has_failure(root, failure) + def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): From b1a797dd4f60a146b0338161972a821b4a0ece93 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 05:20:21 +0700 Subject: [PATCH 28/34] fix(release): inventory workflow image references Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 120 +++++++++++- release/scripts/test_check_debian13_images.py | 171 ++++++++++++++++++ 2 files changed, 290 insertions(+), 1 deletion(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 0133ffc7..a4d51bc4 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -7,6 +7,8 @@ import sys from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[2] @@ -125,6 +127,16 @@ Path("products/notary/docs/security-assurance.md"), ) +WORKFLOW_IMAGE_ALLOWLIST = { + ( + Path(".github/workflows/relay-postgres-conformance.yml"), + "postgres:${{ matrix.postgresql }}-alpine", + ): ( + "External Alpine PostgreSQL state-plane conformance, not a " + "project-owned Debian image." + ), +} + RUST_BUILDER_DOCKERFILES = DOCKERFILES[:3] PREPARATION_DOCKERFILES = DOCKERFILES[3:] RELAY_DOCKERFILES = ( @@ -220,6 +232,95 @@ def read(root: Path, relative: Path, failures: list[str]) -> str: return "" +def discover_workflow_paths(root: Path) -> tuple[Path, ...]: + directory = root / ".github" / "workflows" + if not directory.is_dir(): + return () + return tuple( + path.relative_to(root) + for path in sorted( + ( + path + for pattern in ("*.yml", "*.yaml") + for path in directory.glob(pattern) + if path.is_file() + ), + key=lambda path: path.name, + ) + ) + + +def collect_workflow_image_references( + text: str, + relative: Path, + failures: list[str], +) -> list[tuple[str, str]]: + try: + document = yaml.safe_load(text) + except yaml.YAMLError as error: + failures.append( + f"{relative}: workflow YAML parse failed: {type(error).__name__}" + ) + return [] + if not isinstance(document, dict): + failures.append( + f"{relative}: workflow YAML root must be a mapping, " + f"found {type(document).__name__}" + ) + return [] + + references: list[tuple[str, str]] = [] + active_nodes: set[int] = set() + + def add_image(value: object, location: str) -> None: + if not isinstance(value, str) or not value.strip(): + failures.append( + f"{relative}: unsupported workflow image value at {location}: " + f"expected a non-empty string, found {type(value).__name__}" + ) + return + references.append((location, value)) + + def visit(value: object, location: str) -> None: + if not isinstance(value, (dict, list)): + return + identity = id(value) + if identity in active_nodes: + failures.append( + f"{relative}: unsupported recursive YAML alias at {location}" + ) + return + active_nodes.add(identity) + if isinstance(value, dict): + for key, child in value.items(): + child_location = f"{location}.{key}" + if key == "container": + if isinstance(child, str): + add_image(child, child_location) + elif not isinstance(child, dict) or "image" not in child: + failures.append( + f"{relative}: unsupported workflow image value at " + f"{child_location}: container must be a non-empty " + "string or a mapping with an image key" + ) + elif key == "image": + add_image(child, child_location) + elif ( + key == "uses" + and isinstance(child, str) + and child.startswith("docker://") + ): + add_image(child.removeprefix("docker://"), child_location) + visit(child, child_location) + else: + for index, child in enumerate(value): + visit(child, f"{location}[{index}]") + active_nodes.remove(identity) + + visit(document, "$") + return references + + def require( text: str, needle: str, @@ -328,9 +429,13 @@ def shell_continuation_command( def check_repository(root: Path = ROOT) -> list[str]: failures: list[str] = [] + workflow_paths = discover_workflow_paths(root) + maintained_paths = tuple( + dict.fromkeys((*MAINTAINED_TEXT_PATHS, *workflow_paths)) + ) texts = { relative: read(root, relative, failures) - for relative in MAINTAINED_TEXT_PATHS + for relative in maintained_paths } for relative, text in texts.items(): @@ -341,6 +446,19 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{marker.group(0).casefold()}" ) + for relative in workflow_paths: + references = collect_workflow_image_references( + texts[relative], + relative, + failures, + ) + for location, value in references: + if (relative, value) not in WORKFLOW_IMAGE_ALLOWLIST: + failures.append( + f"{relative}: workflow image reference is not allowlisted " + f"at {location}: {value!r}" + ) + for relative in DOCKERFILES: text = texts[relative] stage_matches = list(FROM_RE.finditer(text)) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 99df20ff..c3a7e58f 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -60,6 +60,12 @@ def fixture(self) -> Path: shutil.copyfile(ROOT / relative, destination) return root + def write_workflow(self, root: Path, name: str, text: str) -> Path: + target = root / ".github" / "workflows" / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + return target + def assert_has_failure( self, root: Path, @@ -141,6 +147,171 @@ def test_retired_marker_does_not_expand_to_slash_or_dot_separators( ) self.assertEqual([], self.module.check_repository(root)) + def test_workflow_image_allowlist_has_one_bounded_exception(self) -> None: + allowed_path = Path( + ".github/workflows/relay-postgres-conformance.yml" + ) + allowed_image = "postgres:${{ matrix.postgresql }}-alpine" + self.assertEqual( + {(allowed_path, allowed_image)}, + set(self.module.WORKFLOW_IMAGE_ALLOWLIST), + ) + self.assertIn( + "not a project-owned Debian image", + self.module.WORKFLOW_IMAGE_ALLOWLIST[ + (allowed_path, allowed_image) + ], + ) + + workflow = ( + "name: External PostgreSQL conformance\n" + "jobs:\n" + " state-plane:\n" + " services:\n" + " postgres:\n" + f' image: "{allowed_image}"\n' + ) + root = self.fixture() + self.write_workflow(root, allowed_path.name, workflow) + self.assertEqual([], self.module.check_repository(root)) + + root = self.fixture() + self.write_workflow(root, "copied-postgres.yml", workflow) + self.assert_has_failure( + root, + "copied-postgres.yml: workflow image reference is not allowlisted", + ) + + def test_dynamic_workflow_images_are_denied_from_structured_forms( + self, + ) -> None: + digest_image = "ghcr.io/example/tool@sha256:" + "a" * 64 + cases = ( + ( + "scalar-container.yaml", + "name: Scalar container\n" + "jobs:\n" + " build:\n" + " container: rust:1.95-trixie\n", + ), + ( + "flow-container.yml", + 'name: Flow container\njobs: {build: {container: "' + + digest_image + + '"}}\n', + ), + ( + "anchored-container.yml", + "name: Anchored container\n" + "x-builder: &builder rust:1.95-trixie\n" + "jobs:\n" + " build:\n" + " container: *builder\n", + ), + ( + "mapping-container.yml", + "name: Mapping container\n" + "jobs:\n" + " build:\n" + f' container: {{image: "{digest_image}"}}\n', + ), + ( + "service-image.yml", + "name: Service image\n" + "jobs:\n" + " build:\n" + " services:\n" + " database:\n" + f' image: "{digest_image}"\n', + ), + ( + "docker-uses.yml", + "name: Docker action\n" + "jobs:\n" + " build:\n" + " steps:\n" + " - uses: docker://alpine:3.22\n", + ), + ) + for name, workflow in cases: + with self.subTest(name=name): + root = self.fixture() + relative = Path(".github/workflows") / name + self.assertNotIn(relative, self.module.MAINTAINED_TEXT_PATHS) + self.write_workflow(root, name, workflow) + self.assert_has_failure( + root, + f"{relative}: workflow image reference is not allowlisted", + ) + + def test_workflow_image_inventory_fails_closed(self) -> None: + cases = ( + ( + "malformed.yml", + "jobs: [\n", + "workflow YAML parse failed", + ), + ( + "root-list.yml", + "- jobs\n", + "workflow YAML root must be a mapping", + ), + ( + "container-list.yml", + "jobs:\n" + " build:\n" + " container:\n" + " - rust:1.95-trixie\n", + "unsupported workflow image value", + ), + ( + "container-without-image.yml", + "jobs:\n" + " build:\n" + " container:\n" + " options: --read-only\n", + "unsupported workflow image value", + ), + ( + "mapping-image.yml", + "jobs:\n" + " build:\n" + " services:\n" + " database:\n" + " image:\n" + " name: rust:1.95-trixie\n", + "unsupported workflow image value", + ), + ( + "empty-docker-uses.yml", + "jobs:\n" + " build:\n" + " steps:\n" + " - uses: docker://\n", + "unsupported workflow image value", + ), + ) + for name, workflow, failure in cases: + with self.subTest(name=name): + root = self.fixture() + self.write_workflow(root, name, workflow) + self.assert_has_failure(root, failure) + + def test_retired_markers_cover_dynamically_discovered_workflows( + self, + ) -> None: + root = self.fixture() + relative = Path(".github/workflows/dynamic-policy.yaml") + self.write_workflow( + root, + relative.name, + "name: Debian v12 compatibility\njobs: {}\n", + ) + self.assert_has_failure( + root, + f"{relative}: retired Debian image generation marker remains", + ) + def test_tutorial_builder_must_match_the_exact_pinned_image(self) -> None: exact = f'BUILDER_IMAGE="{self.module.RUST_BUILDER}"' cases = ( From 5b88e978a3639d52cc718dbb44af2b5d93c5a0af Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 05:43:54 +0700 Subject: [PATCH 29/34] fix(release): close remaining image inventory gaps Signed-off-by: Jeremi Joslin --- .../scripts/run-live-consultation-journey.sh | 2 +- release/scripts/check-debian13-images.py | 139 +++++++++- release/scripts/test_check_debian13_images.py | 245 +++++++++++++++++- 3 files changed, 364 insertions(+), 22 deletions(-) diff --git a/crates/registry-relay/scripts/run-live-consultation-journey.sh b/crates/registry-relay/scripts/run-live-consultation-journey.sh index 72ec2bce..334046d9 100755 --- a/crates/registry-relay/scripts/run-live-consultation-journey.sh +++ b/crates/registry-relay/scripts/run-live-consultation-journey.sh @@ -6,7 +6,7 @@ set +v set -euo pipefail umask 077 -readonly POSTGRES_IMAGE="postgres:16" +readonly POSTGRES_IMAGE="postgres:16.13-alpine" [[ "$#" -le 1 ]] || { printf '%s\n' "usage: $0 [dhis2|rhai|synthetic]" >&2 diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index a4d51bc4..54b458bc 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -43,6 +43,11 @@ RELEASE_BUILDER_CONSUMER = ' "${release_builder_image}" \\' TUTORIAL_BUILDER_CONSUMER = '\t\t"$BUILDER_IMAGE" \\' LIVE_JOURNEY_BUILDER = f" {RUST_BUILDER} \\" +LIVE_JOURNEY_POSTGRES_IMAGE = "postgres:16.13-alpine" +LIVE_JOURNEY_POSTGRES_ASSIGNMENT = ( + f'readonly POSTGRES_IMAGE="{LIVE_JOURNEY_POSTGRES_IMAGE}"' +) +LIVE_JOURNEY_POSTGRES_CONSUMER = ' "$POSTGRES_IMAGE" \\' RELEASE_BUILDER_PREFIX = ( "docker run --rm \\", " --platform linux/amd64 \\", @@ -86,6 +91,23 @@ " --workdir /workspace \\", LIVE_JOURNEY_BUILDER, ) +LIVE_JOURNEY_POSTGRES_SETUP_PREFIX = ( + "docker run --rm \\", + " --user 0:0 \\", + ' --volume "$certificate_volume:/certificates" \\', + ' --volume "$certificate_input:/input:ro" \\', + LIVE_JOURNEY_POSTGRES_CONSUMER, + " sh -eu -c \\", +) +LIVE_JOURNEY_POSTGRES_SERVER_PREFIX = ( + "docker run --detach \\", + ' --name "$container_name" \\', + ' --env-file "$docker_env_file" \\', + " --publish 127.0.0.1::5432 \\", + ' --volume "$certificate_volume:/certificates:ro" \\', + LIVE_JOURNEY_POSTGRES_CONSUMER, + " -c ssl=on \\", +) RELEASE_BUILDER_TAIL = "\n".join( ( *RELEASE_BUILDER_PREFIX[-2:], @@ -127,6 +149,18 @@ Path("products/notary/docs/security-assurance.md"), ) +NOTARY_POSTGRES_WORKFLOW_IMAGES = ( + "postgres:16.13-alpine", + "postgres:16.14-alpine", + "postgres:17.9-alpine", + "postgres:17.10-alpine", + "postgres:18.3-alpine", + "postgres:18.4-alpine", +) +NOTARY_POSTGRES_WORKFLOW_RATIONALE = ( + "External Alpine PostgreSQL migration conformance, not a " + "project-owned Debian image." +) WORKFLOW_IMAGE_ALLOWLIST = { ( Path(".github/workflows/relay-postgres-conformance.yml"), @@ -135,7 +169,15 @@ "External Alpine PostgreSQL state-plane conformance, not a " "project-owned Debian image." ), + **{ + ( + Path(".github/workflows/notary-postgres-conformance.yml"), + image, + ): NOTARY_POSTGRES_WORKFLOW_RATIONALE + for image in NOTARY_POSTGRES_WORKFLOW_IMAGES + }, } +WORKFLOW_IMAGE_KEYS = frozenset(("image", "source_image", "target_image")) RUST_BUILDER_DOCKERFILES = DOCKERFILES[:3] PREPARATION_DOCKERFILES = DOCKERFILES[3:] @@ -148,6 +190,19 @@ Path("products/notary/Dockerfile"), Path("release/docker/Dockerfile.registry-notary"), ) +DOCKERFILE_NAMED_CONTEXTS = { + Path("crates/registry-relay/Dockerfile"): frozenset( + ("registry-platform", "registry-manifest", "crosswalk") + ), + Path("crates/registry-relay/Dockerfile.demo"): frozenset( + ("registry-platform",) + ), + Path("products/notary/Dockerfile"): frozenset( + ("registry-platform", "crosswalk") + ), + Path("release/docker/Dockerfile.registry-notary"): frozenset(), + Path("release/docker/Dockerfile.registry-relay"): frozenset(), +} RELAY_RUNTIME_DIRECTIVES = ( ( @@ -189,6 +244,12 @@ r"[ \t]*(?:#.*)?$", re.IGNORECASE | re.MULTILINE, ) +COPY_FROM_RE = re.compile( + r"^[ \t]*COPY[ \t]+" + r"(?:--[^\s=]+(?:=[^\s]+)?[ \t]+)*" + r"--from=(?P[^\s#]+)", + re.IGNORECASE | re.MULTILINE, +) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( r"\b(?:bookworm|debian[ \t_:-]*v?[ \t_:-]*12)\b", @@ -214,6 +275,9 @@ LIVE_JOURNEY_BUILDER_RE = re.compile( r"^[ \t]+rust:[^ \t#]+[ \t]+\\[ \t]*$" ) +LIVE_JOURNEY_POSTGRES_ASSIGNMENT_RE = re.compile( + r"^[ \t]*(?:(?:export|readonly)[ \t]+)?POSTGRES_IMAGE[ \t]*=" +) CACHE_KEY_RE = re.compile( r"""^[ \t]*(?:key|'key'|"key")[ \t]*:""" ) @@ -303,7 +367,7 @@ def visit(value: object, location: str) -> None: f"{child_location}: container must be a non-empty " "string or a mapping with an image key" ) - elif key == "image": + elif key in WORKFLOW_IMAGE_KEYS: add_image(child, child_location) elif ( key == "uses" @@ -373,14 +437,18 @@ def require_exact_command_prefix( relative: Path, detail: str, failures: list[str], + *, + report_values: bool = True, ) -> None: actual_lines = tuple(command.splitlines()[: len(expected_lines)]) if actual_lines != expected_lines: - failures.append( + failure = ( f"{relative}: {detail} does not match the exact expected " - f"header/options/image prefix: expected {expected_lines!r}; " - f"found {actual_lines!r}" + "header/options/image prefix" ) + if report_values: + failure += f": expected {expected_lines!r}; found {actual_lines!r}" + failures.append(failure) def workflow_step(text: str, name: str) -> str: @@ -456,7 +524,7 @@ def check_repository(root: Path = ROOT) -> list[str]: if (relative, value) not in WORKFLOW_IMAGE_ALLOWLIST: failures.append( f"{relative}: workflow image reference is not allowlisted " - f"at {location}: {value!r}" + f"at {location}" ) for relative in DOCKERFILES: @@ -489,6 +557,22 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{relative}: Dockerfile stage sequence must be exactly " f"{expected_stages!r}; found {stages!r}" ) + stage_aliases = { + alias.casefold() + for _base, alias, _platform in stages + if alias is not None + } + named_contexts = DOCKERFILE_NAMED_CONTEXTS[relative] + for match in COPY_FROM_RE.finditer(text): + source = match.group("source") + if ( + source.casefold() not in stage_aliases + and source not in named_contexts + ): + failures.append( + f"{relative}: COPY --from source is not a declared stage " + "or reviewed named build context" + ) for base, _alias, _platform in stages: if not DIGEST_PIN_RE.search(base): failures.append( @@ -698,11 +782,50 @@ def check_repository(root: Path = ROOT) -> list[str]: live_journey = texts[ Path("crates/registry-relay/scripts/run-live-consultation-journey.sh") ] + live_journey_path = Path( + "crates/registry-relay/scripts/run-live-consultation-journey.sh" + ) + postgres_assignments = [ + line + for line in live_journey.splitlines() + if LIVE_JOURNEY_POSTGRES_ASSIGNMENT_RE.match(line) + ] + if postgres_assignments != [LIVE_JOURNEY_POSTGRES_ASSIGNMENT]: + failures.append( + f"{live_journey_path}: live-journey PostgreSQL image assignment " + "must remain the single reviewed value" + ) + postgres_setup_command = shell_continuation_command( + live_journey, + "docker run --rm", + LIVE_JOURNEY_POSTGRES_CONSUMER, + ) + require_exact_command_prefix( + postgres_setup_command, + LIVE_JOURNEY_POSTGRES_SETUP_PREFIX, + live_journey_path, + "live-journey PostgreSQL certificate setup command", + failures, + report_values=False, + ) + postgres_server_command = shell_continuation_command( + live_journey, + "docker run --detach", + LIVE_JOURNEY_POSTGRES_CONSUMER, + ) + require_exact_command_prefix( + postgres_server_command, + LIVE_JOURNEY_POSTGRES_SERVER_PREFIX, + live_journey_path, + "live-journey PostgreSQL server command", + failures, + report_values=False, + ) require_unique_active_line( live_journey, (LIVE_JOURNEY_BUILDER,), LIVE_JOURNEY_BUILDER_RE, - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + live_journey_path, "pinned Debian 13 live-journey builder", failures, ) @@ -714,14 +837,14 @@ def check_repository(root: Path = ROOT) -> list[str]: require_unique_text( live_builder_command, LIVE_JOURNEY_BUILDER_TAIL, - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + live_journey_path, "live-journey Docker builder command tail", failures, ) require_exact_command_prefix( live_builder_command, LIVE_JOURNEY_BUILDER_PREFIX, - Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), + live_journey_path, "live-journey Docker builder command", failures, ) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index c3a7e58f..d135f681 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -17,6 +17,12 @@ "crates/registry-relay/scripts/run-live-consultation-journey.sh" ) CI_WORKFLOW = Path(".github/workflows/ci.yml") +NOTARY_POSTGRES_WORKFLOW = Path( + ".github/workflows/notary-postgres-conformance.yml" +) +RELAY_POSTGRES_WORKFLOW = Path( + ".github/workflows/relay-postgres-conformance.yml" +) EXPECTED_SURFACES = { CI_WORKFLOW, @@ -147,21 +153,29 @@ def test_retired_marker_does_not_expand_to_slash_or_dot_separators( ) self.assertEqual([], self.module.check_repository(root)) - def test_workflow_image_allowlist_has_one_bounded_exception(self) -> None: - allowed_path = Path( - ".github/workflows/relay-postgres-conformance.yml" - ) - allowed_image = "postgres:${{ matrix.postgresql }}-alpine" + def test_workflow_image_allowlist_has_only_reviewed_postgres_images( + self, + ) -> None: + relay_image = "postgres:${{ matrix.postgresql }}-alpine" + notary_images = { + "postgres:16.13-alpine", + "postgres:16.14-alpine", + "postgres:17.9-alpine", + "postgres:17.10-alpine", + "postgres:18.3-alpine", + "postgres:18.4-alpine", + } self.assertEqual( - {(allowed_path, allowed_image)}, + {(RELAY_POSTGRES_WORKFLOW, relay_image)} + | {(NOTARY_POSTGRES_WORKFLOW, image) for image in notary_images}, set(self.module.WORKFLOW_IMAGE_ALLOWLIST), ) - self.assertIn( - "not a project-owned Debian image", - self.module.WORKFLOW_IMAGE_ALLOWLIST[ - (allowed_path, allowed_image) - ], + self.assertEqual( + {"image", "source_image", "target_image"}, + set(self.module.WORKFLOW_IMAGE_KEYS), ) + for rationale in self.module.WORKFLOW_IMAGE_ALLOWLIST.values(): + self.assertIn("not a project-owned Debian image", rationale) workflow = ( "name: External PostgreSQL conformance\n" @@ -169,10 +183,12 @@ def test_workflow_image_allowlist_has_one_bounded_exception(self) -> None: " state-plane:\n" " services:\n" " postgres:\n" - f' image: "{allowed_image}"\n' + f' image: "{relay_image}"\n' ) root = self.fixture() - self.write_workflow(root, allowed_path.name, workflow) + self.write_workflow(root, RELAY_POSTGRES_WORKFLOW.name, workflow) + notary_target = root / NOTARY_POSTGRES_WORKFLOW + shutil.copyfile(ROOT / NOTARY_POSTGRES_WORKFLOW, notary_target) self.assertEqual([], self.module.check_repository(root)) root = self.fixture() @@ -232,6 +248,22 @@ def test_dynamic_workflow_images_are_denied_from_structured_forms( " steps:\n" " - uses: docker://alpine:3.22\n", ), + ( + "matrix-source-image.yml", + "name: Migration source\n" + "jobs:\n" + " build:\n" + " strategy:\n" + " matrix:\n" + " include:\n" + " - source_image: postgres:16.13-alpine\n", + ), + ( + "flow-target-image.yml", + "name: Migration target\n" + "jobs: {build: {strategy: {matrix: {include: " + '[{target_image: "postgres:16.14-alpine"}]}}}}\n', + ), ) for name, workflow in cases: with self.subTest(name=name): @@ -244,6 +276,36 @@ def test_dynamic_workflow_images_are_denied_from_structured_forms( f"{relative}: workflow image reference is not allowlisted", ) + def test_notary_matrix_images_are_bound_to_the_owning_workflow( + self, + ) -> None: + source = (ROOT / NOTARY_POSTGRES_WORKFLOW).read_text(encoding="utf-8") + for key, reviewed, replacement in ( + ("source_image", "postgres:16.13-alpine", "postgres:16-alpine"), + ("target_image", "postgres:16.14-alpine", "postgres:17-alpine"), + ): + with self.subTest(key=key): + root = self.fixture() + self.write_workflow( + root, + NOTARY_POSTGRES_WORKFLOW.name, + source.replace( + f"{key}: {reviewed}", + f"{key}: {replacement}", + 1, + ), + ) + failures = self.module.check_repository(root) + self.assertTrue( + any( + f"{NOTARY_POSTGRES_WORKFLOW}: workflow image " + "reference is not allowlisted" in failure + for failure in failures + ), + failures, + ) + self.assertNotIn(replacement, "\n".join(failures)) + def test_workflow_image_inventory_fails_closed(self) -> None: cases = ( ( @@ -290,6 +352,16 @@ def test_workflow_image_inventory_fails_closed(self) -> None: " - uses: docker://\n", "unsupported workflow image value", ), + ( + "invalid-source-image.yml", + "jobs:\n" + " build:\n" + " strategy:\n" + " matrix:\n" + " include:\n" + " - source_image: []\n", + "unsupported workflow image value", + ), ) for name, workflow, failure in cases: with self.subTest(name=name): @@ -485,6 +557,73 @@ def test_shell_builder_contracts_allow_only_one_canonical_assignment( ) self.assert_has_failure(root, failure) + def test_live_journey_uses_the_reviewed_postgres_alpine_image( + self, + ) -> None: + exact = self.module.LIVE_JOURNEY_POSTGRES_ASSIGNMENT + self.assertEqual( + 'readonly POSTGRES_IMAGE="postgres:16.13-alpine"', + exact, + ) + cases = ( + 'readonly POSTGRES_IMAGE="postgres:16"', + 'readonly POSTGRES_IMAGE="postgres:16-alpine"', + 'readonly POSTGRES_IMAGE="postgres:17.9-alpine"', + f"{exact}\n" + 'POSTGRES_IMAGE="postgres:16"', + f"# {exact}\n" + 'POSTGRES_IMAGE="postgres:16"', + ) + failure = ( + "live-journey PostgreSQL image assignment must remain the " + "single reviewed value" + ) + for replacement in cases: + with self.subTest(replacement=replacement): + root = self.fixture() + target = root / LIVE_JOURNEY + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + for line in replacement.splitlines(): + if line != exact: + self.assertNotIn(line, "\n".join(failures)) + + def test_live_journey_postgres_consumers_are_bound_to_both_commands( + self, + ) -> None: + consumer = self.module.LIVE_JOURNEY_POSTGRES_CONSUMER + replacement = ' "postgres:17.9-alpine" \\' + failure_fragments = ( + "live-journey PostgreSQL certificate setup command", + "live-journey PostgreSQL server command", + ) + source = (ROOT / LIVE_JOURNEY).read_text(encoding="utf-8") + self.assertEqual(2, source.splitlines().count(consumer)) + indexes = [ + index for index, line in enumerate(source.splitlines()) if line == consumer + ] + for occurrence, failure in zip(indexes, failure_fragments): + with self.subTest(occurrence=occurrence): + root = self.fixture() + target = root / LIVE_JOURNEY + lines = source.splitlines() + lines[occurrence] = replacement + lines.append(consumer) + target.write_text("\n".join(lines) + "\n", encoding="utf-8") + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(replacement, "\n".join(failures)) + def test_builder_handoffs_and_docker_consumers_remain_exact(self) -> None: cases = ( ( @@ -649,6 +788,86 @@ def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: f"{relative}: upstream base is not pinned by immutable digest", ) + def test_dockerfile_copy_sources_are_reviewed_stages_or_contexts( + self, + ) -> None: + self.assertEqual( + set(self.module.DOCKERFILES), + set(self.module.DOCKERFILE_NAMED_CONTEXTS), + ) + for relative in self.module.DOCKERFILES: + with self.subTest(relative=relative): + text = (ROOT / relative).read_text(encoding="utf-8") + aliases = { + match.group("alias").casefold() + for match in self.module.FROM_RE.finditer(text) + if match.group("alias") is not None + } + sources = { + match.group("source") + for match in self.module.COPY_FROM_RE.finditer(text) + } + external_sources = { + source for source in sources if source.casefold() not in aliases + } + self.assertEqual( + self.module.DOCKERFILE_NAMED_CONTEXTS[relative], + external_sources, + ) + + def test_dockerfile_copy_sources_reject_external_images_and_contexts( + self, + ) -> None: + sources = ( + "unreviewed-context", + "alpine:3.22", + "ghcr.io/example/tool@sha256:" + "a" * 64, + ) + failure = ( + "COPY --from source is not a declared stage or reviewed named " + "build context" + ) + for relative in self.module.DOCKERFILES: + for source in sources: + with self.subTest(relative=relative, source=source): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + f"\n copy --chown=65532:65532 --from={source} " + "/bin/tool /bin/tool\n", + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(source, "\n".join(failures)) + + def test_dockerfile_named_context_allowlist_is_path_bounded(self) -> None: + for relative, contexts in self.module.DOCKERFILE_NAMED_CONTEXTS.items(): + for context in contexts: + with self.subTest(relative=relative, context=context): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + exact = f"--from={context}" + self.assertIn(exact, text) + target.write_text( + text.replace( + exact, + f"--from={context}-copy", + 1, + ), + encoding="utf-8", + ) + self.assert_has_failure( + root, + f"{relative}: COPY --from source is not a declared " + "stage or reviewed named build context", + ) + def test_dockerfile_stages_reject_forced_platforms(self) -> None: for relative in self.module.DOCKERFILES: for stage_index in (0, 1): From 16fac88e3e79c100207d91995b20c3e7cbaae715 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 05:54:52 +0700 Subject: [PATCH 30/34] fix(release): inspect multiline Docker copy sources Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 82 +++++++++- release/scripts/test_check_debian13_images.py | 142 +++++++++++++++++- 2 files changed, 217 insertions(+), 7 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 54b458bc..10103b85 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -244,11 +244,17 @@ r"[ \t]*(?:#.*)?$", re.IGNORECASE | re.MULTILINE, ) +DOCKERFILE_PARSER_DIRECTIVE_RE = re.compile( + r"^[ \t]*#[ \t]*(?P[A-Za-z]+)[ \t]*=" + r"[ \t]*(?P\S+)[ \t]*$", + re.IGNORECASE, +) +COPY_INSTRUCTION_RE = re.compile(r"^[ \t]*COPY(?:[ \t]|$)", re.IGNORECASE) COPY_FROM_RE = re.compile( r"^[ \t]*COPY[ \t]+" r"(?:--[^\s=]+(?:=[^\s]+)?[ \t]+)*" r"--from=(?P[^\s#]+)", - re.IGNORECASE | re.MULTILINE, + re.IGNORECASE, ) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( @@ -314,6 +320,73 @@ def discover_workflow_paths(root: Path) -> tuple[Path, ...]: ) +def collect_dockerfile_copy_sources( + text: str, + relative: Path, + failures: list[str], +) -> list[str]: + lines = text.splitlines() + escape_directives = [] + for line in lines: + if not line.strip(): + break + match = DOCKERFILE_PARSER_DIRECTIVE_RE.match(line) + if match is None: + break + key = match.group("key").casefold() + if key not in {"syntax", "escape", "check"}: + break + if key == "escape": + escape_directives.append(match.group("value")) + if len(escape_directives) > 1 or ( + escape_directives + and escape_directives[0] != "\\" + ): + failures.append( + f"{relative}: unsupported Dockerfile escape directive prevents " + "bounded COPY source inspection" + ) + return [] + + sources = [] + index = 0 + while index < len(lines): + line = lines[index] + index += 1 + if not line.strip() or line.lstrip().startswith("#"): + continue + + parts: list[str] = [] + while True: + continued = line.rstrip().endswith("\\") + part = line.rstrip() + if continued: + part = part[:-1] + parts.append(part) + if not continued: + break + + while index < len(lines): + line = lines[index] + index += 1 + if line.strip() and not line.lstrip().startswith("#"): + break + else: + failures.append( + f"{relative}: unterminated Dockerfile line continuation " + "prevents bounded COPY source inspection" + ) + return [] + + instruction = "".join(parts) + if not COPY_INSTRUCTION_RE.match(instruction): + continue + match = COPY_FROM_RE.match(instruction) + if match: + sources.append(match.group("source")) + return sources + + def collect_workflow_image_references( text: str, relative: Path, @@ -563,8 +636,11 @@ def check_repository(root: Path = ROOT) -> list[str]: if alias is not None } named_contexts = DOCKERFILE_NAMED_CONTEXTS[relative] - for match in COPY_FROM_RE.finditer(text): - source = match.group("source") + for source in collect_dockerfile_copy_sources( + text, + relative, + failures, + ): if ( source.casefold() not in stage_aliases and source not in named_contexts diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index d135f681..38262924 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -803,10 +803,15 @@ def test_dockerfile_copy_sources_are_reviewed_stages_or_contexts( for match in self.module.FROM_RE.finditer(text) if match.group("alias") is not None } - sources = { - match.group("source") - for match in self.module.COPY_FROM_RE.finditer(text) - } + failures: list[str] = [] + sources = set( + self.module.collect_dockerfile_copy_sources( + text, + relative, + failures, + ) + ) + self.assertEqual([], failures) external_sources = { source for source in sources if source.casefold() not in aliases } @@ -845,6 +850,135 @@ def test_dockerfile_copy_sources_reject_external_images_and_contexts( ) self.assertNotIn(source, "\n".join(failures)) + def test_multiline_dockerfile_copy_sources_allow_reviewed_sources( + self, + ) -> None: + cases = ( + ( + Path("crates/registry-relay/Dockerfile"), + ( + "COPY --from=registry-platform /Cargo.toml /Cargo.lock " + "/workspace/registry-platform/" + ), + "COPY --chown=0:0 \\\n" + " # Supplied as a named BuildKit context.\n" + " --from=registry-platform \\\n" + " /Cargo.toml /Cargo.lock /workspace/registry-platform/", + ), + ( + Path("release/docker/Dockerfile.registry-relay"), + "COPY --from=runtime-root /workspace/runtime-root/ /", + "COPY \\\n" + " --from=runtime-root \\\n" + " --chown=65532:65532 \\\n" + " /workspace/runtime-root/ /", + ), + ) + for relative, exact, multiline in cases: + with self.subTest(relative=relative): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, multiline, 1), + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + + def test_multiline_dockerfile_copy_sources_reject_external_images( + self, + ) -> None: + relative = Path("crates/registry-relay/Dockerfile") + sources = ( + "alpine:3.22", + "ghcr.io/example/tool@sha256:" + "a" * 64, + ) + instructions = ( + "COPY \\\n" + " --from={source} \\\n" + " /bin/tool /bin/tool\n", + "COPY --chown=65532:65532 \\\n" + " --from={source} \\\n" + " /bin/tool /bin/tool\n", + "COPY --from={source} \\\n" + " --chown=65532:65532 \\\n" + " /bin/tool /bin/tool\n", + "COPY --chown=65532:65532 \\\n" + " # The source option remains part of this instruction.\n" + " --from={source} \\\n" + " /bin/tool /bin/tool\n", + ) + failure = ( + "COPY --from source is not a declared stage or reviewed named " + "build context" + ) + for source in sources: + for instruction in instructions: + with self.subTest(source=source, instruction=instruction): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + "\n" + instruction.format(source=source), + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(source, "\n".join(failures)) + + def test_dockerfile_copy_normalization_fails_closed(self) -> None: + relative = Path("crates/registry-relay/Dockerfile") + cases = ( + ( + "unterminated", + "", + "\nCOPY --chown=65532:65532 \\\n" + " # No continued instruction follows.\n", + "unterminated Dockerfile line continuation", + ), + ( + "alternate-escape", + "# escape=`\n", + "\nCOPY --chown=65532:65532 `\n" + " --from=alpine:3.22 `\n" + " /bin/tool /bin/tool\n", + "unsupported Dockerfile escape directive", + ), + ) + for name, prefix, suffix, failure in cases: + with self.subTest(name=name): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text(prefix + text + suffix, encoding="utf-8") + self.assert_has_failure(root, failure) + + def test_multiline_copy_source_tokens_cannot_shadow_reviewed_names( + self, + ) -> None: + relative = Path("crates/registry-relay/Dockerfile") + for reviewed in ("builder", "registry-platform"): + with self.subTest(reviewed=reviewed): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + + f"\nCOPY --from={reviewed}\\\n" + "-external \\\n" + " /bin/tool /bin/tool\n", + encoding="utf-8", + ) + self.assert_has_failure( + root, + f"{relative}: COPY --from source is not a declared stage " + "or reviewed named build context", + ) + def test_dockerfile_named_context_allowlist_is_path_bounded(self) -> None: for relative, contexts in self.module.DOCKERFILE_NAMED_CONTEXTS.items(): for context in contexts: From 46dc3965aa6ca797d51ece2af03cf7ef10c7ed47 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 06:05:38 +0700 Subject: [PATCH 31/34] fix(release): restrict Docker copy option syntax Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 45 ++++++++++--- release/scripts/test_check_debian13_images.py | 66 +++++++++++++++++++ 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 10103b85..1e6487ed 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -249,13 +249,14 @@ r"[ \t]*(?P\S+)[ \t]*$", re.IGNORECASE, ) -COPY_INSTRUCTION_RE = re.compile(r"^[ \t]*COPY(?:[ \t]|$)", re.IGNORECASE) -COPY_FROM_RE = re.compile( - r"^[ \t]*COPY[ \t]+" - r"(?:--[^\s=]+(?:=[^\s]+)?[ \t]+)*" - r"--from=(?P[^\s#]+)", +COPY_INSTRUCTION_RE = re.compile( + r"^[ \t]*COPY[ \t]+(?P.*)$", re.IGNORECASE, ) +COPY_OPTION_NAMES = frozenset(("from", "chown")) +COPY_OPTION_RE = re.compile( + r"--(?P[a-z][a-z0-9-]*)=(?P[^\s\\\"']+)" +) DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( r"\b(?:bookworm|debian[ \t_:-]*v?[ \t_:-]*12)\b", @@ -378,12 +379,36 @@ def collect_dockerfile_copy_sources( ) return [] - instruction = "".join(parts) - if not COPY_INSTRUCTION_RE.match(instruction): + instruction_match = COPY_INSTRUCTION_RE.match("".join(parts)) + if instruction_match is None: continue - match = COPY_FROM_RE.match(instruction) - if match: - sources.append(match.group("source")) + tokens = instruction_match.group("arguments").split() + seen_options = set() + while tokens and tokens[0].startswith("--"): + option_match = COPY_OPTION_RE.fullmatch(tokens.pop(0)) + if option_match is None: + failures.append( + f"{relative}: unsupported COPY option syntax prevents " + "bounded source inspection" + ) + return [] + name = option_match.group("name") + if name not in COPY_OPTION_NAMES or name in seen_options: + failures.append( + f"{relative}: unsupported COPY option syntax prevents " + "bounded source inspection" + ) + return [] + seen_options.add(name) + if name == "from": + sources.append(option_match.group("value")) + + if not tokens or tokens[0].startswith(("-", "'", '"', "\\")): + failures.append( + f"{relative}: unsupported COPY operand prefix prevents " + "bounded source inspection" + ) + return [] return sources diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 38262924..48096f49 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -791,6 +791,10 @@ def test_every_dockerfile_base_requires_an_immutable_digest(self) -> None: def test_dockerfile_copy_sources_are_reviewed_stages_or_contexts( self, ) -> None: + self.assertEqual( + {"from", "chown"}, + set(self.module.COPY_OPTION_NAMES), + ) self.assertEqual( set(self.module.DOCKERFILES), set(self.module.DOCKERFILE_NAMED_CONTEXTS), @@ -850,6 +854,68 @@ def test_dockerfile_copy_sources_reject_external_images_and_contexts( ) self.assertNotIn(source, "\n".join(failures)) + def test_dockerfile_copy_options_require_canonical_reviewed_syntax( + self, + ) -> None: + relative = Path("crates/registry-relay/Dockerfile") + cases = ( + ("escaped-from-name", r"--fr\om=alpine:3.22"), + ("quoted-from-name", '--fr"om"=alpine:3.22'), + ("unknown-chmod", "--chmod=0755"), + ("unknown-link", "--link=true"), + ("valueless-from", "--from"), + ("quoted-from-value", '--from="builder"'), + ("escaped-from-value", r"--from=buil\der"), + ("uppercase-from", "--FROM=builder"), + ( + "duplicate-from", + "--from=builder --from=registry-platform", + ), + ) + failure = "unsupported COPY option syntax" + for name, options in cases: + with self.subTest(name=name): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + f"\nCOPY {options} /bin/tool /bin/tool\n", + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(options, "\n".join(failures)) + + def test_dockerfile_copy_rejects_encoded_option_like_operands( + self, + ) -> None: + relative = Path("crates/registry-relay/Dockerfile") + for prefix in ( + r'-\-from=alpine:3.22', + r'\--from=alpine:3.22', + '"--from"=alpine:3.22', + ): + with self.subTest(prefix=prefix): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + target.write_text( + text + f"\nCOPY {prefix} /bin/tool /bin/tool\n", + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any( + "unsupported COPY operand prefix" in item + for item in failures + ), + failures, + ) + self.assertNotIn(prefix, "\n".join(failures)) + def test_multiline_dockerfile_copy_sources_allow_reviewed_sources( self, ) -> None: From 63da446cf0378f6accc68f37e33ad5ac2966af91 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 06:31:10 +0700 Subject: [PATCH 32/34] fix(release): enforce remaining image inputs Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 222 ++++++++++++- release/scripts/test_check_debian13_images.py | 305 +++++++++++++++++- 2 files changed, 514 insertions(+), 13 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 1e6487ed..edac19ba 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -134,6 +134,9 @@ Path("release/docker/Dockerfile.registry-notary"), Path("release/docker/Dockerfile.registry-relay"), ) +NOTARY_POSTGRES_CONFORMANCE_SCRIPT = Path( + "products/notary/scripts/postgresql-conformance.sh" +) # These are the maintained image and image-policy surfaces. Historical release # notes are immutable evidence and intentionally are not rewritten by this gate. @@ -147,6 +150,7 @@ Path("crates/registry-relay/scripts/check_docker_build_contract.py"), Path("crates/registry-relay/scripts/run-live-consultation-journey.sh"), Path("products/notary/docs/security-assurance.md"), + NOTARY_POSTGRES_CONFORMANCE_SCRIPT, ) NOTARY_POSTGRES_WORKFLOW_IMAGES = ( @@ -178,6 +182,35 @@ }, } WORKFLOW_IMAGE_KEYS = frozenset(("image", "source_image", "target_image")) +NOTARY_POSTGRES_IMAGE_ASSIGNMENTS = ( + ("default_source_image", '"postgres:16.13-alpine"'), + ("default_target_image", '"postgres:16.14-alpine"'), + ("default_restore_image", '"postgres:17.10-alpine"'), + ("default_source_image", '"postgres:17.9-alpine"'), + ("default_target_image", '"postgres:17.10-alpine"'), + ("default_restore_image", '"postgres:18.4-alpine"'), + ("default_source_image", '"postgres:18.3-alpine"'), + ("default_target_image", '"postgres:18.4-alpine"'), + ("default_restore_image", '"postgres:18.4-alpine"'), + ("unsupported_postgres_image", '"postgres:15.18-alpine"'), +) +NOTARY_POSTGRES_LITERAL_LINES = tuple( + f"{name}={value}" for name, value in NOTARY_POSTGRES_IMAGE_ASSIGNMENTS +) +NOTARY_POSTGRES_FALLBACK_BINDINGS = ( + ( + "source_image", + '"${NOTARY_POSTGRES_SOURCE_IMAGE:-${default_source_image}}"', + ), + ( + "target_image", + '"${NOTARY_POSTGRES_TARGET_IMAGE:-${default_target_image}}"', + ), + ( + "restore_image", + '"${NOTARY_POSTGRES_RESTORE_IMAGE:-${default_restore_image}}"', + ), +) RUST_BUILDER_DOCKERFILES = DOCKERFILES[:3] PREPARATION_DOCKERFILES = DOCKERFILES[3:] @@ -203,6 +236,32 @@ Path("release/docker/Dockerfile.registry-notary"): frozenset(), Path("release/docker/Dockerfile.registry-relay"): frozenset(), } +RELAY_FINAL_STAGE_COPY_INSTRUCTIONS = ( + "COPY --from=builder --chown=65532:65532 /workspace/runtime-root/ /", + ( + "COPY --from=builder /usr/local/bin/registry-relay " + "/usr/local/bin/registry-relay" + ), + ( + "COPY --from=builder /usr/local/bin/registry-relay-rhai-worker " + "/usr/local/bin/registry-relay-rhai-worker" + ), + "COPY LICENSE /licenses/registry-relay/LICENSE", +) +FINAL_STAGE_COPY_INSTRUCTIONS = { + Path("crates/registry-relay/Dockerfile"): RELAY_FINAL_STAGE_COPY_INSTRUCTIONS, + Path("crates/registry-relay/Dockerfile.demo"): RELAY_FINAL_STAGE_COPY_INSTRUCTIONS, + Path("products/notary/Dockerfile"): ( + "COPY --from=builder --chown=65532:65532 /workspace/runtime-root/ /", + "COPY --from=builder /workspace/out/ /usr/local/bin/", + ), + Path("release/docker/Dockerfile.registry-notary"): ( + "COPY --from=runtime-root /workspace/runtime-root/ /", + ), + Path("release/docker/Dockerfile.registry-relay"): ( + "COPY --from=runtime-root /workspace/runtime-root/ /", + ), +} RELAY_RUNTIME_DIRECTIVES = ( ( @@ -254,9 +313,21 @@ re.IGNORECASE, ) COPY_OPTION_NAMES = frozenset(("from", "chown")) -COPY_OPTION_RE = re.compile( +LEADING_OPTION_RE = re.compile( r"--(?P[a-z][a-z0-9-]*)=(?P[^\s\\\"']+)" ) +RUN_INSTRUCTION_RE = re.compile( + r"^[ \t]*RUN[ \t]+(?P.*)$", + re.IGNORECASE, +) +RUN_OPTION_NAMES = frozenset(("mount",)) +RUN_MOUNT_FIELD_RE = re.compile( + r"(?P[a-z][a-z0-9-]*)=(?P[^,\s\\\"']+)" +) +RUN_MOUNT_FIELDS = { + "cache": ("type", "target"), + "bind": ("type", "source", "target"), +} DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( r"\b(?:bookworm|debian[ \t_:-]*v?[ \t_:-]*12)\b", @@ -292,6 +363,15 @@ r"""^[ \t]*(?:restore-keys|'restore-keys'|"restore-keys")[ \t]*:""", re.MULTILINE, ) +NOTARY_POSTGRES_IMAGE_ASSIGNMENT_RE = re.compile( + r"^[ \t]*(?Pdefault_source_image|default_target_image|" + r"default_restore_image|unsupported_postgres_image)[ \t]*=" + r"(?P.*)$" +) +NOTARY_POSTGRES_FALLBACK_RE = re.compile( + r"^[ \t]*(?Psource_image|target_image|restore_image)[ \t]*=" + r"(?P.*)$" +) def read(root: Path, relative: Path, failures: list[str]) -> str: @@ -321,11 +401,11 @@ def discover_workflow_paths(root: Path) -> tuple[Path, ...]: ) -def collect_dockerfile_copy_sources( +def normalize_dockerfile_instructions( text: str, relative: Path, failures: list[str], -) -> list[str]: +) -> tuple[str, ...]: lines = text.splitlines() escape_directives = [] for line in lines: @@ -345,11 +425,11 @@ def collect_dockerfile_copy_sources( ): failures.append( f"{relative}: unsupported Dockerfile escape directive prevents " - "bounded COPY source inspection" + "bounded instruction inspection" ) - return [] + return () - sources = [] + instructions = [] index = 0 while index < len(lines): line = lines[index] @@ -375,17 +455,28 @@ def collect_dockerfile_copy_sources( else: failures.append( f"{relative}: unterminated Dockerfile line continuation " - "prevents bounded COPY source inspection" + "prevents bounded instruction inspection" ) - return [] + return () + + instructions.append("".join(parts)) + return tuple(instructions) - instruction_match = COPY_INSTRUCTION_RE.match("".join(parts)) + +def collect_dockerfile_copy_sources( + instructions: tuple[str, ...], + relative: Path, + failures: list[str], +) -> list[str]: + sources = [] + for instruction in instructions: + instruction_match = COPY_INSTRUCTION_RE.match(instruction) if instruction_match is None: continue tokens = instruction_match.group("arguments").split() seen_options = set() while tokens and tokens[0].startswith("--"): - option_match = COPY_OPTION_RE.fullmatch(tokens.pop(0)) + option_match = LEADING_OPTION_RE.fullmatch(tokens.pop(0)) if option_match is None: failures.append( f"{relative}: unsupported COPY option syntax prevents " @@ -412,6 +503,58 @@ def collect_dockerfile_copy_sources( return sources +def check_dockerfile_run_options( + instructions: tuple[str, ...], + relative: Path, + failures: list[str], +) -> None: + for instruction in instructions: + instruction_match = RUN_INSTRUCTION_RE.match(instruction) + if instruction_match is None: + continue + tokens = instruction_match.group("arguments").split() + while tokens and tokens[0].startswith("--"): + option_match = LEADING_OPTION_RE.fullmatch(tokens.pop(0)) + if ( + option_match is None + or option_match.group("name") not in RUN_OPTION_NAMES + ): + failures.append( + f"{relative}: unsupported RUN option syntax prevents " + "bounded mount inspection" + ) + return + + fields = [] + for field in option_match.group("value").split(","): + field_match = RUN_MOUNT_FIELD_RE.fullmatch(field) + if field_match is None: + failures.append( + f"{relative}: unsupported RUN mount syntax prevents " + "bounded mount inspection" + ) + return + fields.append( + (field_match.group("name"), field_match.group("value")) + ) + + field_names = tuple(name for name, _value in fields) + mount_type = fields[0][1] if fields else "" + if RUN_MOUNT_FIELDS.get(mount_type) != field_names: + failures.append( + f"{relative}: unsupported RUN mount syntax prevents " + "bounded mount inspection" + ) + return + + if not tokens or tokens[0].startswith(("-", "'", '"', "\\")): + failures.append( + f"{relative}: unsupported RUN operand prefix prevents " + "bounded mount inspection" + ) + return + + def collect_workflow_image_references( text: str, relative: Path, @@ -625,8 +768,46 @@ def check_repository(root: Path = ROOT) -> list[str]: f"at {location}" ) + notary_postgres = texts[NOTARY_POSTGRES_CONFORMANCE_SCRIPT] + notary_postgres_assignments = tuple( + (match.group("name"), match.group("value")) + for line in notary_postgres.splitlines() + if (match := NOTARY_POSTGRES_IMAGE_ASSIGNMENT_RE.match(line)) + ) + if notary_postgres_assignments != NOTARY_POSTGRES_IMAGE_ASSIGNMENTS: + failures.append( + f"{NOTARY_POSTGRES_CONFORMANCE_SCRIPT}: PostgreSQL image " + "assignments must match the exact ordered reviewed inventory" + ) + notary_postgres_literal_lines = tuple( + line.strip() + for line in notary_postgres.splitlines() + if not line.lstrip().startswith("#") and "postgres:" in line + ) + if notary_postgres_literal_lines != NOTARY_POSTGRES_LITERAL_LINES: + failures.append( + f"{NOTARY_POSTGRES_CONFORMANCE_SCRIPT}: direct PostgreSQL image " + "literals must match the exact ordered reviewed assignment lines" + ) + notary_postgres_fallbacks = tuple( + (match.group("name"), match.group("value")) + for line in notary_postgres.splitlines() + if (match := NOTARY_POSTGRES_FALLBACK_RE.match(line)) + ) + if notary_postgres_fallbacks != NOTARY_POSTGRES_FALLBACK_BINDINGS: + failures.append( + f"{NOTARY_POSTGRES_CONFORMANCE_SCRIPT}: PostgreSQL environment " + "fallbacks must match the exact ordered reviewed bindings" + ) + for relative in DOCKERFILES: text = texts[relative] + instructions = normalize_dockerfile_instructions( + text, + relative, + failures, + ) + check_dockerfile_run_options(instructions, relative, failures) stage_matches = list(FROM_RE.finditer(text)) if not stage_matches: failures.append(f"{relative}: no FROM instruction found") @@ -662,7 +843,7 @@ def check_repository(root: Path = ROOT) -> list[str]: } named_contexts = DOCKERFILE_NAMED_CONTEXTS[relative] for source in collect_dockerfile_copy_sources( - text, + instructions, relative, failures, ): @@ -674,6 +855,25 @@ def check_repository(root: Path = ROOT) -> list[str]: f"{relative}: COPY --from source is not a declared stage " "or reviewed named build context" ) + final_from_indexes = [ + index + for index, instruction in enumerate(instructions) + if FROM_RE.fullmatch(instruction) + ] + if final_from_indexes: + final_copy_instructions = tuple( + " ".join(instruction.split()) + for instruction in instructions[final_from_indexes[-1] + 1 :] + if COPY_INSTRUCTION_RE.match(instruction) + ) + if ( + final_copy_instructions + != FINAL_STAGE_COPY_INSTRUCTIONS[relative] + ): + failures.append( + f"{relative}: final-stage COPY instructions must match " + "the exact reviewed inventory" + ) for base, _alias, _platform in stages: if not DIGEST_PIN_RE.search(base): failures.append( diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 48096f49..445663d3 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -23,6 +23,9 @@ RELAY_POSTGRES_WORKFLOW = Path( ".github/workflows/relay-postgres-conformance.yml" ) +NOTARY_POSTGRES_CONFORMANCE_SCRIPT = Path( + "products/notary/scripts/postgresql-conformance.sh" +) EXPECTED_SURFACES = { CI_WORKFLOW, @@ -36,6 +39,7 @@ TUTORIAL_CHECK, Path("products/notary/Dockerfile"), Path("products/notary/docs/security-assurance.md"), + NOTARY_POSTGRES_CONFORMANCE_SCRIPT, Path("release/docker/Dockerfile.registry-notary"), Path("release/docker/Dockerfile.registry-relay"), RELEASE_BINARY_RECIPE, @@ -369,6 +373,173 @@ def test_workflow_image_inventory_fails_closed(self) -> None: self.write_workflow(root, name, workflow) self.assert_has_failure(root, failure) + def test_notary_postgres_script_has_exact_reviewed_image_defaults( + self, + ) -> None: + self.assertEqual( + ( + ("default_source_image", '"postgres:16.13-alpine"'), + ("default_target_image", '"postgres:16.14-alpine"'), + ("default_restore_image", '"postgres:17.10-alpine"'), + ("default_source_image", '"postgres:17.9-alpine"'), + ("default_target_image", '"postgres:17.10-alpine"'), + ("default_restore_image", '"postgres:18.4-alpine"'), + ("default_source_image", '"postgres:18.3-alpine"'), + ("default_target_image", '"postgres:18.4-alpine"'), + ("default_restore_image", '"postgres:18.4-alpine"'), + ("unsupported_postgres_image", '"postgres:15.18-alpine"'), + ), + self.module.NOTARY_POSTGRES_IMAGE_ASSIGNMENTS, + ) + self.assertEqual( + ( + ( + "source_image", + '"${NOTARY_POSTGRES_SOURCE_IMAGE:-' + '${default_source_image}}"', + ), + ( + "target_image", + '"${NOTARY_POSTGRES_TARGET_IMAGE:-' + '${default_target_image}}"', + ), + ( + "restore_image", + '"${NOTARY_POSTGRES_RESTORE_IMAGE:-' + '${default_restore_image}}"', + ), + ), + self.module.NOTARY_POSTGRES_FALLBACK_BINDINGS, + ) + + def test_notary_postgres_image_assignments_reject_mutable_defaults( + self, + ) -> None: + cases = ( + ( + ' default_source_image="postgres:16.13-alpine"', + ' default_source_image="postgres:16"', + ), + ( + ' default_restore_image="postgres:17.10-alpine"', + ' default_restore_image="postgres:17"', + ), + ( + 'unsupported_postgres_image="postgres:15.18-alpine"', + 'unsupported_postgres_image="postgres:15"', + ), + ) + failure = ( + "PostgreSQL image assignments must match the exact ordered " + "reviewed inventory" + ) + for exact, replacement in cases: + with self.subTest(replacement=replacement): + root = self.fixture() + target = root / NOTARY_POSTGRES_CONFORMANCE_SCRIPT + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(replacement, "\n".join(failures)) + + def test_notary_postgres_defaults_cannot_be_shadowed_by_comments( + self, + ) -> None: + exact = ' default_restore_image="postgres:17.10-alpine"' + replacement = ( + f" # {exact.strip()}\n" + ' default_restore_image="postgres:17"' + ) + root = self.fixture() + target = root / NOTARY_POSTGRES_CONFORMANCE_SCRIPT + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + self.assert_has_failure( + root, + "PostgreSQL image assignments must match the exact ordered " + "reviewed inventory", + ) + + def test_notary_postgres_rejects_new_direct_image_literals( + self, + ) -> None: + failure = ( + "direct PostgreSQL image literals must match the exact ordered " + "reviewed assignment lines" + ) + for addition in ( + 'direct_image="postgres:16"\n', + "docker pull postgres:16\n", + ): + with self.subTest(addition=addition): + root = self.fixture() + target = root / NOTARY_POSTGRES_CONFORMANCE_SCRIPT + text = target.read_text(encoding="utf-8") + target.write_text(text + addition, encoding="utf-8") + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(addition.strip(), "\n".join(failures)) + + root = self.fixture() + target = root / NOTARY_POSTGRES_CONFORMANCE_SCRIPT + text = target.read_text(encoding="utf-8") + target.write_text( + text + '# example_image="postgres:16"\n', + encoding="utf-8", + ) + self.assertEqual([], self.module.check_repository(root)) + + def test_notary_postgres_env_fallbacks_remain_bound_to_defaults( + self, + ) -> None: + exact = ( + 'source_image="${NOTARY_POSTGRES_SOURCE_IMAGE:-' + '${default_source_image}}"' + ) + replacements = ( + ( + 'source_image="${NOTARY_POSTGRES_SOURCE_IMAGE:-' + 'postgres:16}"' + ), + f"# {exact}\nsource_image=\"postgres:16\"", + f"{exact}\nsource_image=\"postgres:16\"", + ) + failure = ( + "PostgreSQL environment fallbacks must match the exact ordered " + "reviewed bindings" + ) + for replacement in replacements: + with self.subTest(replacement=replacement): + root = self.fixture() + target = root / NOTARY_POSTGRES_CONFORMANCE_SCRIPT + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(replacement, "\n".join(failures)) + def test_retired_markers_cover_dynamically_discovered_workflows( self, ) -> None: @@ -808,9 +979,14 @@ def test_dockerfile_copy_sources_are_reviewed_stages_or_contexts( if match.group("alias") is not None } failures: list[str] = [] + instructions = self.module.normalize_dockerfile_instructions( + text, + relative, + failures, + ) sources = set( self.module.collect_dockerfile_copy_sources( - text, + instructions, relative, failures, ) @@ -936,7 +1112,6 @@ def test_multiline_dockerfile_copy_sources_allow_reviewed_sources( "COPY --from=runtime-root /workspace/runtime-root/ /", "COPY \\\n" " --from=runtime-root \\\n" - " --chown=65532:65532 \\\n" " /workspace/runtime-root/ /", ), ) @@ -1045,6 +1220,132 @@ def test_multiline_copy_source_tokens_cannot_shadow_reviewed_names( "or reviewed named build context", ) + def test_dockerfile_run_mounts_use_the_reviewed_canonical_shapes( + self, + ) -> None: + self.assertEqual({"mount"}, set(self.module.RUN_OPTION_NAMES)) + self.assertEqual( + { + "cache": ("type", "target"), + "bind": ("type", "source", "target"), + }, + self.module.RUN_MOUNT_FIELDS, + ) + for relative in self.module.DOCKERFILES: + with self.subTest(relative=relative): + failures: list[str] = [] + instructions = self.module.normalize_dockerfile_instructions( + (ROOT / relative).read_text(encoding="utf-8"), + relative, + failures, + ) + self.module.check_dockerfile_run_options( + instructions, + relative, + failures, + ) + self.assertEqual([], failures) + + def test_dockerfile_run_mounts_reject_unreviewed_sources_and_syntax( + self, + ) -> None: + relative = Path("release/docker/Dockerfile.registry-relay") + exact = ( + "--mount=type=bind,source=dist/image-bin," + "target=/workspace/image-bin" + ) + cases = ( + ( + "external-from", + "--mount=type=bind,from=debian:trixie-slim," + "source=dist/image-bin,target=/workspace/image-bin", + "unsupported RUN mount syntax", + ), + ( + "escaped-mount-key", + r"--mount=type=bind,fr\om=debian:trixie-slim," + "source=dist/image-bin,target=/workspace/image-bin", + "unsupported RUN option syntax", + ), + ( + "quoted-mount-key", + '--mount=type=bind,fr"om"=debian:trixie-slim,' + "source=dist/image-bin,target=/workspace/image-bin", + "unsupported RUN option syntax", + ), + ( + "escaped-option-name", + r"--mo\unt=type=bind,source=dist/image-bin," + "target=/workspace/image-bin", + "unsupported RUN option syntax", + ), + ( + "quoted-option-name", + '--mo"unt"=type=bind,source=dist/image-bin,' + "target=/workspace/image-bin", + "unsupported RUN option syntax", + ), + ( + "quoted-mount-value", + '--mount="type=bind,source=dist/image-bin,' + 'target=/workspace/image-bin"', + "unsupported RUN option syntax", + ), + ( + "unknown-option", + "--network=none", + "unsupported RUN option syntax", + ), + ( + "cache-with-source", + "--mount=type=cache,source=dist/image-bin," + "target=/workspace/image-bin", + "unsupported RUN mount syntax", + ), + ) + for name, replacement, failure in cases: + with self.subTest(name=name): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + self.assertIn(exact, text) + target.write_text( + text.replace(exact, replacement, 1), + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(replacement, "\n".join(failures)) + + def test_final_stage_copy_inventories_reject_extra_files(self) -> None: + self.assertEqual( + set(self.module.DOCKERFILES), + set(self.module.FINAL_STAGE_COPY_INSTRUCTIONS), + ) + for relative in self.module.DOCKERFILES: + with self.subTest(relative=relative): + root = self.fixture() + target = root / relative + text = target.read_text(encoding="utf-8") + addition = "COPY /bin/bash /bin/bash" + target.write_text( + text + f"\n{addition}\n", + encoding="utf-8", + ) + failures = self.module.check_repository(root) + self.assertTrue( + any( + "final-stage COPY instructions must match the exact " + "reviewed inventory" in item + for item in failures + ), + failures, + ) + self.assertNotIn(addition, "\n".join(failures)) + def test_dockerfile_named_context_allowlist_is_path_bounded(self) -> None: for relative, contexts in self.module.DOCKERFILE_NAMED_CONTEXTS.items(): for context in contexts: From 49fa0b6c96833641ce58695ca616d84036bb804d Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 06:42:43 +0700 Subject: [PATCH 33/34] fix(release): inventory exact Docker run steps Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 212 ++++++++++++------ release/scripts/test_check_debian13_images.py | 157 ++++++++----- 2 files changed, 253 insertions(+), 116 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index edac19ba..196a0498 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -153,7 +153,7 @@ NOTARY_POSTGRES_CONFORMANCE_SCRIPT, ) -NOTARY_POSTGRES_WORKFLOW_IMAGES = ( +NOTARY_POSTGRES_WORKFLOW_SOURCE_TARGET_IMAGES = ( "postgres:16.13-alpine", "postgres:16.14-alpine", "postgres:17.9-alpine", @@ -161,6 +161,18 @@ "postgres:18.3-alpine", "postgres:18.4-alpine", ) +NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES = ( + "postgres:17.10-alpine", + "postgres:18.4-alpine", +) +NOTARY_POSTGRES_WORKFLOW_IMAGES = tuple( + dict.fromkeys( + ( + *NOTARY_POSTGRES_WORKFLOW_SOURCE_TARGET_IMAGES, + *NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES, + ) + ) +) NOTARY_POSTGRES_WORKFLOW_RATIONALE = ( "External Alpine PostgreSQL migration conformance, not a " "project-owned Debian image." @@ -181,7 +193,9 @@ for image in NOTARY_POSTGRES_WORKFLOW_IMAGES }, } -WORKFLOW_IMAGE_KEYS = frozenset(("image", "source_image", "target_image")) +WORKFLOW_IMAGE_KEYS = frozenset( + ("image", "source_image", "target_image", "restore_image") +) NOTARY_POSTGRES_IMAGE_ASSIGNMENTS = ( ("default_source_image", '"postgres:16.13-alpine"'), ("default_target_image", '"postgres:16.14-alpine"'), @@ -262,6 +276,125 @@ "COPY --from=runtime-root /workspace/runtime-root/ /", ), } +DOCKERFILE_RUN_INSTRUCTIONS = { + Path("crates/registry-relay/Dockerfile"): ( + ( + "RUN --mount=type=cache,target=/usr/local/cargo/registry " + "--mount=type=cache,target=/workspace/registry_relay/target " + "find src benches resources -type f -exec touch {} + && " + 'if [ -n "$REGISTRY_RELAY_FEATURES" ]; then ' + "cargo build --release --locked --features " + '"$REGISTRY_RELAY_FEATURES"; ' + "else cargo build --release --locked; fi && " + "cp /workspace/registry_relay/target/release/registry-relay " + "/usr/local/bin/registry-relay && " + "cp /workspace/registry_relay/target/release/" + "registry-relay-rhai-worker " + "/usr/local/bin/registry-relay-rhai-worker && " + "mkdir -p /workspace/runtime-root/etc/registry-relay " + "/workspace/runtime-root/var/lib/registry-relay/cache " + "/workspace/runtime-root/var/lib/registry-relay/data " + "/workspace/runtime-root/var/log/registry-relay && " + "chown -R 65532:65532 /workspace/runtime-root" + ), + ), + Path("crates/registry-relay/Dockerfile.demo"): ( + ( + "RUN --mount=type=cache,target=/usr/local/cargo/registry " + "--mount=type=cache,target=/workspace/registry_relay/target " + "cargo build --release --locked --features " + "spdci-api-standards,standards-cel-mapping," + "attribute-release && " + "cp /workspace/registry_relay/target/release/registry-relay " + "/usr/local/bin/registry-relay && " + "cp /workspace/registry_relay/target/release/" + "registry-relay-rhai-worker " + "/usr/local/bin/registry-relay-rhai-worker && " + "mkdir -p /workspace/runtime-root/etc/registry-relay " + "/workspace/runtime-root/var/lib/registry-relay/cache " + "/workspace/runtime-root/var/lib/registry-relay/data " + "/workspace/runtime-root/var/log/registry-relay && " + "chown -R 65532:65532 /workspace/runtime-root" + ), + ), + Path("products/notary/Dockerfile"): ( + ( + "RUN --mount=type=cache,target=/usr/local/cargo/registry " + "--mount=type=cache,target=/workspace/target " + 'if [ -n "$REGISTRY_NOTARY_FEATURES" ]; then ' + "CARGO_TARGET_DIR=/workspace/target cargo build --release " + "--locked -p registry-notary --features " + '"$REGISTRY_NOTARY_FEATURES"; ' + "else CARGO_TARGET_DIR=/workspace/target cargo build --release " + "--locked -p registry-notary; fi && " + "mkdir -p /workspace/out && " + "cp /workspace/target/release/registry-notary " + "/workspace/out/registry-notary && " + 'case ",$REGISTRY_NOTARY_FEATURES," in ' + "*,registry-notary-cel,*) " + "CARGO_TARGET_DIR=/workspace/target cargo build --release " + "--locked -p registry-notary-server " + "--bin registry-notary-cel-worker --features " + '"$REGISTRY_NOTARY_FEATURES" && ' + "cp /workspace/target/release/registry-notary-cel-worker " + "/workspace/out/registry-notary-cel-worker ;; " + "*) true ;; esac && " + "mkdir -p /workspace/runtime-root/etc/registry-notary " + "/workspace/runtime-root/var/lib/registry-notary " + "/workspace/runtime-root/var/log/registry-notary && " + "chown -R 65532:65532 /workspace/runtime-root" + ), + ), + Path("release/docker/Dockerfile.registry-notary"): ( + ( + "RUN --mount=type=bind,source=dist/image-bin," + "target=/workspace/image-bin " + "mkdir -p /workspace/runtime-root/etc/registry-notary " + "/workspace/runtime-root/usr/local/bin " + "/workspace/runtime-root/var/lib/registry-notary " + "/workspace/runtime-root/var/log/registry-notary && " + "install -m 0755 /workspace/image-bin/registry-notary " + "/workspace/runtime-root/usr/local/bin/registry-notary && " + "install -m 0755 " + "/workspace/image-bin/registry-notary-cel-worker " + "/workspace/runtime-root/usr/local/bin/" + "registry-notary-cel-worker && " + "chown -R 65532:65532 " + "/workspace/runtime-root/etc/registry-notary " + "/workspace/runtime-root/var/lib/registry-notary " + "/workspace/runtime-root/var/log/registry-notary && " + 'find /workspace/runtime-root -exec touch -h ' + '--date="@${SOURCE_DATE_EPOCH}" {} +' + ), + ), + Path("release/docker/Dockerfile.registry-relay"): ( + ( + "RUN --mount=type=bind,source=dist/image-bin," + "target=/workspace/image-bin " + "--mount=type=bind,source=LICENSE,target=/workspace/LICENSE " + "mkdir -p /workspace/runtime-root/etc/registry-relay " + "/workspace/runtime-root/licenses/registry-relay " + "/workspace/runtime-root/usr/local/bin " + "/workspace/runtime-root/var/lib/registry-relay/cache " + "/workspace/runtime-root/var/lib/registry-relay/data " + "/workspace/runtime-root/var/log/registry-relay && " + "install -m 0755 /workspace/image-bin/registry-relay " + "/workspace/runtime-root/usr/local/bin/registry-relay && " + "install -m 0755 " + "/workspace/image-bin/registry-relay-rhai-worker " + "/workspace/runtime-root/usr/local/bin/" + "registry-relay-rhai-worker && " + "install -m 0644 /workspace/LICENSE " + "/workspace/runtime-root/licenses/registry-relay/LICENSE && " + "chown -R 65532:65532 " + "/workspace/runtime-root/etc/registry-relay " + "/workspace/runtime-root/var/lib/registry-relay " + "/workspace/runtime-root/var/log/registry-relay && " + 'find /workspace/runtime-root -exec touch -h ' + '--date="@${SOURCE_DATE_EPOCH}" {} +' + ), + ), +} RELAY_RUNTIME_DIRECTIVES = ( ( @@ -313,21 +446,13 @@ re.IGNORECASE, ) COPY_OPTION_NAMES = frozenset(("from", "chown")) -LEADING_OPTION_RE = re.compile( +COPY_OPTION_RE = re.compile( r"--(?P[a-z][a-z0-9-]*)=(?P[^\s\\\"']+)" ) RUN_INSTRUCTION_RE = re.compile( r"^[ \t]*RUN[ \t]+(?P.*)$", re.IGNORECASE, ) -RUN_OPTION_NAMES = frozenset(("mount",)) -RUN_MOUNT_FIELD_RE = re.compile( - r"(?P[a-z][a-z0-9-]*)=(?P[^,\s\\\"']+)" -) -RUN_MOUNT_FIELDS = { - "cache": ("type", "target"), - "bind": ("type", "source", "target"), -} DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") RETIRED_MARKER_RE = re.compile( r"\b(?:bookworm|debian[ \t_:-]*v?[ \t_:-]*12)\b", @@ -476,7 +601,7 @@ def collect_dockerfile_copy_sources( tokens = instruction_match.group("arguments").split() seen_options = set() while tokens and tokens[0].startswith("--"): - option_match = LEADING_OPTION_RE.fullmatch(tokens.pop(0)) + option_match = COPY_OPTION_RE.fullmatch(tokens.pop(0)) if option_match is None: failures.append( f"{relative}: unsupported COPY option syntax prevents " @@ -503,58 +628,6 @@ def collect_dockerfile_copy_sources( return sources -def check_dockerfile_run_options( - instructions: tuple[str, ...], - relative: Path, - failures: list[str], -) -> None: - for instruction in instructions: - instruction_match = RUN_INSTRUCTION_RE.match(instruction) - if instruction_match is None: - continue - tokens = instruction_match.group("arguments").split() - while tokens and tokens[0].startswith("--"): - option_match = LEADING_OPTION_RE.fullmatch(tokens.pop(0)) - if ( - option_match is None - or option_match.group("name") not in RUN_OPTION_NAMES - ): - failures.append( - f"{relative}: unsupported RUN option syntax prevents " - "bounded mount inspection" - ) - return - - fields = [] - for field in option_match.group("value").split(","): - field_match = RUN_MOUNT_FIELD_RE.fullmatch(field) - if field_match is None: - failures.append( - f"{relative}: unsupported RUN mount syntax prevents " - "bounded mount inspection" - ) - return - fields.append( - (field_match.group("name"), field_match.group("value")) - ) - - field_names = tuple(name for name, _value in fields) - mount_type = fields[0][1] if fields else "" - if RUN_MOUNT_FIELDS.get(mount_type) != field_names: - failures.append( - f"{relative}: unsupported RUN mount syntax prevents " - "bounded mount inspection" - ) - return - - if not tokens or tokens[0].startswith(("-", "'", '"', "\\")): - failures.append( - f"{relative}: unsupported RUN operand prefix prevents " - "bounded mount inspection" - ) - return - - def collect_workflow_image_references( text: str, relative: Path, @@ -807,7 +880,16 @@ def check_repository(root: Path = ROOT) -> list[str]: relative, failures, ) - check_dockerfile_run_options(instructions, relative, failures) + run_instructions = tuple( + " ".join(instruction.split()) + for instruction in instructions + if RUN_INSTRUCTION_RE.match(instruction) + ) + if run_instructions != DOCKERFILE_RUN_INSTRUCTIONS[relative]: + failures.append( + f"{relative}: RUN instructions must match the exact " + "reviewed inventory" + ) stage_matches = list(FROM_RE.finditer(text)) if not stage_matches: failures.append(f"{relative}: no FROM instruction found") diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 445663d3..b9412b59 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -175,9 +175,13 @@ def test_workflow_image_allowlist_has_only_reviewed_postgres_images( set(self.module.WORKFLOW_IMAGE_ALLOWLIST), ) self.assertEqual( - {"image", "source_image", "target_image"}, + {"image", "source_image", "target_image", "restore_image"}, set(self.module.WORKFLOW_IMAGE_KEYS), ) + self.assertEqual( + ("postgres:17.10-alpine", "postgres:18.4-alpine"), + self.module.NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES, + ) for rationale in self.module.WORKFLOW_IMAGE_ALLOWLIST.values(): self.assertIn("not a project-owned Debian image", rationale) @@ -310,6 +314,64 @@ def test_notary_matrix_images_are_bound_to_the_owning_workflow( ) self.assertNotIn(replacement, "\n".join(failures)) + def test_notary_matrix_restore_images_are_exact_and_path_bounded( + self, + ) -> None: + source = (ROOT / NOTARY_POSTGRES_WORKFLOW).read_text( + encoding="utf-8" + ) + additions = ( + ( + " target_image: postgres:16.14-alpine", + " restore_image: postgres:17.10-alpine", + ), + ( + " target_image: postgres:17.10-alpine", + " restore_image: postgres:18.4-alpine", + ), + ( + " target_image: postgres:18.4-alpine", + " restore_image: postgres:18.4-alpine", + ), + ) + for target, restore in additions: + self.assertIn(target, source) + source = source.replace( + target, + f"{target}\n{restore}", + 1, + ) + + root = self.fixture() + self.write_workflow( + root, + NOTARY_POSTGRES_WORKFLOW.name, + source, + ) + self.assertEqual([], self.module.check_repository(root)) + + replacement = " restore_image: postgres:17" + root = self.fixture() + self.write_workflow( + root, + NOTARY_POSTGRES_WORKFLOW.name, + source.replace( + " restore_image: postgres:17.10-alpine", + replacement, + 1, + ), + ) + failures = self.module.check_repository(root) + self.assertTrue( + any( + f"{NOTARY_POSTGRES_WORKFLOW}: workflow image reference " + "is not allowlisted" in failure + for failure in failures + ), + failures, + ) + self.assertNotIn(replacement, "\n".join(failures)) + def test_workflow_image_inventory_fails_closed(self) -> None: cases = ( ( @@ -1220,16 +1282,12 @@ def test_multiline_copy_source_tokens_cannot_shadow_reviewed_names( "or reviewed named build context", ) - def test_dockerfile_run_mounts_use_the_reviewed_canonical_shapes( + def test_dockerfile_run_inventories_match_current_instructions( self, ) -> None: - self.assertEqual({"mount"}, set(self.module.RUN_OPTION_NAMES)) self.assertEqual( - { - "cache": ("type", "target"), - "bind": ("type", "source", "target"), - }, - self.module.RUN_MOUNT_FIELDS, + set(self.module.DOCKERFILES), + set(self.module.DOCKERFILE_RUN_INSTRUCTIONS), ) for relative in self.module.DOCKERFILES: with self.subTest(relative=relative): @@ -1239,71 +1297,68 @@ def test_dockerfile_run_mounts_use_the_reviewed_canonical_shapes( relative, failures, ) - self.module.check_dockerfile_run_options( - instructions, - relative, - failures, + actual = tuple( + " ".join(instruction.split()) + for instruction in instructions + if self.module.RUN_INSTRUCTION_RE.match(instruction) ) self.assertEqual([], failures) + self.assertEqual( + self.module.DOCKERFILE_RUN_INSTRUCTIONS[relative], + actual, + ) - def test_dockerfile_run_mounts_reject_unreviewed_sources_and_syntax( + def test_dockerfile_run_inventories_reject_any_command_change( self, ) -> None: - relative = Path("release/docker/Dockerfile.registry-relay") - exact = ( - "--mount=type=bind,source=dist/image-bin," - "target=/workspace/image-bin" - ) + release_relay = Path("release/docker/Dockerfile.registry-relay") + product_relay = Path("crates/registry-relay/Dockerfile") + runtime_marker = f"FROM {self.module.DISTROLESS_RUNTIME} AS runtime" cases = ( ( - "external-from", - "--mount=type=bind,from=debian:trixie-slim," - "source=dist/image-bin,target=/workspace/image-bin", - "unsupported RUN mount syntax", - ), - ( - "escaped-mount-key", - r"--mount=type=bind,fr\om=debian:trixie-slim," - "source=dist/image-bin,target=/workspace/image-bin", - "unsupported RUN option syntax", + "runtime-root-shell", + product_relay, + " chown -R 65532:65532 /workspace/runtime-root", + " mkdir -p /workspace/runtime-root/bin && \\\n" + " cp /bin/bash /workspace/runtime-root/bin/bash && \\\n" + " chown -R 65532:65532 /workspace/runtime-root", ), ( - "quoted-mount-key", - '--mount=type=bind,fr"om"=debian:trixie-slim,' - "source=dist/image-bin,target=/workspace/image-bin", - "unsupported RUN option syntax", + "bind-source-drift", + release_relay, + "source=dist/image-bin", + "source=dist/other-bin", ), ( - "escaped-option-name", - r"--mo\unt=type=bind,source=dist/image-bin," + "bind-target-drift", + release_relay, "target=/workspace/image-bin", - "unsupported RUN option syntax", + "target=/workspace/other-bin", ), ( - "quoted-option-name", - '--mo"unt"=type=bind,source=dist/image-bin,' - "target=/workspace/image-bin", - "unsupported RUN option syntax", + "external-mount-from", + release_relay, + "type=bind,source=dist/image-bin", + "type=bind,from=debian:trixie-slim," + "source=dist/image-bin", ), ( - "quoted-mount-value", + "quoted-mount", + release_relay, + "--mount=type=bind,source=dist/image-bin," + "target=/workspace/image-bin", '--mount="type=bind,source=dist/image-bin,' 'target=/workspace/image-bin"', - "unsupported RUN option syntax", ), ( - "unknown-option", - "--network=none", - "unsupported RUN option syntax", - ), - ( - "cache-with-source", - "--mount=type=cache,source=dist/image-bin," - "target=/workspace/image-bin", - "unsupported RUN mount syntax", + "extra-run", + product_relay, + runtime_marker, + f"RUN true\n\n{runtime_marker}", ), ) - for name, replacement, failure in cases: + failure = "RUN instructions must match the exact reviewed inventory" + for name, relative, exact, replacement in cases: with self.subTest(name=name): root = self.fixture() target = root / relative From e6289df63559e950108546a89979203e8f23b614 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Fri, 24 Jul 2026 06:50:37 +0700 Subject: [PATCH 34/34] fix(release): bind workflow images to roles Signed-off-by: Jeremi Joslin --- release/scripts/check-debian13-images.py | 71 +++-- release/scripts/test_check_debian13_images.py | 251 ++++++++++++------ 2 files changed, 221 insertions(+), 101 deletions(-) diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index 196a0498..cd6d1ee7 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -153,26 +153,25 @@ NOTARY_POSTGRES_CONFORMANCE_SCRIPT, ) -NOTARY_POSTGRES_WORKFLOW_SOURCE_TARGET_IMAGES = ( +NOTARY_POSTGRES_WORKFLOW_SOURCE_IMAGES = ( "postgres:16.13-alpine", - "postgres:16.14-alpine", "postgres:17.9-alpine", - "postgres:17.10-alpine", "postgres:18.3-alpine", +) +NOTARY_POSTGRES_WORKFLOW_TARGET_IMAGES = ( + "postgres:16.14-alpine", + "postgres:17.10-alpine", "postgres:18.4-alpine", ) NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES = ( "postgres:17.10-alpine", "postgres:18.4-alpine", ) -NOTARY_POSTGRES_WORKFLOW_IMAGES = tuple( - dict.fromkeys( - ( - *NOTARY_POSTGRES_WORKFLOW_SOURCE_TARGET_IMAGES, - *NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES, - ) - ) -) +NOTARY_POSTGRES_WORKFLOW_IMAGES_BY_ROLE = { + "source_image": NOTARY_POSTGRES_WORKFLOW_SOURCE_IMAGES, + "target_image": NOTARY_POSTGRES_WORKFLOW_TARGET_IMAGES, + "restore_image": NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES, +} NOTARY_POSTGRES_WORKFLOW_RATIONALE = ( "External Alpine PostgreSQL migration conformance, not a " "project-owned Debian image." @@ -180,6 +179,7 @@ WORKFLOW_IMAGE_ALLOWLIST = { ( Path(".github/workflows/relay-postgres-conformance.yml"), + "image", "postgres:${{ matrix.postgresql }}-alpine", ): ( "External Alpine PostgreSQL state-plane conformance, not a " @@ -188,14 +188,19 @@ **{ ( Path(".github/workflows/notary-postgres-conformance.yml"), + role, image, ): NOTARY_POSTGRES_WORKFLOW_RATIONALE - for image in NOTARY_POSTGRES_WORKFLOW_IMAGES + for role, images in NOTARY_POSTGRES_WORKFLOW_IMAGES_BY_ROLE.items() + for image in images }, } WORKFLOW_IMAGE_KEYS = frozenset( ("image", "source_image", "target_image", "restore_image") ) +WORKFLOW_DEFAULT_IMAGE_KEYS = frozenset( + ("default_source_image", "default_target_image", "default_restore_image") +) NOTARY_POSTGRES_IMAGE_ASSIGNMENTS = ( ("default_source_image", '"postgres:16.13-alpine"'), ("default_target_image", '"postgres:16.14-alpine"'), @@ -632,7 +637,7 @@ def collect_workflow_image_references( text: str, relative: Path, failures: list[str], -) -> list[tuple[str, str]]: +) -> list[tuple[str, str, str]]: try: document = yaml.safe_load(text) except yaml.YAMLError as error: @@ -647,17 +652,17 @@ def collect_workflow_image_references( ) return [] - references: list[tuple[str, str]] = [] + references: list[tuple[str, str, str]] = [] active_nodes: set[int] = set() - def add_image(value: object, location: str) -> None: + def add_image(value: object, role: str, location: str) -> None: if not isinstance(value, str) or not value.strip(): failures.append( f"{relative}: unsupported workflow image value at {location}: " f"expected a non-empty string, found {type(value).__name__}" ) return - references.append((location, value)) + references.append((role, location, value)) def visit(value: object, location: str) -> None: if not isinstance(value, (dict, list)): @@ -674,21 +679,41 @@ def visit(value: object, location: str) -> None: child_location = f"{location}.{key}" if key == "container": if isinstance(child, str): - add_image(child, child_location) - elif not isinstance(child, dict) or "image" not in child: + add_image(child, "container", child_location) + elif isinstance(child, dict) and "image" in child: + add_image( + child["image"], + "container", + f"{child_location}.image", + ) + else: failures.append( f"{relative}: unsupported workflow image value at " f"{child_location}: container must be a non-empty " "string or a mapping with an image key" ) - elif key in WORKFLOW_IMAGE_KEYS: - add_image(child, child_location) + if isinstance(child, dict): + for nested_key, nested_child in child.items(): + if nested_key != "image": + visit( + nested_child, + f"{child_location}.{nested_key}", + ) + elif isinstance(child, list): + visit(child, child_location) + continue + elif key in WORKFLOW_IMAGE_KEYS or key in WORKFLOW_DEFAULT_IMAGE_KEYS: + add_image(child, str(key), child_location) elif ( key == "uses" and isinstance(child, str) and child.startswith("docker://") ): - add_image(child.removeprefix("docker://"), child_location) + add_image( + child.removeprefix("docker://"), + "uses", + child_location, + ) visit(child, child_location) else: for index, child in enumerate(value): @@ -834,8 +859,8 @@ def check_repository(root: Path = ROOT) -> list[str]: relative, failures, ) - for location, value in references: - if (relative, value) not in WORKFLOW_IMAGE_ALLOWLIST: + for role, location, value in references: + if (relative, role, value) not in WORKFLOW_IMAGE_ALLOWLIST: failures.append( f"{relative}: workflow image reference is not allowlisted " f"at {location}" diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index b9412b59..8c50785f 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -98,6 +98,27 @@ def runtime_directives(self, relative: Path) -> tuple[str, ...]: f'CMD ["--config", "/etc/registry-{product}/config.yaml"]', ) + def notary_workflow_with_restore_images(self) -> str: + source = (ROOT / NOTARY_POSTGRES_WORKFLOW).read_text(encoding="utf-8") + additions = ( + ( + " target_image: postgres:16.14-alpine", + " restore_image: postgres:17.10-alpine", + ), + ( + " target_image: postgres:17.10-alpine", + " restore_image: postgres:18.4-alpine", + ), + ( + " target_image: postgres:18.4-alpine", + " restore_image: postgres:18.4-alpine", + ), + ) + for target, restore in additions: + self.assertIn(target, source) + source = source.replace(target, f"{target}\n{restore}", 1) + return source + def test_current_repository_follows_contract(self) -> None: self.assertEqual([], self.module.check_repository(ROOT)) @@ -161,17 +182,31 @@ def test_workflow_image_allowlist_has_only_reviewed_postgres_images( self, ) -> None: relay_image = "postgres:${{ matrix.postgresql }}-alpine" - notary_images = { + source_images = ( "postgres:16.13-alpine", - "postgres:16.14-alpine", "postgres:17.9-alpine", - "postgres:17.10-alpine", "postgres:18.3-alpine", + ) + target_images = ( + "postgres:16.14-alpine", + "postgres:17.10-alpine", + "postgres:18.4-alpine", + ) + restore_images = ( + "postgres:17.10-alpine", "postgres:18.4-alpine", - } + ) self.assertEqual( - {(RELAY_POSTGRES_WORKFLOW, relay_image)} - | {(NOTARY_POSTGRES_WORKFLOW, image) for image in notary_images}, + {(RELAY_POSTGRES_WORKFLOW, "image", relay_image)} + | { + (NOTARY_POSTGRES_WORKFLOW, role, image) + for role, images in ( + ("source_image", source_images), + ("target_image", target_images), + ("restore_image", restore_images), + ) + for image in images + }, set(self.module.WORKFLOW_IMAGE_ALLOWLIST), ) self.assertEqual( @@ -179,7 +214,23 @@ def test_workflow_image_allowlist_has_only_reviewed_postgres_images( set(self.module.WORKFLOW_IMAGE_KEYS), ) self.assertEqual( - ("postgres:17.10-alpine", "postgres:18.4-alpine"), + { + "default_source_image", + "default_target_image", + "default_restore_image", + }, + set(self.module.WORKFLOW_DEFAULT_IMAGE_KEYS), + ) + self.assertEqual( + source_images, + self.module.NOTARY_POSTGRES_WORKFLOW_SOURCE_IMAGES, + ) + self.assertEqual( + target_images, + self.module.NOTARY_POSTGRES_WORKFLOW_TARGET_IMAGES, + ) + self.assertEqual( + restore_images, self.module.NOTARY_POSTGRES_WORKFLOW_RESTORE_IMAGES, ) for rationale in self.module.WORKFLOW_IMAGE_ALLOWLIST.values(): @@ -284,63 +335,30 @@ def test_dynamic_workflow_images_are_denied_from_structured_forms( f"{relative}: workflow image reference is not allowlisted", ) - def test_notary_matrix_images_are_bound_to_the_owning_workflow( - self, - ) -> None: - source = (ROOT / NOTARY_POSTGRES_WORKFLOW).read_text(encoding="utf-8") - for key, reviewed, replacement in ( - ("source_image", "postgres:16.13-alpine", "postgres:16-alpine"), - ("target_image", "postgres:16.14-alpine", "postgres:17-alpine"), - ): - with self.subTest(key=key): - root = self.fixture() - self.write_workflow( - root, - NOTARY_POSTGRES_WORKFLOW.name, - source.replace( - f"{key}: {reviewed}", - f"{key}: {replacement}", - 1, - ), - ) - failures = self.module.check_repository(root) - self.assertTrue( - any( - f"{NOTARY_POSTGRES_WORKFLOW}: workflow image " - "reference is not allowlisted" in failure - for failure in failures - ), - failures, - ) - self.assertNotIn(replacement, "\n".join(failures)) - - def test_notary_matrix_restore_images_are_exact_and_path_bounded( + def test_notary_matrix_images_match_exact_roles( self, ) -> None: - source = (ROOT / NOTARY_POSTGRES_WORKFLOW).read_text( - encoding="utf-8" + source = self.notary_workflow_with_restore_images() + failures: list[str] = [] + references = self.module.collect_workflow_image_references( + source, + NOTARY_POSTGRES_WORKFLOW, + failures, ) - additions = ( - ( - " target_image: postgres:16.14-alpine", - " restore_image: postgres:17.10-alpine", - ), - ( - " target_image: postgres:17.10-alpine", - " restore_image: postgres:18.4-alpine", - ), - ( - " target_image: postgres:18.4-alpine", - " restore_image: postgres:18.4-alpine", - ), + self.assertEqual([], failures) + self.assertEqual( + { + ("source_image", "postgres:16.13-alpine"), + ("source_image", "postgres:17.9-alpine"), + ("source_image", "postgres:18.3-alpine"), + ("target_image", "postgres:16.14-alpine"), + ("target_image", "postgres:17.10-alpine"), + ("target_image", "postgres:18.4-alpine"), + ("restore_image", "postgres:17.10-alpine"), + ("restore_image", "postgres:18.4-alpine"), + }, + {(role, value) for role, _location, value in references}, ) - for target, restore in additions: - self.assertIn(target, source) - source = source.replace( - target, - f"{target}\n{restore}", - 1, - ) root = self.fixture() self.write_workflow( @@ -350,27 +368,104 @@ def test_notary_matrix_restore_images_are_exact_and_path_bounded( ) self.assertEqual([], self.module.check_repository(root)) - replacement = " restore_image: postgres:17" - root = self.fixture() - self.write_workflow( - root, - NOTARY_POSTGRES_WORKFLOW.name, - source.replace( - " restore_image: postgres:17.10-alpine", - replacement, - 1, + def test_notary_matrix_images_reject_cross_role_and_mutable_values( + self, + ) -> None: + source = self.notary_workflow_with_restore_images() + cases = ( + ( + "target-as-source", + "source_image: postgres:16.13-alpine", + "source_image: postgres:16.14-alpine", + "postgres:16.14-alpine", ), - ) - failures = self.module.check_repository(root) - self.assertTrue( - any( - f"{NOTARY_POSTGRES_WORKFLOW}: workflow image reference " - "is not allowlisted" in failure - for failure in failures + ( + "source-as-target", + "target_image: postgres:16.14-alpine", + "target_image: postgres:16.13-alpine", + "postgres:16.13-alpine", ), - failures, + ( + "source-as-restore", + "restore_image: postgres:17.10-alpine", + "restore_image: postgres:16.13-alpine", + "postgres:16.13-alpine", + ), + ( + "target-only-as-restore", + "restore_image: postgres:17.10-alpine", + "restore_image: postgres:16.14-alpine", + "postgres:16.14-alpine", + ), + ( + "mutable-source", + "source_image: postgres:16.13-alpine", + "source_image: postgres:16-alpine", + "postgres:16-alpine", + ), + ( + "mutable-target", + "target_image: postgres:16.14-alpine", + "target_image: postgres:16-alpine", + "postgres:16-alpine", + ), + ( + "mutable-restore", + "restore_image: postgres:17.10-alpine", + "restore_image: postgres:17", + "postgres:17", + ), + ) + failure = ( + f"{NOTARY_POSTGRES_WORKFLOW}: workflow image reference " + "is not allowlisted" + ) + for name, exact, replacement, leaked_value in cases: + with self.subTest(name=name): + self.assertIn(exact, source) + root = self.fixture() + self.write_workflow( + root, + NOTARY_POSTGRES_WORKFLOW.name, + source.replace(exact, replacement, 1), + ) + failures = self.module.check_repository(root) + self.assertTrue( + any(failure in item for item in failures), + failures, + ) + self.assertNotIn(leaked_value, "\n".join(failures)) + + def test_relay_image_exception_does_not_authorize_other_roles(self) -> None: + source = (ROOT / RELAY_POSTGRES_WORKFLOW).read_text(encoding="utf-8") + relay_image = "postgres:${{ matrix.postgresql }}-alpine" + exact = f" image: {relay_image}" + replacements = ( + f" container: {relay_image}", + " container:\n" + f" image: {relay_image}", + f" uses: docker://{relay_image}", + f" default_source_image: {relay_image}", ) - self.assertNotIn(replacement, "\n".join(failures)) + self.assertIn(exact, source) + for replacement in replacements: + with self.subTest(replacement=replacement): + root = self.fixture() + self.write_workflow( + root, + RELAY_POSTGRES_WORKFLOW.name, + source.replace(exact, replacement, 1), + ) + failures = self.module.check_repository(root) + self.assertTrue( + any( + f"{RELAY_POSTGRES_WORKFLOW}: workflow image reference " + "is not allowlisted" in item + for item in failures + ), + failures, + ) + self.assertNotIn(relay_image, "\n".join(failures)) def test_workflow_image_inventory_fails_closed(self) -> None: cases = (