From d482160f0a07dd7a494818c9e36ff919a053831a Mon Sep 17 00:00:00 2001 From: brandom Date: Fri, 7 Aug 2026 15:23:28 -0700 Subject: [PATCH 1/4] Add public validation health dashboard pilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/render-validation-health.py | 289 ++++++++++++++++++ .../test/test-render-validation-health.py | 232 ++++++++++++++ .github/validation-health-pilot.json | 8 + .github/workflows/scripts-selftest.yml | 13 + 4 files changed, 542 insertions(+) create mode 100644 .github/scripts/render-validation-health.py create mode 100644 .github/scripts/test/test-render-validation-health.py create mode 100644 .github/validation-health-pilot.json diff --git a/.github/scripts/render-validation-health.py b/.github/scripts/render-validation-health.py new file mode 100644 index 000000000..eea941ce9 --- /dev/null +++ b/.github/scripts/render-validation-health.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Render the public validation health dashboard pilot as Markdown.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlparse + + +STATUS_DISPLAY = { + "pass": "✅ Pass", + "failure": "❌ Failed", + "error": "⚠️ Warning", +} +LEVELS = ("l3", "l4") +SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + + +class ContractError(ValueError): + """Raised when dashboard configuration or result data is invalid.""" + + +def load_json(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ContractError(f"{label} file not found: {path}") from exc + except json.JSONDecodeError as exc: + raise ContractError(f"{label} is not valid JSON: {exc}") from exc + if not isinstance(value, dict): + raise ContractError(f"{label} must be a JSON object") + return value + + +def validate_path(path: Any) -> str: + if not isinstance(path, str) or not path.startswith("samples/"): + raise ContractError(f"sample path must be a string under samples/: {path!r}") + parts = Path(path).parts + if Path(path).is_absolute() or ".." in parts or "." in parts: + raise ContractError(f"sample path must be repository-relative: {path!r}") + if any(character in path for character in ("|", "`", "\r", "\n")): + raise ContractError(f"sample path contains unsupported Markdown characters: {path!r}") + return Path(path).as_posix() + + +def load_config(path: Path, repo_root: Path) -> tuple[str, list[str]]: + config = load_json(path, "pilot config") + if config.get("schema_version") != 1: + raise ContractError("pilot config schema_version must be 1") + + repository = config.get("repository") + if not isinstance(repository, str) or not REPOSITORY_PATTERN.fullmatch(repository): + raise ContractError("pilot config repository must be owner/name") + + raw_samples = config.get("samples") + if not isinstance(raw_samples, list) or not raw_samples: + raise ContractError("pilot config samples must be a non-empty list") + + samples = [validate_path(sample) for sample in raw_samples] + if len(samples) != len(set(samples)): + raise ContractError("pilot config samples must not contain duplicates") + if samples != sorted(samples): + raise ContractError("pilot config samples must be sorted") + + for sample in samples: + if not (repo_root / sample).is_dir(): + raise ContractError(f"configured sample directory does not exist: {sample}") + + return repository, samples + + +def parse_utc_timestamp(value: Any, field: str) -> datetime: + if not isinstance(value, str) or not value.endswith("Z"): + raise ContractError(f"{field} must be an ISO-8601 UTC timestamp ending in Z") + try: + parsed = datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise ContractError(f"{field} is not a valid timestamp: {value!r}") from exc + if parsed.utcoffset() != timezone.utc.utcoffset(parsed): + raise ContractError(f"{field} must use UTC") + return parsed + + +def validate_url(value: Any, field: str) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ContractError(f"{field} must be a string") + try: + parsed = urlparse(value) + except ValueError as exc: + raise ContractError(f"{field} must be an absolute HTTP(S) URL") from exc + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ContractError(f"{field} must be an absolute HTTP(S) URL") + if any(ord(character) < 32 for character in value): + raise ContractError(f"{field} must not contain control characters") + return value + + +def markdown_url(value: str) -> str: + return quote(value, safe=":/?#@!$&'*+,;=%._~-") + + +def load_results(path: Path | None, selected_samples: set[str]) -> tuple[str | None, dict[str, Any]]: + if path is None: + return None, {} + + payload = load_json(path, "results") + if payload.get("schema_version") != 1: + raise ContractError("results schema_version must be 1") + + source_sha = payload.get("source_sha") + if not isinstance(source_sha, str) or not SHA_PATTERN.fullmatch(source_sha): + raise ContractError("results source_sha must be a full lowercase Git SHA") + + raw_results = payload.get("results") + if not isinstance(raw_results, dict): + raise ContractError("results must be a JSON object") + + selected: dict[str, Any] = {} + for sample in selected_samples: + if sample not in raw_results: + continue + sample_result = raw_results[sample] + if not isinstance(sample_result, dict): + raise ContractError(f"result for {sample} must be a JSON object") + unknown_levels = set(sample_result) - set(LEVELS) + if unknown_levels: + raise ContractError( + f"result for {sample} has unsupported levels: {sorted(unknown_levels)}" + ) + + selected[sample] = {} + for level in LEVELS: + if level not in sample_result: + continue + level_result = sample_result[level] + if not isinstance(level_result, dict): + raise ContractError(f"{sample}.{level} must be a JSON object") + unknown_fields = set(level_result) - {"status", "run_at", "evidence_url"} + if unknown_fields: + raise ContractError( + f"{sample}.{level} has unsupported fields: {sorted(unknown_fields)}" + ) + status = level_result.get("status") + if not isinstance(status, str) or status not in STATUS_DISPLAY: + raise ContractError( + f"{sample}.{level}.status must be one of {sorted(STATUS_DISPLAY)}" + ) + run_at = parse_utc_timestamp(level_result.get("run_at"), f"{sample}.{level}.run_at") + evidence_url = validate_url( + level_result.get("evidence_url"), f"{sample}.{level}.evidence_url" + ) + selected[sample][level] = { + "status": status, + "run_at": run_at, + "evidence_url": evidence_url, + } + + return source_sha, selected + + +def resolve_source_sha(repo_root: Path, result_sha: str | None, argument_sha: str | None) -> str: + if result_sha is not None: + if argument_sha is not None and argument_sha != result_sha: + raise ContractError("--source-sha does not match results source_sha") + return result_sha + if argument_sha is not None: + if not SHA_PATTERN.fullmatch(argument_sha): + raise ContractError("--source-sha must be a full lowercase Git SHA") + return argument_sha + try: + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise ContractError("could not resolve source SHA from the repository") from exc + + +def render_status(result: dict[str, Any] | None) -> tuple[str, str]: + if result is None: + return "⚪ Never run", "—" + display = STATUS_DISPLAY[result["status"]] + evidence_url = result["evidence_url"] + if evidence_url: + display = f"[{display}]({markdown_url(evidence_url)})" + run_at = result["run_at"].strftime("%Y-%m-%d %H:%M UTC") + return display, run_at + + +def render_markdown( + repository: str, + samples: list[str], + results: dict[str, Any], + source_sha: str, + generated_at: datetime, +) -> str: + lines = [ + "# Validation Health Dashboard", + "", + ( + f"_Generated {generated_at.strftime('%Y-%m-%d %H:%M UTC')} from " + f"[`{source_sha[:12]}`](https://github.com/{repository}/commit/{source_sha}). " + f"Pilot scope: {len(samples)} samples._" + ), + "", + "| Sample | L3 | Last L3 run | L4 | Last L4 run |", + "|---|---|---|---|---|", + ] + + for sample in samples: + link = f"https://github.com/{repository}/tree/main/{quote(sample, safe='/')}" + sample_result = results.get(sample, {}) + l3_status, l3_run = render_status(sample_result.get("l3")) + l4_status, l4_run = render_status(sample_result.get("l4")) + lines.append( + f"| [`{sample}`]({link}) | {l3_status} | {l3_run} | {l4_status} | {l4_run} |" + ) + + lines.extend( + [ + "", + "**Legend:** ✅ pass · ❌ sample failure · ⚠️ infrastructure/error · ⚪ never run", + "", + "> This pilot uses public validation results only. A missing result means " + "“never run,” not “pass.”", + "", + ] + ) + return "\n".join(lines) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", + type=Path, + default=Path(".github/validation-health-pilot.json"), + help="pilot scope JSON", + ) + parser.add_argument("--results", type=Path, help="optional normalized result JSON") + parser.add_argument("--output", type=Path, required=True, help="Markdown output path") + parser.add_argument("--repo-root", type=Path, default=Path("."), help="repository root") + parser.add_argument("--source-sha", help="source SHA when no results document is supplied") + parser.add_argument( + "--generated-at", + help="override generation timestamp for deterministic tests (ISO-8601 UTC)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + repo_root = args.repo_root.resolve() + try: + repository, samples = load_config(args.config, repo_root) + result_sha, results = load_results(args.results, set(samples)) + source_sha = resolve_source_sha(repo_root, result_sha, args.source_sha) + if not SHA_PATTERN.fullmatch(source_sha): + raise ContractError("resolved source SHA must be a full lowercase Git SHA") + generated_at = ( + parse_utc_timestamp(args.generated_at, "--generated-at") + if args.generated_at + else datetime.now(timezone.utc) + ) + markdown = render_markdown(repository, samples, results, source_sha, generated_at) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(markdown, encoding="utf-8", newline="\n") + except (ContractError, OSError) as exc: + print(f"render-validation-health: {exc}", file=sys.stderr) + return 1 + print(f"render-validation-health: wrote {args.output}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test/test-render-validation-health.py b/.github/scripts/test/test-render-validation-health.py new file mode 100644 index 000000000..813ad7c2f --- /dev/null +++ b/.github/scripts/test/test-render-validation-health.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Fixture tests for render-validation-health.py.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "render-validation-health.py" +SHA = "0123456789abcdef0123456789abcdef01234567" +GENERATED_AT = "2026-08-07T20:30:00Z" +C_SHARP = "samples/csharp/quickstart/chat-with-agent" +PYTHON = "samples/python/quickstart/chat-with-agent" + + +class RendererTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + for sample in (C_SHARP, PYTHON): + (self.root / sample).mkdir(parents=True) + self.config = self.root / "config.json" + self.config.write_text( + json.dumps( + { + "schema_version": 1, + "repository": "microsoft-foundry/foundry-samples", + "samples": [C_SHARP, PYTHON], + } + ), + encoding="utf-8", + ) + + def tearDown(self) -> None: + self.temp.cleanup() + + def run_renderer( + self, + results: dict | str | None = None, + *, + config: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + output = self.root / "dashboard.md" + command = [ + sys.executable, + str(SCRIPT), + "--config", + str(config or self.config), + "--repo-root", + str(self.root), + "--output", + str(output), + "--source-sha", + SHA, + "--generated-at", + GENERATED_AT, + ] + if results is not None: + results_path = self.root / "results.json" + if isinstance(results, str): + results_path.write_text(results, encoding="utf-8") + else: + results_path.write_text(json.dumps(results), encoding="utf-8") + command.extend(["--results", str(results_path)]) + completed = subprocess.run(command, capture_output=True, text=True) + completed.output_path = output # type: ignore[attr-defined] + return completed + + def test_never_run_view_has_two_linked_rows(self) -> None: + completed = self.run_renderer() + self.assertEqual(completed.returncode, 0, completed.stderr) + markdown = completed.output_path.read_text(encoding="utf-8") + self.assertEqual(markdown.count("| [`samples/"), 2) + self.assertEqual(markdown.count("⚪ Never run"), 4) + self.assertIn( + f"https://github.com/microsoft-foundry/foundry-samples/tree/main/{C_SHARP}", + markdown, + ) + self.assertIn( + f"https://github.com/microsoft-foundry/foundry-samples/tree/main/{PYTHON}", + markdown, + ) + self.assertLess(markdown.index(C_SHARP), markdown.index(PYTHON)) + + def test_status_mapping_dates_and_evidence(self) -> None: + results = { + "schema_version": 1, + "source_sha": SHA, + "results": { + C_SHARP: { + "l3": { + "status": "pass", + "run_at": "2026-08-06T01:02:03Z", + "evidence_url": ( + "https://github.com/example/actions/runs/1" + "?name=a|b&label=
" + ), + }, + "l4": { + "status": "failure", + "run_at": "2026-08-06T02:03:04Z", + }, + }, + PYTHON: { + "l3": { + "status": "error", + "run_at": "2026-08-06T03:04:05Z", + } + }, + }, + } + completed = self.run_renderer(results) + self.assertEqual(completed.returncode, 0, completed.stderr) + markdown = completed.output_path.read_text(encoding="utf-8") + self.assertIn( + ( + "[✅ Pass](https://github.com/example/actions/runs/1" + "?name=a%7Cb&label=%3Cdetails%20open%3E)" + ), + markdown, + ) + self.assertIn("❌ Failed", markdown) + self.assertIn("⚠️ Warning", markdown) + self.assertIn("⚪ Never run", markdown) + self.assertIn("2026-08-06 01:02 UTC", markdown) + self.assertIn("2026-08-06 02:03 UTC", markdown) + self.assertIn("2026-08-06 03:04 UTC", markdown) + + def test_invalid_selected_status_fails(self) -> None: + results = { + "schema_version": 1, + "source_sha": SHA, + "results": { + PYTHON: { + "l3": { + "status": "pending", + "run_at": "2026-08-06T03:04:05Z", + } + } + }, + } + completed = self.run_renderer(results) + self.assertNotEqual(completed.returncode, 0) + self.assertIn("status must be one of", completed.stderr) + + def test_non_string_selected_status_fails_cleanly(self) -> None: + results = { + "schema_version": 1, + "source_sha": SHA, + "results": { + PYTHON: { + "l3": { + "status": [], + "run_at": "2026-08-06T03:04:05Z", + } + } + }, + } + completed = self.run_renderer(results) + self.assertNotEqual(completed.returncode, 0) + self.assertNotIn("Traceback", completed.stderr) + self.assertIn("status must be one of", completed.stderr) + + def test_malformed_evidence_url_fails_cleanly(self) -> None: + results = { + "schema_version": 1, + "source_sha": SHA, + "results": { + PYTHON: { + "l3": { + "status": "pass", + "run_at": "2026-08-06T03:04:05Z", + "evidence_url": "http://[", + } + } + }, + } + completed = self.run_renderer(results) + self.assertNotEqual(completed.returncode, 0) + self.assertNotIn("Traceback", completed.stderr) + self.assertIn("absolute HTTP(S) URL", completed.stderr) + + def test_invalid_selected_timestamp_fails(self) -> None: + results = { + "schema_version": 1, + "source_sha": SHA, + "results": { + PYTHON: { + "l3": { + "status": "pass", + "run_at": "yesterday", + } + } + }, + } + completed = self.run_renderer(results) + self.assertNotEqual(completed.returncode, 0) + self.assertIn("ISO-8601 UTC", completed.stderr) + + def test_unselected_results_are_ignored(self) -> None: + results = { + "schema_version": 1, + "source_sha": SHA, + "results": { + "samples/rust/not-in-pilot": { + "l3": {"status": "not-a-real-status", "run_at": "not-a-date"} + } + }, + } + completed = self.run_renderer(results) + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_missing_configured_sample_fails(self) -> None: + missing = self.root / PYTHON + missing.rmdir() + completed = self.run_renderer() + self.assertNotEqual(completed.returncode, 0) + self.assertIn("configured sample directory does not exist", completed.stderr) + + def test_malformed_results_json_fails(self) -> None: + completed = self.run_renderer("{") + self.assertNotEqual(completed.returncode, 0) + self.assertIn("results is not valid JSON", completed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/validation-health-pilot.json b/.github/validation-health-pilot.json new file mode 100644 index 000000000..e4277e829 --- /dev/null +++ b/.github/validation-health-pilot.json @@ -0,0 +1,8 @@ +{ + "schema_version": 1, + "repository": "microsoft-foundry/foundry-samples", + "samples": [ + "samples/csharp/quickstart/chat-with-agent", + "samples/python/quickstart/chat-with-agent" + ] +} diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 9a799b5f7..017b20b8d 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -19,6 +19,7 @@ on: workflow_dispatch: pull_request: paths: + - '.github/validation-health-pilot.json' - '.github/scripts/**' - '.github/workflows/validate.yml' - '.github/workflows/scripts-selftest.yml' @@ -27,6 +28,18 @@ permissions: contents: read jobs: + # --- Health dashboard renderer: fixture-only, no status API or issue writes -------------------- + health-dashboard-harness: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Validation health dashboard renderer exit gate + run: python .github/scripts/test/test-render-validation-health.py + # --- Required-gate structure: hermetic Bash assertions, no credentials ------------------------- workflow-structure-harness: runs-on: ubuntu-latest From 24c5b17553282f2f987e80511b90b9a4d0f2d2e5 Mon Sep 17 00:00:00 2001 From: brandom Date: Fri, 7 Aug 2026 15:51:57 -0700 Subject: [PATCH 2/4] Wire pilot validation results to dashboard Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/render-validation-health.py | 26 ++ .../scripts/run-validation-health-pilot.py | 258 ++++++++++++++++++ .../test/test-render-validation-health.py | 9 + .../test/test-run-validation-health-pilot.py | 232 ++++++++++++++++ .github/scripts/validation_health_state.py | 43 +++ .github/workflows/scripts-selftest.yml | 3 + .github/workflows/validation-health-pilot.yml | 134 +++++++++ 7 files changed, 705 insertions(+) create mode 100644 .github/scripts/run-validation-health-pilot.py create mode 100644 .github/scripts/test/test-run-validation-health-pilot.py create mode 100644 .github/scripts/validation_health_state.py create mode 100644 .github/workflows/validation-health-pilot.yml diff --git a/.github/scripts/render-validation-health.py b/.github/scripts/render-validation-health.py index eea941ce9..621709735 100644 --- a/.github/scripts/render-validation-health.py +++ b/.github/scripts/render-validation-health.py @@ -13,6 +13,8 @@ from typing import Any from urllib.parse import quote, urlparse +from validation_health_state import encode_state + STATUS_DISPLAY = { "pass": "✅ Pass", @@ -237,6 +239,30 @@ def render_markdown( "> This pilot uses public validation results only. A missing result means " "“never run,” not “pass.”", "", + encode_state( + { + "schema_version": 1, + "source_sha": source_sha, + "results": { + sample: { + level: { + "status": level_result["status"], + "run_at": level_result["run_at"].strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + **( + {"evidence_url": level_result["evidence_url"]} + if level_result["evidence_url"] + else {} + ), + } + for level, level_result in sample_result.items() + } + for sample, sample_result in results.items() + }, + } + ), + "", ] ) return "\n".join(lines) diff --git a/.github/scripts/run-validation-health-pilot.py b/.github/scripts/run-validation-health-pilot.py new file mode 100644 index 000000000..4d7bc3b3c --- /dev/null +++ b/.github/scripts/run-validation-health-pilot.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Run the configured dashboard pilot samples and emit normalized results.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from validation_health_state import StateError, extract_state + + +LANGUAGES = { + "csharp": "csharp", + "python": "python", + "typescript": "typescript", + "javascript": "typescript", + "java": "java", + "go": "go", +} +SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") + + +class PilotError(ValueError): + """Raised when the pilot cannot produce a trustworthy result document.""" + + +def load_config(path: Path, repo_root: Path) -> list[str]: + try: + config = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise PilotError(f"pilot config not found: {path}") from exc + except json.JSONDecodeError as exc: + raise PilotError(f"pilot config is not valid JSON: {exc}") from exc + if not isinstance(config, dict) or config.get("schema_version") != 1: + raise PilotError("pilot config must be a schema_version 1 JSON object") + samples = config.get("samples") + if not isinstance(samples, list) or not samples: + raise PilotError("pilot config samples must be a non-empty list") + if any(not isinstance(sample, str) for sample in samples): + raise PilotError("pilot config sample paths must be strings") + if samples != sorted(set(samples)): + raise PilotError("pilot config sample paths must be sorted and unique") + for sample in samples: + if not sample.startswith("samples/") or not (repo_root / sample).is_dir(): + raise PilotError(f"configured sample directory does not exist: {sample}") + language_dir = sample.split("/", 2)[1] + if language_dir not in LANGUAGES: + raise PilotError(f"configured sample language is unsupported: {sample}") + return samples + + +def load_previous_results(path: Path | None, samples: set[str]) -> dict[str, Any]: + if path is None: + return {} + try: + body = path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + raise PilotError(f"previous dashboard body not found: {path}") from exc + try: + payload = extract_state(body) + except StateError as exc: + raise PilotError(str(exc)) from exc + if payload is None: + return {} + if payload.get("schema_version") != 1 or not isinstance(payload.get("results"), dict): + raise PilotError("previous dashboard hidden state has an unsupported contract") + return { + sample: deepcopy(payload["results"][sample]) + for sample in samples + if sample in payload["results"] + } + + +def resolve_sha(repo_root: Path, argument: str | None) -> str: + if argument is not None: + if not SHA_PATTERN.fullmatch(argument): + raise PilotError("--source-sha must be a full lowercase Git SHA") + return argument + try: + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise PilotError("could not resolve source SHA") from exc + if not SHA_PATTERN.fullmatch(sha): + raise PilotError("resolved source SHA is invalid") + return sha + + +def probe_l4(yq: list[str], repo_root: Path, sample: str) -> bool: + yaml_path = repo_root / sample / "sample.yaml" + completed = subprocess.run( + [*yq, "eval", 'has("l4")', yaml_path.as_posix()], + cwd=repo_root, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + raise PilotError(f"could not inspect L4 declaration: {sample}") + value = completed.stdout.strip() + if value not in ("true", "false"): + raise PilotError(f"unexpected L4 declaration probe for {sample}: {value!r}") + return value == "true" + + +def run_validator( + bash: str, + validator: Path, + repo_root: Path, + sample: str, + level: str, +) -> int: + try: + validator_argument = validator.relative_to(repo_root).as_posix() + except ValueError: + validator_argument = validator.as_posix() + command = [bash, validator_argument, "--level", level, "--sample-dir", sample] + if level == "3": + language_dir = sample.split("/", 2)[1] + command.extend(["--language", LANGUAGES[language_dir]]) + completed = subprocess.run( + command, + cwd=repo_root, + capture_output=True, + text=True, + env=os.environ.copy(), + ) + print(f"===== {sample} L{level} (exit={completed.returncode}) =====") + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + return completed.returncode + + +def status_for_exit_code(exit_code: int) -> str: + if exit_code == 0: + return "pass" + if exit_code == 1: + return "failure" + return "error" + + +def result_record(status: str, run_at: str, evidence_url: str | None) -> dict[str, str]: + record = {"status": status, "run_at": run_at} + if evidence_url: + record["evidence_url"] = evidence_url + return record + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", + type=Path, + default=Path(".github/validation-health-pilot.json"), + ) + parser.add_argument("--repo-root", type=Path, default=Path(".")) + parser.add_argument( + "--validator", + type=Path, + default=Path(".github/scripts/validate-sample.sh"), + ) + parser.add_argument("--bash", default="bash") + parser.add_argument("--yq", nargs="+", default=["yq"]) + parser.add_argument("--output", type=Path) + parser.add_argument("--previous-body", type=Path) + parser.add_argument("--source-sha") + parser.add_argument("--evidence-url") + parser.add_argument("--run-at", help="fixed ISO-8601 UTC timestamp for tests") + parser.add_argument("--detect-l4-only", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + repo_root = args.repo_root.resolve() + try: + samples = load_config(args.config, repo_root) + declarations = { + sample: probe_l4(args.yq, repo_root, sample) for sample in samples + } + if args.detect_l4_only: + print("true" if any(declarations.values()) else "false") + return 0 + if args.output is None: + raise PilotError("--output is required unless --detect-l4-only is used") + if os.environ.get("SKIP_PROVISION") != "false": + raise PilotError("SKIP_PROVISION must be exactly false for the pilot run") + + source_sha = resolve_sha(repo_root, args.source_sha) + run_at = args.run_at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", run_at): + raise PilotError("--run-at must use YYYY-MM-DDTHH:MM:SSZ") + + results = load_previous_results(args.previous_body, set(samples)) + overall_failed = False + validator = (repo_root / args.validator).resolve() + if not validator.is_file(): + raise PilotError(f"validator not found: {validator}") + + for sample in samples: + sample_result = results.setdefault(sample, {}) + if not isinstance(sample_result, dict): + raise PilotError(f"previous result for {sample} is malformed") + + l3_exit = run_validator(args.bash, validator, repo_root, sample, "3") + l3_status = status_for_exit_code(l3_exit) + sample_result["l3"] = result_record( + l3_status, run_at, args.evidence_url + ) + if l3_exit != 0: + overall_failed = True + + if not declarations[sample]: + sample_result.pop("l4", None) + elif l3_exit == 0: + l4_exit = run_validator(args.bash, validator, repo_root, sample, "4") + l4_status = status_for_exit_code(l4_exit) + sample_result["l4"] = result_record( + l4_status, run_at, args.evidence_url + ) + if l4_exit != 0: + overall_failed = True + + payload = { + "schema_version": 1, + "source_sha": source_sha, + "results": results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(f"{args.output}.tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + temporary.replace(args.output) + except (PilotError, OSError) as exc: + print(f"run-validation-health-pilot: {exc}", file=sys.stderr) + return 2 + return 1 if overall_failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test/test-render-validation-health.py b/.github/scripts/test/test-render-validation-health.py index 813ad7c2f..448210668 100644 --- a/.github/scripts/test/test-render-validation-health.py +++ b/.github/scripts/test/test-render-validation-health.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import re import subprocess import sys import tempfile @@ -86,6 +87,10 @@ def test_never_run_view_has_two_linked_rows(self) -> None: markdown, ) self.assertLess(markdown.index(C_SHARP), markdown.index(PYTHON)) + self.assertRegex( + markdown, + r"", + ) def test_status_mapping_dates_and_evidence(self) -> None: results = { @@ -130,6 +135,10 @@ def test_status_mapping_dates_and_evidence(self) -> None: self.assertIn("2026-08-06 01:02 UTC", markdown) self.assertIn("2026-08-06 02:03 UTC", markdown) self.assertIn("2026-08-06 03:04 UTC", markdown) + self.assertEqual( + len(re.findall(r"validation-health-state-v1:", markdown)), + 1, + ) def test_invalid_selected_status_fails(self) -> None: results = { diff --git a/.github/scripts/test/test-run-validation-health-pilot.py b/.github/scripts/test/test-run-validation-health-pilot.py new file mode 100644 index 000000000..51a23def1 --- /dev/null +++ b/.github/scripts/test/test-run-validation-health-pilot.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Fixture tests for run-validation-health-pilot.py.""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "run-validation-health-pilot.py" +RENDERER = Path(__file__).resolve().parents[1] / "render-validation-health.py" +WORKFLOW = Path(__file__).resolve().parents[2] / "workflows" / "validation-health-pilot.yml" +SHA = "0123456789abcdef0123456789abcdef01234567" +C_SHARP = "samples/csharp/quickstart/chat-with-agent" +PYTHON = "samples/python/quickstart/chat-with-agent" + + +def state_marker(payload: dict) -> str: + encoded = base64.urlsafe_b64encode( + json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() + ).decode() + return f"" + + +class PilotRunnerTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + for sample in (C_SHARP, PYTHON): + sample_dir = self.root / sample + sample_dir.mkdir(parents=True) + (sample_dir / "sample.yaml").write_text("name: fixture\n", encoding="utf-8") + self.config = self.root / "config.json" + self.config.write_text( + json.dumps( + { + "schema_version": 1, + "repository": "microsoft-foundry/foundry-samples", + "samples": [C_SHARP, PYTHON], + } + ), + encoding="utf-8", + ) + self.validator = self.root / "validator.sh" + self.yq = self.root / "yq.sh" + + def tearDown(self) -> None: + self.temp.cleanup() + + def write_tools(self, *, l3_csharp: int, l3_python: int, csharp_l4: bool, l4: int) -> None: + self.validator.write_text( + f"""#!/usr/bin/env bash +set -u +args="$*" +if [[ "$args" == *"--level 4"* ]]; then + exit {l4} +fi +if [[ "$args" == *"{C_SHARP}"* ]]; then + exit {l3_csharp} +fi +if [[ "$args" == *"{PYTHON}"* ]]; then + exit {l3_python} +fi +exit 2 +""", + encoding="utf-8", + newline="\n", + ) + self.yq = self.root / "fake_yq.py" + self.yq.write_text( + ( + "import sys\n" + f"print('true' if {C_SHARP!r} in sys.argv[-1] else 'false')\n" + if csharp_l4 + else "print('false')\n" + ), + encoding="utf-8", + newline="\n", + ) + + def run_pilot(self, previous: dict | str | None = None) -> subprocess.CompletedProcess[str]: + output = self.root / "results.json" + command = [ + sys.executable, + str(SCRIPT), + "--config", + str(self.config), + "--repo-root", + str(self.root), + "--validator", + str(self.validator), + "--yq", + sys.executable, + str(self.yq), + "--output", + str(output), + "--source-sha", + SHA, + "--run-at", + "2026-08-07T21:00:00Z", + "--evidence-url", + "https://github.com/example/actions/runs/1", + ] + if previous is not None: + body = self.root / "previous.md" + if isinstance(previous, str): + body.write_text(previous, encoding="utf-8") + else: + body.write_text(state_marker(previous), encoding="utf-8") + command.extend(["--previous-body", str(body)]) + environment = os.environ.copy() + environment["SKIP_PROVISION"] = "false" + completed = subprocess.run( + command, capture_output=True, text=True, env=environment + ) + completed.output_path = output # type: ignore[attr-defined] + return completed + + def test_results_include_completed_levels_and_clear_undeclared_l4(self) -> None: + self.write_tools(l3_csharp=0, l3_python=1, csharp_l4=True, l4=2) + previous = { + "schema_version": 1, + "source_sha": SHA, + "results": { + PYTHON: { + "l4": { + "status": "pass", + "run_at": "2026-08-01T00:00:00Z", + } + } + }, + } + completed = self.run_pilot(previous) + self.assertEqual(completed.returncode, 1, completed.stderr) + payload = json.loads(completed.output_path.read_text(encoding="utf-8")) + self.assertEqual(payload["results"][C_SHARP]["l3"]["status"], "pass") + self.assertEqual(payload["results"][C_SHARP]["l4"]["status"], "error") + self.assertEqual(payload["results"][PYTHON]["l3"]["status"], "failure") + self.assertNotIn("l4", payload["results"][PYTHON]) + + def test_previous_l4_is_preserved_when_declared_but_l3_blocks_run(self) -> None: + self.write_tools(l3_csharp=1, l3_python=0, csharp_l4=True, l4=0) + previous_l4 = { + "status": "pass", + "run_at": "2026-08-01T00:00:00Z", + } + previous = { + "schema_version": 1, + "source_sha": SHA, + "results": {C_SHARP: {"l4": previous_l4}}, + } + completed = self.run_pilot(previous) + self.assertEqual(completed.returncode, 1, completed.stderr) + payload = json.loads(completed.output_path.read_text(encoding="utf-8")) + self.assertEqual(payload["results"][C_SHARP]["l4"], previous_l4) + + def test_all_pass_with_no_l4_declarations_exits_zero(self) -> None: + self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) + completed = self.run_pilot() + self.assertEqual(completed.returncode, 0, completed.stderr) + payload = json.loads(completed.output_path.read_text(encoding="utf-8")) + self.assertNotIn("l4", payload["results"][C_SHARP]) + self.assertNotIn("l4", payload["results"][PYTHON]) + dashboard = self.root / "dashboard.md" + rendered = subprocess.run( + [ + sys.executable, + str(RENDERER), + "--config", + str(self.config), + "--repo-root", + str(self.root), + "--results", + str(completed.output_path), + "--output", + str(dashboard), + "--generated-at", + "2026-08-07T21:01:00Z", + ], + capture_output=True, + text=True, + ) + self.assertEqual(rendered.returncode, 0, rendered.stderr) + markdown = dashboard.read_text(encoding="utf-8") + self.assertEqual(markdown.count("✅ Pass"), 2) + self.assertEqual(markdown.count("⚪ Never run"), 2) + self.assertIn("validation-health-state-v1:", markdown) + + def test_malformed_hidden_state_fails_without_overwriting(self) -> None: + self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) + completed = self.run_pilot( + "" + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("hidden state marker is invalid", completed.stderr) + self.assertFalse(completed.output_path.exists()) + + def test_corrupted_hidden_state_prefix_fails_without_overwriting(self) -> None: + self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) + completed = self.run_pilot( + "" + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("hidden state marker is invalid", completed.stderr) + self.assertFalse(completed.output_path.exists()) + + def test_workflow_is_manual_main_only_and_preserves_verdict(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + self.assertIn("workflow_dispatch:", workflow) + self.assertNotIn("schedule:", workflow) + self.assertNotIn("pull_request:", workflow) + self.assertIn("ref: main", workflow) + self.assertIn("if: github.ref == 'refs/heads/main'", workflow) + self.assertIn("SKIP_PROVISION: 'false'", workflow) + self.assertIn("issues: write", workflow) + self.assertNotIn("id-token: write", workflow) + self.assertNotIn("azure/login", workflow) + self.assertNotIn("environment: L4-validation", workflow) + self.assertIn("Refuse undeclared credential boundary", workflow) + self.assertIn("gh issue edit", workflow) + self.assertIn("if: always()", workflow) + self.assertIn('exit "$VALIDATION_RC"', workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validation_health_state.py b/.github/scripts/validation_health_state.py new file mode 100644 index 000000000..0663fc2e9 --- /dev/null +++ b/.github/scripts/validation_health_state.py @@ -0,0 +1,43 @@ +"""Encode and decode the validation dashboard's hidden issue state.""" + +from __future__ import annotations + +import base64 +import binascii +import json +import re +from typing import Any + + +MARKER_PREFIX = "", re.DOTALL) +ENCODED_STATE_PATTERN = re.compile(r"^[A-Za-z0-9_=-]+$") + + +class StateError(ValueError): + """Raised when hidden dashboard state is malformed.""" + + +def encode_state(payload: dict[str, Any]) -> str: + serialized = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + encoded = base64.urlsafe_b64encode(serialized).decode("ascii") + return f"" + + +def extract_state(markdown: str) -> dict[str, Any] | None: + marker_count = markdown.count(MARKER_PREFIX) + if marker_count == 0: + return None + if marker_count != 1: + raise StateError("dashboard body must contain at most one hidden state marker") + match = MARKER_PATTERN.search(markdown) + if match is None or not ENCODED_STATE_PATTERN.fullmatch(match.group(1)): + raise StateError("dashboard hidden state marker is invalid") + try: + decoded = base64.urlsafe_b64decode(match.group(1).encode("ascii")) + payload = json.loads(decoded) + except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise StateError("dashboard hidden state marker is invalid") from exc + if not isinstance(payload, dict): + raise StateError("dashboard hidden state must be a JSON object") + return payload diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 017b20b8d..fc714cd8b 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -22,6 +22,7 @@ on: - '.github/validation-health-pilot.json' - '.github/scripts/**' - '.github/workflows/validate.yml' + - '.github/workflows/validation-health-pilot.yml' - '.github/workflows/scripts-selftest.yml' permissions: @@ -39,6 +40,8 @@ jobs: python-version: '3.12' - name: Validation health dashboard renderer exit gate run: python .github/scripts/test/test-render-validation-health.py + - name: Validation health pilot runner exit gate + run: python .github/scripts/test/test-run-validation-health-pilot.py # --- Required-gate structure: hermetic Bash assertions, no credentials ------------------------- workflow-structure-harness: diff --git a/.github/workflows/validation-health-pilot.yml b/.github/workflows/validation-health-pilot.yml new file mode 100644 index 000000000..c832a4e04 --- /dev/null +++ b/.github/workflows/validation-health-pilot.yml @@ -0,0 +1,134 @@ +name: Validation Health Pilot + +# Manual-only pilot. The production daily cadence remains out of scope until +# P5's transition manifest v1 is reviewed and the full target set is explicit. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: validation-health-pilot + cancel-in-progress: false + +env: + DASHBOARD_ISSUE: '892' + SKIP_PROVISION: 'false' + +jobs: + validate-and-publish: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + issues: write + steps: + - name: Checkout public main + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 1 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install yq + run: | + sudo wget -qO /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + yq --version + + - name: Detect selected L4 declarations + id: l4 + run: | + set -euo pipefail + has_l4="$(python .github/scripts/run-validation-health-pilot.py \ + --detect-l4-only \ + --repo-root "$GITHUB_WORKSPACE")" + echo "has_l4=$has_l4" >> "$GITHUB_OUTPUT" + echo "Selected sample declares L4: $has_l4" + + - name: Refuse undeclared credential boundary + if: steps.l4.outputs.has_l4 == 'true' + run: | + echo "::error::A selected sample now declares L4, but this pilot intentionally has no credentialed L4 boundary." + echo "::error::Add a reviewed main-only credential boundary before running declared L4." + exit 1 + + - name: Read current dashboard state + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + gh issue view "$DASHBOARD_ISSUE" \ + --repo "$GITHUB_REPOSITORY" \ + --json body \ + --jq '.body' > "$RUNNER_TEMP/previous-dashboard.md" + + - name: Validate pilot samples + id: validate + env: + EVIDENCE_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -uo pipefail + source_sha="$(git rev-parse HEAD)" + python .github/scripts/run-validation-health-pilot.py \ + --repo-root "$GITHUB_WORKSPACE" \ + --previous-body "$RUNNER_TEMP/previous-dashboard.md" \ + --source-sha "$source_sha" \ + --evidence-url "$EVIDENCE_URL" \ + --output "$RUNNER_TEMP/validation-health-results.json" + validation_rc=$? + echo "validation_rc=$validation_rc" >> "$GITHUB_OUTPUT" + echo "Pilot validation exit code: $validation_rc" + exit 0 + + - name: Render dashboard + run: | + set -euo pipefail + python .github/scripts/render-validation-health.py \ + --repo-root "$GITHUB_WORKSPACE" \ + --results "$RUNNER_TEMP/validation-health-results.json" \ + --output "$RUNNER_TEMP/validation-health.md" + + - name: Update pinned dashboard issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + gh issue edit "$DASHBOARD_ISSUE" \ + --repo "$GITHUB_REPOSITORY" \ + --body-file "$RUNNER_TEMP/validation-health.md" + + - name: Upload pilot result evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: validation-health-pilot-${{ github.run_id }} + path: | + ${{ runner.temp }}/validation-health-results.json + ${{ runner.temp }}/validation-health.md + if-no-files-found: warn + retention-days: 30 + + - name: Enforce pilot validation verdict + if: always() + env: + VALIDATION_RC: ${{ steps.validate.outputs.validation_rc }} + run: | + set -euo pipefail + if [[ ! "$VALIDATION_RC" =~ ^[0-9]+$ ]]; then + echo "::error::Pilot validation did not produce a result exit code." + exit 1 + fi + exit "$VALIDATION_RC" From 408aaf84c8aed104b7542ead209e6bbe9a404207 Mon Sep 17 00:00:00 2001 From: brandom Date: Fri, 7 Aug 2026 16:10:03 -0700 Subject: [PATCH 3/4] Fail closed on missing dashboard state Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 320063cb-69d2-49f6-8105-adc392a67da6 --- .../scripts/run-validation-health-pilot.py | 22 +++++++--- .../test/test-run-validation-health-pilot.py | 41 ++++++++++++++++++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/.github/scripts/run-validation-health-pilot.py b/.github/scripts/run-validation-health-pilot.py index 4d7bc3b3c..6ad24d265 100644 --- a/.github/scripts/run-validation-health-pilot.py +++ b/.github/scripts/run-validation-health-pilot.py @@ -57,9 +57,7 @@ def load_config(path: Path, repo_root: Path) -> list[str]: return samples -def load_previous_results(path: Path | None, samples: set[str]) -> dict[str, Any]: - if path is None: - return {} +def load_previous_results(path: Path, samples: set[str]) -> dict[str, Any]: try: body = path.read_text(encoding="utf-8") except FileNotFoundError as exc: @@ -69,7 +67,7 @@ def load_previous_results(path: Path | None, samples: set[str]) -> dict[str, Any except StateError as exc: raise PilotError(str(exc)) from exc if payload is None: - return {} + raise PilotError("previous dashboard body is missing hidden state marker") if payload.get("schema_version") != 1 or not isinstance(payload.get("results"), dict): raise PilotError("previous dashboard hidden state has an unsupported contract") return { @@ -177,6 +175,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--yq", nargs="+", default=["yq"]) parser.add_argument("--output", type=Path) parser.add_argument("--previous-body", type=Path) + parser.add_argument( + "--bootstrap", + action="store_true", + help="start without previous dashboard state", + ) parser.add_argument("--source-sha") parser.add_argument("--evidence-url") parser.add_argument("--run-at", help="fixed ISO-8601 UTC timestamp for tests") @@ -197,6 +200,10 @@ def main() -> int: return 0 if args.output is None: raise PilotError("--output is required unless --detect-l4-only is used") + if args.bootstrap == (args.previous_body is not None): + raise PilotError( + "exactly one of --previous-body or --bootstrap is required" + ) if os.environ.get("SKIP_PROVISION") != "false": raise PilotError("SKIP_PROVISION must be exactly false for the pilot run") @@ -205,7 +212,12 @@ def main() -> int: if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", run_at): raise PilotError("--run-at must use YYYY-MM-DDTHH:MM:SSZ") - results = load_previous_results(args.previous_body, set(samples)) + if args.bootstrap: + results = {} + else: + if args.previous_body is None: + raise PilotError("--previous-body is required outside bootstrap mode") + results = load_previous_results(args.previous_body, set(samples)) overall_failed = False validator = (repo_root / args.validator).resolve() if not validator.is_file(): diff --git a/.github/scripts/test/test-run-validation-health-pilot.py b/.github/scripts/test/test-run-validation-health-pilot.py index 51a23def1..73b931a9a 100644 --- a/.github/scripts/test/test-run-validation-health-pilot.py +++ b/.github/scripts/test/test-run-validation-health-pilot.py @@ -84,7 +84,12 @@ def write_tools(self, *, l3_csharp: int, l3_python: int, csharp_l4: bool, l4: in newline="\n", ) - def run_pilot(self, previous: dict | str | None = None) -> subprocess.CompletedProcess[str]: + def run_pilot( + self, + previous: dict | str | None = None, + *, + include_bootstrap: bool | None = None, + ) -> subprocess.CompletedProcess[str]: output = self.root / "results.json" command = [ sys.executable, @@ -107,6 +112,10 @@ def run_pilot(self, previous: dict | str | None = None) -> subprocess.CompletedP "--evidence-url", "https://github.com/example/actions/runs/1", ] + if include_bootstrap is None: + include_bootstrap = previous is None + if include_bootstrap: + command.append("--bootstrap") if previous is not None: body = self.root / "previous.md" if isinstance(previous, str): @@ -201,6 +210,36 @@ def test_malformed_hidden_state_fails_without_overwriting(self) -> None: self.assertIn("hidden state marker is invalid", completed.stderr) self.assertFalse(completed.output_path.exists()) + def test_missing_hidden_state_fails_without_overwriting(self) -> None: + self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) + completed = self.run_pilot("# Validation health dashboard\n") + self.assertEqual(completed.returncode, 2) + self.assertIn("missing hidden state marker", completed.stderr) + self.assertFalse(completed.output_path.exists()) + + def test_state_mode_is_required(self) -> None: + self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) + completed = self.run_pilot(include_bootstrap=False) + self.assertEqual(completed.returncode, 2) + self.assertIn( + "exactly one of --previous-body or --bootstrap is required", + completed.stderr, + ) + self.assertFalse(completed.output_path.exists()) + + def test_bootstrap_and_previous_body_are_mutually_exclusive(self) -> None: + self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) + completed = self.run_pilot( + {"schema_version": 1, "results": {}}, + include_bootstrap=True, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn( + "exactly one of --previous-body or --bootstrap is required", + completed.stderr, + ) + self.assertFalse(completed.output_path.exists()) + def test_corrupted_hidden_state_prefix_fails_without_overwriting(self) -> None: self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) completed = self.run_pilot( From a79a43870c1844652a2d22312f54de4d4d0f3936 Mon Sep 17 00:00:00 2001 From: brandom Date: Fri, 7 Aug 2026 16:17:24 -0700 Subject: [PATCH 4/4] Keep dashboard pilot presentation-only Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 320063cb-69d2-49f6-8105-adc392a67da6 --- .github/scripts/render-validation-health.py | 26 -- .../scripts/run-validation-health-pilot.py | 270 ----------------- .../test/test-render-validation-health.py | 9 - .../test/test-run-validation-health-pilot.py | 271 ------------------ .github/scripts/validation_health_state.py | 43 --- .github/workflows/scripts-selftest.yml | 3 - .github/workflows/validation-health-pilot.yml | 134 --------- 7 files changed, 756 deletions(-) delete mode 100644 .github/scripts/run-validation-health-pilot.py delete mode 100644 .github/scripts/test/test-run-validation-health-pilot.py delete mode 100644 .github/scripts/validation_health_state.py delete mode 100644 .github/workflows/validation-health-pilot.yml diff --git a/.github/scripts/render-validation-health.py b/.github/scripts/render-validation-health.py index 621709735..eea941ce9 100644 --- a/.github/scripts/render-validation-health.py +++ b/.github/scripts/render-validation-health.py @@ -13,8 +13,6 @@ from typing import Any from urllib.parse import quote, urlparse -from validation_health_state import encode_state - STATUS_DISPLAY = { "pass": "✅ Pass", @@ -239,30 +237,6 @@ def render_markdown( "> This pilot uses public validation results only. A missing result means " "“never run,” not “pass.”", "", - encode_state( - { - "schema_version": 1, - "source_sha": source_sha, - "results": { - sample: { - level: { - "status": level_result["status"], - "run_at": level_result["run_at"].strftime( - "%Y-%m-%dT%H:%M:%SZ" - ), - **( - {"evidence_url": level_result["evidence_url"]} - if level_result["evidence_url"] - else {} - ), - } - for level, level_result in sample_result.items() - } - for sample, sample_result in results.items() - }, - } - ), - "", ] ) return "\n".join(lines) diff --git a/.github/scripts/run-validation-health-pilot.py b/.github/scripts/run-validation-health-pilot.py deleted file mode 100644 index 6ad24d265..000000000 --- a/.github/scripts/run-validation-health-pilot.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 -"""Run the configured dashboard pilot samples and emit normalized results.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -from copy import deepcopy -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from validation_health_state import StateError, extract_state - - -LANGUAGES = { - "csharp": "csharp", - "python": "python", - "typescript": "typescript", - "javascript": "typescript", - "java": "java", - "go": "go", -} -SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") - - -class PilotError(ValueError): - """Raised when the pilot cannot produce a trustworthy result document.""" - - -def load_config(path: Path, repo_root: Path) -> list[str]: - try: - config = json.loads(path.read_text(encoding="utf-8")) - except FileNotFoundError as exc: - raise PilotError(f"pilot config not found: {path}") from exc - except json.JSONDecodeError as exc: - raise PilotError(f"pilot config is not valid JSON: {exc}") from exc - if not isinstance(config, dict) or config.get("schema_version") != 1: - raise PilotError("pilot config must be a schema_version 1 JSON object") - samples = config.get("samples") - if not isinstance(samples, list) or not samples: - raise PilotError("pilot config samples must be a non-empty list") - if any(not isinstance(sample, str) for sample in samples): - raise PilotError("pilot config sample paths must be strings") - if samples != sorted(set(samples)): - raise PilotError("pilot config sample paths must be sorted and unique") - for sample in samples: - if not sample.startswith("samples/") or not (repo_root / sample).is_dir(): - raise PilotError(f"configured sample directory does not exist: {sample}") - language_dir = sample.split("/", 2)[1] - if language_dir not in LANGUAGES: - raise PilotError(f"configured sample language is unsupported: {sample}") - return samples - - -def load_previous_results(path: Path, samples: set[str]) -> dict[str, Any]: - try: - body = path.read_text(encoding="utf-8") - except FileNotFoundError as exc: - raise PilotError(f"previous dashboard body not found: {path}") from exc - try: - payload = extract_state(body) - except StateError as exc: - raise PilotError(str(exc)) from exc - if payload is None: - raise PilotError("previous dashboard body is missing hidden state marker") - if payload.get("schema_version") != 1 or not isinstance(payload.get("results"), dict): - raise PilotError("previous dashboard hidden state has an unsupported contract") - return { - sample: deepcopy(payload["results"][sample]) - for sample in samples - if sample in payload["results"] - } - - -def resolve_sha(repo_root: Path, argument: str | None) -> str: - if argument is not None: - if not SHA_PATTERN.fullmatch(argument): - raise PilotError("--source-sha must be a full lowercase Git SHA") - return argument - try: - sha = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=repo_root, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - except (OSError, subprocess.CalledProcessError) as exc: - raise PilotError("could not resolve source SHA") from exc - if not SHA_PATTERN.fullmatch(sha): - raise PilotError("resolved source SHA is invalid") - return sha - - -def probe_l4(yq: list[str], repo_root: Path, sample: str) -> bool: - yaml_path = repo_root / sample / "sample.yaml" - completed = subprocess.run( - [*yq, "eval", 'has("l4")', yaml_path.as_posix()], - cwd=repo_root, - capture_output=True, - text=True, - ) - if completed.returncode != 0: - sys.stdout.write(completed.stdout) - sys.stderr.write(completed.stderr) - raise PilotError(f"could not inspect L4 declaration: {sample}") - value = completed.stdout.strip() - if value not in ("true", "false"): - raise PilotError(f"unexpected L4 declaration probe for {sample}: {value!r}") - return value == "true" - - -def run_validator( - bash: str, - validator: Path, - repo_root: Path, - sample: str, - level: str, -) -> int: - try: - validator_argument = validator.relative_to(repo_root).as_posix() - except ValueError: - validator_argument = validator.as_posix() - command = [bash, validator_argument, "--level", level, "--sample-dir", sample] - if level == "3": - language_dir = sample.split("/", 2)[1] - command.extend(["--language", LANGUAGES[language_dir]]) - completed = subprocess.run( - command, - cwd=repo_root, - capture_output=True, - text=True, - env=os.environ.copy(), - ) - print(f"===== {sample} L{level} (exit={completed.returncode}) =====") - sys.stdout.write(completed.stdout) - sys.stderr.write(completed.stderr) - return completed.returncode - - -def status_for_exit_code(exit_code: int) -> str: - if exit_code == 0: - return "pass" - if exit_code == 1: - return "failure" - return "error" - - -def result_record(status: str, run_at: str, evidence_url: str | None) -> dict[str, str]: - record = {"status": status, "run_at": run_at} - if evidence_url: - record["evidence_url"] = evidence_url - return record - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--config", - type=Path, - default=Path(".github/validation-health-pilot.json"), - ) - parser.add_argument("--repo-root", type=Path, default=Path(".")) - parser.add_argument( - "--validator", - type=Path, - default=Path(".github/scripts/validate-sample.sh"), - ) - parser.add_argument("--bash", default="bash") - parser.add_argument("--yq", nargs="+", default=["yq"]) - parser.add_argument("--output", type=Path) - parser.add_argument("--previous-body", type=Path) - parser.add_argument( - "--bootstrap", - action="store_true", - help="start without previous dashboard state", - ) - parser.add_argument("--source-sha") - parser.add_argument("--evidence-url") - parser.add_argument("--run-at", help="fixed ISO-8601 UTC timestamp for tests") - parser.add_argument("--detect-l4-only", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - repo_root = args.repo_root.resolve() - try: - samples = load_config(args.config, repo_root) - declarations = { - sample: probe_l4(args.yq, repo_root, sample) for sample in samples - } - if args.detect_l4_only: - print("true" if any(declarations.values()) else "false") - return 0 - if args.output is None: - raise PilotError("--output is required unless --detect-l4-only is used") - if args.bootstrap == (args.previous_body is not None): - raise PilotError( - "exactly one of --previous-body or --bootstrap is required" - ) - if os.environ.get("SKIP_PROVISION") != "false": - raise PilotError("SKIP_PROVISION must be exactly false for the pilot run") - - source_sha = resolve_sha(repo_root, args.source_sha) - run_at = args.run_at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", run_at): - raise PilotError("--run-at must use YYYY-MM-DDTHH:MM:SSZ") - - if args.bootstrap: - results = {} - else: - if args.previous_body is None: - raise PilotError("--previous-body is required outside bootstrap mode") - results = load_previous_results(args.previous_body, set(samples)) - overall_failed = False - validator = (repo_root / args.validator).resolve() - if not validator.is_file(): - raise PilotError(f"validator not found: {validator}") - - for sample in samples: - sample_result = results.setdefault(sample, {}) - if not isinstance(sample_result, dict): - raise PilotError(f"previous result for {sample} is malformed") - - l3_exit = run_validator(args.bash, validator, repo_root, sample, "3") - l3_status = status_for_exit_code(l3_exit) - sample_result["l3"] = result_record( - l3_status, run_at, args.evidence_url - ) - if l3_exit != 0: - overall_failed = True - - if not declarations[sample]: - sample_result.pop("l4", None) - elif l3_exit == 0: - l4_exit = run_validator(args.bash, validator, repo_root, sample, "4") - l4_status = status_for_exit_code(l4_exit) - sample_result["l4"] = result_record( - l4_status, run_at, args.evidence_url - ) - if l4_exit != 0: - overall_failed = True - - payload = { - "schema_version": 1, - "source_sha": source_sha, - "results": results, - } - args.output.parent.mkdir(parents=True, exist_ok=True) - temporary = Path(f"{args.output}.tmp") - temporary.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - newline="\n", - ) - temporary.replace(args.output) - except (PilotError, OSError) as exc: - print(f"run-validation-health-pilot: {exc}", file=sys.stderr) - return 2 - return 1 if overall_failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/scripts/test/test-render-validation-health.py b/.github/scripts/test/test-render-validation-health.py index 448210668..813ad7c2f 100644 --- a/.github/scripts/test/test-render-validation-health.py +++ b/.github/scripts/test/test-render-validation-health.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import re import subprocess import sys import tempfile @@ -87,10 +86,6 @@ def test_never_run_view_has_two_linked_rows(self) -> None: markdown, ) self.assertLess(markdown.index(C_SHARP), markdown.index(PYTHON)) - self.assertRegex( - markdown, - r"", - ) def test_status_mapping_dates_and_evidence(self) -> None: results = { @@ -135,10 +130,6 @@ def test_status_mapping_dates_and_evidence(self) -> None: self.assertIn("2026-08-06 01:02 UTC", markdown) self.assertIn("2026-08-06 02:03 UTC", markdown) self.assertIn("2026-08-06 03:04 UTC", markdown) - self.assertEqual( - len(re.findall(r"validation-health-state-v1:", markdown)), - 1, - ) def test_invalid_selected_status_fails(self) -> None: results = { diff --git a/.github/scripts/test/test-run-validation-health-pilot.py b/.github/scripts/test/test-run-validation-health-pilot.py deleted file mode 100644 index 73b931a9a..000000000 --- a/.github/scripts/test/test-run-validation-health-pilot.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 -"""Fixture tests for run-validation-health-pilot.py.""" - -from __future__ import annotations - -import base64 -import json -import os -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - - -SCRIPT = Path(__file__).resolve().parents[1] / "run-validation-health-pilot.py" -RENDERER = Path(__file__).resolve().parents[1] / "render-validation-health.py" -WORKFLOW = Path(__file__).resolve().parents[2] / "workflows" / "validation-health-pilot.yml" -SHA = "0123456789abcdef0123456789abcdef01234567" -C_SHARP = "samples/csharp/quickstart/chat-with-agent" -PYTHON = "samples/python/quickstart/chat-with-agent" - - -def state_marker(payload: dict) -> str: - encoded = base64.urlsafe_b64encode( - json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() - ).decode() - return f"" - - -class PilotRunnerTests(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory() - self.root = Path(self.temp.name) - for sample in (C_SHARP, PYTHON): - sample_dir = self.root / sample - sample_dir.mkdir(parents=True) - (sample_dir / "sample.yaml").write_text("name: fixture\n", encoding="utf-8") - self.config = self.root / "config.json" - self.config.write_text( - json.dumps( - { - "schema_version": 1, - "repository": "microsoft-foundry/foundry-samples", - "samples": [C_SHARP, PYTHON], - } - ), - encoding="utf-8", - ) - self.validator = self.root / "validator.sh" - self.yq = self.root / "yq.sh" - - def tearDown(self) -> None: - self.temp.cleanup() - - def write_tools(self, *, l3_csharp: int, l3_python: int, csharp_l4: bool, l4: int) -> None: - self.validator.write_text( - f"""#!/usr/bin/env bash -set -u -args="$*" -if [[ "$args" == *"--level 4"* ]]; then - exit {l4} -fi -if [[ "$args" == *"{C_SHARP}"* ]]; then - exit {l3_csharp} -fi -if [[ "$args" == *"{PYTHON}"* ]]; then - exit {l3_python} -fi -exit 2 -""", - encoding="utf-8", - newline="\n", - ) - self.yq = self.root / "fake_yq.py" - self.yq.write_text( - ( - "import sys\n" - f"print('true' if {C_SHARP!r} in sys.argv[-1] else 'false')\n" - if csharp_l4 - else "print('false')\n" - ), - encoding="utf-8", - newline="\n", - ) - - def run_pilot( - self, - previous: dict | str | None = None, - *, - include_bootstrap: bool | None = None, - ) -> subprocess.CompletedProcess[str]: - output = self.root / "results.json" - command = [ - sys.executable, - str(SCRIPT), - "--config", - str(self.config), - "--repo-root", - str(self.root), - "--validator", - str(self.validator), - "--yq", - sys.executable, - str(self.yq), - "--output", - str(output), - "--source-sha", - SHA, - "--run-at", - "2026-08-07T21:00:00Z", - "--evidence-url", - "https://github.com/example/actions/runs/1", - ] - if include_bootstrap is None: - include_bootstrap = previous is None - if include_bootstrap: - command.append("--bootstrap") - if previous is not None: - body = self.root / "previous.md" - if isinstance(previous, str): - body.write_text(previous, encoding="utf-8") - else: - body.write_text(state_marker(previous), encoding="utf-8") - command.extend(["--previous-body", str(body)]) - environment = os.environ.copy() - environment["SKIP_PROVISION"] = "false" - completed = subprocess.run( - command, capture_output=True, text=True, env=environment - ) - completed.output_path = output # type: ignore[attr-defined] - return completed - - def test_results_include_completed_levels_and_clear_undeclared_l4(self) -> None: - self.write_tools(l3_csharp=0, l3_python=1, csharp_l4=True, l4=2) - previous = { - "schema_version": 1, - "source_sha": SHA, - "results": { - PYTHON: { - "l4": { - "status": "pass", - "run_at": "2026-08-01T00:00:00Z", - } - } - }, - } - completed = self.run_pilot(previous) - self.assertEqual(completed.returncode, 1, completed.stderr) - payload = json.loads(completed.output_path.read_text(encoding="utf-8")) - self.assertEqual(payload["results"][C_SHARP]["l3"]["status"], "pass") - self.assertEqual(payload["results"][C_SHARP]["l4"]["status"], "error") - self.assertEqual(payload["results"][PYTHON]["l3"]["status"], "failure") - self.assertNotIn("l4", payload["results"][PYTHON]) - - def test_previous_l4_is_preserved_when_declared_but_l3_blocks_run(self) -> None: - self.write_tools(l3_csharp=1, l3_python=0, csharp_l4=True, l4=0) - previous_l4 = { - "status": "pass", - "run_at": "2026-08-01T00:00:00Z", - } - previous = { - "schema_version": 1, - "source_sha": SHA, - "results": {C_SHARP: {"l4": previous_l4}}, - } - completed = self.run_pilot(previous) - self.assertEqual(completed.returncode, 1, completed.stderr) - payload = json.loads(completed.output_path.read_text(encoding="utf-8")) - self.assertEqual(payload["results"][C_SHARP]["l4"], previous_l4) - - def test_all_pass_with_no_l4_declarations_exits_zero(self) -> None: - self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) - completed = self.run_pilot() - self.assertEqual(completed.returncode, 0, completed.stderr) - payload = json.loads(completed.output_path.read_text(encoding="utf-8")) - self.assertNotIn("l4", payload["results"][C_SHARP]) - self.assertNotIn("l4", payload["results"][PYTHON]) - dashboard = self.root / "dashboard.md" - rendered = subprocess.run( - [ - sys.executable, - str(RENDERER), - "--config", - str(self.config), - "--repo-root", - str(self.root), - "--results", - str(completed.output_path), - "--output", - str(dashboard), - "--generated-at", - "2026-08-07T21:01:00Z", - ], - capture_output=True, - text=True, - ) - self.assertEqual(rendered.returncode, 0, rendered.stderr) - markdown = dashboard.read_text(encoding="utf-8") - self.assertEqual(markdown.count("✅ Pass"), 2) - self.assertEqual(markdown.count("⚪ Never run"), 2) - self.assertIn("validation-health-state-v1:", markdown) - - def test_malformed_hidden_state_fails_without_overwriting(self) -> None: - self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) - completed = self.run_pilot( - "" - ) - self.assertEqual(completed.returncode, 2) - self.assertIn("hidden state marker is invalid", completed.stderr) - self.assertFalse(completed.output_path.exists()) - - def test_missing_hidden_state_fails_without_overwriting(self) -> None: - self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) - completed = self.run_pilot("# Validation health dashboard\n") - self.assertEqual(completed.returncode, 2) - self.assertIn("missing hidden state marker", completed.stderr) - self.assertFalse(completed.output_path.exists()) - - def test_state_mode_is_required(self) -> None: - self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) - completed = self.run_pilot(include_bootstrap=False) - self.assertEqual(completed.returncode, 2) - self.assertIn( - "exactly one of --previous-body or --bootstrap is required", - completed.stderr, - ) - self.assertFalse(completed.output_path.exists()) - - def test_bootstrap_and_previous_body_are_mutually_exclusive(self) -> None: - self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) - completed = self.run_pilot( - {"schema_version": 1, "results": {}}, - include_bootstrap=True, - ) - self.assertEqual(completed.returncode, 2) - self.assertIn( - "exactly one of --previous-body or --bootstrap is required", - completed.stderr, - ) - self.assertFalse(completed.output_path.exists()) - - def test_corrupted_hidden_state_prefix_fails_without_overwriting(self) -> None: - self.write_tools(l3_csharp=0, l3_python=0, csharp_l4=False, l4=0) - completed = self.run_pilot( - "" - ) - self.assertEqual(completed.returncode, 2) - self.assertIn("hidden state marker is invalid", completed.stderr) - self.assertFalse(completed.output_path.exists()) - - def test_workflow_is_manual_main_only_and_preserves_verdict(self) -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - self.assertIn("workflow_dispatch:", workflow) - self.assertNotIn("schedule:", workflow) - self.assertNotIn("pull_request:", workflow) - self.assertIn("ref: main", workflow) - self.assertIn("if: github.ref == 'refs/heads/main'", workflow) - self.assertIn("SKIP_PROVISION: 'false'", workflow) - self.assertIn("issues: write", workflow) - self.assertNotIn("id-token: write", workflow) - self.assertNotIn("azure/login", workflow) - self.assertNotIn("environment: L4-validation", workflow) - self.assertIn("Refuse undeclared credential boundary", workflow) - self.assertIn("gh issue edit", workflow) - self.assertIn("if: always()", workflow) - self.assertIn('exit "$VALIDATION_RC"', workflow) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/scripts/validation_health_state.py b/.github/scripts/validation_health_state.py deleted file mode 100644 index 0663fc2e9..000000000 --- a/.github/scripts/validation_health_state.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Encode and decode the validation dashboard's hidden issue state.""" - -from __future__ import annotations - -import base64 -import binascii -import json -import re -from typing import Any - - -MARKER_PREFIX = "", re.DOTALL) -ENCODED_STATE_PATTERN = re.compile(r"^[A-Za-z0-9_=-]+$") - - -class StateError(ValueError): - """Raised when hidden dashboard state is malformed.""" - - -def encode_state(payload: dict[str, Any]) -> str: - serialized = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") - encoded = base64.urlsafe_b64encode(serialized).decode("ascii") - return f"" - - -def extract_state(markdown: str) -> dict[str, Any] | None: - marker_count = markdown.count(MARKER_PREFIX) - if marker_count == 0: - return None - if marker_count != 1: - raise StateError("dashboard body must contain at most one hidden state marker") - match = MARKER_PATTERN.search(markdown) - if match is None or not ENCODED_STATE_PATTERN.fullmatch(match.group(1)): - raise StateError("dashboard hidden state marker is invalid") - try: - decoded = base64.urlsafe_b64decode(match.group(1).encode("ascii")) - payload = json.loads(decoded) - except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise StateError("dashboard hidden state marker is invalid") from exc - if not isinstance(payload, dict): - raise StateError("dashboard hidden state must be a JSON object") - return payload diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index fc714cd8b..017b20b8d 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -22,7 +22,6 @@ on: - '.github/validation-health-pilot.json' - '.github/scripts/**' - '.github/workflows/validate.yml' - - '.github/workflows/validation-health-pilot.yml' - '.github/workflows/scripts-selftest.yml' permissions: @@ -40,8 +39,6 @@ jobs: python-version: '3.12' - name: Validation health dashboard renderer exit gate run: python .github/scripts/test/test-render-validation-health.py - - name: Validation health pilot runner exit gate - run: python .github/scripts/test/test-run-validation-health-pilot.py # --- Required-gate structure: hermetic Bash assertions, no credentials ------------------------- workflow-structure-harness: diff --git a/.github/workflows/validation-health-pilot.yml b/.github/workflows/validation-health-pilot.yml deleted file mode 100644 index c832a4e04..000000000 --- a/.github/workflows/validation-health-pilot.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: Validation Health Pilot - -# Manual-only pilot. The production daily cadence remains out of scope until -# P5's transition manifest v1 is reviewed and the full target set is explicit. -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: validation-health-pilot - cancel-in-progress: false - -env: - DASHBOARD_ISSUE: '892' - SKIP_PROVISION: 'false' - -jobs: - validate-and-publish: - if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - issues: write - steps: - - name: Checkout public main - uses: actions/checkout@v4 - with: - ref: main - fetch-depth: 1 - - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install yq - run: | - sudo wget -qO /usr/local/bin/yq \ - https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 - sudo chmod +x /usr/local/bin/yq - yq --version - - - name: Detect selected L4 declarations - id: l4 - run: | - set -euo pipefail - has_l4="$(python .github/scripts/run-validation-health-pilot.py \ - --detect-l4-only \ - --repo-root "$GITHUB_WORKSPACE")" - echo "has_l4=$has_l4" >> "$GITHUB_OUTPUT" - echo "Selected sample declares L4: $has_l4" - - - name: Refuse undeclared credential boundary - if: steps.l4.outputs.has_l4 == 'true' - run: | - echo "::error::A selected sample now declares L4, but this pilot intentionally has no credentialed L4 boundary." - echo "::error::Add a reviewed main-only credential boundary before running declared L4." - exit 1 - - - name: Read current dashboard state - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - gh issue view "$DASHBOARD_ISSUE" \ - --repo "$GITHUB_REPOSITORY" \ - --json body \ - --jq '.body' > "$RUNNER_TEMP/previous-dashboard.md" - - - name: Validate pilot samples - id: validate - env: - EVIDENCE_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - set -uo pipefail - source_sha="$(git rev-parse HEAD)" - python .github/scripts/run-validation-health-pilot.py \ - --repo-root "$GITHUB_WORKSPACE" \ - --previous-body "$RUNNER_TEMP/previous-dashboard.md" \ - --source-sha "$source_sha" \ - --evidence-url "$EVIDENCE_URL" \ - --output "$RUNNER_TEMP/validation-health-results.json" - validation_rc=$? - echo "validation_rc=$validation_rc" >> "$GITHUB_OUTPUT" - echo "Pilot validation exit code: $validation_rc" - exit 0 - - - name: Render dashboard - run: | - set -euo pipefail - python .github/scripts/render-validation-health.py \ - --repo-root "$GITHUB_WORKSPACE" \ - --results "$RUNNER_TEMP/validation-health-results.json" \ - --output "$RUNNER_TEMP/validation-health.md" - - - name: Update pinned dashboard issue - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - gh issue edit "$DASHBOARD_ISSUE" \ - --repo "$GITHUB_REPOSITORY" \ - --body-file "$RUNNER_TEMP/validation-health.md" - - - name: Upload pilot result evidence - if: always() - uses: actions/upload-artifact@v4 - with: - name: validation-health-pilot-${{ github.run_id }} - path: | - ${{ runner.temp }}/validation-health-results.json - ${{ runner.temp }}/validation-health.md - if-no-files-found: warn - retention-days: 30 - - - name: Enforce pilot validation verdict - if: always() - env: - VALIDATION_RC: ${{ steps.validate.outputs.validation_rc }} - run: | - set -euo pipefail - if [[ ! "$VALIDATION_RC" =~ ^[0-9]+$ ]]; then - echo "::error::Pilot validation did not produce a result exit code." - exit 1 - fi - exit "$VALIDATION_RC"