From 6ababec10b8f9399f969af1e58835ebadbcacdb3 Mon Sep 17 00:00:00 2001 From: mrjf Date: Wed, 27 May 2026 20:38:47 -0700 Subject: [PATCH 1/4] ci: restore basic and migration checks --- .github/workflows/ci.yml | 84 ++++++++++++ .github/workflows/migration-ci.yml | 138 +++++++++++++++++++ scripts/ci/migration_cli_benchmark.py | 187 ++++++++++++++++++++++++++ 3 files changed, 409 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/migration-ci.yml create mode 100644 scripts/ci/migration_cli_benchmark.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..13d14263 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,84 @@ +name: CI + +on: + pull_request: + branches: [main] + merge_group: + branches: [main] + types: [checks_requested] + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.12" + +jobs: + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Ruff lint + run: uv run --extra dev ruff check src/ tests/ + + - name: Ruff format check + run: uv run --extra dev ruff format --check src/ tests/ + + python-tests: + name: Python Unit Tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --extra dev + + - name: Run unit tests + run: uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal + + go-tests: + name: Go Tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - uses: actions/setup-go@v5 + if: hashFiles('go.mod') != '' + with: + go-version-file: go.mod + cache: true + + - name: Install Python reference CLI + if: hashFiles('go.mod') != '' + run: | + uv sync --extra dev + test -x "$GITHUB_WORKSPACE/.venv/bin/apm" + echo "APM_PYTHON_BIN=$GITHUB_WORKSPACE/.venv/bin/apm" >> "$GITHUB_ENV" + + - name: Run Go tests + if: hashFiles('go.mod') != '' + run: go test ./... + diff --git a/.github/workflows/migration-ci.yml b/.github/workflows/migration-ci.yml new file mode 100644 index 00000000..1f83c940 --- /dev/null +++ b/.github/workflows/migration-ci.yml @@ -0,0 +1,138 @@ +name: Migration Parity and Benchmarks + +on: + pull_request: + branches: [main] + paths: + - ".crane/**" + - ".github/workflows/migration-ci.yml" + - "cmd/**" + - "internal/**" + - "pkg/**" + - "go.mod" + - "go.sum" + - "pyproject.toml" + - "scripts/ci/**" + - "src/**" + - "tests/benchmarks/**" + - "tests/unit/test_crane_score.py" + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.12" + +jobs: + parity: + name: Python-vs-Go Parity Gate + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install Python reference CLI + run: | + uv sync --extra dev + test -x "$GITHUB_WORKSPACE/.venv/bin/apm" + echo "APM_PYTHON_BIN=$GITHUB_WORKSPACE/.venv/bin/apm" >> "$GITHUB_ENV" + + - name: Run Go parity tests + run: go test ./... + + - name: Compute migration score + run: | + go test -json ./... | tee "$RUNNER_TEMP/go-test-events.json" | go run .crane/scripts/score.go | tee "$RUNNER_TEMP/migration-score.json" + python - "$RUNNER_TEMP/migration-score.json" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as fh: + score = json.load(fh) + + print(json.dumps(score, indent=2, sort_keys=True)) + if score.get("migration_score") != 1.0: + raise SystemExit("migration_score must be 1.0 for completion parity") + PY + + - name: Upload parity evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: migration-parity-evidence + path: | + ${{ runner.temp }}/go-test-events.json + ${{ runner.temp }}/migration-score.json + if-no-files-found: ignore + retention-days: 14 + + benchmarks: + name: Migration Benchmarks + needs: [parity] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install dependencies + run: uv sync --extra dev + + - name: Build Go CLI + run: go build -o "$RUNNER_TEMP/apm-go" ./cmd/apm + + - name: Run Python performance guards + run: | + uv run pytest tests/benchmarks/test_scaling_guards.py -v + uv run pytest tests/benchmarks -v --tb=short -m benchmark + + - name: Run Python-vs-Go CLI benchmark + run: | + python scripts/ci/migration_cli_benchmark.py \ + --python-bin "$GITHUB_WORKSPACE/.venv/bin/apm" \ + --go-bin "$RUNNER_TEMP/apm-go" \ + --json-out "$RUNNER_TEMP/migration-cli-benchmark.json" \ + --markdown-out "$RUNNER_TEMP/migration-cli-benchmark.md" \ + --max-ratio 5.0 + + - name: Add benchmark summary + if: always() + run: | + if [ -f "$RUNNER_TEMP/migration-cli-benchmark.md" ]; then + cat "$RUNNER_TEMP/migration-cli-benchmark.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload benchmark evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: migration-benchmark-evidence + path: | + ${{ runner.temp }}/migration-cli-benchmark.json + ${{ runner.temp }}/migration-cli-benchmark.md + if-no-files-found: ignore + retention-days: 14 diff --git a/scripts/ci/migration_cli_benchmark.py b/scripts/ci/migration_cli_benchmark.py new file mode 100644 index 00000000..b8cf7cf3 --- /dev/null +++ b/scripts/ci/migration_cli_benchmark.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Compare Python and Go CLI latency for migration smoke commands.""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import subprocess +import tempfile +import time +from pathlib import Path + +COMMANDS: list[tuple[str, list[str], bool]] = [ + ("help", ["--help"], False), + ("version", ["--version"], False), + ("compile-help", ["compile", "--help"], False), + ("install-help", ["install", "--help"], False), + ("pack-help", ["pack", "--help"], False), + ("audit-help", ["audit", "--help"], False), + ("init-yes", ["init", "--yes"], True), +] + + +def _run_once(binary: str, args: list[str], cwd: Path, env: dict[str, str]) -> dict[str, object]: + start = time.perf_counter() + proc = subprocess.run( # noqa: S603 -- benchmark intentionally executes supplied CLIs. + [binary, *args], + cwd=cwd, + env=env, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + elapsed = time.perf_counter() - start + return { + "elapsed_seconds": elapsed, + "returncode": proc.returncode, + "stdout_bytes": len(proc.stdout.encode("utf-8")), + "stderr_bytes": len(proc.stderr.encode("utf-8")), + } + + +def _workspace(base: Path, name: str, run_index: int) -> Path: + workdir = base / name / str(run_index) + workdir.mkdir(parents=True, exist_ok=True) + (workdir / "README.md").write_text("# Benchmark fixture\n", encoding="utf-8") + return workdir + + +def _measure( + *, + binary: str, + args: list[str], + mutates_workspace: bool, + repeats: int, + base: Path, + label: str, + env: dict[str, str], +) -> dict[str, object]: + samples: list[dict[str, object]] = [] + for index in range(repeats): + cwd = _workspace(base, label, index) if mutates_workspace else base + samples.append(_run_once(binary, args, cwd, env)) + + elapsed = [float(sample["elapsed_seconds"]) for sample in samples] + return { + "median_seconds": statistics.median(elapsed), + "min_seconds": min(elapsed), + "max_seconds": max(elapsed), + "returncodes": sorted({int(sample["returncode"]) for sample in samples}), + "samples": samples, + } + + +def _markdown(results: list[dict[str, object]], max_ratio: float) -> str: + lines = [ + "## Migration CLI Benchmark", + "", + f"Max allowed Go/Python median ratio: `{max_ratio:.2f}`", + "", + "| Command | Python median | Go median | Go/Python | Return codes |", + "|---|---:|---:|---:|---|", + ] + for row in results: + lines.append( + "| {command} | {python:.4f}s | {go:.4f}s | {ratio:.2f}x | {codes} |".format( + command=row["command"], + python=row["python_median_seconds"], + go=row["go_median_seconds"], + ratio=row["ratio"], + codes=row["returncodes"], + ) + ) + lines.append("") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--python-bin", required=True) + parser.add_argument("--go-bin", required=True) + parser.add_argument("--json-out", required=True) + parser.add_argument("--markdown-out", required=True) + parser.add_argument("--max-ratio", type=float, default=5.0) + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + + env = os.environ.copy() + env.update( + { + "NO_COLOR": "1", + "TERM": "dumb", + "PYTHONUNBUFFERED": "1", + } + ) + + results: list[dict[str, object]] = [] + failures: list[str] = [] + with tempfile.TemporaryDirectory(prefix="apm-migration-bench-") as tmp: + base = Path(tmp) + for command, command_args, mutates_workspace in COMMANDS: + python_result = _measure( + binary=args.python_bin, + args=command_args, + mutates_workspace=mutates_workspace, + repeats=args.repeats, + base=base / "python" / command, + label=command, + env=env, + ) + go_result = _measure( + binary=args.go_bin, + args=command_args, + mutates_workspace=mutates_workspace, + repeats=args.repeats, + base=base / "go" / command, + label=command, + env=env, + ) + + python_median = float(python_result["median_seconds"]) + go_median = float(go_result["median_seconds"]) + ratio = go_median / max(python_median, 0.000001) + returncodes = { + "python": python_result["returncodes"], + "go": go_result["returncodes"], + } + + row = { + "command": " ".join(command_args), + "python": python_result, + "go": go_result, + "python_median_seconds": python_median, + "go_median_seconds": go_median, + "ratio": ratio, + "returncodes": returncodes, + } + results.append(row) + + if python_result["returncodes"] != go_result["returncodes"]: + failures.append(f"{command}: return codes differ: {returncodes}") + if ratio > args.max_ratio: + failures.append( + f"{command}: Go median {ratio:.2f}x slower than Python " + f"(limit {args.max_ratio:.2f}x)" + ) + + json_path = Path(args.json_out) + markdown_path = Path(args.markdown_out) + json_path.write_text( + json.dumps({"results": results, "failures": failures}, indent=2), encoding="utf-8" + ) + markdown_path.write_text(_markdown(results, args.max_ratio), encoding="utf-8") + + print(markdown_path.read_text(encoding="utf-8")) + if failures: + for failure in failures: + print(f"::error::{failure}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6cba7878fdbc4e2bef8a02f4d8650417993f3f1d Mon Sep 17 00:00:00 2001 From: mrjf Date: Wed, 27 May 2026 20:40:23 -0700 Subject: [PATCH 2/4] ci: skip migration gate for workflow-only changes --- .github/workflows/migration-ci.yml | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/migration-ci.yml b/.github/workflows/migration-ci.yml index 1f83c940..37e305c8 100644 --- a/.github/workflows/migration-ci.yml +++ b/.github/workflows/migration-ci.yml @@ -25,8 +25,40 @@ env: PYTHON_VERSION: "3.12" jobs: + detect-changes: + name: Detect Migration Changes + runs-on: ubuntu-24.04 + outputs: + should-run: ${{ steps.filter.outputs.should-run }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check changed paths + id: filter + shell: bash + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "should-run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git diff --name-only \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + | tee "$RUNNER_TEMP/changed-files.txt" + + if grep -Eq '^(\.crane/|cmd/|internal/|pkg/|go\.mod$|go\.sum$|pyproject\.toml$|src/|tests/benchmarks/|tests/unit/test_crane_score\.py$)' "$RUNNER_TEMP/changed-files.txt"; then + echo "should-run=true" >> "$GITHUB_OUTPUT" + else + echo "should-run=false" >> "$GITHUB_OUTPUT" + fi + parity: name: Python-vs-Go Parity Gate + needs: [detect-changes] + if: needs.detect-changes.outputs.should-run == 'true' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 From bc1b6e8adce42fa5462a6d33c463200c5141f3de Mon Sep 17 00:00:00 2001 From: mrjf Date: Wed, 27 May 2026 20:42:00 -0700 Subject: [PATCH 3/4] ci: set up python for lint job --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13d14263..f1680543 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,10 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@v6 with: enable-cache: true @@ -81,4 +85,3 @@ jobs: - name: Run Go tests if: hashFiles('go.mod') != '' run: go test ./... - From cfe9db1d93903fb574651f481cd3d25d0abe8d1b Mon Sep 17 00:00:00 2001 From: mrjf Date: Wed, 27 May 2026 20:43:34 -0700 Subject: [PATCH 4/4] test: format crane workflow prompt test --- tests/unit/test_crane_workflow_prompt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_crane_workflow_prompt.py b/tests/unit/test_crane_workflow_prompt.py index 3db3a487..b754623f 100644 --- a/tests/unit/test_crane_workflow_prompt.py +++ b/tests/unit/test_crane_workflow_prompt.py @@ -29,4 +29,3 @@ def test_crane_commit_guidance_provides_structured_summary_fallback() -> None: assert "Changes:" in text assert "Run: {run_url}" in text assert text.index("Changes:") < text.index("Run: {run_url}") -