diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 0000000..b1f718b --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,158 @@ +name: Detect CV changes +description: Detect bump type, fetch registry version, compute next version, generate changelog + +inputs: + base_sha: + description: Base SHA to diff against + required: true + github_token: + description: GitHub token for registry API calls + required: true + fetch_universe: + description: Whether to fetch universe version from registry + required: false + default: "false" + +outputs: + bump_type: + description: "none, patch, minor, or major" + value: ${{ steps.bump.outputs.bump_type }} + warning: + description: Warning message (e.g. for spec changes) + value: ${{ steps.bump.outputs.warning }} + current_version: + description: Current version from registry + value: ${{ steps.registry.outputs.current_version }} + next_version: + description: Computed next version + value: ${{ steps.next.outputs.next_version }} + universe_version: + description: Universe version from registry (only if fetch_universe=true) + value: ${{ steps.registry.outputs.universe_version }} + changelog: + description: Markdown changelog + value: ${{ steps.changelog.outputs.changelog }} + +runs: + using: composite + steps: + # --------------------------------------------------------------- + # 1. Detect bump type from diff + # --------------------------------------------------------------- + - name: Detect bump type + id: bump + shell: bash + env: + BASE_SHA: ${{ inputs.base_sha }} + run: | + set -e + + echo "Comparing ${BASE_SHA}..HEAD" + + CHANGED=$(git diff --name-only --diff-filter=ACDMR "${BASE_SHA}..HEAD" -- \ + ':(exclude)_src' ':(exclude)_tests' ':(exclude)_archive' ':(exclude)scripts' \ + ':(exclude)*.md' ':(exclude)*.toml' ':(exclude)*.lock' ':(exclude)*.py' \ + ':(exclude).github' ':(exclude)LICENSE*' ':(exclude).gitignore' \ + ':(exclude).pre-commit-config.yaml' ':(exclude)esgvoc_manifest.yaml' || true) + + if [ -z "${CHANGED}" ]; then + echo "No CV files changed" + echo "bump_type=none" >> "$GITHUB_OUTPUT" + echo "warning=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Changed files:" + echo "${CHANGED}" + + BUMP="patch" + WARN="" + + # New collections → major + NEW_CONTEXTS=$(git diff --name-only --diff-filter=A "${BASE_SHA}..HEAD" -- '*/000_context.jsonld' || true) + if [ -n "${NEW_CONTEXTS}" ]; then + BUMP="major" + fi + + # Spec changes → minor minimum + if [ "${BUMP}" != "major" ]; then + SPEC_CHANGES=$(git diff --name-only --diff-filter=ACDMR "${BASE_SHA}..HEAD" -- '*_specs.yaml' || true) + if [ -n "${SPEC_CHANGES}" ]; then + BUMP="minor" + WARN="Spec files changed — at minimum a minor bump, could warrant major if it breaks downstream tooling." + fi + fi + + # Modified collection models → minor + if [ "${BUMP}" = "patch" ]; then + MOD_CONTEXTS=$(git diff --name-only --diff-filter=M "${BASE_SHA}..HEAD" -- '*/000_context.jsonld' || true) + if [ -n "${MOD_CONTEXTS}" ]; then + BUMP="minor" + fi + fi + + echo "bump_type=${BUMP}" >> "$GITHUB_OUTPUT" + echo "warning=${WARN}" >> "$GITHUB_OUTPUT" + echo "Detected bump type: ${BUMP}" + + # --------------------------------------------------------------- + # 2. Fetch current version from registry + # --------------------------------------------------------------- + - name: Fetch registry version + if: steps.bump.outputs.bump_type != 'none' + id: registry + shell: bash + env: + GH_TOKEN: ${{ inputs.github_token }} + FETCH_UNIVERSE: ${{ inputs.fetch_universe }} + run: | + set -e + + CURRENT=$(gh api repos/WCRP-CMIP/esgvoc_registry/contents/obs4ref.json \ + --jq '.content' | base64 -d | python3 .github/scripts/ci_version.py fetch-current) + + echo "current_version=${CURRENT}" >> "$GITHUB_OUTPUT" + echo "Current registry version: ${CURRENT}" + + if [ "${FETCH_UNIVERSE}" = "true" ]; then + UNIVERSE=$(gh api repos/WCRP-CMIP/esgvoc_registry/contents/universe.json \ + --jq '.content' | base64 -d | python3 .github/scripts/ci_version.py fetch-universe) + echo "universe_version=${UNIVERSE}" >> "$GITHUB_OUTPUT" + echo "Universe version: ${UNIVERSE}" + fi + + # --------------------------------------------------------------- + # 3. Compute next version + # --------------------------------------------------------------- + - name: Compute next version + if: steps.bump.outputs.bump_type != 'none' + id: next + shell: bash + env: + CURRENT: ${{ steps.registry.outputs.current_version }} + BUMP: ${{ steps.bump.outputs.bump_type }} + run: | + NEXT=$(python3 .github/scripts/ci_version.py compute-next --current "${CURRENT}" --bump "${BUMP}") + echo "next_version=${NEXT}" >> "$GITHUB_OUTPUT" + echo "Next version: ${NEXT}" + + # --------------------------------------------------------------- + # 4. Generate changelog + # --------------------------------------------------------------- + - name: Generate changelog + if: steps.bump.outputs.bump_type != 'none' + id: changelog + shell: bash + env: + BASE_SHA: ${{ inputs.base_sha }} + run: | + CHANGELOG=$(python3 .github/scripts/ci_changelog.py "${BASE_SHA}") + + { + echo "changelog<> "$GITHUB_OUTPUT" + + echo "Changelog:" + echo "${CHANGELOG}" diff --git a/.github/scripts/ci_changelog.py b/.github/scripts/ci_changelog.py new file mode 100644 index 0000000..7b22483 --- /dev/null +++ b/.github/scripts/ci_changelog.py @@ -0,0 +1,130 @@ +""" +CI helper: generate a markdown changelog from git diff. + +Usage: + python .github/scripts/ci_changelog.py +""" + +import argparse +import collections +import subprocess +import sys + +# Paths excluded from CV change detection (infrastructure, docs, etc.) +EXCLUDE_PATTERNS = [ + ":(exclude)_src", + ":(exclude)_tests", + ":(exclude)_archive", + ":(exclude)scripts", + ":(exclude)*.md", + ":(exclude)*.toml", + ":(exclude)*.lock", + ":(exclude)*.py", + ":(exclude).github", + ":(exclude)LICENSE*", + ":(exclude).gitignore", + ":(exclude).pre-commit-config.yaml", + ":(exclude)esgvoc_manifest.yaml", +] + + +def get_diff(base_sha): + """Run git diff and return the name-status output.""" + cmd = [ + "git", "diff", "--name-status", "--diff-filter=ACDMR", + f"{base_sha}..HEAD", "--", + ] + EXCLUDE_PATTERNS + + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return result.stdout.strip() + + +def parse_diff(diff_output): + """Parse git diff name-status output into categorized changes.""" + changes = collections.defaultdict(lambda: {"A": [], "M": [], "D": []}) + model_changes = [] + spec_changes = [] + + for line in diff_output.split("\n"): + if not line.strip(): + continue + + parts = line.split("\t") + + if parts[0].startswith(("R", "C")): + # Renames/copies: "R100\told/path\tnew/path" — treat as add of the new path + if len(parts) < 3: + continue + status = "A" + path = parts[2] + elif len(parts) >= 2: + status = parts[0] + path = parts[1] + else: + continue + + segments = path.split("/") + filename = segments[-1] + + # Spec files at root level + if len(segments) == 1 and filename.endswith("_specs.yaml"): + action = "new" if status == "A" else "updated" + spec_changes.append((filename, action)) + continue + + if len(segments) < 2: + continue + + collection = segments[0] + + if filename == "000_context.jsonld": + action = "new" if status == "A" else "updated" + model_changes.append((collection, action)) + else: + term_id = filename.replace(".json", "") + changes[collection][status[0]].append(term_id) + + return spec_changes, model_changes, changes + + +def format_changelog(spec_changes, model_changes, term_changes): + """Format changes into markdown.""" + lines = [] + + for filename, action in spec_changes: + lines.append(f"- {filename}: {action}") + + for collection, action in model_changes: + lines.append(f"- {collection}: {action} collection model") + + for collection in sorted(term_changes): + parts = [] + counts = term_changes[collection] + if counts["A"]: + parts.append(f'{len(counts["A"])} added') + if counts["M"]: + parts.append(f'{len(counts["M"])} modified') + if counts["D"]: + parts.append(f'{len(counts["D"])} removed') + if parts: + lines.append(f"- {collection}: {', '.join(parts)}") + + return "\n".join(lines) if lines else "No CV changes." + + +def main(): + parser = argparse.ArgumentParser(description="Generate changelog from git diff") + parser.add_argument("base_sha", help="Base SHA to diff against") + args = parser.parse_args() + + diff_output = get_diff(args.base_sha) + if not diff_output: + print("No CV changes.") + sys.exit(0) + + spec_changes, model_changes, term_changes = parse_diff(diff_output) + print(format_changelog(spec_changes, model_changes, term_changes)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/ci_update_manifest.py b/.github/scripts/ci_update_manifest.py new file mode 100644 index 0000000..61b9f9f --- /dev/null +++ b/.github/scripts/ci_update_manifest.py @@ -0,0 +1,63 @@ +""" +CI helper: update esgvoc_manifest.yaml with new version info. + +Usage: + python .github/scripts/ci_update_manifest.py \ + --version 1.2.0 \ + --universe-version 1.0.3 \ + --changelog-file changelog.md +""" + +import argparse +import sys + +import yaml + + +def _literal_str_representer(dumper, data): + """Force multi-line strings to use YAML literal block scalar (|).""" + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + +yaml.add_representer(str, _literal_str_representer) + + +def main(): + parser = argparse.ArgumentParser(description="Update esgvoc manifest") + parser.add_argument("--version", required=True, help="New CV version") + parser.add_argument("--universe-version", required=True, help="Universe version") + parser.add_argument( + "--changelog-file", + default="-", + help="Path to changelog file, or - for stdin (default: -)", + ) + parser.add_argument( + "--manifest", + default="esgvoc_manifest.yaml", + help="Path to manifest file (default: esgvoc_manifest.yaml)", + ) + args = parser.parse_args() + + if args.changelog_file == "-": + changelog = sys.stdin.read().strip() + else: + with open(args.changelog_file) as f: + changelog = f.read().strip() + + with open(args.manifest) as f: + manifest = yaml.safe_load(f) + + manifest["cv_version"] = args.version + manifest["universe_version"] = args.universe_version + manifest["release_notes"] = changelog + "\n" + + with open(args.manifest, "w") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + print(f"Updated manifest to version {args.version}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/ci_version.py b/.github/scripts/ci_version.py new file mode 100644 index 0000000..142252d --- /dev/null +++ b/.github/scripts/ci_version.py @@ -0,0 +1,79 @@ +""" +CI helper: version management for CV releases. + +Subcommands: + fetch-current Read registry JSON from stdin, print current stable version. + fetch-universe Read registry JSON from stdin, print current universe version. + compute-next Given --current and --bump, print the next semver version. +""" + +import argparse +import json +import sys + + +def fetch_version(data): + """Extract the latest non-prerelease, non-dev version from a registry index.""" + for release in data.get("releases", []): + version = release.get("version", "") + if version == "dev-latest" or release.get("is_prerelease"): + continue + return version.lstrip("v") + return "0.0.0" + + +def cmd_fetch_current(args): + data = json.load(sys.stdin) + print(fetch_version(data)) + + +def cmd_fetch_universe(args): + data = json.load(sys.stdin) + print(fetch_version(data)) + + +def cmd_compute_next(args): + parts = [int(x) for x in args.current.split(".")] + if len(parts) != 3: + print(f"error: invalid version '{args.current}'", file=sys.stderr) + sys.exit(1) + + if args.bump == "major": + parts[0] += 1 + parts[1] = 0 + parts[2] = 0 + elif args.bump == "minor": + parts[1] += 1 + parts[2] = 0 + elif args.bump == "patch": + parts[2] += 1 + else: + print(f"error: invalid bump type '{args.bump}'", file=sys.stderr) + sys.exit(1) + + print(".".join(str(p) for p in parts)) + + +def main(): + parser = argparse.ArgumentParser(description="CI version helper") + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("fetch-current", help="Read registry JSON from stdin, print current version") + sub.add_parser("fetch-universe", help="Read registry JSON from stdin, print universe version") + + next_parser = sub.add_parser("compute-next", help="Compute next semver version") + next_parser.add_argument("--current", required=True, help="Current version (e.g. 1.2.3)") + next_parser.add_argument("--bump", required=True, choices=["major", "minor", "patch"]) + + args = parser.parse_args() + + commands = { + "fetch-current": cmd_fetch_current, + "fetch-universe": cmd_fetch_universe, + "compute-next": cmd_compute_next, + } + commands[args.command](args) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/publish-prod.yaml b/.github/workflows/publish-prod.yaml new file mode 100644 index 0000000..ed7fc9b --- /dev/null +++ b/.github/workflows/publish-prod.yaml @@ -0,0 +1,169 @@ +name: Prepare release (merge to main) + +on: + push: + branches: [main] + +permissions: + contents: write + pull-requests: write + +jobs: + prepare-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + # Skip if this is a manifest-only commit (from a previous run) + - name: Check if manifest-only commit + id: skip_check + run: | + CHANGED=$(git diff --name-only HEAD~1..HEAD) + if [ "$CHANGED" = "esgvoc_manifest.yaml" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "Manifest-only commit detected — skipping release." + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - uses: ./.github/actions/setup-esgvoc + if: steps.skip_check.outputs.skip != 'true' + with: + extra-packages: pyyaml + + - name: Test obs4ref build + if: steps.skip_check.outputs.skip != 'true' + env: + BRANCH: ${{ github.ref_name }} + run: esgvoc test run obs4ref --branch "$BRANCH" --universe-branch main + + - name: Get last published source SHA + if: steps.skip_check.outputs.skip != 'true' + id: base + env: + GH_TOKEN: ${{ github.token }} + run: | + set -e + # Fetch source_sha of the latest stable release from the registry + SOURCE_SHA=$(gh api repos/WCRP-CMIP/esgvoc_registry/contents/obs4ref.json \ + --jq '.content' | base64 -d | python3 -c " + import json, sys + idx = json.load(sys.stdin) + for r in idx['releases']: + v = r.get('version', '') + if v != 'dev-latest' and not r.get('is_prerelease'): + print(r.get('source_sha', '')) + break + else: + print('') + ") + + if [ -z "${SOURCE_SHA}" ]; then + # No source_sha in registry — first release, diff from root commit + echo "::warning::No source_sha in registry, falling back to root commit (first release)" + SOURCE_SHA=$(git rev-list --max-parents=0 HEAD | tail -1) + fi + + echo "sha=${SOURCE_SHA}" >> "$GITHUB_OUTPUT" + echo "Base SHA: ${SOURCE_SHA}" + + - name: Detect changes + if: steps.skip_check.outputs.skip != 'true' + id: changes + uses: ./.github/actions/detect-changes + with: + base_sha: ${{ steps.base.outputs.sha }} + github_token: ${{ github.token }} + fetch_universe: "true" + + - name: Update manifest + if: steps.skip_check.outputs.skip != 'true' && steps.changes.outputs.bump_type != 'none' + env: + NEXT_VERSION: ${{ steps.changes.outputs.next_version }} + UNIVERSE_VERSION: ${{ steps.changes.outputs.universe_version }} + CHANGELOG: ${{ steps.changes.outputs.changelog }} + run: | + echo "${CHANGELOG}" | python3 .github/scripts/ci_update_manifest.py \ + --version "${NEXT_VERSION}" \ + --universe-version "${UNIVERSE_VERSION}" \ + --changelog-file - + + echo "Updated manifest:" + cat esgvoc_manifest.yaml + + - name: Create manifest PR + id: manifest_pr + if: steps.skip_check.outputs.skip != 'true' && steps.changes.outputs.bump_type != 'none' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEXT_VERSION: ${{ steps.changes.outputs.next_version }} + CURRENT_VERSION: ${{ steps.changes.outputs.current_version }} + BUMP_TYPE: ${{ steps.changes.outputs.bump_type }} + CHANGELOG: ${{ steps.changes.outputs.changelog }} + run: | + BRANCH="chore/manifest-${NEXT_VERSION}" + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + git checkout -b "${BRANCH}" + git add esgvoc_manifest.yaml + git diff --cached --quiet && echo "No manifest changes" && exit 0 + + git commit -m "chore: update manifest to ${NEXT_VERSION}" + git push origin "${BRANCH}" + + BODY=$(cat <> "$GITHUB_OUTPUT" + echo "Manifest PR created: ${PR_URL}" + + # Post manifest PR link on the PR that triggered this workflow + - name: Notify source PR + if: steps.manifest_pr.outputs.pr_url + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + COMMIT_SHA: ${{ github.sha }} + PR_URL: ${{ steps.manifest_pr.outputs.pr_url }} + run: | + # Find the PR that was just merged (by merge commit SHA) + SOURCE_PR=$(gh api "repos/${REPO}/commits/${COMMIT_SHA}/pulls" --jq '.[0].number // empty') + + if [ -n "${SOURCE_PR}" ]; then + gh pr comment "${SOURCE_PR}" --body "Manifest PR created: ${PR_URL}" + fi + + - name: No CV changes + if: steps.skip_check.outputs.skip != 'true' && steps.changes.outputs.bump_type == 'none' + run: echo "No CV files changed — no release needed." diff --git a/.github/workflows/test-prod.yaml b/.github/workflows/test-prod.yaml new file mode 100644 index 0000000..59c9f06 --- /dev/null +++ b/.github/workflows/test-prod.yaml @@ -0,0 +1,87 @@ +name: Test release (PR) + +on: + pull_request: + branches: [main] + +permissions: + contents: read + pull-requests: write + +jobs: + test-and-preview: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-esgvoc + + - name: Test obs4ref build + env: + BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref }} + REPO: ${{ github.event.pull_request.head.repo.full_name }} + run: | + REPO_FLAG="" + if [ -n "$REPO" ]; then + REPO_FLAG="--repo $REPO" + fi + esgvoc test run obs4ref $REPO_FLAG --branch "$BRANCH" --universe-branch main + + - name: Detect changes + id: changes + uses: ./.github/actions/detect-changes + with: + base_sha: ${{ github.event.pull_request.base.sha }} + github_token: ${{ github.token }} + + - name: Post release preview + if: steps.changes.outputs.bump_type != 'none' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + CHANGELOG: ${{ steps.changes.outputs.changelog }} + BUMP_TYPE: ${{ steps.changes.outputs.bump_type }} + CURRENT_VERSION: ${{ steps.changes.outputs.current_version }} + NEXT_VERSION: ${{ steps.changes.outputs.next_version }} + WARNING: ${{ steps.changes.outputs.warning }} + run: | + # Clean up previous bot comments + gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq '.[] | select(.user.login == "github-actions[bot]") | select(.body | contains("Release preview")) | .id' \ + | xargs -I {} gh api -X DELETE "repos/${REPO}/issues/comments/{}" 2>/dev/null || true + + WARN_BLOCK="" + if [ -n "${WARNING}" ]; then + WARN_BLOCK="> **Warning:** ${WARNING}" + fi + + BODY=$(cat <