From 6758b469184754d1f93adbcc27f0ac78488fec0e Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:21:22 -0700 Subject: [PATCH] Add daily CFS feed warm-up and make azpysdk tool pins scannable Move the hardcoded static-analysis tool versions (mypy, pylint, pyright, sphinx, black, bandit, breaking) out of the azpysdk check modules and into eng/tool_requirements/*.txt as a single, machine-scannable source of truth, loaded at runtime via azpysdk._tool_reqs. Add eng/scripts/warm_cfs_feed.py, which scans every declared dependency in the repo (dev_requirements.txt, pyproject.toml, shared eng/*.txt files, and the new tool-requirement files) and runs 'pip download' with transitive dependencies against the CFS feed. Add eng/pipelines/warm-cfs-feed.yml to run it daily, authenticated, so unpinned transitive deps of tools like mypy are pulled through and cached before unauthenticated PR pipelines need them. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 15 + eng/pipelines/warm-cfs-feed.yml | 99 +++++ eng/scripts/warm_cfs_feed.py | 396 ++++++++++++++++++ eng/tool_requirements/README.md | 26 ++ eng/tool_requirements/bandit.txt | 3 + eng/tool_requirements/black.txt | 2 + eng/tool_requirements/breaking.txt | 3 + eng/tool_requirements/mypy.txt | 6 + eng/tool_requirements/mypy_next.txt | 6 + eng/tool_requirements/pylint.txt | 3 + eng/tool_requirements/pylint_next.txt | 3 + eng/tool_requirements/pyright.txt | 2 + eng/tool_requirements/pyright_next.txt | 2 + eng/tool_requirements/sphinx.txt | 5 + eng/tool_requirements/sphinx_next.txt | 5 + .../azure-sdk-tools/azpysdk/_tool_reqs.py | 96 +++++ eng/tools/azure-sdk-tools/azpysdk/bandit.py | 7 +- eng/tools/azure-sdk-tools/azpysdk/black.py | 7 +- eng/tools/azure-sdk-tools/azpysdk/breaking.py | 7 +- eng/tools/azure-sdk-tools/azpysdk/mypy.py | 23 +- eng/tools/azure-sdk-tools/azpysdk/pylint.py | 30 +- eng/tools/azure-sdk-tools/azpysdk/pyright.py | 14 +- eng/tools/azure-sdk-tools/azpysdk/sphinx.py | 44 +- .../azure-sdk-tools/azpysdk/verifytypes.py | 7 +- .../azure-sdk-tools/tests/test_tool_reqs.py | 38 ++ 25 files changed, 770 insertions(+), 79 deletions(-) create mode 100644 eng/pipelines/warm-cfs-feed.yml create mode 100644 eng/scripts/warm_cfs_feed.py create mode 100644 eng/tool_requirements/README.md create mode 100644 eng/tool_requirements/bandit.txt create mode 100644 eng/tool_requirements/black.txt create mode 100644 eng/tool_requirements/breaking.txt create mode 100644 eng/tool_requirements/mypy.txt create mode 100644 eng/tool_requirements/mypy_next.txt create mode 100644 eng/tool_requirements/pylint.txt create mode 100644 eng/tool_requirements/pylint_next.txt create mode 100644 eng/tool_requirements/pyright.txt create mode 100644 eng/tool_requirements/pyright_next.txt create mode 100644 eng/tool_requirements/sphinx.txt create mode 100644 eng/tool_requirements/sphinx_next.txt create mode 100644 eng/tools/azure-sdk-tools/azpysdk/_tool_reqs.py create mode 100644 eng/tools/azure-sdk-tools/tests/test_tool_reqs.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15181020168c..b9e16804c2b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -174,6 +174,21 @@ export PIP_INDEX_URL="https://your-azure-username:your-pat-token@pkgs.dev.azure. uv pip install --index-url https://pypi.org/simple/ ``` +#### Keeping the CFS feed warm + +Because CFS only serves versions it has already cached (and unauthenticated CI +runs cannot trigger an upstream pull-through), a scheduled pipeline keeps the feed +warm. `eng/scripts/warm_cfs_feed.py` scans every declared dependency in the repo — +all `dev_requirements.txt` files, every `pyproject.toml`, the shared `eng/*.txt` +requirement files, and the `azpysdk` tool pins in `eng/tool_requirements/` — and +runs `pip download` (including transitive dependencies) against CFS so the latest +versions are cached before an unauthenticated build needs them. + +The static-analysis tools that `azpysdk` installs at runtime (mypy, pylint, +pyright, sphinx, black, bandit, ...) are pinned in `eng/tool_requirements/*.txt`. +That is the single source of truth for those versions; bump a tool by editing the +relevant file there rather than hardcoding a version in the check modules. + ### Dev Feed Daily dev build version of Azure sdk packages for python are available and are uploaded to Azure devops feed daily. Below is the link to Azure devops feed. [`https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-python`](https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-python) diff --git a/eng/pipelines/warm-cfs-feed.yml b/eng/pipelines/warm-cfs-feed.yml new file mode 100644 index 000000000000..0622a9ea945d --- /dev/null +++ b/eng/pipelines/warm-cfs-feed.yml @@ -0,0 +1,99 @@ +# Daily job that keeps the Central Feed Services (CFS) feed warm. +# +# CFS is an upstream pull-through cache for PyPI. Authenticated requests for a +# not-yet-cached package version pull it through from PyPI and cache it forever; +# unauthenticated pipelines (e.g. fork PR CI) can only read what is already +# cached. This job runs authenticated on a daily schedule and `pip download`s the +# full transitive closure of every dependency declared in the repo (including the +# azpysdk static-analysis tool pins in eng/tool_requirements/), so the latest +# transitive versions are cached before an unauthenticated CI run needs them. +# +# See eng/scripts/warm_cfs_feed.py for the scanning/download logic. + +trigger: none +pr: none + +schedules: + - cron: "0 8 * * *" + displayName: Daily CFS feed warm-up (08:00 UTC) + branches: + include: + - main + always: true + +parameters: + - name: dryRun + displayName: Dry Run (discover dependencies without downloading) + type: boolean + default: false + - name: failOnError + displayName: Fail the job if any dependency could not be warmed + type: boolean + default: false + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - stage: WarmCfsFeed + displayName: Warm CFS Feed + + jobs: + - job: WarmCfsFeedJob + timeoutInMinutes: 180 + displayName: Scan dependencies and warm CFS feed + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - name: dryRunArg + ${{ if eq(parameters.dryRun, true) }}: + value: '--dry-run' + ${{ else }}: + value: '' + - name: failOnErrorArg + ${{ if eq(parameters.failOnError, true) }}: + value: '--fail-on-error' + ${{ else }}: + value: '' + + pool: + name: azsdk-pool + image: ubuntu-24.04 + os: linux + + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/cfs-warm-report' + artifactName: 'cfs-warm-report' + condition: succeededOrFailed() + sbomEnabled: false + + steps: + - checkout: self + + - template: /eng/pipelines/templates/steps/use-python-version.yml + parameters: + versionSpec: '3.11' + + # Authenticate to the feed so pip download can pull-through from PyPI upstream. + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + EnableTwineAuth: false + EnableUvAuth: false + + - script: | + python -m pip install --upgrade pip + python -m pip install "eng/tools/azure-sdk-tools" + displayName: 'Prep Environment' + + - script: | + mkdir -p "$(Build.ArtifactStagingDirectory)/cfs-warm-report" + python eng/scripts/warm_cfs_feed.py \ + $(dryRunArg) \ + $(failOnErrorArg) \ + --report "$(Build.ArtifactStagingDirectory)/cfs-warm-report/report.json" + displayName: 'Warm CFS feed' + env: + # PIP_INDEX_URL is set (with embedded credentials) by the auth step above; + # warm_cfs_feed.py defaults --index-url to it. + PIP_INDEX_URL: $(PIP_INDEX_URL) diff --git a/eng/scripts/warm_cfs_feed.py b/eng/scripts/warm_cfs_feed.py new file mode 100644 index 000000000000..e33cd1da2229 --- /dev/null +++ b/eng/scripts/warm_cfs_feed.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Warm the Azure SDK Central Feed Services (CFS) feed with every third-party +dependency declared anywhere in this repository. + +Background +---------- +This repository installs all packages from the CFS feed +(``https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/``) +instead of directly from PyPI. CFS is an *upstream pull-through* cache: an +**authenticated** request for a package version that CFS has not seen yet causes +CFS to fetch (and permanently cache) it from PyPI. **Unauthenticated** pipelines +(for example CI runs for pull requests from forks) can only read versions that +CFS has *already* cached -- they cannot trigger a pull-through. + +That asymmetry is the failure mode this script exists to prevent. A tool such as +``mypy`` has transitive dependencies that are not strictly pinned. When one of +those transitive dependencies publishes a new release, an unauthenticated CI job +that resolves ``mypy`` to that brand-new (uncached) transitive version fails, +because CFS does not have it and the job cannot authenticate to pull it through. + +Running this script on a daily authenticated schedule keeps CFS warm: for every +dependency declared in the repo we run ``pip download`` (which resolves and +downloads the **full transitive closure**) against the CFS feed. Because we do +*not* pass ``--no-deps``, the current-latest transitive versions get pulled +through and cached, so the next unauthenticated CI run finds them already present. + +What counts as a "declared dependency" +-------------------------------------- +The script aggregates requirement specifiers from: + +* every ``dev_requirements.txt`` in the repo, +* every package's ``pyproject.toml`` (``[project].dependencies`` and + ``[project.optional-dependencies]``), +* the shared/engineering requirement files + (``shared_requirements.txt``, ``eng/ci_tools.txt``, ``eng/test_tools.txt``, + ``eng/dependency_tools.txt``, ``eng/release_requirements.txt``), +* the ``azpysdk`` static-analysis tool pins in ``eng/tool_requirements/*.txt`` + (mypy, pylint, pyright, sphinx, black, bandit, ... -- this is precisely the set + that used to be invisible because it was hardcoded in Python). + +First-party / local entries (editable installs, relative paths, and the +first-party ``azure-*`` packages that live in this repo) are skipped: they are +built and published by our own pipelines and are not pulled from PyPI upstream. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from typing import Dict, Iterable, List, Optional, Set, Tuple + +from packaging.requirements import InvalidRequirement, Requirement + +from ci_tools.functions import discover_targeted_packages +from ci_tools.parsing import ParsedSetup +from ci_tools.parsing.parse_functions import get_pyproject_dict +from ci_tools.variables import discover_repo_root + +CFS_INDEX_URL = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + +# Requirement files at well-known locations that are not attached to a single package. +SHARED_REQUIREMENT_FILES = [ + "shared_requirements.txt", + os.path.join("eng", "ci_tools.txt"), + os.path.join("eng", "test_tools.txt"), + os.path.join("eng", "dependency_tools.txt"), + os.path.join("eng", "release_requirements.txt"), +] + +# Directory names that never contain declared dependencies we care about. +_PRUNE_DIRS = { + ".git", + ".venv", + ".tox", + "node_modules", + "build", + "dist", + ".eggs", + "__pycache__", + ".mypy_cache", + ".pytest_cache", +} + + +class RequirementSource: + """A single requirement specifier and where it came from (for reporting).""" + + __slots__ = ("spec", "name", "origin") + + def __init__(self, spec: str, name: str, origin: str) -> None: + self.spec = spec + self.name = name + self.origin = origin + + +def _looks_like_local(line: str) -> bool: + """Return True for editable installs, path installs, and pip control flags. + + These are first-party or local references that are not pulled from PyPI and + therefore should not be sent to ``pip download`` against the CFS feed. + """ + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return True + # editable installs / nested-or-constraint file includes + if stripped.startswith(("-e ", "--editable")): + return True + if stripped.startswith(("-r ", "--requirement", "-c ", "--constraint", "-f ", "--find-links")): + return True + # bare pip options (e.g. --index-url=...); the caller controls the index + if stripped.startswith("-"): + return True + # relative / absolute path references (../../core/azure-core, ./foo, C:\...) + if stripped.startswith((".", "/")) or (len(stripped) > 1 and stripped[1] == ":"): + return True + # URL / VCS installs + if "://" in stripped: + return True + return False + + +def _parse_requirement_line(line: str) -> Optional[Requirement]: + """Parse a single requirements-file line into a Requirement, or None to skip.""" + # strip inline comments and environment-marker-safe trailing whitespace + content = line.split(" #", 1)[0].strip() + if _looks_like_local(content): + return None + try: + return Requirement(content) + except InvalidRequirement: + return None + + +def _is_first_party(name: str, first_party: Set[str]) -> bool: + return name.lower() in first_party + + +def iter_requirement_files(repo_root: str) -> Iterable[str]: + """Yield the absolute path of every ``dev_requirements.txt`` in the repo.""" + for current, dirnames, filenames in os.walk(repo_root): + dirnames[:] = [d for d in dirnames if d not in _PRUNE_DIRS] + if "dev_requirements.txt" in filenames: + yield os.path.join(current, "dev_requirements.txt") + + +def collect_from_requirement_file(path: str, origin: str) -> List[RequirementSource]: + sources: List[RequirementSource] = [] + try: + with open(path, "r", encoding="utf-8") as handle: + for line in handle: + requirement = _parse_requirement_line(line) + if requirement is not None: + sources.append(RequirementSource(str(requirement), requirement.name, origin)) + except OSError as exc: + print(f"[warn] could not read {path}: {exc}", file=sys.stderr) + return sources + + +def collect_from_pyproject(package_dir: str) -> List[RequirementSource]: + """Collect [project].dependencies and [project.optional-dependencies].""" + sources: List[RequirementSource] = [] + pyproject_path = os.path.join(package_dir, "pyproject.toml") + if not os.path.exists(pyproject_path): + return sources + try: + pyproject = get_pyproject_dict(pyproject_path) + except Exception as exc: # pragma: no cover - malformed toml + print(f"[warn] could not parse {pyproject_path}: {exc}", file=sys.stderr) + return sources + + project = pyproject.get("project", {}) if isinstance(pyproject, dict) else {} + if not project: + return sources + origin = pyproject_path + + def _add(specifiers: Iterable[str]) -> None: + for spec in specifiers: + requirement = _parse_requirement_line(spec) + if requirement is not None: + sources.append(RequirementSource(str(requirement), requirement.name, origin)) + + _add(project.get("dependencies", []) or []) + for extra_specs in (project.get("optional-dependencies", {}) or {}).values(): + _add(extra_specs or []) + return sources + + +def discover_first_party_names(repo_root: str) -> Set[str]: + """Return the lowercased names of every package built from this repo. + + These are skipped because they are published by our own pipelines, not pulled + from PyPI upstream. + """ + names: Set[str] = set() + sdk_root = os.path.join(repo_root, "sdk") + search_root = sdk_root if os.path.isdir(sdk_root) else repo_root + try: + for package_dir in discover_targeted_packages("azure*", search_root, compatibility_filter=False): + try: + names.add(ParsedSetup.from_path(package_dir).name.lower()) + except Exception: # pragma: no cover - best effort discovery + continue + except Exception as exc: # pragma: no cover - best effort discovery + print(f"[warn] first-party discovery failed: {exc}", file=sys.stderr) + return names + + +def collect_all_sources(repo_root: str) -> List[RequirementSource]: + sources: List[RequirementSource] = [] + + # 1) dev_requirements.txt everywhere + for req_file in iter_requirement_files(repo_root): + sources.extend(collect_from_requirement_file(req_file, req_file)) + + # 2) pyproject.toml dependencies for every discovered package + sdk_root = os.path.join(repo_root, "sdk") + search_root = sdk_root if os.path.isdir(sdk_root) else repo_root + try: + package_dirs = discover_targeted_packages("azure*", search_root, compatibility_filter=False) + except Exception as exc: # pragma: no cover - best effort discovery + print(f"[warn] package discovery failed: {exc}", file=sys.stderr) + package_dirs = [] + for package_dir in package_dirs: + sources.extend(collect_from_pyproject(package_dir)) + + # 3) shared / engineering requirement files + for relative in SHARED_REQUIREMENT_FILES: + absolute = os.path.join(repo_root, relative) + if os.path.exists(absolute): + sources.extend(collect_from_requirement_file(absolute, absolute)) + + # 4) azpysdk static-analysis tool pins (formerly hardcoded in Python) + tool_requirements_dir = os.path.join(repo_root, "eng", "tool_requirements") + if os.path.isdir(tool_requirements_dir): + for entry in sorted(os.listdir(tool_requirements_dir)): + if entry.endswith(".txt"): + path = os.path.join(tool_requirements_dir, entry) + sources.extend(collect_from_requirement_file(path, path)) + + return sources + + +def dedupe_specs(sources: List[RequirementSource], first_party: Set[str]) -> Tuple[Dict[str, List[str]], List[str]]: + """Return (spec -> [origins]) for third-party specs, and the skipped first-party names. + + Specs are de-duplicated on their exact text so that distinct version + constraints for the same distribution are all warmed. + """ + spec_to_origins: Dict[str, List[str]] = {} + skipped_first_party: Set[str] = set() + for source in sources: + if _is_first_party(source.name, first_party): + skipped_first_party.add(source.name.lower()) + continue + spec_to_origins.setdefault(source.spec, []) + if source.origin not in spec_to_origins[source.spec]: + spec_to_origins[source.spec].append(source.origin) + return spec_to_origins, sorted(skipped_first_party) + + +def pip_download(spec: str, dest: str, index_url: str, python_executable: str) -> Tuple[bool, str]: + """Download *spec* and its full transitive closure into *dest* from *index_url*. + + Dependencies are intentionally included (no ``--no-deps``) so the whole + closure is pulled through into the CFS cache. + """ + command = [ + python_executable, + "-m", + "pip", + "download", + spec, + "--dest", + dest, + "--index-url", + index_url, + # allow pre-releases; some tool pins (and their deps) are pre-release + "--pre", + ] + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode == 0: + return True, "" + return False, (result.stderr or result.stdout).strip() + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--index-url", + default=os.environ.get("PIP_INDEX_URL", CFS_INDEX_URL), + help="Feed to warm. Defaults to $PIP_INDEX_URL (set by the pipeline auth step) or the public CFS URL.", + ) + parser.add_argument( + "--dest", + default=None, + help="Directory to download into. Defaults to a temporary directory that is removed afterwards.", + ) + parser.add_argument( + "--report", + default=None, + help="Optional path to write a JSON summary report (published as a pipeline artifact).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Discover and print the requirement set without downloading anything.", + ) + parser.add_argument( + "--fail-on-error", + action="store_true", + help="Exit non-zero if any requirement failed to download. By default the daily job exits 0 and just reports.", + ) + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + args = build_arg_parser().parse_args(argv) + repo_root = discover_repo_root() + + print(f"[info] scanning repository for declared dependencies: {repo_root}") + first_party = discover_first_party_names(repo_root) + print(f"[info] discovered {len(first_party)} first-party packages (these are skipped)") + + sources = collect_all_sources(repo_root) + spec_to_origins, skipped_first_party = dedupe_specs(sources, first_party) + specs = sorted(spec_to_origins) + + print(f"[info] collected {len(sources)} requirement references") + print(f"[info] {len(specs)} unique third-party specifiers to warm") + + if args.dry_run: + for spec in specs: + print(f" {spec}") + print(f"[info] dry-run: skipped {len(skipped_first_party)} first-party distributions") + return 0 + + dest_context: Optional[tempfile.TemporaryDirectory] = None + if args.dest: + os.makedirs(args.dest, exist_ok=True) + dest = args.dest + else: + dest_context = tempfile.TemporaryDirectory(prefix="cfs-warm-") + dest = dest_context.name + + succeeded: List[str] = [] + failed: List[Dict[str, str]] = [] + try: + for index, spec in enumerate(specs, start=1): + print(f"[{index}/{len(specs)}] pip download {spec}") + ok, error = pip_download(spec, dest, args.index_url, sys.executable) + if ok: + succeeded.append(spec) + else: + print(f"[warn] failed to warm {spec}: {error.splitlines()[-1] if error else 'unknown error'}") + failed.append({"spec": spec, "origins": spec_to_origins[spec], "error": error}) + finally: + if dest_context is not None: + dest_context.cleanup() + + summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "index_url": args.index_url, + "total_unique_specs": len(specs), + "succeeded": len(succeeded), + "failed": len(failed), + "skipped_first_party": len(skipped_first_party), + "failures": failed, + } + + print( + f"[info] done: {len(succeeded)} warmed, {len(failed)} failed, " + f"{len(skipped_first_party)} first-party skipped" + ) + + if args.report: + with open(args.report, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2) + print(f"[info] wrote report to {args.report}") + + if failed and args.fail_on_error: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eng/tool_requirements/README.md b/eng/tool_requirements/README.md new file mode 100644 index 000000000000..97e47b7682b9 --- /dev/null +++ b/eng/tool_requirements/README.md @@ -0,0 +1,26 @@ +# Static-analysis tool pins (`eng/tool_requirements/`) + +Pinned versions of the third-party tools that the `azpysdk` checks install at +runtime (mypy, pylint, pyright, sphinx, black, bandit, breaking-change checker). + +Historically these versions were hardcoded as constants inside each check module +(for example `MYPY_VERSION = "1.19.1"` in `azpysdk/mypy.py`). That made them +invisible to any tool that scans the repository's declared dependencies, so the +daily CFS warm-up (`eng/scripts/warm_cfs_feed.py`) could not pre-cache the tools +or — more importantly — their transitive dependencies. When an unpinned +transitive dependency released a new version, unauthenticated PR pipelines (which +can only read what CFS has already cached) failed. + +These files are the single source of truth for those pins: + +- The `azpysdk` checks load them via `azpysdk._tool_reqs`. +- The daily warm-up script scans them (alongside every `dev_requirements.txt` and + `pyproject.toml`) and runs `pip download` so the full transitive closure is + pulled through into the CFS feed. + +Each file is an ordinary pip requirements file (one requirement per line, `#` +comments allowed). Files suffixed `_next` hold the "next"/vNext tool versions +tested by the `next-*` checks. + +To bump a tool version, edit the relevant file here — do not add the version back +into the Python check modules. diff --git a/eng/tool_requirements/bandit.txt b/eng/tool_requirements/bandit.txt new file mode 100644 index 000000000000..85b92010b7a0 --- /dev/null +++ b/eng/tool_requirements/bandit.txt @@ -0,0 +1,3 @@ +# Pins for the azpysdk `bandit` check. Single source of truth (see README.md). +bandit==1.6.2 +pbr diff --git a/eng/tool_requirements/black.txt b/eng/tool_requirements/black.txt new file mode 100644 index 000000000000..e4efe8e4dd6d --- /dev/null +++ b/eng/tool_requirements/black.txt @@ -0,0 +1,2 @@ +# Pins for the azpysdk `black` check. Single source of truth (see README.md). +black==24.4.0 diff --git a/eng/tool_requirements/breaking.txt b/eng/tool_requirements/breaking.txt new file mode 100644 index 000000000000..d27c5900ec80 --- /dev/null +++ b/eng/tool_requirements/breaking.txt @@ -0,0 +1,3 @@ +# Pins for the azpysdk `breaking` change check. Single source of truth (see README.md). +# The breaking-change checker itself is installed from a local path (`-e`) in code. +jsondiff==1.2.0 diff --git a/eng/tool_requirements/mypy.txt b/eng/tool_requirements/mypy.txt new file mode 100644 index 000000000000..5ce8ad843785 --- /dev/null +++ b/eng/tool_requirements/mypy.txt @@ -0,0 +1,6 @@ +# Pins for the azpysdk `mypy` check. Single source of truth (see README.md). +mypy==1.19.1 +types-chardet==5.0.4.6 +types-requests==2.31.0.6 +types-six==1.16.21.9 +types-redis==4.6.0.7 diff --git a/eng/tool_requirements/mypy_next.txt b/eng/tool_requirements/mypy_next.txt new file mode 100644 index 000000000000..9e0a300ab437 --- /dev/null +++ b/eng/tool_requirements/mypy_next.txt @@ -0,0 +1,6 @@ +# Pins for the azpysdk `next-mypy` check. Single source of truth (see README.md). +mypy==2.1.0 +types-chardet==5.0.4.6 +types-requests==2.31.0.6 +types-six==1.16.21.9 +types-redis==4.6.0.7 diff --git a/eng/tool_requirements/pylint.txt b/eng/tool_requirements/pylint.txt new file mode 100644 index 000000000000..0f0fa481fdfd --- /dev/null +++ b/eng/tool_requirements/pylint.txt @@ -0,0 +1,3 @@ +# Pins for the azpysdk `pylint` check. Single source of truth (see README.md). +pylint==4.0.4 +azure-pylint-guidelines-checker==0.5.7 diff --git a/eng/tool_requirements/pylint_next.txt b/eng/tool_requirements/pylint_next.txt new file mode 100644 index 000000000000..90d134c60c4b --- /dev/null +++ b/eng/tool_requirements/pylint_next.txt @@ -0,0 +1,3 @@ +# Pins for the azpysdk `next-pylint` check. Single source of truth (see README.md). +pylint==4.0.6 +azure-pylint-guidelines-checker==0.5.9 diff --git a/eng/tool_requirements/pyright.txt b/eng/tool_requirements/pyright.txt new file mode 100644 index 000000000000..d2c6799188d7 --- /dev/null +++ b/eng/tool_requirements/pyright.txt @@ -0,0 +1,2 @@ +# Pins for the azpysdk `pyright` and `verifytypes` checks. Single source of truth (see README.md). +pyright==1.1.407 diff --git a/eng/tool_requirements/pyright_next.txt b/eng/tool_requirements/pyright_next.txt new file mode 100644 index 000000000000..c3a3d707ca1c --- /dev/null +++ b/eng/tool_requirements/pyright_next.txt @@ -0,0 +1,2 @@ +# Pins for the azpysdk `next-pyright` check. Single source of truth (see README.md). +pyright==1.1.411 diff --git a/eng/tool_requirements/sphinx.txt b/eng/tool_requirements/sphinx.txt new file mode 100644 index 000000000000..3a4f015b5ad1 --- /dev/null +++ b/eng/tool_requirements/sphinx.txt @@ -0,0 +1,5 @@ +# Pins for the azpysdk `sphinx` check. Single source of truth (see README.md). +sphinx==8.2.0 +sphinx_rtd_theme==3.0.2 +myst_parser==4.0.1 +sphinxcontrib-jquery==4.1 diff --git a/eng/tool_requirements/sphinx_next.txt b/eng/tool_requirements/sphinx_next.txt new file mode 100644 index 000000000000..143e3ec4fb22 --- /dev/null +++ b/eng/tool_requirements/sphinx_next.txt @@ -0,0 +1,5 @@ +# Pins for the azpysdk `next-sphinx` check. Single source of truth (see README.md). +sphinx==8.2.0 +sphinx_rtd_theme==3.0.2 +myst_parser==4.0.1 +sphinxcontrib-jquery==4.1 diff --git a/eng/tools/azure-sdk-tools/azpysdk/_tool_reqs.py b/eng/tools/azure-sdk-tools/azpysdk/_tool_reqs.py new file mode 100644 index 000000000000..b79efcb62865 --- /dev/null +++ b/eng/tools/azure-sdk-tools/azpysdk/_tool_reqs.py @@ -0,0 +1,96 @@ +"""Helpers for loading the pinned versions of the third-party tools that the +``azpysdk`` checks install at runtime (mypy, pylint, pyright, sphinx, ...). + +The pins live in ``eng/tool_requirements/.txt`` so they are a single, +machine-scannable source of truth. Keeping them out of the Python modules lets +the daily CFS warm-up (``eng/scripts/warm_cfs_feed.py``) discover the tools and +pre-cache their full transitive dependency closures in the CFS feed, which is +what prevents unauthenticated PR pipelines from failing when an unpinned +transitive dependency releases a new version. + +Each requirements file is an ordinary pip requirements file: one requirement per +line, ``#`` comments and blank lines allowed. +""" + +import os +from typing import List + +from ci_tools.variables import discover_repo_root + +REPO_ROOT = discover_repo_root() + +# Folder that holds the pinned tool requirement files. +TOOL_REQUIREMENTS_DIR = os.path.join(REPO_ROOT, "eng", "tool_requirements") + + +def requirements_path(name: str) -> str: + """Return the absolute path to ``eng/tool_requirements/.txt``. + + :param str name: The requirements file stem (e.g. ``"mypy"`` or ``"pylint_next"``). + :rtype: str + """ + return os.path.join(TOOL_REQUIREMENTS_DIR, f"{name}.txt") + + +def load_requirements(name: str) -> List[str]: + """Load the requirement specifiers from ``eng/tool_requirements/.txt``. + + Comments (``#``) and blank lines are stripped. The returned list is suitable + for passing directly to :func:`ci_tools.functions.install_into_venv`. + + :param str name: The requirements file stem (e.g. ``"mypy"`` or ``"pylint_next"``). + :return: The list of requirement specifiers, in file order. + :rtype: List[str] + """ + path = requirements_path(name) + if not os.path.exists(path): + raise FileNotFoundError(f"No tool requirements file found at {path}") + + specifiers: List[str] = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + stripped = line.split("#", 1)[0].strip() + if stripped: + specifiers.append(stripped) + return specifiers + + +def _requirement_name(specifier: str) -> str: + """Return the lowercased distribution name from a requirement specifier.""" + from packaging.requirements import Requirement + + return Requirement(specifier).name.lower() + + +def pin(name: str, package: str) -> str: + """Return the single requirement specifier for ``package`` from ``.txt``. + + Useful when a check installs the packages from one file across multiple pip + invocations (e.g. pylint installs ``azure-pylint-guidelines-checker`` before + building the target package and ``pylint`` itself afterwards). + + :param str name: The requirements file stem (e.g. ``"pylint"``). + :param str package: The distribution name to look up (e.g. ``"pylint"``). + :return: The full requirement specifier (e.g. ``"pylint==4.0.4"``). + :rtype: str + """ + target = package.lower() + for specifier in load_requirements(name): + if _requirement_name(specifier) == target: + return specifier + raise KeyError(f"'{package}' is not listed in tool requirements file '{name}.txt'") + + +def pinned_version(name: str, package: str) -> str: + """Return just the pinned version string for ``package`` from ``.txt``. + + :param str name: The requirements file stem (e.g. ``"mypy"``). + :param str package: The distribution name to look up (e.g. ``"mypy"``). + :return: The pinned version (e.g. ``"1.19.1"``), or empty string if unpinned. + :rtype: str + """ + from packaging.requirements import Requirement + + requirement = Requirement(pin(name, package)) + specifiers = list(requirement.specifier) + return specifiers[0].version if specifiers else "" diff --git a/eng/tools/azure-sdk-tools/azpysdk/bandit.py b/eng/tools/azure-sdk-tools/azpysdk/bandit.py index 157073fe2b7b..33a13abac9ae 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/bandit.py +++ b/eng/tools/azure-sdk-tools/azpysdk/bandit.py @@ -6,12 +6,15 @@ from subprocess import check_call, CalledProcessError from .Check import Check +from ._tool_reqs import load_requirements, pinned_version from ci_tools.environment_exclusions import is_check_enabled from ci_tools.variables import in_ci, set_envvar_defaults from ci_tools.logging import logger from ci_tools.functions import install_into_venv, get_pip_command -BANDIT_VERSION = "1.6.2" +# Tool version is pinned in eng/tool_requirements/bandit.txt (single source of +# truth). Constant is derived for backwards compatibility. +BANDIT_VERSION = pinned_version("bandit", "bandit") class bandit(Check): @@ -55,7 +58,7 @@ def run(self, args: argparse.Namespace) -> int: try: # pbr is required by the pinned version of bandit - install_into_venv(executable, [f"bandit=={BANDIT_VERSION}", "pbr"], package_dir) + install_into_venv(executable, load_requirements("bandit"), package_dir) except CalledProcessError as e: logger.error(f"Failed to install bandit and dependencies: {e}") return e.returncode diff --git a/eng/tools/azure-sdk-tools/azpysdk/black.py b/eng/tools/azure-sdk-tools/azpysdk/black.py index e74c7d4445ac..e44dd846431c 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/black.py +++ b/eng/tools/azure-sdk-tools/azpysdk/black.py @@ -12,8 +12,11 @@ from ci_tools.logging import logger from .Check import Check +from ._tool_reqs import load_requirements, pinned_version -BLACK_VERSION = "24.4.0" +# Tool version is pinned in eng/tool_requirements/black.txt (single source of +# truth). Constant is derived for backwards compatibility. +BLACK_VERSION = pinned_version("black", "black") REPO_ROOT = discover_repo_root() @@ -115,7 +118,7 @@ def format_directory( issues without modifying files. """ try: - install_into_venv(executable, [f"black=={BLACK_VERSION}"], target_dir) + install_into_venv(executable, load_requirements("black"), target_dir) except CalledProcessError as e: logger.error(f"Failed to install black, skipping formatting: {e}") return None diff --git a/eng/tools/azure-sdk-tools/azpysdk/breaking.py b/eng/tools/azure-sdk-tools/azpysdk/breaking.py index 83f289963cdd..a1a932acd745 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/breaking.py +++ b/eng/tools/azure-sdk-tools/azpysdk/breaking.py @@ -7,12 +7,15 @@ from subprocess import CalledProcessError, check_call from .Check import Check +from ._tool_reqs import load_requirements, pinned_version from ci_tools.functions import install_into_venv from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import discover_repo_root, set_envvar_defaults from ci_tools.logging import logger -JSONDIFF_VERSION = "1.2.0" +# Tool version is pinned in eng/tool_requirements/breaking.txt (single source of +# truth). Constant is derived for backwards compatibility. +JSONDIFF_VERSION = pinned_version("breaking", "jsondiff") REPO_ROOT = discover_repo_root() BREAKING_CHECKER_PATH = os.path.join(REPO_ROOT, "scripts", "breaking_changes_checker") @@ -141,7 +144,7 @@ def run(self, args: argparse.Namespace) -> int: try: install_into_venv( executable, - [f"jsondiff=={JSONDIFF_VERSION}", "-e", BREAKING_CHECKER_PATH], + load_requirements("breaking") + ["-e", BREAKING_CHECKER_PATH], package_dir, ) except CalledProcessError as e: diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py index 2862e1f3f1bc..64b3f6e08604 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -7,20 +7,17 @@ from subprocess import CalledProcessError, check_call from .Check import Check +from ._tool_reqs import load_requirements, pinned_version from ci_tools.functions import install_into_venv from ci_tools.variables import in_ci, set_envvar_defaults from ci_tools.environment_exclusions import is_check_enabled, is_typing_ignored from ci_tools.logging import logger PYTHON_VERSION = "3.10" -MYPY_VERSION = "1.19.1" -NEXT_MYPY_VERSION = "2.1.0" -ADDITIONAL_LOCKED_DEPENDENCIES = [ - "types-chardet==5.0.4.6", - "types-requests==2.31.0.6", - "types-six==1.16.21.9", - "types-redis==4.6.0.7", -] +# Tool versions are pinned in eng/tool_requirements/{mypy,mypy_next}.txt (single +# source of truth). Constants are derived for backwards compatibility. +MYPY_VERSION = pinned_version("mypy", "mypy") +NEXT_MYPY_VERSION = pinned_version("mypy_next", "mypy") class mypy(Check): @@ -52,7 +49,6 @@ def run(self, args: argparse.Namespace) -> int: os.chdir(parsed.folder) package_dir = parsed.folder package_name = parsed.name - additional_requirements = ADDITIONAL_LOCKED_DEPENDENCIES executable, staging_directory = self.get_executable( args.isolate, @@ -66,13 +62,10 @@ def run(self, args: argparse.Namespace) -> int: # # need to install dev_requirements to ensure that type-hints properly resolve self.install_dev_reqs(executable, args, package_dir) - # install mypy + # install mypy (and locked type stubs) from the pinned requirements file try: - if args.next: - # use latest version of mypy - install_into_venv(executable, [f"mypy=={NEXT_MYPY_VERSION}"] + additional_requirements, package_dir) - else: - install_into_venv(executable, [f"mypy=={MYPY_VERSION}"] + additional_requirements, package_dir) + requirements = load_requirements("mypy_next" if args.next else "mypy") + install_into_venv(executable, requirements, package_dir) except CalledProcessError as e: logger.error(f"Failed to install mypy: {e}") return e.returncode diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 92255ad263a7..94080acd38da 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -7,6 +7,7 @@ from subprocess import CalledProcessError, check_call from .Check import Check +from ._tool_reqs import pin, pinned_version from ci_tools.functions import install_into_venv from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults @@ -14,10 +15,12 @@ from ci_tools.logging import logger, run_logged REPO_ROOT = discover_repo_root() -PYLINT_VERSION = "4.0.4" -PYLINT_GUIDELINES_CHECKER_VERSION = "0.5.7" -NEXT_PYLINT_VERSION = "4.0.6" -NEXT_PYLINT_GUIDELINES_CHECKER_VERSION = "0.5.9" +# Tool versions are pinned in eng/tool_requirements/{pylint,pylint_next}.txt +# (single source of truth). Constants are derived for backwards compatibility. +PYLINT_VERSION = pinned_version("pylint", "pylint") +PYLINT_GUIDELINES_CHECKER_VERSION = pinned_version("pylint", "azure-pylint-guidelines-checker") +NEXT_PYLINT_VERSION = pinned_version("pylint_next", "pylint") +NEXT_PYLINT_GUIDELINES_CHECKER_VERSION = pinned_version("pylint_next", "azure-pylint-guidelines-checker") # README snippet files can contain independent code blocks, so imports may be # repeated or appear after executable statements when the blocks share a file. SNIPPET_SAMPLE_IMPORT_DISABLES = ( @@ -100,15 +103,10 @@ def run(self, args: argparse.Namespace) -> int: # install dependencies self.install_dev_reqs(executable, args, package_dir) try: - if args.next: - # use latest version of azure-pylint-guidelines-checker for next pylint checks - cmds = [ - f"azure-pylint-guidelines-checker=={NEXT_PYLINT_GUIDELINES_CHECKER_VERSION}", - ] - else: - cmds = [ - f"azure-pylint-guidelines-checker=={PYLINT_GUIDELINES_CHECKER_VERSION}", - ] + req_file = "pylint_next" if args.next else "pylint" + cmds = [ + pin(req_file, "azure-pylint-guidelines-checker"), + ] cmds.append( "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" ) @@ -136,11 +134,7 @@ def run(self, args: argparse.Namespace) -> int: # install pylint try: - if args.next: - # use latest version of pylint - install_into_venv(executable, [f"pylint=={NEXT_PYLINT_VERSION}"], package_dir) - else: - install_into_venv(executable, [f"pylint=={PYLINT_VERSION}"], package_dir) + install_into_venv(executable, [pin(req_file, "pylint")], package_dir) except CalledProcessError as e: logger.error(f"Failed to install pylint: {e}") return e.returncode diff --git a/eng/tools/azure-sdk-tools/azpysdk/pyright.py b/eng/tools/azure-sdk-tools/azpysdk/pyright.py index 78dcd9790ce5..f4729fa63e80 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pyright.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pyright.py @@ -7,6 +7,7 @@ from subprocess import CalledProcessError from .Check import Check +from ._tool_reqs import load_requirements, pinned_version from ci_tools.functions import install_into_venv from ci_tools.variables import in_ci, set_envvar_defaults, discover_repo_root from ci_tools.environment_exclusions import is_check_enabled, is_typing_ignored @@ -14,8 +15,10 @@ from ci_tools.logging import logger -PYRIGHT_VERSION = "1.1.407" -NEXT_PYRIGHT_VERSION = "1.1.411" +# Tool versions are pinned in eng/tool_requirements/{pyright,pyright_next}.txt +# (single source of truth). Constants are derived for backwards compatibility. +PYRIGHT_VERSION = pinned_version("pyright", "pyright") +NEXT_PYRIGHT_VERSION = pinned_version("pyright_next", "pyright") REPO_ROOT = discover_repo_root() @@ -89,11 +92,8 @@ def run(self, args: argparse.Namespace) -> int: logger.info(f"Processing {package_name} for pyright check") try: - if args.next: - # use latest version of pyright - install_into_venv(executable, [f"pyright=={NEXT_PYRIGHT_VERSION}"], package_dir) - else: - install_into_venv(executable, [f"pyright=={PYRIGHT_VERSION}"], package_dir) + requirements = load_requirements("pyright_next" if args.next else "pyright") + install_into_venv(executable, requirements, package_dir) except CalledProcessError as e: logger.error("Failed to install pyright:", e) return e.returncode diff --git a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py index dead8176eca3..3c274de41b9e 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py +++ b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py @@ -11,21 +11,23 @@ from pathlib import Path from .Check import Check +from ._tool_reqs import load_requirements, pinned_version from ci_tools.functions import install_into_venv, unzip_file_to_directory from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import in_ci, set_envvar_defaults, discover_repo_root, in_analyze_weekly from ci_tools.logging import logger -# dependencies -SPHINX_VERSION = "8.2.0" -NEXT_SPHINX_VERSION = "8.2.0" -SPHINX_RTD_THEME_VERSION = "3.0.2" -NEXT_SPHINX_RTD_THEME_VERSION = "3.0.2" -MYST_PARSER_VERSION = "4.0.1" -NEXT_MYST_PARSER_VERSION = "4.0.1" -SPHINX_CONTRIB_JQUERY_VERSION = "4.1" -NEXT_SPHINX_CONTRIB_JQUERY_VERSION = "4.1" +# Tool versions are pinned in eng/tool_requirements/{sphinx,sphinx_next}.txt +# (single source of truth). Constants are derived for backwards compatibility. +SPHINX_VERSION = pinned_version("sphinx", "sphinx") +NEXT_SPHINX_VERSION = pinned_version("sphinx_next", "sphinx") +SPHINX_RTD_THEME_VERSION = pinned_version("sphinx", "sphinx_rtd_theme") +NEXT_SPHINX_RTD_THEME_VERSION = pinned_version("sphinx_next", "sphinx_rtd_theme") +MYST_PARSER_VERSION = pinned_version("sphinx", "myst_parser") +NEXT_MYST_PARSER_VERSION = pinned_version("sphinx_next", "myst_parser") +SPHINX_CONTRIB_JQUERY_VERSION = pinned_version("sphinx", "sphinxcontrib-jquery") +NEXT_SPHINX_CONTRIB_JQUERY_VERSION = pinned_version("sphinx_next", "sphinxcontrib-jquery") RST_EXTENSION_FOR_INDEX = """ @@ -257,28 +259,8 @@ def run(self, args: argparse.Namespace) -> int: # install sphinx try: - if args.next: - install_into_venv( - executable, - [ - f"sphinx=={NEXT_SPHINX_VERSION}", - f"sphinx_rtd_theme=={NEXT_SPHINX_RTD_THEME_VERSION}", - f"myst_parser=={NEXT_MYST_PARSER_VERSION}", - f"sphinxcontrib-jquery=={NEXT_SPHINX_CONTRIB_JQUERY_VERSION}", - ], - package_dir, - ) - else: - install_into_venv( - executable, - [ - f"sphinx=={SPHINX_VERSION}", - f"sphinx_rtd_theme=={SPHINX_RTD_THEME_VERSION}", - f"myst_parser=={MYST_PARSER_VERSION}", - f"sphinxcontrib-jquery=={SPHINX_CONTRIB_JQUERY_VERSION}", - ], - package_dir, - ) + requirements = load_requirements("sphinx_next" if args.next else "sphinx") + install_into_venv(executable, requirements, package_dir) except CalledProcessError as e: logger.error(f"Failed to install sphinx: {e}") return e.returncode diff --git a/eng/tools/azure-sdk-tools/azpysdk/verifytypes.py b/eng/tools/azure-sdk-tools/azpysdk/verifytypes.py index a6c25465ad04..ffcb9d194d20 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/verifytypes.py +++ b/eng/tools/azure-sdk-tools/azpysdk/verifytypes.py @@ -11,6 +11,7 @@ from subprocess import CalledProcessError from .Check import Check +from ._tool_reqs import load_requirements, pinned_version from ci_tools.functions import install_into_venv from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults @@ -18,7 +19,9 @@ from ci_tools.functions import get_pip_command from ci_tools.logging import logger -PYRIGHT_VERSION = "1.1.407" +# verifytypes uses the same pyright pin as the pyright check; the version is +# pinned in eng/tool_requirements/pyright.txt (single source of truth). +PYRIGHT_VERSION = pinned_version("pyright", "pyright") REPO_ROOT = discover_repo_root() @@ -64,7 +67,7 @@ def run(self, args: argparse.Namespace) -> int: # install pyright try: - install_into_venv(executable, [f"pyright=={PYRIGHT_VERSION}"], package_dir) + install_into_venv(executable, load_requirements("pyright"), package_dir) except CalledProcessError as e: logger.error(f"Failed to install pyright: {e}") return e.returncode diff --git a/eng/tools/azure-sdk-tools/tests/test_tool_reqs.py b/eng/tools/azure-sdk-tools/tests/test_tool_reqs.py new file mode 100644 index 000000000000..eafc1a8b8e51 --- /dev/null +++ b/eng/tools/azure-sdk-tools/tests/test_tool_reqs.py @@ -0,0 +1,38 @@ +from azpysdk import _tool_reqs + + +def test_load_requirements_strips_comments_and_blanks(): + specs = _tool_reqs.load_requirements("mypy") + assert "mypy==1.19.1" in specs + # the locked type stubs travel with the mypy pin + assert "types-requests==2.31.0.6" in specs + # comment lines and blank lines are not returned + assert all(not spec.startswith("#") for spec in specs) + assert all(spec.strip() for spec in specs) + + +def test_pin_returns_single_specifier(): + assert _tool_reqs.pin("pylint", "pylint") == "pylint==4.0.4" + assert _tool_reqs.pin("pylint", "azure-pylint-guidelines-checker") == "azure-pylint-guidelines-checker==0.5.7" + + +def test_pin_is_case_insensitive(): + assert _tool_reqs.pin("pylint", "PyLint") == "pylint==4.0.4" + + +def test_pin_missing_package_raises(): + try: + _tool_reqs.pin("black", "not-a-real-tool") + except KeyError: + return + raise AssertionError("expected KeyError for a package not listed in the file") + + +def test_pinned_version_returns_version_only(): + assert _tool_reqs.pinned_version("mypy", "mypy") == "1.19.1" + assert _tool_reqs.pinned_version("sphinx", "sphinxcontrib-jquery") == "4.1" + + +def test_verifytypes_and_pyright_share_a_single_pin(): + # verifytypes intentionally reuses pyright.txt so the pin lives in one place + assert _tool_reqs.load_requirements("pyright") == ["pyright==1.1.407"]