diff --git a/.gitignore b/.gitignore index 2f3c741..ed9987f 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ cython_debug/ src/spdx_python_model/bindings/ gen/**/*.py gen/**/*.pyi + +# Generated at build time by gen/generate-protocols +src/spdx_python_model/protocols/_domain.py diff --git a/README.md b/README.md index 4cf44ca..995e4d9 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,63 @@ to get started with spdx-python-model. [tutorial]: https://spdx.github.io/spdx-python-model/tutorial/using-spdx3.html +### Version-agnostic types + +While SPDX 3 keeps backward compatibility across minor versions, internally in +`spdx-python-model` each minor version has its own set of generated types, so +`v3_0_1.SHACLObjectSet` and `v3_1.SHACLObjectSet` are technically distinct. +To assist strict type checking, the package exposes version-neutral Python +[`Protocol`](https://docs.python.org/3/library/typing.html#typing.Protocol) +structural types you can use to write functions that work with any 3.x version +and still pass strict type checking: + +```python +from spdx_python_model import SpdxObjectSet, load + +def count_persons(objset: SpdxObjectSet) -> int: + return sum(1 for _ in objset.foreach_type("Person")) + +model, objset = load(path) # objset is typed as SpdxObjectSet +print(count_persons(objset)) +``` + +Three core structural types are available: + +- `SpdxObjectSet` — a collection of SPDX objects (`SHACLObjectSet`) +- `SpdxObject` — a single SPDX object (`SHACLObject`) +- `SpdxModelModule` — a version submodule (e.g. the `model` returned by `load()`) + +In addition, the `spdx_python_model.protocols` submodule provides a +version-agnostic protocol for every SPDX class (`protocols.Element`, +`protocols.Relationship`, `protocols.CreationInfo`, …). A function annotated with +these reads any property (typed) and writes scalar or object-reference properties, +for any 3.x version: + +```python +from typing import Optional +from spdx_python_model import protocols + +def relabel(e: protocols.Element, ci: protocols.CreationInfo) -> Optional[str]: + e.name = "renamed" # write a scalar property + e.creationInfo = ci # write an object-reference property + return e.name # read it back (typed) +``` + +These are for static typing only. Construct objects using a concrete version: + +```python +# From a known version module +p = v3_0_1.Person() + +# Or from the model returned by load() +model, objset = load(path) +p = model.Person() +``` + +When writing object-reference properties, the assigned value is accepted as +`Any` — the value must belong to the same SPDX version as the object it is added +to (this is enforced at runtime). Construction always uses a concrete version. + ## Testing This repository has support for running tests against the bindings using `pytest`. diff --git a/gen/generate-protocols b/gen/generate-protocols new file mode 100755 index 0000000..c10b28e --- /dev/null +++ b/gen/generate-protocols @@ -0,0 +1,177 @@ +#! /usr/bin/env python3 +# +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# +# Generate version-agnostic domain Protocols (_domain.py) from the baseline +# version's model.pyi stub. Run after generate-bindings, from the gen/ dir. +# +# SPDX 3 keeps minor versions backward compatible, so the baseline (oldest) +# version's surface is the common subset every newer version satisfies. One +# Protocol is emitted per SPDX domain class, named identically to the binding +# class (so callers write e.g. ``protocols.Element``): +# - scalar properties -> plain attribute (read + write) +# - object-ref / list props -> typed @property getter (covariant read) plus an +# Any setter (version-agnostic write) + +from __future__ import annotations + +import ast +import re +import sys +from pathlib import Path + +# Machinery base classes that map to the hand-written core SpdxObject protocol. +_BASE_TO_CORE = {"SHACLObject", "SHACLExtensibleObject"} + +HEADER = '''\ +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# +# Generated from {baseline}/model.pyi at build time. DO NOT EDIT. +"""Version-agnostic domain Protocols, one per SPDX 3 class. + +Each protocol is satisfied by the corresponding class of every supported SPDX 3 +minor version. Object-reference properties expose a typed getter (read) and an +Any setter (write); construct objects with a concrete version. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Iterable, Optional, Protocol, Union + +from .core import SpdxObject +''' + + +def _kwonly_props(cls: ast.ClassDef) -> list[str]: + """Property names = the __init__ keyword-only arguments of this class. + + Using __init__ (rather than the body annotations) reliably excludes ClassVars + and named-individual string constants, which are not constructor arguments. + """ + for node in cls.body: + if isinstance(node, ast.FunctionDef) and node.name == "__init__": + return [a.arg for a in node.args.kwonlyargs] + return [] + + +def _body_read_types(cls: ast.ClassDef) -> dict[str, str]: + """Map each property name to its READ-type annotation (as source).""" + types: dict[str, str] = {} + for node in cls.body: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + ann = ast.unparse(node.annotation) + if ann.startswith("ClassVar["): + continue + types[node.target.id] = ann + elif isinstance(node, ast.FunctionDef) and node.returns is not None: + if any( + isinstance(d, ast.Name) and d.id == "property" + for d in node.decorator_list + ): + types[node.name] = ast.unparse(node.returns) + return types + + +def _collect(tree: ast.Module) -> dict[str, ast.ClassDef]: + return {n.name: n for n in tree.body if isinstance(n, ast.ClassDef)} + + +def _domain_classes(classes: dict[str, ast.ClassDef]) -> list[str]: + """Names of classes that (transitively) derive from SHACLObject, in source + order, excluding the machinery bases themselves.""" + out: list[str] = [] + for name, cls in classes.items(): + if name in _BASE_TO_CORE: + continue + seen: set[str] = set() + stack = list(cls.bases) + while stack: + b = stack.pop() + if not isinstance(b, ast.Name) or b.id in seen: + continue + seen.add(b.id) + if b.id in _BASE_TO_CORE: + out.append(name) + break + parent = classes.get(b.id) + if parent is not None: + stack.extend(parent.bases) + return out + + +def _map_base(name: str) -> str: + return "SpdxObject" if name in _BASE_TO_CORE else name + + +def _is_object_ref(ann: str, domain: set[str]) -> bool: + if "ListProxy[" in ann: + return True + return bool(set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", ann)) & domain) + + +def _emit_class(cls: ast.ClassDef, domain: set[str]) -> str: + bases = [_map_base(b.id) for b in cls.bases if isinstance(b, ast.Name)] or [ + "SpdxObject" + ] + base_list = ", ".join(dict.fromkeys(bases + ["Protocol"])) + + read_types = _body_read_types(cls) + lines = [f"class {cls.name}({base_list}):"] + body: list[str] = [] + for p in _kwonly_props(cls): + ann = read_types.get(p) + if ann is None: + continue + # The read view of a list property is an Iterable, not the ListProxy. + ann = ann.replace("ListProxy[", "Iterable[") + if _is_object_ref(ann, domain): + body.append(" @property") + body.append(f" def {p}(self) -> {ann}: ...") + body.append(f" @{p}.setter") + body.append(f" def {p}(self, value: Any) -> None: ...") + else: + body.append(f" {p}: {ann}") + if not body: + body.append(" pass") + return "\n".join(lines + body) + + +def generate(stub_src: str, baseline: str) -> str: + tree = ast.parse(stub_src) + classes = _collect(tree) + domain = _domain_classes(classes) + domain_set = set(domain) + blocks = [_emit_class(classes[name], domain_set) for name in domain] + all_block = "".join(f' "{d}",\n' for d in domain) + return ( + HEADER.format(baseline=baseline) + + f"\n__all__ = [\n{all_block}]\n\n\n" + + "\n\n\n".join(blocks) + + "\n" + ) + + +def _find_baseline(gen_dir: Path) -> Path: + """The oldest generated version directory (lexicographically first vN_...).""" + candidates = sorted( + p for p in gen_dir.iterdir() if p.is_dir() and re.match(r"v\d", p.name) + ) + for p in candidates: + if (p / "model.pyi").exists(): + return p + raise SystemExit("generate-protocols: no baseline model.pyi found") + + +def main() -> int: + gen_dir = Path(__file__).resolve().parent + baseline = _find_baseline(gen_dir) + out = generate((baseline / "model.pyi").read_text(), baseline.name) + (gen_dir / "_domain.py").write_text(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 94b511c..38ecb34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,20 @@ artifacts = [ "*.pyi", ] +# Generate the version-agnostic domain Protocols from the baseline bindings. +# Runs after generate-bindings (reads its baseline model.pyi). clean_out_dir is +# false so the hand-written core.py / __init__.py in the package are preserved. +[[tool.hatch.build.hooks.build-scripts.scripts]] +out_dir = "src/spdx_python_model/protocols" +work_dir = "gen" +clean_out_dir = false +commands = [ + "./generate-protocols" +] +artifacts = [ + "_domain.py", +] + [tool.hatch.version] path = "src/spdx_python_model/version.py" @@ -83,5 +97,8 @@ addopts = [ ] [tool.pylint.main] -# The generated binding fails style checks. Ignore it. -ignore-paths = ['^src/spdx_python_model/bindings/v3_0_1/.*$'] +# The generated binding and domain protocols fail style checks. Ignore them. +ignore-paths = [ + '^src/spdx_python_model/bindings/v3_0_1/.*$', + '^src/spdx_python_model/protocols/_domain.py$', +] diff --git a/src/spdx_python_model/__init__.py b/src/spdx_python_model/__init__.py index 51d39bb..539c72b 100644 --- a/src/spdx_python_model/__init__.py +++ b/src/spdx_python_model/__init__.py @@ -8,9 +8,10 @@ import json from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any, List, Tuple, cast from .bindings import _CONTEXT_TABLE +from .protocols import SpdxModelModule, SpdxObject, SpdxObjectSet from .version import VERSION from .version import VERSION as __version__ @@ -22,6 +23,9 @@ __all__ = [ "bindings", # generated # noqa: F405 "LoadError", + "SpdxModelModule", + "SpdxObject", + "SpdxObjectSet", "VERSION", "__version__", "load", @@ -47,14 +51,17 @@ def __dir__() -> List[str]: return sorted(set(globals()) | _VERSION_MODULES) -def load_data(data: Any) -> Tuple[ModuleType, Any]: +def load_data(data: Any) -> Tuple[ModuleType, "SpdxObjectSet"]: """ Automatically load a SPDX 3 JSON document with the correct model based on its context :param data: The decoded JSON data as a Python dict - :returns: A tuple that contains the model and the decoded SHACLObjectSet + :returns: A tuple that contains the model and the decoded SHACLObjectSet. + The object set is typed as the version-agnostic + :class:`~spdx_python_model.protocols.SpdxObjectSet`; the model is the + concrete version submodule (a ``ModuleType``). :raises LoadError: If the data is missing a context or if the context is not recognized @@ -92,17 +99,20 @@ def load_data(data: Any) -> Tuple[ModuleType, Any]: d.deserialize_data(data, objset) - return model, objset + return model, cast("SpdxObjectSet", objset) -def load(path: Path) -> Tuple[ModuleType, Any]: +def load(path: Path) -> Tuple[ModuleType, "SpdxObjectSet"]: """ Automatically load a SPDX 3 JSON document with the correct model based on its context :param path: The path to the SPDX 3 JSON file - :returns: A tuple that contains the model and the decoded SHACLObjectSet + :returns: A tuple that contains the model and the decoded SHACLObjectSet. + The object set is typed as the version-agnostic + :class:`~spdx_python_model.protocols.SpdxObjectSet`; the model is the + concrete version submodule (a ``ModuleType``). :raises LoadError: If the data is missing a context or if the context is not recognized diff --git a/src/spdx_python_model/protocols/__init__.py b/src/spdx_python_model/protocols/__init__.py new file mode 100644 index 0000000..b70f9c8 --- /dev/null +++ b/src/spdx_python_model/protocols/__init__.py @@ -0,0 +1,25 @@ +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Version-agnostic structural types (PEP 544 Protocols) for the SPDX 3 bindings. + +- Core protocols (:mod:`.core`, hand-written): ``SpdxObject``, ``SpdxObjectSet``, + ``SpdxModelModule`` — the version-neutral machinery surface. +- Domain protocols (:mod:`._domain`, generated at build time from the baseline + version's stub): one per SPDX class, e.g. ``protocols.Element``, + ``protocols.CreationInfo``. + +See the package README for guidance on when to annotate with a protocol versus a +concrete version type. +""" + +from . import _domain +from ._domain import * # noqa: F401, F403 +from .core import SpdxModelModule, SpdxObject, SpdxObjectSet + +__all__ = [ + "SpdxModelModule", + "SpdxObject", + "SpdxObjectSet", + *_domain.__all__, +] diff --git a/src/spdx_python_model/protocols/core.py b/src/spdx_python_model/protocols/core.py new file mode 100644 index 0000000..b92181a --- /dev/null +++ b/src/spdx_python_model/protocols/core.py @@ -0,0 +1,71 @@ +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Version-agnostic structural types (PEP 544 Protocols) for the SPDX 3 bindings. + +Each SPDX 3 minor version (``v3_0_1``, ``v3_1``, …) is generated as a distinct +set of nominal types. Because SPDX 3 keeps minor versions backward compatible, +they are structurally compatible: these protocols capture the shared surface so +one function or class can work with any 3.x version under ``mypy --strict``. + +Static typing only (no ``@runtime_checkable``); no runtime imports from the +bindings. +""" + +from __future__ import annotations + +from typing import Any, Iterable, Iterator, Optional, Protocol, Set, Tuple + +__all__ = [ + "SpdxObject", + "SpdxObjectSet", + "SpdxModelModule", +] + + +class SpdxObject(Protocol): + """Version-agnostic view of a single SPDX object (the ``SHACLObject`` base).""" + + def get_id(self) -> Optional[str]: ... + def set_id(self, value: Optional[str]) -> None: ... + def get_type(self) -> str: ... + def get_compact_type(self) -> Optional[str]: ... + def property_keys(self) -> Iterator[Tuple[Optional[str], str, Optional[str]]]: ... + + +class SpdxObjectSet(Protocol): + """Version-agnostic view of a ``SHACLObjectSet`` (read / query / link surface). + + Omits the raw index attributes (``objects``, ``obj_by_id``, ``missing_ids``, + ``context``) and the version-coupled methods (``encode``/``decode``, + ``merge``); use a concrete version type for those. + """ + + def find_by_id(self, _id: str, default: Any = None) -> Optional[SpdxObject]: ... + def foreach(self) -> Iterable[SpdxObject]: ... + def foreach_type( + self, typ: str, *, match_subclass: bool = True + ) -> Iterable[SpdxObject]: ... + def link(self) -> Set[str]: ... + + # Object parameters are ``Any``: no version-neutral type is a supertype of + # every version's ``SHACLObject``. + def add(self, obj: Any) -> Any: ... + def remove(self, obj: Any) -> None: ... + def update(self, *others: Iterable[Any]) -> None: ... + def __contains__(self, item: Any) -> bool: ... + + +class SpdxModelModule(Protocol): + """Version-agnostic view of a bindings module (e.g. the model returned by + :func:`spdx_python_model.load`). + + Factories are declared as methods (covariant return) so every version's + module satisfies the protocol; ``__getattr__`` exposes version-specific + classes such as ``Person`` as ``Any``. + """ + + def SHACLObjectSet(self) -> SpdxObjectSet: ... + def JSONLDDeserializer(self) -> Any: ... + def JSONLDSerializer(self) -> Any: ... + def __getattr__(self, name: str) -> Any: ... diff --git a/tests/test_protocols.py b/tests/test_protocols.py new file mode 100644 index 0000000..2b67df4 --- /dev/null +++ b/tests/test_protocols.py @@ -0,0 +1,194 @@ +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# +# Every generated version's concrete classes must satisfy the protocols under +# strict mypy, and all versions must unify under the one structural type. The set +# of generated versions varies by build (draft versions such as v3_1 are not +# always present), so the probe is built from the versions actually present. + +from __future__ import annotations + +from pathlib import Path + +PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml" +DATA_DIR = Path(__file__).parent / "data" + +# Each version's instances of these classes must satisfy SpdxObject. +_CLASSES = [ + "Person", + "Element", + "ElementCollection", + "SpdxDocument", + "CreationInfo", + "Relationship", + "RelationshipType", + "IndividualElement", + "software_Package", + "software_Sbom", + "simplelicensing_AnyLicenseInfo", + "expandedlicensing_IndividualLicensingInfo", + "expandedlicensing_License", +] + +# Named-individual IRI constants (str) that must exist, as (owning class, name). +_NAMED_INDIVIDUALS = [ + ("IndividualElement", "NoneElement"), + ("IndividualElement", "NoAssertionElement"), + ("expandedlicensing_IndividualLicensingInfo", "NoAssertionLicense"), + ("expandedlicensing_IndividualLicensingInfo", "NoneLicense"), +] + + +def _available_versions() -> list[str]: + """Module names (e.g. "v3_0_1") of every generated version binding.""" + import spdx_python_model.bindings as bindings + + pkg_dir = Path(bindings.__file__).resolve().parent + return sorted( + p.name + for p in pkg_dir.iterdir() + if p.is_dir() and p.name.startswith("v") and (p / "model.py").exists() + ) + + +def _domain_protocol_names() -> list[str]: + """Names of the generated domain protocols (== their binding class names).""" + from spdx_python_model.protocols import _domain + + return list(_domain.__all__) + + +def _build_domain_probe(versions: list[str], names: list[str]) -> str: + """Every generated domain protocol must be satisfied by the matching concrete + class of every version. Protocol names match the binding class names, so + `protocols.Element` is satisfied by `vX.Element()`.""" + header = ( + "from spdx_python_model import protocols\n" + + "".join(f"from spdx_python_model.bindings import {v}\n" for v in versions) + + "\n" + ) + checks = "" + for v in versions: + for i, name in enumerate(names): + checks += f"d{i}_{v}: protocols.{name} = {v}.{name}()\n" + checks += "\n" + return header + checks + + +# A fully-typed, version-agnostic function and class a library user could write: +# reads and writes a scalar and an object-reference property, accepted for every +# version. This is the headline Phase 2 capability. +_USER_SCENARIO = """\ +from typing import Optional +from spdx_python_model import protocols +{imports} + +def relabel(e: protocols.Element, ci: protocols.CreationInfo) -> Optional[str]: + e.name = "renamed" # write scalar + e.creationInfo = ci # write object reference + got = e.creationInfo # read object reference (typed) + if isinstance(got, str) or got is None: + return None + c: Optional[str] = got.comment + return c + + +class Report: + def __init__(self, root: protocols.Element) -> None: + self.root: protocols.Element = root + + def title(self) -> Optional[str]: + return self.root.name + +{calls} +""" + + +def _build_user_scenario(versions: list[str]) -> str: + imports = "\n".join(f"from spdx_python_model.bindings import {v}" for v in versions) + calls = "\n".join( + f"relabel({v}.Person(), {v}.CreationInfo())\nReport({v}.Person()).title()" + for v in versions + ) + return _USER_SCENARIO.format(imports=imports, calls=calls) + + +def _build_probe(versions: list[str]) -> str: + header = ( + "from typing import List\n" + "from spdx_python_model import SpdxModelModule, SpdxObject, SpdxObjectSet\n" + + "".join(f"from spdx_python_model.bindings import {v}\n" for v in versions) + + "\n\n" + "def count(s: SpdxObjectSet) -> int:\n" + " return sum(1 for _ in s.foreach())\n\n\n" + ) + + checks = "" + for v in versions: + checks += ( + # Assignment to protocol type forces the static conformance check. + f"module_{v}: SpdxModelModule = {v}\n" + f"objset_{v}: SpdxObjectSet = {v}.SHACLObjectSet()\n" + f"count(objset_{v})\n" + f"count(module_{v}.SHACLObjectSet())\n" + f"module_{v}.SHACLObjectSet().add(module_{v}.Person())\n" + ) + for i, cls in enumerate(_CLASSES): + checks += f"c{i}_{v}: SpdxObject = {v}.{cls}()\n" + for i, (cls, name) in enumerate(_NAMED_INDIVIDUALS): + checks += f"n{i}_{v}: str = {v}.{cls}.{name}\n" + checks += "\n" + + # All versions' object sets coexist in one List[SpdxObjectSet]. + all_sets = ", ".join(f"objset_{v}" for v in versions) + checks += ( + f"every: List[SpdxObjectSet] = [{all_sets}]\n" + "for _s in every:\n" + " count(_s)\n" + ) + return header + checks + + +def _assert_strict_mypy_clean(tmp_path, source: str) -> None: + from mypy import api + + probe = tmp_path / "probe.py" + probe.write_text(source) + stdout, stderr, status = api.run( + ["--config-file", str(PYPROJECT), "--strict", str(probe)] + ) + assert status == 0, stdout + stderr + + +def test_strict_mypy_core_protocols_satisfied_by_all_versions(tmp_path): + versions = _available_versions() + assert versions, "no generated version bindings found" + _assert_strict_mypy_clean(tmp_path, _build_probe(versions)) + + +def test_strict_mypy_domain_protocols_satisfied_by_all_versions(tmp_path): + versions = _available_versions() + names = _domain_protocol_names() + assert names, "no generated domain protocols found" + _assert_strict_mypy_clean(tmp_path, _build_domain_probe(versions, names)) + + +def test_strict_mypy_user_scenario_works_across_versions(tmp_path): + versions = _available_versions() + _assert_strict_mypy_clean(tmp_path, _build_user_scenario(versions)) + + +def test_loaded_objset_is_usable_through_protocol(): + import spdx_python_model + from spdx_python_model import SpdxObjectSet + + _model, objset = spdx_python_model.load( + DATA_DIR / "3.0.1" / "example.spdx3.json" + ) + + def names(s: SpdxObjectSet) -> list: + return [o.get_id() for o in s.foreach_type("Person")] + + assert isinstance(list(objset.foreach()), list) + assert names(objset) is not None + assert objset.find_by_id("does-not-exist", None) is None