diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6c95994..1b9b1c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,22 +44,11 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - uses: compas-dev/compas-actions/setup-python@v1 + - uses: compas-dev/compas-actions/release-assets@v1 with: python-version: "3.12" management-tool: uv extras: dev - - - name: Test asset creation - run: invoke pre-build - - - name: Upload test artifacts - uses: actions/upload-artifact@v7.0.1 - with: - name: protobuf-test-assets - path: dist/proto/*.zip - if-no-files-found: error + invoke-tasks: pre-build + paths: dist/proto/*.zip + artifact-name: protobuf-test-assets diff --git a/.gitignore b/.gitignore index 9181ddb..7228f48 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,8 @@ cSharp/**/bin/* cSharp/**/obj/* *.bin + +# Transient per-language protobuf output: create_class_assets generates each language, +# zips it for release upload, then removes the folder. The Python bindings beside them +# are committed on purpose. +src/compas_pb/generated/*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 34b9f0b..19a902a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Added `create-proto-bundle`, which zips the `.proto` schemas for upload as a release asset, so downstream generators can pin a schema version instead of reading files out of this repository. +* Added TypeScript to the generated bindings, via `@bufbuild/protoc-gen-es`. `PROTO_PLUGIN_LANGUAGES` maps a language to the flag its protoc plugin registers, so javascript and go can be added the same way. +* Added an Architecture page describing how one domain model reaches many languages, and who owns `.proto` files versus who implements a runtime. +* Added a guide for implementing a `compas_pb` runtime in a new language. + ### Changed +* Changed the asset tasks to take their package name and output folder from the invoke configuration, so any package that owns `.proto` files can reuse them. `base_folder` and `proto_folder` now accept a string or a `Path`. +* Changed assets built by a protoc plugin to carry the plugin version rather than the protoc version, since the plugin is what shapes the generated API. +* Changed `generate-proto-classes` to raise on an unsupported target language, instead of warning and then running protoc with an empty output flag. +* Changed the asset job to use `compas-dev/compas-actions/release-assets@v1`. + ### Removed diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..7a919b1 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,257 @@ +# Architecture + +`compas_pb` is the Python reference implementation of a cross-language serialization +architecture. This page documents that architecture so sibling implementations +(`compas_pb_ts` for TypeScript, `compas_pb_csharp` for C#) and packages that own their own +domain models can follow the same contract. + +The goal is a single wire format that carries **domain model objects** — not just the +generated `*Data` protobuf messages — between Python, TypeScript, C# and any other language +that gains a runtime. A `Frame` sent from Grasshopper arrives in a browser agent as a +`Frame`, and an Antikythera `TaskAssignmentMessage` arrives in Python as a +`TaskAssignmentMessage`, without either side knowing anything about the other's language. + +## Two kinds of package + +Every package in this architecture is one of two things. Keeping the distinction sharp is +what makes the system extensible. + +| | Domain model owner | Language runtime | +| --- | --- | --- | +| **Examples** | `compas_pb`[^1], `antikythera`, `compas_timber` | `compas_pb`, `compas_pb_ts`, `compas_pb_csharp` | +| **Owns** | `.proto` files and the domain classes they mirror | registry, discovery, recursive codec | +| **Publishes** | proto bundle + generated bindings, every release | a serialization library for one language | +| **Knows about** | its own types only | no domain types at all | + +[^1]: + `compas_pb` is both, exceptionally. It owns the `.proto` files for COMPAS core types + (`Point`, `Frame`, `Mesh`, `Graph`, …) because protobuf support has not been upstreamed + into `compas` core yet. For every practical purpose, treat `compas_pb`'s ownership of + those schemas as if it were core's own. + +A language runtime never imports a domain package, and a domain package never implements +codec logic. They meet at the registry. + +## How one domain model reaches many languages + +The same three-part structure repeats in every language a domain owner supports: generated +bindings, domain classes that mirror the owner's own model, and a conversions module that +registers the mapping between them. Only the registration mechanism differs. + +```mermaid +flowchart TB + proto["antikythera/proto/*.proto
single source of truth"] + rel["Release vX.Y.Z
proto bundle · typescript · c# · c++ bindings"] + proto ==> rel + + subgraph ts["TypeScript — proposed"] + direction TB + tsm["antikythera_ts
TaskAssignmentMessage
mirrors the Python model"] + tsc["antikythera_ts/conversions
registerSerializer · registerDeserializer"] + tsr["compas_pb_ts
registry · discovery · codec"] + tsm --> tsc + tsc -->|"registers via
register()"| tsr + end + + subgraph py["Python — shipping today"] + direction TB + pym["antikythera.models
TaskAssignmentMessage
subclasses compas.data.Data"] + pyc["antikythera.models.conversions
@pb_serializer · @pb_deserializer"] + pyr["compas_pb
registry · discovery · codec"] + pym --> pyc + pyc -->|"registers via
entry point"| pyr + end + + rel -.->|"bindings"| pym + rel -.->|"bindings"| tsm + + wire["MessageData bytes
one wire format, both directions"] + pyr ==> wire + tsr ==> wire + + classDef ghost stroke-dasharray:4 3 + class tsm,tsc,tsr ghost +``` + +/// caption +One domain model, two languages. Solid borders exist today; dashed borders are proposed. +The proto bundle is what keeps the two columns describing the same wire — without a +published artifact, each language's bindings drift independently. +/// + +The registration arrow is the whole point of the design: `compas_pb` contains no reference +to Antikythera, and Antikythera contains no codec logic, yet +`pb_dump_bts(TaskAssignmentMessage(...))` works. + +## The wire format + +Every message is a `MessageData` envelope carrying a version tag and exactly one `AnyData`. +`AnyData` is a `oneof`, so a value occupies exactly one arm. Which arm it occupies decides +what the reader gets back. + +```mermaid +flowchart LR + md["MessageData"] --> ver["version: string
e.g. 1.1.4"] + md --> any["AnyData
oneof data"] + + any --> m["message: Any
a registered type"] + any --> v["value: Value
null · bool · str · bytes"] + any --> f["fallback: FallbackData"] + any --> iv["int_value: int64"] + any --> dv["double_value: double"] + any --> dict["dict_value: DictData"] + any --> list["list_value: ListData"] + + dict -.->|"map<string, AnyData>
recurses"| any + list -.->|"repeated AnyData
recurses"| any + f -->|"DictData"| decoder["runs DataDecoder
reconstructs the object"] + dict --> plain["stays a plain dict"] + + classDef hot stroke-width:3px + class f,decoder hot +``` + +/// caption +The seven arms of `AnyData`. `dict_value` and `list_value` recurse back into `AnyData`, +which is what makes arbitrary nesting work. The thick path is the one that surprises +people: `fallback` is the **only** arm whose decode runs `DataDecoder` and reconstructs a +COMPAS object. An envelope-shaped dict sent as `dict_value` arrives as a bare dict. +/// + +Two consequences worth stating explicitly, because implementations get them wrong: + +- **`int_value` and `double_value` exist so numbers survive a round trip.** A + `google.protobuf.Value` coerces everything to double, so `3` returns as `3.0`. An integer + must use `int_value`; an integral float must use `double_value`. +- **`bytes` travel as a string** prefixed `base64:` inside `value`, since + `google.protobuf.Value` has no bytes kind. + +## Recursive dispatch + +A runtime's encoder is one recursive function. It is worth reading as a decision tree, +because every arm above corresponds to exactly one branch. + +```mermaid +flowchart TB + start(["serialize(obj)
re-entered by each recursion"]) --> islist{"list
or tuple?"} + islist -->|yes| lv["list_value
← recurse per item"] + islist -->|no| isdict{"dict?"} + isdict -->|yes| dv["dict_value
← recurse per value"] + isdict -->|no| reg{"registered
in registry?"} + reg -->|yes| msg["message: Any
pack with type_url"] + reg -->|no| isdata{"a domain object?
Python: isinstance Data"} + isdata -->|yes| fb["fallback
← serialize its dict form"] + isdata -->|no| prim{"primitive?"} + prim -->|"int"| iv["int_value"] + prim -->|"float"| ddv["double_value"] + prim -->|"bool · str · bytes · None"| val["value"] + prim -->|no| err(["raise TypeError"]) + + + classDef term stroke-width:2px + class lv,dv,msg,fb,iv,ddv,val term +``` + +/// caption +`compas_pb.core._serializer_any` as a decision tree. Registry lookup sits *between* the +container arms and the fallback arm — a registered type gets its native protobuf message, +and only an unregistered domain object degrades to `fallback`. The three container arms +re-enter at the top, once per item. + +Decoding is the mirror image, dispatching on which `oneof` arm is set. +/// + +The registry lookup in the middle is where plugins take effect. In Python the lookup walks +the type's MRO, so registering a serializer for a base class covers its subclasses. + +## Distributing schemas + +A domain model owner publishes its `.proto` bundle and generated bindings as release +artifacts. Consumers pin a version and download it. This replaces the pattern that grew up +by default, where every consumer wrote its own script to scrape `.proto` files out of a git +repository. + +```mermaid +flowchart LR + subgraph before["Today — each consumer rolls its own fetch"] + direction TB + g1["compas_pb
git repo"] + g2["antikythera
git repo"] + c1["compas_pb_ts
proto-sync.mjs"] + c2["compas_pb_csharp
fetch_compas_pb.py"] + c3["antikythera-frontend
update-protos.js"] + g1 -->|"pinned commit"| c1 + g1 -->|"release zip"| c2 + g1 -->|"tag v1.1.4"| c3 + g2 ==>|"branch main
unpinned"| c3 + end + + subgraph after["Proposed — one artifact contract"] + direction TB + r1["compas_pb
Release vX.Y.Z"] + r2["antikythera
Release vX.Y.Z"] + d1["compas_pb_ts"] + d2["compas_pb_csharp"] + d3["antikythera-frontend"] + r1 -->|"pinned"| d1 + r1 -->|"pinned"| d2 + r1 -->|"pinned"| d3 + r2 -->|"pinned"| d3 + end + + before ==>|"consolidate"| after +``` + +/// caption +Three consumers, three hand-written fetchers, three different pinning strategies — one of +which pins nothing at all. Consolidating on published artifacts makes the schema version an +explicit, auditable dependency. +/// + +Rules for the artifact set: + +- **Publish the `.proto` bundle.** It is what every downstream generator needs, and it is + the one artifact that lets a language without an official binding get started. +- **Publish generated bindings per supported language.** Priority languages are TypeScript, + C# and C++. +- **Skip a Python bindings artifact.** The generated `_pb2` modules already ship inside the + wheel, so a separate archive would duplicate what `pip` delivers. +- **Pin by version, never by branch.** A consumer tracking a branch silently adopts wire + changes at build time. + +## The contract for a language runtime + +To call itself a `compas_pb` runtime, a library must provide three things. + +1. **A unified entry point.** `pb_dump_bts` / `pb_load_bts` in Python, adapted to local + naming conventions elsewhere (`pbDump` / `pbLoad`). Callers pass a domain object and get + bytes, or pass bytes and get a domain object. They never branch on type. +2. **Recursive resolution.** Encoding and decoding must recurse through `dict_value` and + `list_value` so arbitrarily nested structures work, and must handle every arm of + `AnyData` — including writing `fallback`, not only reading it. +3. **A registration mechanism.** Third-party packages must be able to add types without + modifying the runtime. Automatic discovery is preferred where the language supports it; + explicit registration is acceptable where it does not. + +On the third point, languages differ in what they can honestly offer. Python enumerates +installed plugins through package metadata, so discovery is genuinely automatic and lazy. +JavaScript has no equivalent registry, and a bundled browser application cannot inspect its +own dependency tree at runtime — so TypeScript uses an explicit `register()` call. Keeping +the registration API separate from the discovery mechanism means a future build-time +discovery step can be added without changing how plugins declare themselves. + +## Status + +| Capability | Python | TypeScript | C# | +| --- | --- | --- | --- | +| Decode, recursive | yes | yes | yes | +| Encode, recursive | yes | **missing** | yes | +| Writes `fallback` | yes | **missing** | yes | +| Extensible registry | yes | **hardcoded map** | partial | +| Third-party registration | yes | **missing** | no | +| Automatic discovery | yes | not possible | no | + +`compas_pb_ts` can encode a single registered wrapper into an envelope via `pbDumpBytes`, +and every wrapper class can serialize itself to its own `*Data` bytes. What it lacks is the +recursive layer above them — the equivalent of `_serializer_any` — so it cannot yet encode a +dict, a list, a nested structure, or a `fallback`. diff --git a/docs/implementing-a-runtime.md b/docs/implementing-a-runtime.md new file mode 100644 index 0000000..5964cfe --- /dev/null +++ b/docs/implementing-a-runtime.md @@ -0,0 +1,163 @@ +# Implementing a runtime for a new language + +COMPAS models are written in Python, but more and more of what uses them is not: a browser +agent running a session, a Rhino plugin in C#, a controller in Rust. `compas_pb` is what +lets all of those pass the same objects around, instead of inventing a JSON format for +every new pair of tools. A `Frame` sent from Grasshopper shows up in the browser as a +`Frame`. This page explains how to add a language to that set. + +Have a look at [Architecture](architecture.md) first, for who owns what. The short version: +a **language runtime** implements the registry and the codec for one language, and knows +nothing about any domain model. **Domain-model owners** own `.proto` files and register +their types with the runtimes they care about. + +## What your runtime has to do + +1. **One entry point in, one out.** `pb_dump_bts` and `pb_load_bts`, named to suit your + language. People hand you an object and get bytes, or hand you bytes and get an object. + They should never have to check the type themselves. +2. **Handle nesting.** Walk into `dict_value` and `list_value`, and support every arm of + `AnyData` — including *writing* `fallback`, not just reading it. +3. **Let other packages register types**, without them having to edit your code. + +## The wire format + +A message is a `MessageData`: a version string plus one `AnyData`. + +Check the version before you trust anything else. Don't compare the versions directly — +compare their compatibility keys. Under `0.x` the key is `MAJOR.MINOR`, and from `1.0` +onwards it is just `MAJOR`. So `1.0.0` can read `1.2.9`, and `0.5.1` can read `0.5.7` but +not `0.6.0`. `_wire_compat_key` in `compas_pb.core` has the logic to copy. If there is no +version at all, that is an error — don't assume one. + +`AnyData` is a `oneof`, so exactly one of these is set. Look at which one it is and switch +on that, rather than testing each field to see if it has a value. + +| Arm | Holds | Worth knowing | +| --- | --- | --- | +| `message` | `google.protobuf.Any` | A registered type. Older data also puts lists and dicts here | +| `value` | `google.protobuf.Value` | null, bool and string. Bytes live here too, base64-encoded behind a `base64:` prefix | +| `fallback` | `FallbackData` | The only arm that rebuilds a real object rather than a dict | +| `int_value` | `int64` | Whole numbers | +| `double_value` | `double` | Floats, even when the value is round, like `3.0` | +| `dict_value` | `DictData` | Nests | +| `list_value` | `ListData` | Nests | + +Four rules that are easy to miss: + +- If your language has separate integer and float types, keep them apart. If it does not + (JavaScript), you cannot fully honour this — pick a rule and write it down. +- Decode lists and dicts in both shapes, old and new, but only ever encode the new ones. +- Match type URLs on everything after the last `/`, not on the last piece after a dot. + Short names seem fine until two packages both register a class called `TaskError`. +- An object with no registered serializer has to go out as `fallback`, or it arrives as a + plain dict. Python knows it has an object because it holds a live `Data` instance; + TypeScript never does, so it looks for the `{dtype, data}` shape instead. Decide what + that question means in your language, and ask it in the same place: after the registry + lookup, before "this is just a dictionary". + +## Registering types, and finding them + +These are two separate jobs. Registering is a package saying "this class maps to that +protobuf message". Discovery is your runtime finding those declarations on its own. Split +them, and you can add discovery later without changing how packages declare things. + +Store **functions**, not a required class shape. Python keeps a serializer per type and a +deserializer per protobuf name, which means a domain model never has to know protobuf +exists. If your registry instead insists that every registered class has, say, a `bytes` +property, then every package has to wrap its own model just to register it. When you look a +type up on the way out, follow your language's inheritance chain, so registering a base +class also covers everything below it. + +For discovery, do whatever your language can actually support. Python lists installed +plugins through packaging entry points. Rust can register at link time with `inventory` or +`linkme`. C# can scan the assemblies it has loaded. TypeScript gets a plain +`register()` call you make yourself, because a bundler can quietly drop a side-effect +import — and a call you can see beats one that vanishes. + +## Getting the schemas + +Don't copy `.proto` files into your repo, and don't generate them yourself. Whoever owns a +schema publishes the `.proto` bundle and the generated bindings for each language on every +release. Pin a version, download the artifact, done. Pulling files straight out of git +works right up until you follow a branch and your build quietly picks up a wire change. + +**Shared types have to come from the runtime package.** A domain owner's `.proto` imports +`compas_pb`'s, so its generated code refers to `compas_pb.data` types. Those need to be the +same types your runtime uses. In TypeScript this really matters: protobuf-es matches file +descriptors by identity, so a second copy registers a rival definition and the two halves +stop agreeing about types they are meant to share. Two Rust crates that each generate +`compas_pb.data` end up with two types that will not talk to each other. Depend on the +runtime package for them. + +## What changes everywhere else + +```mermaid +flowchart TB + new["compas_pb_rust
new runtime crate"] + + subgraph owner["compas_pb — one change"] + direction TB + lang["add the language to
PROTO_TARGET_LANGUAGES
or PROTO_PLUGIN_LANGUAGES"] + end + + subgraph downstream["Every domain-model owner — no change"] + direction TB + akt["antikythera"] + tmb["compas_timber"] + end + + lang ==>|"bindings now built
on every release"| new + lang -.->|"same task machinery,
so they publish it too"| akt + lang -.-> tmb + + akt ==>|"opt in by writing
a conversions module"| new + tmb ==>|"opt in"| new + + classDef ghost stroke-dasharray:4 3 + class new ghost +``` + +/// caption +Adding a language touches one place. Domain-model owners share compas_pb's build tasks, so +they start publishing bindings for the new language on their own — but a domain model only +*reaches* that language once someone writes its conversions module. +/// + +In **`compas_pb`**, add your language to the generation task. If `protoc` can emit it +directly, add it to `PROTO_TARGET_LANGUAGES`. If it needs a plugin, add it to +`PROTO_PLUGIN_LANGUAGES` with the flag its plugin uses, and cache the binary the way +`setup_protoc_gen_es` does. Rust needs `protoc-gen-prost`. Languages that need a plugin are +tagged with the *plugin* version in the asset name, since the plugin is what decides how +the generated code looks. + +In **domain-model owners**, nothing has to change for the bindings to start appearing. For +a domain model to actually reach your language, someone has to write the conversions module +— the equivalent of `antikythera.models.conversions`. + +In **other runtimes**, nothing at all. They share a wire format, not code. + +## Testing it + +Test against real bytes, not just your own round trips. A round trip still passes if you +get both directions wrong in the same way. + +- Take a payload from Python's `pb_dump_bts`, commit it as a base64 string, and check you + decode it correctly. This is the single most useful test you can write. +- Then go the other way: have Python read something you wrote. +- Cover every arm, plus a type registered by a package other than your own. + +## Checklist + +- [ ] Version written, and checked with the compatibility key +- [ ] All seven arms read and written, switching on the one that is set +- [ ] Whole numbers and floats stay apart, or the limitation is written down +- [ ] Bytes go through the `base64:` prefix +- [ ] `fallback` is written, not only read +- [ ] Old `Any`-wrapped lists and dicts still decode +- [ ] Type URLs matched on the full name after the last `/` +- [ ] Registry holds functions, and other packages can register types +- [ ] One entry point each way, with no type checking left to callers +- [ ] Bindings come from a pinned release, not copied into the repo +- [ ] Shared `compas_pb` types come from the runtime package +- [ ] Tested both directions against bytes Python produced diff --git a/mkdocs.yml b/mkdocs.yml index 956dd19..6b9ffad 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,7 +75,11 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.snippets: check_paths: true - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - toc: permalink: "¤" @@ -146,6 +150,8 @@ nav: - compas_pb.core: reference/compas_pb.core.md - compas_pb.registry: reference/compas_pb.registry.md - compas_pb.conversions: reference/compas_pb.conversions.md + - Architecture: architecture.md + - Implementing a Runtime: implementing-a-runtime.md - Protobuf Definitions: protobuf.md - Other: - changelog.md diff --git a/src/compas_pb/invocations.py b/src/compas_pb/invocations.py index 13e72ee..f96c41c 100644 --- a/src/compas_pb/invocations.py +++ b/src/compas_pb/invocations.py @@ -11,9 +11,18 @@ PROTOC_VERSION = "31.1" PROTOC_GEN_DOCS_VERSION = "1.5.1" -# typescript,javascript, and go need other compiler plugins +PROTOC_GEN_ES_VERSION = "2.14.0" + +# Languages protoc emits natively, with no extra plugin. PROTO_TARGET_LANGUAGES = ["cpp", "csharp", "java", "objc", "php", "ruby"] +# Languages that need a third-party protoc plugin. Maps the language name used in asset +# names to the protoc flag prefix the plugin registers (``--es_out`` for protobuf-es), so +# javascript and go can be added here the same way. +PROTO_PLUGIN_LANGUAGES = {"typescript": "es"} + +ALL_PROTO_TARGET_LANGUAGES = PROTO_TARGET_LANGUAGES + list(PROTO_PLUGIN_LANGUAGES) + _PROTOC_ARCH_MAPPING = { "x86_64": "x86_64", "aarch64": "aarch_64", @@ -97,6 +106,42 @@ def _download_and_extract_protoc(url, extract_path): archive_path.unlink() +def _get_cached_protoc_gen_es_path(): + cache_dir = Path.home() / ".cache" / "protoc-gen-es" / PROTOC_GEN_ES_VERSION + plugin_bin = cache_dir / "node_modules" / ".bin" / "protoc-gen-es" + if platform.system() == "Windows": + plugin_bin = plugin_bin.with_suffix(".cmd") + + return plugin_bin, cache_dir + + +def setup_protoc_gen_es(ctx): + """Install the protobuf-es code generator, and return the path to its executable. + + protoc has no native TypeScript output, so TypeScript bindings come from + ``@bufbuild/protoc-gen-es``. It is an npm package, so this needs node on PATH. The + install is cached per version alongside the protoc cache. + """ + plugin_bin, cache_dir = _get_cached_protoc_gen_es_path() + + if plugin_bin.exists(): + print(f"Using cached protoc-gen-es at: {plugin_bin}") + return plugin_bin + + print(f"protoc-gen-es not found in cache. Installing to: {cache_dir}") + cache_dir.mkdir(parents=True, exist_ok=True) + ctx.run( + f'npm install --silent --no-package-lock --prefix "{cache_dir}" ' + f"@bufbuild/protoc-gen-es@{PROTOC_GEN_ES_VERSION}" + ) + + if not plugin_bin.exists(): + raise FileNotFoundError(f"Failed to find protoc-gen-es after install: {plugin_bin}") + + print("Install complete.") + return plugin_bin + + def setup_protoc(): protoc_bin, cache_dir = _get_cached_protoc_path() plugin_executable = protoc_bin.parent / "protoc-gen-doc" @@ -135,20 +180,33 @@ def setup_protoc(): def generate_proto_classes(ctx, target_language: str = "python"): protoc_path, _ = setup_protoc() - proto_out_folder = "" if target_language == "python": proto_out_folder = Path(ctx.proto_out_folder) - elif target_language in PROTO_TARGET_LANGUAGES: - proto_out_folder = Path(ctx.proto_out_folder) / "compas_pb" / "generated" / target_language + elif target_language in ALL_PROTO_TARGET_LANGUAGES: + proto_out_folder = _generated_root(ctx) / target_language proto_out_folder.mkdir(parents=True, exist_ok=True) else: - print(f"Target language '{target_language}' not supported.") + supported = ", ".join(["python"] + ALL_PROTO_TARGET_LANGUAGES) + raise ValueError(f"Target language '{target_language}' not supported. Choose one of: {supported}") + + # Plugin-backed languages replace the --_out flag with the plugin's own flag, + # and need the plugin binary passed explicitly since it is not on PATH. + plugin_flag = PROTO_PLUGIN_LANGUAGES.get(target_language) + plugin_path = setup_protoc_gen_es(ctx) if plugin_flag == "es" else None - for idl_file in ctx.proto_folder.glob("*.proto"): + for idl_file in Path(ctx.proto_folder).glob("*.proto"): cmd = f"{protoc_path} " cmd += " ".join(f"--proto_path={p}" for p in ctx.proto_include_paths) - cmd += f" --{target_language}_out={proto_out_folder} {idl_file}" + if plugin_flag: + cmd += f' --plugin=protoc-gen-{plugin_flag}="{plugin_path}"' + cmd += f" --{plugin_flag}_out={proto_out_folder}" + if plugin_flag == "es": + # Emit TypeScript rather than JavaScript + .d.ts. + cmd += f" --{plugin_flag}_opt=target=ts" + cmd += f" {idl_file}" + else: + cmd += f" --{target_language}_out={proto_out_folder} {idl_file}" if target_language == "python": cmd += f" --pyi_out={proto_out_folder}" @@ -157,23 +215,79 @@ def generate_proto_classes(ctx, target_language: str = "python"): ctx.run(cmd) +def _package_name(ctx) -> str: + """Distribution name used to label release assets. + + Defaults to compas_pb, so any package that owns .proto files can reuse these tasks by + setting ``package_name`` in its invoke configuration. + """ + return ctx.config.get("package_name", "compas_pb") + + +def _generated_root(ctx) -> Path: + """Folder the per-language bindings are generated into before being zipped.""" + configured = ctx.config.get("generated_folder", None) + if configured: + return Path(configured) + return Path(ctx.proto_out_folder) / _package_name(ctx) / "generated" + + +def _asset_version(language: str) -> str: + """Version marker for a generated-bindings asset. + + Natively-emitted languages are pinned by the protoc that produced them. Plugin-backed + languages are pinned by the plugin version instead, since that is what shapes the + generated API — protoc only feeds it a descriptor. + """ + if language in PROTO_PLUGIN_LANGUAGES: + return PROTOC_GEN_ES_VERSION if PROTO_PLUGIN_LANGUAGES[language] == "es" else PROTOC_VERSION + return PROTOC_VERSION + + +@task() +def create_proto_bundle(ctx): + """Zip the .proto schemas themselves for release upload. + + Every downstream generator needs the schemas, not just the bindings compas_pb happens + to generate. Publishing them is what lets a consumer pin a schema version instead of + scraping this repository at some ref. + """ + dist_dir = Path(ctx.base_folder) / "dist" / "proto" + dist_dir.mkdir(parents=True, exist_ok=True) + + zip_path = dist_dir / f"{_package_name(ctx)}-proto.zip" + zip_path.unlink(missing_ok=True) + + proto_root = Path(ctx.proto_include_paths[0]) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: + for item in sorted(proto_root.rglob("*.proto")): + arcname = str(item.relative_to(proto_root)) + zipf.write(item, arcname) + print(f"Added {arcname}") + + print(f"Proto bundle ready: {zip_path}") + return zip_path + + @task() def create_class_assets(ctx): - base_dir = ctx.base_folder + base_dir = Path(ctx.base_folder) dist_dir = base_dir / "dist" dist_dir.mkdir(parents=True, exist_ok=True) - for existing_file in dist_dir.glob("compas_pb-generated-*.zip"): + # Assets are written to dist/proto/, so clean there — globbing dist/ itself never + # matched anything and left stale zips behind for the release upload to pick up. + for existing_file in (dist_dir / "proto").glob(f"{_package_name(ctx)}-generated-*.zip"): existing_file.unlink() print(f"Removed existing asset: {existing_file}") - class_assests = [] + class_assests = [create_proto_bundle(ctx)] - for language in PROTO_TARGET_LANGUAGES: + for language in ALL_PROTO_TARGET_LANGUAGES: generate_proto_classes(ctx, target_language=language) - generated_dir = base_dir / "src" / "compas_pb" / "generated" / language - zip_path = dist_dir / "proto" / f"compas_pb-generated-{language}-{PROTOC_VERSION}.zip" + generated_dir = _generated_root(ctx) / language + zip_path = dist_dir / "proto" / f"{_package_name(ctx)}-generated-{language}-{_asset_version(language)}.zip" zip_path.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: for item in generated_dir.rglob("*"): diff --git a/tasks.py b/tasks.py index 7d032b2..db04466 100644 --- a/tasks.py +++ b/tasks.py @@ -7,6 +7,7 @@ from invoke.tasks import task from compas_pb.invocations import create_class_assets +from compas_pb.invocations import create_proto_bundle from compas_pb.invocations import generate_proto_classes from compas_pb.invocations import proto_docs @@ -28,6 +29,7 @@ def pre_build(ctx): pre_build, generate_proto_classes, create_class_assets, + create_proto_bundle, proto_docs, )