WIP: support draft APIs

.github/workflows/ci.yml

@@ -26,3 +26,28 @@ jobs:
           fi
       - name: Check for OpenAPI path conflicts
         run: go run ./cmd/check-path-conflicts/main.go openapi/openapiv2.json
+
+  stable-api-generated:
+    name: Verify stable-api is generated
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: actions/setup-go@7b8cf10d4e4a01d4992d18a89f4d7dc5a3e6d6f4 # v4.3.0
+        with:
+          go-version: '^1.25'
+      - name: Install buf
+        run: make buf-install
+      - name: Test protoc-gen-stable-api
+        run: make stable-api-test
+      - name: Build protoc-gen-stable-api
+        run: make stable-api-plugin
+      - name: Regenerate temporal/api from temporal/api_next
+        run: make stable-api
+      - name: Fail if temporal/api is stale
+        run: |
+          if [[ -n $(git status --porcelain -- temporal/api) ]]; then
+            echo "temporal/api is out of date. Run 'make stable-api' and commit the result."
+            git status --porcelain -- temporal/api
+            git diff -- temporal/api
+            exit 1
+          fi

.gitignore

@@ -5,3 +5,4 @@
 *~
 *.swp
 *.swo
+bin/

Makefile

@@ -4,6 +4,8 @@ $(VERBOSE).SILENT:
 ############################# Main targets #############################
 ci-build: install proto http-api-docs
 
+.PHONY: stable-api stable-api-plugin stable-api-test
+
 # Install dependencies.
 install: grpc-install api-linter-install buf-install
 
@@ -25,7 +27,9 @@ STAMPDIR := .stamp
 COLOR := "\e[1;36m%s\e[0m\n"
 
 PROTO_ROOT := .
-PROTO_FILES = $(shell find temporal -name "*.proto")
+# exclude api_next files for api-linter
+PROTO_FILES = $(shell find temporal -name "*.proto" -not -path "temporal/api_next/*")
+PROTO_FILES_NEXT = $(shell find temporal/api_next -name "*.proto")
 PROTO_DIRS = $(sort $(dir $(PROTO_FILES)))
 PROTO_OUT := .gen
 PROTO_IMPORTS = \
@@ -40,8 +44,20 @@ NEX_GEN ?= nex-gen
 $(PROTO_OUT):
 	mkdir $(PROTO_OUT)
 
+stable-api: stable-api-plugin
+	rm -rf temporal/api
+	buf generate --config buf.next.yaml --template buf.stable.gen.yaml
+	buf format -w temporal/api
+
+stable-api-plugin:
+	@cd cmd/generate-stable && go build -o $(GOBIN)/protoc-gen-stable-api .
+
+stable-api-test:
+	printf $(COLOR) "Run generate-stable tests..."
+	@cd cmd/generate-stable && go test ./...
+
 ##### Compile proto files for go #####
-grpc: buf-lint api-linter buf-breaking clean go-grpc fix-path
+grpc: stable-api buf-lint api-linter buf-breaking clean go-grpc fix-path
 
 go-grpc: clean $(PROTO_OUT)
 	printf $(COLOR) "Compile for go-gRPC..."
@@ -50,6 +66,7 @@ go-grpc: clean $(PROTO_OUT)
 		--output=$(PROTO_OUT) \
 		--exclude=internal \
 		--exclude=proto/api/google \
+		--exclude=temporal/api_next \
 		-I $(PROTO_ROOT) \
 		-p go-grpc_out=$(PROTO_PATHS) \
 		-p grpc-gateway_out=allow_patch_feature=false,$(PROTO_PATHS) \
@@ -105,9 +122,13 @@ sync-nexus-annotations:
 	buf export buf.build/temporalio/nexus-annotations --output .
 
 ##### Linters #####
+API_LINTER_FMT = 'map(select(.problems != []) | . as $$file | .problems[] | {rule: .rule_doc_uri, location: "\($$file.file_path):\(.location.start_position.line_number)"}) | group_by(.rule) | .[] | .[0].rule + ":\n" + (map("\t" + .location) | join("\n"))'
+
 api-linter:
-	printf $(COLOR) "Run api-linter..."
-	@api-linter --set-exit-status $(PROTO_IMPORTS) --config $(PROTO_ROOT)/api-linter.yaml --output-format json $(PROTO_FILES) | gojq -r 'map(select(.problems != []) | . as $$file | .problems[] | {rule: .rule_doc_uri, location: "\($$file.file_path):\(.location.start_position.line_number)"}) | group_by(.rule) | .[] | .[0].rule + ":\n" + (map("\t" + .location) | join("\n"))'
+	printf $(COLOR) "Run api-linter on temporal/api_next..."
+	@api-linter --set-exit-status $(PROTO_IMPORTS) --config $(PROTO_ROOT)/api-linter.yaml --output-format json $(PROTO_FILES_NEXT) | gojq -r $(API_LINTER_FMT)
+	printf $(COLOR) "Run api-linter on temporal/api..."
+	@api-linter --set-exit-status $(PROTO_IMPORTS) --config $(PROTO_ROOT)/api-linter.yaml --output-format json $(PROTO_FILES) | gojq -r $(API_LINTER_FMT)
 
 $(STAMPDIR):
 	mkdir $@
@@ -122,8 +143,10 @@ buf-lint: $(STAMPDIR)/buf-mod-prune
 	(cd $(PROTO_ROOT) && buf lint)
 
 buf-breaking:
-	@printf $(COLOR) "Run buf breaking changes check against main branch..."
+	@printf $(COLOR) "Run buf breaking changes check for stable API against main branch..."
 	@(cd $(PROTO_ROOT) && buf breaking --against 'https://github.com/temporalio/api.git#branch=main')
+	@printf $(COLOR) "Run buf breaking changes check for api_next against main branch..."
+	@(cd $(PROTO_ROOT) && buf breaking --config buf.next.yaml --against 'https://github.com/temporalio/api.git#branch=main' --against-config buf.yaml)
 
 nexus-rpc-yaml: nexus-rpc-yaml-install
 	printf $(COLOR) "Generate nexus/temporal-proto-models-nexusrpc.yaml..."

README.md

@@ -8,7 +8,7 @@ Install as git submodule to the project.
 
 ## Contribution
 
-Make your change to the temporal/proto files, and run `make` to update the openapi definitions.
+Make your change to the temporal/api_next/proto files, and run `make` to update the openapi definitions.
 Rust is also required because `make` installs and runs `nex-gen` when regenerating system Nexus WIT files.
 
 ## Breaking changes

api-linter.yaml

@@ -64,3 +64,10 @@
     - "google/**/*.proto"
   disabled_rules:
     - "all"
+
+- included_paths:
+    - "temporal/api_next/**/*.proto"
+  disabled_rules:
+    # api_next files declare the `temporal.api.*` package they are projected
+    # into, so their directory deliberately does not match their package.
+    - "core::0191::proto-package" # https://linter.aip.dev/191/proto-package

buf.next.yaml

@@ -0,0 +1,26 @@
+# buf config for generating api_next 
+version: v1
+name: buf.build/temporalio/api-next
+deps:
+  - buf.build/grpc-ecosystem/grpc-gateway
+  - buf.build/googleapis/googleapis
+  - buf.build/temporalio/nexus-annotations
+build:
+  excludes:
+    # Buf won't accept a local dependency on the google protos but we need them
+    # to run api-linter, so just tell buf it ignore it
+    - google
+    # Same for nexusannotations - local copy for api-linter, BSR dep for buf
+    - nexusannotations
+    - temporal/api
+breaking:
+  use:
+    - WIRE_JSON
+  ignore:
+    - google
+lint:
+  use:
+    - DEFAULT
+  ignore:
+    - google
+    - cmd

buf.stable.gen.yaml

@@ -0,0 +1,5 @@
+version: v1
+plugins:
+  - plugin: stable-api
+    out: .
+    strategy: all

buf.yaml

@@ -1,3 +1,4 @@
+# buf config for generating the stable proto definitions 
 version: v1
 name: buf.build/temporalio/api
 deps:
@@ -11,6 +12,7 @@ build:
     - google
     # Same for nexusannotations - local copy for api-linter, BSR dep for buf
     - nexusannotations
+    - temporal/api_next
 breaking:
   use:
     - WIRE_JSON

cmd/generate-stable/README.md

@@ -0,0 +1,17 @@
+# Stable API Plugin
+
+`protoc-gen-stable-api` generates the stable set of proto definitions from the
+full `api_next` schema. It uses descriptors for filtering and `protoprint` to
+emit the resulting `.proto` files.
+
+`api_next` is the source of truth. `api` is generated output and must not be
+manually edited.
+
+Run the complete generation flow from the repository root:
+
+```bash
+make stable-api
+```
+
+The Make target builds this plugin, runs it through Buf using
+`buf.stable.gen.yaml`, and formats the generated stable tree.