From aea59b7f2e069d52b4417cbf79f43394d5370131 Mon Sep 17 00:00:00 2001 From: Alek Date: Thu, 16 Jul 2026 21:05:59 -0700 Subject: [PATCH 1/2] Exclude Pyodide wheel from PyPI uploads --- .github/workflows/release.yml | 13 ++++++++++--- scripts/verify_ci_workflow.py | 2 ++ tests/test_verify_ci_workflow.py | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74d36d0..2d8d26a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -174,7 +174,10 @@ jobs: python scripts/pyodide_load_smoke.py "$whl" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: dist-pyodide + # Keep this outside the dist-* namespace: PyPI rejects Pyodide's + # pyodide_* platform tags, so this wheel is retained as a workflow + # artifact but must not enter the PyPI upload directory. + name: pyodide-wheel path: dist/*.whl sdist: @@ -229,8 +232,9 @@ jobs: publish: name: Publish to PyPI - # Publishing requires the runtime-verified Pyodide wheel alongside every native - # wheel and the sdist; the dist-* download pattern collects all of their artifacts. + # Publishing still requires the runtime-verified Pyodide job to pass, but + # only native wheels and the sdist use dist-* artifact names. PyPI does not + # accept Pyodide platform tags, so its wheel remains a separate artifact. needs: [wheels, sdist, wasm] runs-on: ubuntu-latest environment: pypi @@ -265,3 +269,6 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.dry_run != 'true' with: packages-dir: dist/ + # A failed upload can leave a partial release on immutable PyPI. + # Retrying the same tag should skip files that were already accepted. + skip-existing: true diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index 4bb022f..c884a7d 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -481,6 +481,7 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "scripts/pyodide_load_smoke.py", "scripts/verify_wheel.py", "--expect-native", + "name: pyodide-wheel", ) wasm_job = jobs.get("wasm", "") if "continue-on-error:" in wasm_job: @@ -519,6 +520,7 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "dry_run", "pypa/gh-action-pypi-publish@", "packages-dir: dist/", + "skip-existing: true", ) _require_workflow_contains( errors, diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py index e20a98d..bf10826 100644 --- a/tests/test_verify_ci_workflow.py +++ b/tests/test_verify_ci_workflow.py @@ -483,6 +483,19 @@ def test_release_workflow_rejects_nonblocking_pyodide_probe(tmp_path: Path) -> N assert any("wasm job must block publishing" in error for error in errors) +def test_release_workflow_rejects_pyodide_artifact_in_pypi_batch(tmp_path: Path) -> None: + workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + path = tmp_path / "release.yml" + path.write_text( + workflow.replace(" name: pyodide-wheel\n", " name: dist-pyodide\n"), + encoding="utf-8", + ) + + errors = verify_ci_workflow.validate_release_workflow(path) + + assert any("release wasm job" in error and "pyodide-wheel" in error for error in errors) + + def test_release_workflow_rejects_missing_sdist_norust_smoke(tmp_path: Path) -> None: workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") path = tmp_path / "release.yml" @@ -550,3 +563,13 @@ def test_release_workflow_rejects_ungated_pypi_publish_step(tmp_path: Path) -> N errors = verify_ci_workflow.validate_release_workflow(path) assert any("has no if: condition of its own" in error for error in errors) + + +def test_release_workflow_rejects_non_retryable_pypi_publish(tmp_path: Path) -> None: + workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + path = tmp_path / "release.yml" + path.write_text(workflow.replace(" skip-existing: true\n", ""), encoding="utf-8") + + errors = verify_ci_workflow.validate_release_workflow(path) + + assert any("release publish job" in error and "skip-existing" in error for error in errors) From 27b38ef67aec6c9af551c8ea9e378a6aed4720e4 Mon Sep 17 00:00:00 2001 From: Alek Date: Thu, 16 Jul 2026 22:59:49 -0700 Subject: [PATCH 2/2] Publish Pyodide wheel as a release asset --- .github/workflows/release.yml | 52 ++++++++++++++++++++++++ docs/engineering/production-readiness.md | 7 +++- scripts/pyodide_load_smoke.py | 32 ++++++++++----- scripts/verify_ci_workflow.py | 25 +++++++++++- tests/test_pyodide_load_smoke.py | 46 +++++++++++++++++++++ tests/test_verify_ci_workflow.py | 13 ++++++ 6 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 tests/test_pyodide_load_smoke.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d8d26a..71d49ed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -272,3 +272,55 @@ jobs: # A failed upload can leave a partial release on immutable PyPI. # Retrying the same tag should skip files that were already accepted. skip-existing: true + + publish-pyodide: + name: Publish Pyodide wheel + if: github.event_name != 'workflow_dispatch' || github.event.inputs.dry_run != 'true' + needs: [wheels, sdist, wasm] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.2.2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: pyodide-wheel + path: dist-pyodide + # PyPI rejects Pyodide platform tags. Publish the runtime-verified wheel + # as a release asset instead; micropip supports installing binary WASM + # wheels directly from HTTPS URLs. + - name: Upload Pyodide wheel to GitHub Release + id: publish + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + tag="v${version}" + if [[ "$EVENT_NAME" == "push" && "$REF_NAME" != "$tag" ]]; then + echo "tag $REF_NAME does not match project version $tag" >&2 + exit 1 + fi + python3 scripts/check_release_version.py --tag "$tag" + wheel=$(find dist-pyodide -maxdepth 1 -name '*.whl' -print -quit) + test -n "$wheel" + if gh release view "$tag" >/dev/null 2>&1; then + gh release upload "$tag" "$wheel" --clobber + else + gh release create "$tag" "$wheel" --verify-tag --generate-notes --title "$tag" + fi + url="https://github.com/${GH_REPO}/releases/download/${tag}/$(basename "$wheel")" + echo "wheel_url=$url" >> "$GITHUB_OUTPUT" + echo "Pyodide wheel: $url" + - name: Verify published Pyodide wheel from its HTTPS URL + env: + WHEEL_URL: ${{ steps.publish.outputs.wheel_url }} + run: | + npm i --no-save pyodide@0.29.4 + python3 scripts/pyodide_load_smoke.py "$WHEEL_URL" diff --git a/docs/engineering/production-readiness.md b/docs/engineering/production-readiness.md index d27a751..91a645e 100644 --- a/docs/engineering/production-readiness.md +++ b/docs/engineering/production-readiness.md @@ -330,8 +330,11 @@ Before tagging a release: `__cpp_exception` WebAssembly tag that Pyodide's main module did not provide. `scripts/pyodide_load_smoke.py` installs the built artifact with micropip, loads the C ABI through `ctypes`, verifies `xy_abi_version`, and calls the - native `min_max` kernel. The wasm job is release-blocking so an ABI or - toolchain drift cannot silently ship a build-only, unloadable artifact. + native `min_max` kernel. PyPI does not accept Pyodide platform tags, so the + release workflow publishes this wheel as a GitHub Release asset and repeats + the same runtime smoke against its public HTTPS URL. The wasm job is + release-blocking so an ABI or toolchain drift cannot silently ship a + build-only, unloadable artifact. - Confirm the no-Rust install job passed (it must build, install, and then raise a clear ImportError on first compute — never a silent fallback). - Confirm the sdist verifier passed and the source archive contains the expected diff --git a/scripts/pyodide_load_smoke.py b/scripts/pyodide_load_smoke.py index 9fea20c..b16af88 100644 --- a/scripts/pyodide_load_smoke.py +++ b/scripts/pyodide_load_smoke.py @@ -7,7 +7,7 @@ the release wheel's exception-free Rust build: it installs the exact artifact, checks the ABI version, calls a native kernel, and exits non-zero on failure. -Usage: python scripts/pyodide_load_smoke.py path/to/xy-...-pyodide_....whl +Usage: python scripts/pyodide_load_smoke.py Requires: node with the `pyodide` npm package resolvable from CWD. """ @@ -22,15 +22,22 @@ import { loadPyodide } from "pyodide"; import fs from "node:fs"; const wheelPath = process.argv[2]; -const name = wheelPath.split("/").pop(); +const isUrl = wheelPath.startsWith("https://") || wheelPath.startsWith("http://"); +const name = isUrl + ? new URL(wheelPath).pathname.split("/").pop() + : wheelPath.split("/").pop(); const out = (o) => console.log("RESULT " + JSON.stringify(o)); try { const py = await loadPyodide(); await py.loadPackage(["numpy", "micropip"]); const micropip = py.pyimport("micropip"); - py.FS.mkdirTree("/wheels"); - py.FS.writeFile("/wheels/" + name, fs.readFileSync(wheelPath)); - await micropip.install("emfs:/wheels/" + name); + if (isUrl) { + await micropip.install(wheelPath); + } else { + py.FS.mkdirTree("/wheels"); + py.FS.writeFile("/wheels/" + name, fs.readFileSync(wheelPath)); + await micropip.install("emfs:/wheels/" + name); + } const r = await py.runPythonAsync(` import xy.kernels as k import numpy as np @@ -51,10 +58,15 @@ def main() -> int: if len(sys.argv) != 2: print("usage: pyodide_load_smoke.py ", file=sys.stderr) return 2 - wheel = Path(sys.argv[1]).resolve() - if not wheel.exists(): - print(f"wheel not found: {wheel}", file=sys.stderr) - return 2 + source = sys.argv[1] + if source.startswith(("https://", "http://")): + wheel_source = source + else: + wheel = Path(source).resolve() + if not wheel.exists(): + print(f"wheel not found: {wheel}", file=sys.stderr) + return 2 + wheel_source = str(wheel) # Write the driver into CWD so its `import "pyodide"` resolves against the # node_modules of the directory where `npm install pyodide` was run (node @@ -63,7 +75,7 @@ def main() -> int: driver.write_text(_DRIVER, encoding="utf-8") try: proc = subprocess.run( - ["node", str(driver), str(wheel)], + ["node", str(driver), wheel_source], capture_output=True, text=True, check=False, diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index c884a7d..8b5d8d7 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -33,7 +33,7 @@ "install_without_rust", } REQUIRED_CODSPEED_JOBS = {"benchmarks"} -REQUIRED_RELEASE_JOBS = {"wheels", "sdist", "publish", "wasm"} +REQUIRED_RELEASE_JOBS = {"wheels", "sdist", "publish", "publish-pyodide", "wasm"} def _job_blocks(text: str) -> dict[str, str]: @@ -533,6 +533,29 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "type: boolean", "default: true", ) + _require_job_contains( + errors, + jobs, + "publish-pyodide", + "release", + "GitHub Release publication of the runtime-verified Pyodide wheel", + "needs: [wheels, sdist, wasm]", + "contents: write", + "actions/setup-node@", + 'node-version: "22"', + "actions/download-artifact@", + "name: pyodide-wheel", + "dry_run", + "scripts/check_release_version.py", + "GH_TOKEN", + "gh release upload", + "--clobber", + "gh release create", + "--verify-tag", + "steps.publish.outputs.wheel_url", + "pyodide@0.29.4", + "scripts/pyodide_load_smoke.py", + ) publish = jobs.get("publish", "") if "password:" in publish or "api-token" in publish: diff --git a/tests/test_pyodide_load_smoke.py b/tests/test_pyodide_load_smoke.py new file mode 100644 index 0000000..a047d28 --- /dev/null +++ b/tests/test_pyodide_load_smoke.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + + +def _load_smoke_module(): + path = Path(__file__).resolve().parents[1] / "scripts" / "pyodide_load_smoke.py" + spec = importlib.util.spec_from_file_location("pyodide_load_smoke", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +pyodide_load_smoke = _load_smoke_module() + + +def test_remote_wheel_url_is_installed_directly(monkeypatch, tmp_path: Path) -> None: + url = ( + "https://github.com/reflex-dev/xy/releases/download/v0.0.1/" + "xy-0.0.1-py3-none-pyodide_2025_0_wasm32.whl" + ) + observed: dict[str, object] = {} + + def fake_run(command, **kwargs): + observed["command"] = command + observed["driver"] = Path(command[1]).read_text(encoding="utf-8") + return subprocess.CompletedProcess( + command, + 0, + stdout='RESULT {"ok":true,"backend":"native","abi":34,"min":1,"max":3}\n', + stderr="", + ) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(pyodide_load_smoke.subprocess, "run", fake_run) + monkeypatch.setattr(sys, "argv", ["pyodide_load_smoke.py", url]) + + assert pyodide_load_smoke.main() == 0 + assert observed["command"][-1] == url + assert "await micropip.install(wheelPath)" in observed["driver"] + assert not (tmp_path / "_pyodide_load_driver.mjs").exists() diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py index bf10826..98f38c6 100644 --- a/tests/test_verify_ci_workflow.py +++ b/tests/test_verify_ci_workflow.py @@ -496,6 +496,19 @@ def test_release_workflow_rejects_pyodide_artifact_in_pypi_batch(tmp_path: Path) assert any("release wasm job" in error and "pyodide-wheel" in error for error in errors) +def test_release_workflow_rejects_missing_pyodide_release_publisher(tmp_path: Path) -> None: + workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + path = tmp_path / "release.yml" + path.write_text( + workflow.replace(" publish-pyodide:\n", " publish-pyodide-removed:\n"), + encoding="utf-8", + ) + + errors = verify_ci_workflow.validate_release_workflow(path) + + assert any("missing required release job 'publish-pyodide'" in error for error in errors) + + def test_release_workflow_rejects_missing_sdist_norust_smoke(tmp_path: Path) -> None: workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") path = tmp_path / "release.yml"