diff --git a/.cargo/config.toml b/.cargo/config.toml index 7b7b2eb4e1f..80bc08b5671 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,7 +3,8 @@ rustflags = ["--cfg", "tokio_unstable"] [alias] llm = "run --package xtask-llm-benchmark --bin llm_benchmark --" -ci = "run -p ci --" +ci = "run -vv -p ci --" +spacetime = "ci other-workflows run-spacetime" regen = "run -p regen --" smoketest = "ci smoketests --" smoketests = "smoketest" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56dd41aa0c..8dc50285dc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,14 +74,24 @@ jobs: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Upload build artifacts (Linux) - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux EXE_SUFFIX: "" steps: &upload-build-artifact-steps + - &set-cargo-target-dir + name: Set Cargo target directory + shell: bash + run: | + if [[ "${RUNNER_OS}" == "Linux" ]]; then + cargo_target_dir="${HOME}/actions-runner/_work/target" + else + cargo_target_dir="${GITHUB_WORKSPACE}/target" + fi + echo "CARGO_TARGET_DIR=${cargo_target_dir}" >>"${GITHUB_ENV}" + - &find-git-ref name: Find Git ref env: @@ -101,172 +111,71 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} + # Full history lets git restore-mtime recover each file's actual + # last-change timestamp and makes the warmed commit available for + # content-based cache invalidation below. + fetch-depth: 0 - - uses: dsherret/rust-toolchain-file@v1 - - &set-default-rust-toolchain - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - - &verify-openssl-assembler - name: Verify OpenSSL assembler - if: runner.os == 'Windows' - shell: pwsh - run: nasm -v - - # These keys intentionally spell out stable native-build inputs instead of - # hashing the workspace. Keep the versions in sync with Cargo.lock and bump - # v1 if a fixed feature set or build configuration changes. This manual - # versioning is temporary until a better caching mechanism replaces it. - # - V8 uses its default features and has profile-specific native output. - # - OpenSSL is the vendored static build with legacy enabled. Windows uses - # NASM, which is represented by a separate cache-key suffix. - # - jemalloc is an optimized static PIC build with profiling, stats, and - # its default background-thread support. - # - Zstd is an optimized static build with legacy support and dictionary - # APIs. It is reused across Cargo profiles. - - &set-native-cache-keys - name: Set native cache keys - id: native-cache-keys + # cargo has a behavior that's a bit unfortunate for us here. It does not know the mtime of the files that + # were used to build its artifacts; it just knows the mtime of the artifact. + # So, if you have a sequence of events like: + # 1. cache source commit X happens at time X + # 2. PR commit Y happens at time Y > X + # 3. cache is warmed from commit X at time Z > Y + # + # cargo then sees "artifact built at time Z is more recent than time Y, so it must be up to date". + # + # So here we restore committed mtimes to reuse warm artifacts, then touch files that + # differ from the warmed commit so they will look more recent than the artifact, + # and cargo will properly rebuild anything depending on those files. + - &restore-source-mtimes + name: Restore source mtimes and invalidate changed files + if: runner.os == 'Linux' shell: bash run: | - target="$(rustc -vV | sed -n 's/^host: //p')" - openssl_suffix="" - if [[ "${OPENSSL_RUST_USE_NASM:-}" == "1" ]]; then - openssl_suffix="-nasm" - fi - echo "v8-release-key=rusty-v8-v1-145.0.0-${target}" >>"$GITHUB_OUTPUT" - echo "v8-debug-key=rusty-v8-debug-v1-145.0.0-${target}" >>"$GITHUB_OUTPUT" - echo "openssl-key=openssl-v1-300.5.3+3.5.4-${target}${openssl_suffix}" >>"$GITHUB_OUTPUT" - if [[ "$RUNNER_OS" == "Linux" ]]; then - cc="${CC:-cc}" - cc_hash="$("$cc" --version | sha256sum | cut -c1-16)" - echo "jemalloc-key=jemalloc-v1-0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7-${target}-${cc_hash}" >>"$GITHUB_OUTPUT" - echo "zstd-key=zstd-v1-2.0.16+zstd.1.5.7-${target}-${cc_hash}" >>"$GITHUB_OUTPUT" + cached_commit_file="${CARGO_TARGET_DIR}/.warm-cache-commit-SpacetimeDB" + if [[ -f "${cached_commit_file}" ]]; then + cached_commit="$(<"${cached_commit_file}")" + else + cached_commit="" fi - - &cache-rusty-v8 - name: Cache rusty_v8 - uses: actions/cache@v4 - with: &v8-release-cache - path: ${{ env.CARGO_TARGET_DIR }}/release/gn_out/obj - key: ${{ steps.native-cache-keys.outputs.v8-release-key }} - - - &cache-openssl - name: Cache OpenSSL - id: cache-openssl - uses: actions/cache@v4 - with: &openssl-cache - path: ${{ github.workspace }}/.ci-cache/openssl - key: ${{ steps.native-cache-keys.outputs.openssl-key }} - - - &configure-cached-openssl - name: Configure cached OpenSSL - if: steps.cache-openssl.outputs.cache-hit == 'true' - shell: bash - run: | - # Reuse the cached vendored output as a prebuilt OpenSSL installation. - echo "OPENSSL_NO_VENDOR=1" >>"$GITHUB_ENV" - echo "OPENSSL_DIR=${{ github.workspace }}/.ci-cache/openssl" >>"$GITHUB_ENV" + if [[ -n "${cached_commit}" ]] && git cat-file -e "${cached_commit}^{commit}" 2>/dev/null; then + git restore-mtime --quiet + echo "Invalidating files changed since cached commit ${cached_commit}" + while IFS= read -r -d '' path; do + if [[ -L "${path}" ]]; then + touch -h -- "${path}" + elif [[ -e "${path}" ]]; then + touch -- "${path}" + fi + done < <(git diff --name-only -z "${cached_commit}" HEAD --) + else + echo "::warning::Warm-cache commit is unavailable; leaving checkout mtimes unchanged" + fi - - name: Cache jemalloc - id: cache-jemalloc - if: runner.os == 'Linux' - uses: actions/cache@v4 - with: &jemalloc-cache - path: ${{ github.workspace }}/.ci-cache/jemalloc - key: ${{ steps.native-cache-keys.outputs.jemalloc-key }} - - - &configure-cached-jemalloc - name: Configure cached jemalloc - if: runner.os == 'Linux' && steps.cache-jemalloc.outputs.cache-hit == 'true' - shell: bash - run: echo "JEMALLOC_OVERRIDE=${{ github.workspace }}/.ci-cache/jemalloc/libjemalloc_pic.a" >>"$GITHUB_ENV" + - uses: dsherret/rust-toolchain-file@v1 + - &set-default-rust-toolchain + name: Set default rust toolchain + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - name: Cache Zstd - id: cache-zstd - if: runner.os == 'Linux' - uses: actions/cache@v4 - with: &zstd-cache - path: ${{ github.workspace }}/.ci-cache/zstd - key: ${{ steps.native-cache-keys.outputs.zstd-key }} - - - &configure-cached-zstd - name: Configure cached Zstd - if: runner.os == 'Linux' && steps.cache-zstd.outputs.cache-hit == 'true' - shell: bash - run: | - echo "ZSTD_SYS_USE_PKG_CONFIG=1" >>"$GITHUB_ENV" - echo "PKG_CONFIG_PATH=${{ github.workspace }}/.ci-cache/zstd/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >>"$GITHUB_ENV" + - name: print dir + run: pwd -P - name: Build CLI and standalone shell: bash run: | - cargo build --timings --release \ + cargo build -vv --timings --release \ -p spacetimedb-cli \ -p spacetimedb-standalone \ --features spacetimedb-standalone/allow_loopback_http_for_tests - - &prepare-openssl-cache - name: Prepare OpenSSL cache - if: steps.cache-openssl.outputs.cache-hit != 'true' - shell: bash - run: | - shopt -s nullglob - openssl_installs=(target/release/build/openssl-sys-*/out/openssl-build/install) - if (( ${#openssl_installs[@]} != 1 )); then - echo "Expected one vendored OpenSSL installation, found ${#openssl_installs[@]}." - exit 1 - fi - mkdir -p .ci-cache/openssl - cp -R "${openssl_installs[0]}/." .ci-cache/openssl/ - - - name: Prepare jemalloc cache - if: runner.os == 'Linux' && steps.cache-jemalloc.outputs.cache-hit != 'true' - shell: bash - run: | - shopt -s nullglob - jemalloc_archives=(target/release/build/tikv-jemalloc-sys-*/out/lib/libjemalloc_pic.a) - if (( ${#jemalloc_archives[@]} != 1 )); then - echo "Expected one jemalloc archive, found ${#jemalloc_archives[@]}." - exit 1 - fi - mkdir -p .ci-cache/jemalloc - cp "${jemalloc_archives[0]}" .ci-cache/jemalloc/ - echo "JEMALLOC_OVERRIDE=${{ github.workspace }}/.ci-cache/jemalloc/libjemalloc_pic.a" >>"$GITHUB_ENV" - - - name: Prepare Zstd cache - if: runner.os == 'Linux' && steps.cache-zstd.outputs.cache-hit != 'true' - shell: bash - run: | - shopt -s nullglob - zstd_outputs=(target/release/build/zstd-sys-*/out) - if (( ${#zstd_outputs[@]} != 1 )); then - echo "Expected one vendored Zstd build, found ${#zstd_outputs[@]}." - exit 1 - fi - mkdir -p .ci-cache/zstd/lib/pkgconfig .ci-cache/zstd/include - cp "${zstd_outputs[0]}/libzstd.a" .ci-cache/zstd/lib/ - cp "${zstd_outputs[0]}"/include/*.h .ci-cache/zstd/include/ - printf '%s\n' \ - 'prefix=${pcfiledir}/../..' \ - 'libdir=${prefix}/lib' \ - 'includedir=${prefix}/include' \ - '' \ - 'Name: libzstd' \ - 'Description: Zstandard compression library' \ - 'Version: 1.5.7' \ - 'Libs: -L${libdir} -lzstd' \ - 'Cflags: -I${includedir}' \ - > .ci-cache/zstd/lib/pkgconfig/libzstd.pc - echo "ZSTD_SYS_USE_PKG_CONFIG=1" >>"$GITHUB_ENV" - echo "PKG_CONFIG_PATH=${{ github.workspace }}/.ci-cache/zstd/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >>"$GITHUB_ENV" - - name: Package build artifacts shell: bash run: | - tar -czf build-support.tar.gz \ - "target/release/spacetimedb-cli${EXE_SUFFIX}" \ - "target/release/spacetimedb-standalone${EXE_SUFFIX}" + tar -C "${CARGO_TARGET_DIR}" -czf build-support.tar.gz \ + "release/spacetimedb-cli${EXE_SUFFIX}" \ + "release/spacetimedb-standalone${EXE_SUFFIX}" - name: Upload Cargo timing reports if: always() @@ -289,12 +198,11 @@ jobs: upload-build-artifacts-windows: needs: [merge_queue_noop, lints] - if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + if: false name: Upload build artifacts (Windows) runs-on: spacetimedb-windows-runner timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: windows EXE_SUFFIX: .exe @@ -304,48 +212,41 @@ jobs: smoketest_build_linux: needs: [upload-build-artifacts-linux] name: Build smoketests (Linux) - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux EXE_SUFFIX: "" SMOKETEST_SUITE: standalone steps: &smoketest-build-steps - - *find-git-ref + - *set-cargo-target-dir + - *find-git-ref - *checkout-sources + - *restore-source-mtimes + - uses: dsherret/rust-toolchain-file@v1 - *set-default-rust-toolchain - - *set-native-cache-keys - - *verify-openssl-assembler - - &restore-openssl - name: Restore OpenSSL - id: cache-openssl - uses: actions/cache/restore@v4 - with: *openssl-cache - - *configure-cached-openssl - - name: Install cargo-nextest uses: taiki-e/install-action@nextest - name: Build smoketest dependencies and archive test binaries shell: bash run: | - cargo run --timings --package ci-smoketests -- --suite "${SMOKETEST_SUITE}" archive --archive-file smoketest-nextest.tar.zst + cargo run -vv --timings --package ci -- smoketests --suite "${SMOKETEST_SUITE}" archive --archive-file smoketest-nextest.tar.zst shopt -s nullglob - precompiled_modules=(target/wasm32-unknown-unknown/release/smoketest_module_*.wasm) + precompiled_modules=(${CARGO_TARGET_DIR}/wasm32-unknown-unknown/release/smoketest_module_*.wasm) if (( ${#precompiled_modules[@]} == 0 )); then echo "No precompiled smoketest modules were produced." exit 1 fi tar -czf smoketest-support.tar.gz \ - "target/debug/ci-smoketests${EXE_SUFFIX}" \ + "${CARGO_TARGET_DIR}/debug/ci-smoketests${EXE_SUFFIX}" \ "${precompiled_modules[@]}" - name: Upload Cargo timing reports @@ -390,19 +291,21 @@ jobs: fail-fast: false matrix: partition: [1] - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging timeout-minutes: 30 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp ARTIFACT_SUFFIX: linux PARTITION_COUNT: 1 steps: &smoketest-partition-steps - - *find-git-ref + - *set-cargo-target-dir + - *find-git-ref - *checkout-sources + - *restore-source-mtimes + - uses: dsherret/rust-toolchain-file@v1 - *set-default-rust-toolchain @@ -472,22 +375,26 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - - name: Download build artifacts + - &download-build-artifacts + name: Download build artifacts uses: actions/download-artifact@v4 with: name: build-artifacts-${{ env.ARTIFACT_SUFFIX }} path: build-artifacts - - name: Extract build artifacts + - &extract-build-artifacts + name: Extract build artifacts shell: bash run: | - tar -xzf build-artifacts/build-support.tar.gz exe_suffix="" if [[ "${RUNNER_OS}" == "Windows" ]]; then exe_suffix=".exe" fi + mkdir -p "${CARGO_TARGET_DIR}" + tar -C "${CARGO_TARGET_DIR}" -xzf build-artifacts/build-support.tar.gz test -f "${CARGO_TARGET_DIR}/release/spacetimedb-cli${exe_suffix}" test -f "${CARGO_TARGET_DIR}/release/spacetimedb-standalone${exe_suffix}" + echo "SPACETIME_BIN=${CARGO_TARGET_DIR}/release/spacetimedb-cli${exe_suffix}" >>"${GITHUB_ENV}" - name: Download smoketest build uses: actions/download-artifact@v4 @@ -506,7 +413,7 @@ jobs: if [ -f ~/emsdk/emsdk_env.sh ]; then source ~/emsdk/emsdk_env.sh fi - ./target/debug/ci-smoketests run-archive \ + cargo ci smoketests run-archive \ --archive-file smoketest-nextest.tar.zst \ -- \ --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} @@ -519,7 +426,7 @@ jobs: if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") { & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null } - .\target\debug\ci-smoketests.exe run-archive ` + cargo ci smoketests run-archive ` --archive-file smoketest-nextest.tar.zst ` -- ` --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} @@ -570,15 +477,17 @@ jobs: run: echo "Merge queue commit has the same tree as the PR head; smoketest already ran for the PR." test: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Test Suite - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux steps: + - *set-cargo-target-dir + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -595,38 +504,15 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} + fetch-depth: 0 + + - *restore-source-mtimes - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - &restore-jemalloc - name: Restore jemalloc - id: cache-jemalloc - if: runner.os == 'Linux' - uses: actions/cache/restore@v4 - with: *jemalloc-cache - - *configure-cached-jemalloc - - &restore-zstd - name: Restore Zstd - id: cache-zstd - if: runner.os == 'Linux' - uses: actions/cache/restore@v4 - with: *zstd-cache - - *configure-cached-zstd - - &restore-rusty-v8-debug - name: Restore rusty_v8 (debug) - uses: actions/cache/restore@v4 - with: &v8-debug-cache - path: ${{ env.CARGO_TARGET_DIR }}/debug/gn_out/obj - key: ${{ steps.native-cache-keys.outputs.v8-debug-key }} - - &restore-rusty-v8 - name: Restore rusty_v8 - uses: actions/cache/restore@v4 - with: *v8-release-cache - - *restore-openssl - - *configure-cached-openssl - + - *download-build-artifacts + - *extract-build-artifacts - uses: actions/setup-dotnet@v3 with: global-json-file: global.json @@ -669,7 +555,7 @@ jobs: INSTALLED_WASM_BINDGEN_VERSION="$(wasm-bindgen --version 2>/dev/null | awk '{print $2}' || true)" if [ "${INSTALLED_WASM_BINDGEN_VERSION}" != "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then - cargo install --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" + cargo install -vv --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" fi wasm-bindgen --version @@ -678,7 +564,7 @@ jobs: - name: Run tests run: | source ~/emsdk/emsdk_env.sh - cargo ci test + cargo ci test --no-build - name: Upload timing reports if: always() @@ -690,7 +576,7 @@ jobs: index_scan_bench: needs: [merge_queue_noop, lints] - if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + if: false name: Index Scan Bench runs-on: spacetimedb-benchmark-runner concurrency: @@ -722,46 +608,31 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - name: Run index scan benchmark regression check - run: cargo run --release -p spacetimedb-index-scan-gate + run: cargo run -vv --release -p spacetimedb-index-scan-gate lints: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Lints - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - name: Checkout sources uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - *restore-source-mtimes - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - run: echo ::add-matcher::.github/workflows/rust_matcher.json - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - name: Cache rusty_v8 (debug) - uses: actions/cache@v4 - with: *v8-debug-cache - - *restore-openssl - - *configure-cached-openssl - - uses: actions/setup-dotnet@v3 with: global-json-file: global.json @@ -789,19 +660,22 @@ jobs: codeowners_check: if: ${{ github.event_name == 'pull_request' }} name: CODEOWNERS check - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging permissions: contents: read pull-requests: read env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: + - *set-cargo-target-dir + - uses: actions/checkout@v4 with: fetch-depth: 0 + - *restore-source-mtimes + - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) @@ -820,58 +694,48 @@ jobs: path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 - wasm_bindings: - needs: [merge_queue_noop] + module_latest_deps: + needs: [merge_queue_noop, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Build and test wasm bindings - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux steps: - - uses: actions/checkout@v3 + - *set-cargo-target-dir + + - *find-git-ref + - *checkout-sources + - *restore-source-mtimes - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + - *set-default-rust-toolchain - run: echo ::add-matcher::.github/workflows/rust_matcher.json - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-openssl - - *configure-cached-openssl - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: ./.github/actions/setup-pnpm - - - name: Run bindgen tests - run: cargo ci wasm-bindings + - *download-build-artifacts + - *extract-build-artifacts - - name: Upload timing reports - if: always() - uses: actions/upload-artifact@v4 - with: - name: cargo-timings-wasm-bindings - path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ - retention-days: 14 + - name: Build module with latest compatible dependencies + run: cargo ci module-latest-deps --no-build publish_checks: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check that packages are publishable - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging permissions: read-all env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - *restore-source-mtimes + - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) @@ -883,7 +747,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-publish-checks - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 update: @@ -901,18 +765,17 @@ jobs: env: RUST_BACKTRACE: full steps: - - name: Checkout - uses: actions/checkout@v3 + - *set-cargo-target-dir + + - *find-git-ref + - *checkout-sources + - *restore-source-mtimes - name: Install Rust uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-zstd - - *configure-cached-zstd - - name: Install rust target run: rustup target add ${{ matrix.target }} @@ -926,7 +789,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-update-flow-${{ matrix.target }} - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 # this is a no-op version of the above check with a trivially-passing body. @@ -952,7 +815,7 @@ jobs: name: Unreal Engine Tests # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use # a custom runner. - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging # Disable the tests because they are very flaky at the moment. # TODO: Remove this line and re-enable the `if` line just below here. if: false @@ -1038,7 +901,7 @@ jobs: cd "$GITHUB_WORKSPACE/sdks/unreal" cargo --version - cargo test -- --test-threads=1 + cargo test -vv -- --test-threads=1 ' cli_docs: @@ -1046,11 +909,12 @@ jobs: if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check CLI docs permissions: read-all - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1068,6 +932,9 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} + fetch-depth: 0 + + - *restore-source-mtimes - name: Set up Node.js uses: actions/setup-node@v4 @@ -1082,14 +949,6 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-openssl - - *configure-cached-openssl - - name: Check for docs change run: | cargo ci cli-docs @@ -1103,10 +962,10 @@ jobs: retention-days: 14 unity-testsuite: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] # Skip if this is an external contribution. # The license secrets will be empty, so the step would fail anyway. - if: ${{ needs.merge_queue_noop.outputs.skip != 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + if: false permissions: contents: read checks: write @@ -1115,11 +974,11 @@ jobs: env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux UNITY_VERSION: 2022.3.32f1 steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 + - *find-git-ref + - *checkout-sources # Keep this before generation/hydration so ignored build outputs do not affect hashFiles. - name: Restore Unity Library @@ -1168,23 +1027,15 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl + - *download-build-artifacts + - *extract-build-artifacts - - name: Install SpacetimeDB CLI from the local checkout + - &expose-spacetime-cli + name: Expose SpacetimeDB CLI on PATH + shell: bash run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + ln -sf spacetimedb-cli "${CARGO_TARGET_DIR}/release/spacetime" + echo "${CARGO_TARGET_DIR}/release" >>"${GITHUB_PATH}" - name: Generate client bindings working-directory: demo/Blackholio/server-rust @@ -1247,18 +1098,20 @@ jobs: key: ${{ steps.restore-unity-library.outputs.cache-primary-key }} godot-testsuite: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} permissions: contents: read - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target + ARTIFACT_SUFFIX: linux UseLocalBsatnRuntime: true steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 + - *set-cargo-target-dir + + - *find-git-ref + - *checkout-sources + - *restore-source-mtimes - name: Setup dotnet uses: actions/setup-dotnet@v3 @@ -1295,23 +1148,9 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - - name: Install SpacetimeDB CLI from the local checkout - run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + - *download-build-artifacts + - *extract-build-artifacts + - *expose-spacetime-cli - name: Generate client bindings working-directory: demo/Blackholio/server-rust @@ -1361,17 +1200,19 @@ jobs: run: godot --headless --scene res://tests/GodotPlayModeTests.tscn csharp-testsuite: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging timeout-minutes: 30 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 + - *set-cargo-target-dir + + - *find-git-ref + - *checkout-sources + - *restore-source-mtimes - name: Setup dotnet uses: actions/setup-dotnet@v3 @@ -1419,24 +1260,9 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8-debug - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - - name: Install SpacetimeDB CLI from the local checkout - run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --features allow_loopback_http_for_tests --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + - *download-build-artifacts + - *extract-build-artifacts + - *expose-spacetime-cli - name: Check quickstart-chat bindings are up to date run: | @@ -1490,12 +1316,14 @@ jobs: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Verify global.json files are symlinks - runs-on: ubuntu-latest + runs-on: spacetimedb-new-runner-2-staging permissions: contents: read env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - name: Find Git ref env: PR_NUMBER: ${{ github.event.inputs.pr_number }} @@ -1511,6 +1339,9 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} + fetch-depth: 0 + + - *restore-source-mtimes - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain @@ -1524,19 +1355,21 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-global-json-policy - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 smoketests_mod_rs_complete: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check smoketests/mod.rs is complete - runs-on: ubuntu-latest + runs-on: spacetimedb-new-runner-2-staging permissions: contents: read env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1553,24 +1386,33 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} + fetch-depth: 0 + + - *restore-source-mtimes + - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Verify smoketest module lists and suite constraints - run: | - cargo run -p ci-smoketest-checks + run: cargo ci smoketests check-mod-list docs-build: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Docs build - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - name: Checkout repository uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - *restore-source-mtimes - name: Set up Node.js uses: actions/setup-node@v3 @@ -1593,20 +1435,23 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-docs-build - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 typescript-test: - needs: [merge_queue_noop] + needs: [merge_queue_noop, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: TypeScript - Tests - runs-on: spacetimedb-new-runner-2 + runs-on: spacetimedb-new-runner-2-staging env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux steps: - - name: Checkout repository - uses: actions/checkout@v4 + - *set-cargo-target-dir + + - *find-git-ref + - *checkout-sources + - *restore-source-mtimes - name: Set up Node.js uses: actions/setup-node@v4 @@ -1617,65 +1462,23 @@ jobs: with: run_install: true - # - name: Extract SpacetimeDB branch name from file - # id: extract-branch - # run: | - # # Define the path to the branch file - # BRANCH_FILE=".github/spacetimedb-branch.txt" - - # # Default to master if file doesn't exist - # if [ ! -f "$BRANCH_FILE" ]; then - # echo "::notice::No SpacetimeDB branch file found, using 'master'" - # echo "branch=master" >> $GITHUB_OUTPUT - # exit 0 - # fi - - # # Read and trim whitespace from the file - # branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - - # # Fallback to master if empty - # if [ -z "$branch" ]; then - # echo "::warning::SpacetimeDB branch file is empty, using 'master'" - # branch="master" - # fi - - # echo "branch=$branch" >> $GITHUB_OUTPUT - # echo "Using SpacetimeDB branch from file: $branch" - - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8-debug - - *restore-openssl - - *configure-cached-openssl - - # - name: Install SpacetimeDB CLI from the local checkout - # run: | - # export CARGO_HOME="$HOME/.cargo" - # echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - # cargo install --force --path crates/cli --locked --message-format=short - # cargo install --force --path crates/standalone --locked --message-format=short - # # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - # ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - # # Clear any existing information - # spacetime server clear -y + - *download-build-artifacts + - *extract-build-artifacts - name: Run TypeScript tests - run: cargo ci typescript-test + run: cargo ci typescript-test --no-build - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 with: name: cargo-timings-typescript-test - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 # - name: Run quickstart-chat tests diff --git a/.github/workflows/internal-tests.yml b/.github/workflows/internal-tests.yml index e2301d3da96..57198e22011 100644 --- a/.github/workflows/internal-tests.yml +++ b/.github/workflows/internal-tests.yml @@ -25,8 +25,7 @@ jobs: name: Internal Tests # Skip if not a PR or a push to master # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway. - if: ${{ (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master')) - && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + if: false runs-on: ubuntu-latest env: TARGET_OWNER: clockworklabs diff --git a/Cargo.lock b/Cargo.lock index 22cd022fb09..d16dc421a4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ci-module-latest-deps" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-common", + "clap 4.5.50", + "duct", +] + [[package]] name = "ci-publish-checks" version = "0.1.0" @@ -970,6 +980,14 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-run-spacetime" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", +] + [[package]] name = "ci-smoketest-checks" version = "0.1.0" @@ -1025,16 +1043,6 @@ dependencies = [ "duct", ] -[[package]] -name = "ci-wasm-bindings" -version = "0.1.0" -dependencies = [ - "anyhow", - "ci-common", - "clap 4.5.50", - "duct", -] - [[package]] name = "ci-workflow-watch" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 44858f71b9c..28529189980 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ members = [ "tools/ci", "tools/ci/commands/test", "tools/ci/commands/lint", - "tools/ci/commands/wasm-bindings", + "tools/ci/commands/module-latest-deps", "tools/ci/commands/smoketests", "tools/ci/commands/smoketest-checks", "tools/ci/commands/keynote-bench", @@ -78,6 +78,7 @@ members = [ "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", "tools/ci/commands/publish-checks", + "tools/ci/commands/run-spacetime", "tools/ci/commands/typescript-test", "tools/ci/commands/version-upgrade-check", "tools/ci/commands/docs", diff --git a/crates/bindings-typescript/test-app/package.json b/crates/bindings-typescript/test-app/package.json index 8ba058dad9d..7c431dd7a54 100644 --- a/crates/bindings-typescript/test-app/package.json +++ b/crates/bindings-typescript/test-app/package.json @@ -12,7 +12,7 @@ "format": "prettier . --write --ignore-path ../../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../../.prettierignore", "preview": "vite preview", - "generate": "cargo run -p gen-bindings -- --replacement ../../../src/index && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --replacement ../../../src/index && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --project-path server", "spacetime:start": "spacetime start", "spacetime:publish:local": "spacetime publish game --project-path server --server local", diff --git a/crates/bindings-typescript/test-react-router-app/package.json b/crates/bindings-typescript/test-react-router-app/package.json index db0f2d504d2..4b165faa9c2 100644 --- a/crates/bindings-typescript/test-react-router-app/package.json +++ b/crates/bindings-typescript/test-react-router-app/package.json @@ -12,7 +12,7 @@ "format": "prettier . --write --ignore-path ../../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../../.prettierignore", "preview": "vite preview", - "generate": "cargo run -p gen-bindings -- --replacement ../../../src/index && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --replacement ../../../src/index && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path server", "spacetime:start": "spacetime start", "spacetime:publish:local": "spacetime publish game --module-path server --server local", diff --git a/crates/bindings-typescript/test-solid-router/package.json b/crates/bindings-typescript/test-solid-router/package.json index f5dc7bb5842..01a0af04b3a 100644 --- a/crates/bindings-typescript/test-solid-router/package.json +++ b/crates/bindings-typescript/test-solid-router/package.json @@ -9,7 +9,7 @@ "dev": "vite", "build": "vite build", "serve": "vite preview", - "generate": "cargo run -p gen-bindings -- --replacement ../../../src/index && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --replacement ../../../src/index && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path server", "spacetime:start": "spacetime start", "spacetime:publish:local": "spacetime publish game --module-path server --server local", diff --git a/modules/benchmarks-ts/package.json b/modules/benchmarks-ts/package.json index 27c251622dc..3dd18348bb2 100644 --- a/modules/benchmarks-ts/package.json +++ b/modules/benchmarks-ts/package.json @@ -4,9 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "keywords": [], "author": "", diff --git a/modules/module-test-ts/package.json b/modules/module-test-ts/package.json index 4e964319a75..c5ed04bbe20 100644 --- a/modules/module-test-ts/package.json +++ b/modules/module-test-ts/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-case-conversion-ts/package.json b/modules/sdk-test-case-conversion-ts/package.json index 326c97ec7fc..f18f60c3e5c 100644 --- a/modules/sdk-test-case-conversion-ts/package.json +++ b/modules/sdk-test-case-conversion-ts/package.json @@ -4,9 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "keywords": [], "author": "", diff --git a/modules/sdk-test-connect-disconnect-ts/package.json b/modules/sdk-test-connect-disconnect-ts/package.json index cd7b60b0e47..10b1b1d76d6 100644 --- a/modules/sdk-test-connect-disconnect-ts/package.json +++ b/modules/sdk-test-connect-disconnect-ts/package.json @@ -4,9 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "keywords": [], "author": "", diff --git a/modules/sdk-test-procedural-view-pk-ts/package.json b/modules/sdk-test-procedural-view-pk-ts/package.json index ed3cefd9de5..c9ec6933973 100644 --- a/modules/sdk-test-procedural-view-pk-ts/package.json +++ b/modules/sdk-test-procedural-view-pk-ts/package.json @@ -4,9 +4,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-procedure-ts/package.json b/modules/sdk-test-procedure-ts/package.json index c8b4bc4ba89..97f104e94cc 100644 --- a/modules/sdk-test-procedure-ts/package.json +++ b/modules/sdk-test-procedure-ts/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-ts/package.json b/modules/sdk-test-ts/package.json index 4e964319a75..c5ed04bbe20 100644 --- a/modules/sdk-test-ts/package.json +++ b/modules/sdk-test-ts/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-view-pk-ts/package.json b/modules/sdk-test-view-pk-ts/package.json index a03509611e5..28918205874 100644 --- a/modules/sdk-test-view-pk-ts/package.json +++ b/modules/sdk-test-view-pk-ts/package.json @@ -4,9 +4,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/sdks/csharp/tools~/gen-quickstart.sh b/sdks/csharp/tools~/gen-quickstart.sh index e683271bf53..834dcbcd111 100755 --- a/sdks/csharp/tools~/gen-quickstart.sh +++ b/sdks/csharp/tools~/gen-quickstart.sh @@ -7,6 +7,8 @@ SDK_PATH="$(realpath "$SDK_PATH")" STDB_PATH="$SDK_PATH/../.." DOTNET_VERSION="${1:-}" +cd "$STDB_PATH" + GLOBAL_JSON_BACKUPS=() backup_global_json_once() { @@ -66,5 +68,4 @@ if [ -n "$DOTNET_VERSION" ]; then BUILD_OPTIONS+=("--build-options=--dotnet-version $DOTNET_VERSION") fi -cargo build --manifest-path "$STDB_PATH/crates/standalone/Cargo.toml" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$STDB_PATH/templates/chat-console-cs/module_bindings" --module-path "$STDB_PATH/templates/chat-console-cs/spacetimedb" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$STDB_PATH/templates/chat-console-cs/module_bindings" --module-path "$STDB_PATH/templates/chat-console-cs/spacetimedb" "${BUILD_OPTIONS[@]}" diff --git a/sdks/csharp/tools~/gen-regression-tests.sh b/sdks/csharp/tools~/gen-regression-tests.sh index 54852ff03fb..d7bd005cf45 100755 --- a/sdks/csharp/tools~/gen-regression-tests.sh +++ b/sdks/csharp/tools~/gen-regression-tests.sh @@ -7,6 +7,8 @@ SDK_PATH="$(realpath "$SDK_PATH")" STDB_PATH="$SDK_PATH/../.." DOTNET_VERSION="${1:-}" +cd "$STDB_PATH" + GLOBAL_JSON_BACKUPS=() expected_global_json_symlink_target() { @@ -81,7 +83,6 @@ if [ -n "$DOTNET_VERSION" ]; then BUILD_OPTIONS+=("--build-options=--dotnet-version $DOTNET_VERSION") fi -cargo build --manifest-path "$STDB_PATH/crates/standalone/Cargo.toml" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/server" "${BUILD_OPTIONS[@]}" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/republishing/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/republishing/server-republish" "${BUILD_OPTIONS[@]}" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/procedure-client/module_bindings" --module-path "$STDB_PATH/modules/sdk-test-procedure" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/server" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/republishing/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/republishing/server-republish" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/procedure-client/module_bindings" --module-path "$STDB_PATH/modules/sdk-test-procedure" "${BUILD_OPTIONS[@]}" diff --git a/sdks/csharp/tools~/run-regression-tests.sh b/sdks/csharp/tools~/run-regression-tests.sh index fb3aca90f4d..8415f319ee7 100644 --- a/sdks/csharp/tools~/run-regression-tests.sh +++ b/sdks/csharp/tools~/run-regression-tests.sh @@ -9,6 +9,8 @@ SDK_PATH="$(realpath "$SDK_PATH")" STDB_PATH="$SDK_PATH/../.." SPACETIMEDB_SERVER_URL="${SPACETIMEDB_SERVER_URL:-local}" +cd "$STDB_PATH" + DOTNET_VERSIONS=("$@") if [ ${#DOTNET_VERSIONS[@]} -eq 0 ]; then DOTNET_VERSIONS=(8 10) @@ -99,9 +101,6 @@ run_client() { trap restore_global_jsons EXIT -# Build and run SpacetimeDB server -cargo build --manifest-path "$STDB_PATH/crates/standalone/Cargo.toml" - for dotnet_version in "${DOTNET_VERSIONS[@]}"; do echo "Running C# regression tests with .NET $dotnet_version" @@ -111,13 +110,13 @@ for dotnet_version in "${DOTNET_VERSIONS[@]}"; do "$SDK_PATH/tools~/gen-regression-tests.sh" "$dotnet_version" # Publish module for btree test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/server" btree-repro + cargo spacetime publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/server" btree-repro # Publish module for republishing module test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-initial" republish-test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 1 - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-republish" --break-clients republish-test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 2 + cargo spacetime publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-initial" republish-test + cargo spacetime call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 1 + cargo spacetime publish --dotnet-version "$dotnet_version" --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-republish" --break-clients republish-test + cargo spacetime call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 2 echo "Cleanup obj~ folders generated in $SDK_PATH/examples~/regression-tests/procedure-client" # There is a bug in the code generator that creates obj~ folders in the output directory using a Rust project. @@ -125,7 +124,7 @@ for dotnet_version in "${DOTNET_VERSIONS[@]}"; do rm -rf "$SDK_PATH/examples~/regression-tests/procedure-client/module_bindings"/*/obj~ # Publish module for procedure tests - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$STDB_PATH/modules/sdk-test-procedure" procedure-tests + cargo spacetime publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$STDB_PATH/modules/sdk-test-procedure" procedure-tests # Run clients against the modules published with this .NET version. run_client "$SDK_PATH/examples~/regression-tests/client" "$dotnet_version" diff --git a/templates/angular-ts/package.json b/templates/angular-ts/package.json index b0cd069cf89..ccda26c5120 100644 --- a/templates/angular-ts/package.json +++ b/templates/angular-ts/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "node scripts/dev.mjs", "build": "ng build", - "generate": "cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/astro-ts/package.json b/templates/astro-ts/package.json index 1e194115bb8..0888e22265a 100644 --- a/templates/astro-ts/package.json +++ b/templates/astro-ts/package.json @@ -9,7 +9,7 @@ "preview": "astro preview", "start": "node ./dist/server/entry.mjs", "astro": "astro", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/chat-react-ts/package.json b/templates/chat-react-ts/package.json index a5a081caed2..b4177a49475 100644 --- a/templates/chat-react-ts/package.json +++ b/templates/chat-react-ts/package.json @@ -10,7 +10,7 @@ "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "preview": "vite preview", "test": "vitest run", - "generate": "cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path server --server local", "spacetime:publish": "spacetime publish --module-path server --server maincloud" diff --git a/templates/chat-react-ts/spacetimedb/package.json b/templates/chat-react-ts/spacetimedb/package.json index c8a3cdbdaa4..1662934e7a3 100644 --- a/templates/chat-react-ts/spacetimedb/package.json +++ b/templates/chat-react-ts/spacetimedb/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/templates/hangman-react-ts/package.json b/templates/hangman-react-ts/package.json index 4076ae402cf..ccb5f1c5ad0 100644 --- a/templates/hangman-react-ts/package.json +++ b/templates/hangman-react-ts/package.json @@ -9,7 +9,7 @@ "format": "prettier . --write --ignore-path ../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "preview": "vite preview", - "generate": "cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/money-exchange-react-ts/package.json b/templates/money-exchange-react-ts/package.json index c42682d1744..86e199644b9 100644 --- a/templates/money-exchange-react-ts/package.json +++ b/templates/money-exchange-react-ts/package.json @@ -10,7 +10,7 @@ "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "preview": "vite preview", "test": "vitest run", - "generate": "cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/nextjs-ts/package.json b/templates/nextjs-ts/package.json index 9ec40873f49..92ee1799d47 100644 --- a/templates/nextjs-ts/package.json +++ b/templates/nextjs-ts/package.json @@ -8,7 +8,7 @@ "build": "next build", "start": "next start", "lint": "next lint", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/nuxt-ts/package.json b/templates/nuxt-ts/package.json index 5c220f3bcc8..6034481ec97 100644 --- a/templates/nuxt-ts/package.json +++ b/templates/nuxt-ts/package.json @@ -7,7 +7,7 @@ "dev": "nuxt dev", "build": "nuxt build", "preview": "nuxt preview", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir module_bindings --module-path spacetimedb && prettier --write module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir module_bindings --module-path spacetimedb && prettier --write module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/react-ts/package.json b/templates/react-ts/package.json index 00765c9e27a..b847cda21ac 100644 --- a/templates/react-ts/package.json +++ b/templates/react-ts/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "tsc -b && vite build", "preview": "vite preview", - "generate": "cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path server --server local", "spacetime:publish": "spacetime publish --module-path server --server maincloud" diff --git a/templates/remix-ts/package.json b/templates/remix-ts/package.json index 155dd879d04..4d9f0ee1f0d 100644 --- a/templates/remix-ts/package.json +++ b/templates/remix-ts/package.json @@ -7,7 +7,7 @@ "dev": "npx remix vite:dev", "build": "npx remix vite:build", "start": "npx remix-serve ./build/server/index.js", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/solid-ts/package.json b/templates/solid-ts/package.json index ed5cf238373..973486cce1c 100644 --- a/templates/solid-ts/package.json +++ b/templates/solid-ts/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/svelte-ts/package.json b/templates/svelte-ts/package.json index fb909102b0e..9e0a32f9b07 100644 --- a/templates/svelte-ts/package.json +++ b/templates/svelte-ts/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/templates/vue-ts/package.json b/templates/vue-ts/package.json index 8a41c1f0ef3..5621895c30b 100644 --- a/templates/vue-ts/package.json +++ b/templates/vue-ts/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "vue-tsc -b && vite build", "preview": "vite preview", - "generate": "pnpm --dir spacetimedb install && cargo run -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", + "generate": "pnpm --dir spacetimedb install && cargo run -vv -p gen-bindings -- --out-dir src/module_bindings --module-path spacetimedb && prettier --write src/module_bindings", "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", "spacetime:publish:local": "spacetime publish --module-path spacetimedb --server local", "spacetime:publish": "spacetime publish --module-path spacetimedb --server maincloud" diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index 956d4f44ce4..10bb7c61125 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -308,6 +308,7 @@ fn main() -> Result<()> { cmd!( "cargo", "clippy", + "-vv", "--timings", "--all", "--tests", @@ -320,6 +321,7 @@ fn main() -> Result<()> { cmd!( "cargo", "clippy", + "-vv", "--timings", "--no-default-features", "--features=browser", @@ -336,16 +338,16 @@ fn main() -> Result<()> { .dir("crates/bindings-csharp") .run()?; pnpm(["lint"]).run()?; - cmd!("cargo", "test", "--doc", "--target", "wasm32-unknown-unknown") + cmd!("cargo", "test", "-vv", "--doc", "--target", "wasm32-unknown-unknown") .dir("crates/bindings") .run()?; - cmd!("cargo", "test", "--doc").dir("crates/bindings").run()?; + cmd!("cargo", "test", "-vv", "--doc").dir("crates/bindings").run()?; // `bindings` is the only crate we care strongly about documenting, // since we link to its docs.rs from our website. // We won't pass `--no-deps`, though, // since we want everything reachable through it to also work. // This includes `sats` and `lib`. - cmd!("cargo", "doc") + cmd!("cargo", "doc", "-vv") .dir("crates/bindings") // Make `cargo doc` exit with error on warnings, most notably broken links .env("RUSTDOCFLAGS", "--deny warnings") diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/module-latest-deps/Cargo.toml similarity index 84% rename from tools/ci/commands/wasm-bindings/Cargo.toml rename to tools/ci/commands/module-latest-deps/Cargo.toml index e807f25ea7e..c360db5b600 100644 --- a/tools/ci/commands/wasm-bindings/Cargo.toml +++ b/tools/ci/commands/module-latest-deps/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ci-wasm-bindings" +name = "ci-module-latest-deps" version = "0.1.0" edition.workspace = true diff --git a/tools/ci/commands/module-latest-deps/src/main.rs b/tools/ci/commands/module-latest-deps/src/main.rs new file mode 100644 index 00000000000..83d8851c4f2 --- /dev/null +++ b/tools/ci/commands/module-latest-deps/src/main.rs @@ -0,0 +1,48 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{ensure, Result}; +use clap::Parser; +use duct::cmd; +use std::{env, path::PathBuf}; + +/// Checks that a module builds with the latest compatible dependencies. +#[derive(Parser)] +struct Cli { + /// Do not build the CLI; use the binary selected by SPACETIME_BIN. + #[arg(long)] + no_build: bool, +} + +fn target_dir() -> PathBuf { + match env::var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + Some(path) if path.is_absolute() => path, + Some(path) => ci_common::repo_root().join(path), + None => ci_common::repo_root().join("target"), + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + // Build the CLI before updating the lockfile so a newly published incompatible dependency + // cannot prevent us from exercising the fresh module dependency graph. + let cli_path = if cli.no_build { + ci_common::require_spacetime_bin()? + } else { + ensure!( + env::var_os("SPACETIME_BIN").is_none(), + "SPACETIME_BIN requires --no-build" + ); + cmd!("cargo", "build", "-vv", "-p", "spacetimedb-cli").run()?; + target_dir() + .join("debug/spacetimedb-cli") + .with_extension(env::consts::EXE_EXTENSION) + }; + + // A fresh module gets the latest versions permitted by its dependency constraints rather + // than the exact versions pinned in this repository's committed lockfile. + cmd!("cargo", "update", "-vv").run()?; + cmd!(cli_path, "build", "--module-path", "modules/module-test").run()?; + + Ok(()) +} diff --git a/tools/ci/commands/run-spacetime/Cargo.toml b/tools/ci/commands/run-spacetime/Cargo.toml new file mode 100644 index 00000000000..3ec38ddf291 --- /dev/null +++ b/tools/ci/commands/run-spacetime/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-run-spacetime" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true + +[lints] +workspace = true diff --git a/tools/ci/commands/run-spacetime/src/main.rs b/tools/ci/commands/run-spacetime/src/main.rs new file mode 100644 index 00000000000..183ba415b78 --- /dev/null +++ b/tools/ci/commands/run-spacetime/src/main.rs @@ -0,0 +1,78 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{bail, ensure, Result}; +use duct::cmd; +use std::{ + env, + ffi::OsString, + path::{Path, PathBuf}, +}; + +const CLI_NAME: &str = "spacetimedb-cli"; +const STANDALONE_NAME: &str = "spacetimedb-standalone"; + +fn executable_name(name: &str) -> OsString { + if env::consts::EXE_EXTENSION.is_empty() { + name.into() + } else { + format!("{name}.{}", env::consts::EXE_EXTENSION).into() + } +} + +fn validate_runtime(cli: &Path) -> Result<()> { + ensure!( + cli.is_file(), + "SpacetimeDB CLI binary does not exist at {}", + cli.display() + ); + + let standalone = cli.with_file_name(executable_name(STANDALONE_NAME)); + ensure!( + standalone.is_file(), + "SpacetimeDB standalone binary does not exist beside the CLI at {}", + standalone.display() + ); + + Ok(()) +} + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(4) + .expect("run-spacetime package should be nested beneath the workspace root") + .to_owned() +} + +fn local_cli() -> Result { + cmd!("cargo", "build", "-vv", "-p", CLI_NAME, "-p", STANDALONE_NAME).run()?; + + let target_dir = env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| repo_root().join("target")); + Ok(target_dir.join("debug").join(executable_name(CLI_NAME))) +} + +fn main() -> Result<()> { + let args = env::args_os().skip(1).collect::>(); + if args.is_empty() { + bail!("Usage: cargo spacetime [args...]"); + } + + let cli = match env::var_os("SPACETIME_BIN") { + Some(path) => { + let path = PathBuf::from(path); + ensure!( + path.is_absolute(), + "SPACETIME_BIN must be an absolute path, got {}", + path.display() + ); + path + } + None => local_cli()?, + }; + + validate_runtime(&cli)?; + cmd(cli, args).run()?; + Ok(()) +} diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index 3dbf2087fe7..67909ca2de8 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -120,7 +120,7 @@ fn main() -> Result<()> { fn build_cli() -> Result<()> { let mut cmd = Command::new("cargo"); - cmd.args(["build", "--timings", "--release", "-p", "spacetimedb-cli"]); + cmd.args(["build", "-vv", "--timings", "--release", "-p", "spacetimedb-cli"]); run_binary_build(cmd, "Failed to build CLI") } @@ -128,6 +128,7 @@ fn build_standalone() -> Result<()> { let mut cmd = Command::new("cargo"); cmd.args([ "build", + "-vv", "--timings", "--release", "-p", @@ -176,6 +177,7 @@ fn build_precompiled_modules() -> Result<()> { let status = Command::new("cargo") .args([ "build", + "-vv", "--timings", "--workspace", "--release", diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 373a30ef49a..3f68623cfc4 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,5 +1,5 @@ #![allow(clippy::disallowed_macros)] -use anyhow::Result; +use anyhow::{ensure, Result}; use ci_common::pnpm; use clap::Parser; use duct::cmd; @@ -10,10 +10,23 @@ use duct::cmd; /// This does not include Unreal tests. /// This expects to run in a clean git state. #[derive(Parser)] -struct Cli {} +struct Cli { + /// Do not build CLI and standalone; use the binaries selected by SPACETIME_BIN. + #[arg(long)] + no_build: bool, +} fn main() -> Result<()> { - Cli::parse(); + let cli = Cli::parse(); + + if cli.no_build { + ci_common::require_runtime()?; + } else { + ensure!( + std::env::var_os("SPACETIME_BIN").is_none(), + "SPACETIME_BIN requires --no-build" + ); + } pnpm(["build"]).dir("crates/bindings-typescript").run()?; @@ -24,6 +37,7 @@ fn main() -> Result<()> { cmd!( "cargo", "test", + "-vv", "--all", "--exclude", "spacetimedb-smoketests", @@ -43,6 +57,7 @@ fn main() -> Result<()> { cmd!( "cargo", "test", + "-vv", "-p", "spacetimedb", "--features", @@ -53,22 +68,26 @@ fn main() -> Result<()> { .run()?; // The SDK test harness uses the same child-process server guard as smoketests, // which expects release CLI/standalone binaries to already exist. - cmd!( - "cargo", - "build", - "--release", - "-p", - "spacetimedb-cli", - "-p", - "spacetimedb-standalone", - "--features", - "spacetimedb-standalone/allow_loopback_http_for_tests", - ) - .run()?; + if !cli.no_build { + cmd!( + "cargo", + "build", + "-vv", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ) + .run()?; + } // SDK procedure tests intentionally make localhost HTTP requests. cmd!( "cargo", "test", + "-vv", "-p", "spacetimedb-sdk", "--features", @@ -83,6 +102,7 @@ fn main() -> Result<()> { cmd!( "cargo", "test", + "-vv", "-p", "spacetimedb-sdk", "--features", @@ -101,6 +121,7 @@ fn main() -> Result<()> { cmd!( "cargo", "test", + "-vv", "-p", "spacetimedb-durability", "--features", @@ -113,6 +134,7 @@ fn main() -> Result<()> { cmd!( "cargo", "run", + "-vv", "-p", "spacetimedb-codegen", "--example", diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index f18a3b82bfb..8fc8d4ce34f 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -1,20 +1,31 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Result}; use ci_common::pnpm; use clap::Parser; -use duct::cmd; /// Runs TypeScript workspace tests and template build checks. #[derive(Parser)] -struct Cli {} +struct Cli { + /// Do not build CLI and standalone; use the binaries selected by SPACETIME_BIN. + #[arg(long)] + no_build: bool, +} fn main() -> Result<()> { - Cli::parse(); + let cli = Cli::parse(); + if cli.no_build { + ci_common::require_runtime()?; + } else { + ensure!( + std::env::var_os("SPACETIME_BIN").is_none(), + "SPACETIME_BIN requires --no-build" + ); + } pnpm(["build"]).dir("crates/bindings-typescript").run()?; pnpm(["test"]).dir("crates/bindings-typescript").run()?; pnpm(["generate"]).dir("templates/chat-react-ts").run()?; - let diff_status = cmd!( + let diff_status = duct::cmd!( "bash", "tools/check-diff.sh", "templates/chat-react-ts/src/module_bindings" diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs index 273e5f26efc..d861b3ce20f 100644 --- a/tools/ci/commands/update-flow/src/main.rs +++ b/tools/ci/commands/update-flow/src/main.rs @@ -40,7 +40,7 @@ fn main() -> Result<()> { cmd( "cargo", - ["build", "-p", "spacetimedb-update"] + ["build", "-vv", "-p", "spacetimedb-update"] .into_iter() .chain(common_args.clone()), ) @@ -53,7 +53,7 @@ fn main() -> Result<()> { let root_arg = format!("--root-dir={}", root_dir_string); cmd( "cargo", - ["run", "-p", "spacetimedb-update"] + ["run", "-vv", "-p", "spacetimedb-update"] .into_iter() .chain(common_args.clone()) .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs deleted file mode 100644 index aa1946d11ac..00000000000 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ /dev/null @@ -1,41 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::Result; -use ci_common::pnpm; -use clap::Parser; -use duct::cmd; - -/// Tests Wasm bindings -/// -/// Runs tests for the codegen crate and builds a test module with the wasm bindings. -#[derive(Parser)] -struct Cli {} - -fn main() -> Result<()> { - Cli::parse(); - - pnpm([ - "install", - "--filter", - "./crates/bindings-typescript...", - "--filter", - "./modules/module-test-ts...", - ]) - .run()?; - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; - // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. - cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; - // Make sure the `Cargo.lock` file reflects the latest available versions. - // This is what users would end up with on a fresh module, so we want to - // catch any compile errors arising from a different transitive closure - // of dependencies than what is in the workspace lock file. - // - // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - cmd!("cargo", "update").run()?; - let cli_path = ci_common::repo_root() - .join("target/debug/spacetimedb-cli") - .with_extension(std::env::consts::EXE_EXTENSION); - cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; - - Ok(()) -} diff --git a/tools/ci/common/src/lib.rs b/tools/ci/common/src/lib.rs index 4dfafe8e03b..063383cd0ba 100644 --- a/tools/ci/common/src/lib.rs +++ b/tools/ci/common/src/lib.rs @@ -1,5 +1,6 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Context, Result}; use duct::{cmd, Expression}; +use std::env; use std::ffi::OsStr; use std::path::Path; use std::path::PathBuf; @@ -20,6 +21,36 @@ pub fn repo_root() -> PathBuf { .to_path_buf() } +pub fn require_spacetime_bin() -> Result { + let path = env::var_os("SPACETIME_BIN") + .map(PathBuf::from) + .context("--no-build requires SPACETIME_BIN")?; + ensure!( + path.is_absolute(), + "SPACETIME_BIN must be an absolute path, got {}", + path.display() + ); + ensure!( + path.is_file(), + "SpacetimeDB CLI binary does not exist at {}", + path.display() + ); + Ok(path) +} + +pub fn require_runtime() -> Result<()> { + let cli = require_spacetime_bin()?; + let standalone = cli + .with_file_name("spacetimedb-standalone") + .with_extension(env::consts::EXE_EXTENSION); + ensure!( + standalone.is_file(), + "SpacetimeDB standalone binary does not exist beside the CLI at {}", + standalone.display() + ); + Ok(()) +} + pub fn pnpm(args: I) -> Expression where I: IntoIterator, diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index bb0878277d9..e75a539619c 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -21,8 +21,8 @@ const COMMANDS: &[Command] = &[ package: "ci-lint", }, Command { - path: &["wasm-bindings"], - package: "ci-wasm-bindings", + path: &["module-latest-deps"], + package: "ci-module-latest-deps", }, Command { path: &["smoketests"], @@ -80,6 +80,10 @@ const COMMANDS: &[Command] = &[ path: &["other-workflows", "watch"], package: "ci-workflow-watch", }, + Command { + path: &["other-workflows", "run-spacetime"], + package: "ci-run-spacetime", + }, ]; fn print_help() { @@ -112,7 +116,7 @@ fn is_skipped(command: &Command, skips: &[String]) -> bool { } fn run_package(package: &str, args: &[String]) -> Result<()> { - let mut cargo_args = vec!["run", "--timings", "--package", package, "--"]; + let mut cargo_args = vec!["run", "-vv", "--timings", "--package", package, "--"]; cargo_args.extend(args.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; Ok(()) diff --git a/tools/gen-bindings/src/main.rs b/tools/gen-bindings/src/main.rs index 88fe7c7cc51..2c36792470b 100644 --- a/tools/gen-bindings/src/main.rs +++ b/tools/gen-bindings/src/main.rs @@ -46,26 +46,17 @@ fn run_inherit(cmd: impl AsRef, args: &[&str], cwd: Option<&Path>) -> Res fn main() -> Result<()> { let args = Cli::parse(); - let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap(); - - // 1) Build prerequisite - run_inherit("cargo", &["build"], Some(workspace_dir))?; - - // 2) Ensure output directory exists + // 1) Ensure output directory exists if !Path::new(&args.out_dir).exists() { fs::create_dir_all(&args.out_dir).context("create output directory")?; } - // 3) Generate TS client from project + // 2) Generate TS client from project run_inherit( - workspace_dir - .join("target/debug/spacetimedb-cli") - .with_extension(std::env::consts::EXE_EXTENSION), + "cargo", &[ + "-vv", + "spacetime", "generate", "-y", "--lang", diff --git a/tools/generate-client-api/src/main.rs b/tools/generate-client-api/src/main.rs index e9b794f93a2..0626a36c199 100644 --- a/tools/generate-client-api/src/main.rs +++ b/tools/generate-client-api/src/main.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Context, Result}; use replace_spacetimedb::{replace_in_tree, ReplaceOptions}; use std::ffi::OsStr; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use tempfile::NamedTempFile; @@ -25,11 +25,20 @@ fn run_inherit(cmd: impl AsRef, args: &[&str], cwd: Option<&Path>) -> Res Ok(()) } +fn cargo_target_dir(workspace_dir: &Path) -> PathBuf { + match std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + Some(path) if path.is_absolute() => path, + Some(path) => workspace_dir.join(path), + None => workspace_dir.join("target"), + } +} + /// Run a command and return captured stdout as UTF-8 string. fn run_capture(cmd: &str, args: &[&str]) -> Result { let out = Command::new(cmd) .args(args) .stdin(Stdio::null()) + .stderr(Stdio::inherit()) .output() .with_context(|| format!("Failed to start {cmd}"))?; if !out.status.success() { @@ -50,7 +59,7 @@ fn main() -> Result<()> { .unwrap(); // 1) Build prerequisite - run_inherit("cargo", &["build"], Some(workspace_dir))?; + run_inherit("cargo", &["build", "-vv"], Some(workspace_dir))?; // 2) Get schema to a temp file (auto-cleaned) let mut tmp_schema = NamedTempFile::new().context("create temp schema file")?; @@ -58,6 +67,7 @@ fn main() -> Result<()> { "cargo", &[ "run", + "-vv", "-p", "spacetimedb-client-api-messages", "--example", @@ -75,8 +85,8 @@ fn main() -> Result<()> { // 4) Generate TS client run_inherit( - workspace_dir - .join("target/debug/spacetimedb-cli") + cargo_target_dir(workspace_dir) + .join("debug/spacetimedb-cli") .with_extension(std::env::consts::EXE_EXTENSION), &[ "generate",