diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3a72a61..743325b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,7 @@ jobs: - .build* - vcpkg.json - .github/workflows/ci.yml + - .github/workflows/data-track-publish-many.yml - .github/workflows/tests.yml - .github/workflows/nightly.yml docs: diff --git a/.github/workflows/data-track-publish-many.yml b/.github/workflows/data-track-publish-many.yml new file mode 100644 index 00000000..3c26eee9 --- /dev/null +++ b/.github/workflows/data-track-publish-many.yml @@ -0,0 +1,30 @@ +name: Data Track Publish-Many Reproduction + +on: + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + branches: ["main"] + workflow_dispatch: + +permissions: + contents: read + actions: read + packages: read + +jobs: + macos-x64: + name: macOS x64 Reproduction + uses: ./.github/workflows/tests.yml + with: + build_type: debug + unit_repeat: 0 + integration_repeat: 20 + integration_filter: "DataTrackE2ETest.PublishManyTracks" + integration_timeout_minutes: 15 + run_stress_tests: false + artifact_retention_days: 14 + run_coverage: false + rust_log: "livekit_datatrack=debug,livekit=debug,livekit_ffi=debug,metrics=debug" + test_matrix: >- + {"include":[{"os":"macos-26-large","name":"macos-x64","macos_arch":"x86_64","e2e-testing":true}]} + secrets: inherit diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index daf4bb45..8f2307f8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,4 +26,5 @@ jobs: stress_timeout_minutes: 120 artifact_retention_days: 14 run_coverage: false + rust_log: info secrets: inherit diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 887c5a58..16b41c1b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,6 +54,22 @@ on: required: false type: boolean default: true + rust_log: + description: Optional RUST_LOG value for test jobs (e.g. info). + required: false + type: string + default: "" + integration_filter: + description: GoogleTest filter for integration tests. + required: false + type: string + default: "-PlatformAudioIntegrationTest.*" + test_matrix: + description: JSON test matrix; override to target a subset of platforms. + required: false + type: string + default: >- + {"include":[{"os":"ubuntu-latest","name":"linux-x64","e2e-testing":true},{"os":"ubuntu-24.04-arm","name":"linux-arm64","e2e-testing":true},{"os":"macos-26-xlarge","name":"macos-arm64","e2e-testing":true},{"os":"macos-26-large","name":"macos-x64","macos_arch":"x86_64","e2e-testing":true},{"os":"windows-2022","name":"windows-x64","e2e-testing":true}]} workflow_dispatch: inputs: build_type: @@ -109,6 +125,22 @@ on: required: false type: boolean default: true + rust_log: + description: Optional RUST_LOG value for test jobs (e.g. info). + required: false + type: string + default: "" + integration_filter: + description: GoogleTest filter for integration tests. + required: false + type: string + default: "-PlatformAudioIntegrationTest.*" + test_matrix: + description: JSON test matrix; override to target a subset of platforms. + required: false + type: string + default: >- + {"include":[{"os":"ubuntu-latest","name":"linux-x64","e2e-testing":true},{"os":"ubuntu-24.04-arm","name":"linux-arm64","e2e-testing":true},{"os":"macos-26-xlarge","name":"macos-arm64","e2e-testing":true},{"os":"macos-26-large","name":"macos-x64","macos_arch":"x86_64","e2e-testing":true},{"os":"windows-2022","name":"windows-x64","e2e-testing":true}]} permissions: contents: read @@ -118,6 +150,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full + RUST_LOG: ${{ inputs.rust_log }} # Disable Cargo incremental artifacts. The Rust FFI is built once per # submodule SHA and cached whole (see the cargo target cache below), so # incremental dirs add nothing but bloat the cached target/ directory and @@ -137,25 +170,7 @@ jobs: test: strategy: fail-fast: false - matrix: - include: - - os: ubuntu-latest - name: linux-x64 - e2e-testing: true - - os: ubuntu-24.04-arm - name: linux-arm64 - e2e-testing: true - - os: macos-26-xlarge - name: macos-arm64 - e2e-testing: true - - os: macos-26-large - name: macos-x64 - macos_arch: x86_64 - e2e-testing: true - # Pinned to Windows 2022 for current VS 17 implementation - - os: windows-2022 - name: windows-x64 - e2e-testing: true + matrix: ${{ fromJSON(inputs.test_matrix) }} name: Test (${{ matrix.name }}) runs-on: ${{ matrix.os }} @@ -319,7 +334,7 @@ jobs: # ---------- Run unit tests ---------- - name: Run unit tests (Unix) - if: runner.os != 'Windows' + if: runner.os != 'Windows' && inputs.unit_repeat > 0 timeout-minutes: ${{ inputs.unit_timeout_minutes }} shell: bash run: | @@ -330,7 +345,7 @@ jobs: --gtest_output=xml:${{ env.BUILD_DIR }}/unit-test-results.xml - name: Run unit tests (Windows) - if: runner.os == 'Windows' + if: runner.os == 'Windows' && inputs.unit_repeat > 0 timeout-minutes: ${{ inputs.unit_timeout_minutes }} shell: pwsh run: | @@ -395,7 +410,8 @@ jobs: timeout-minutes: ${{ inputs.integration_timeout_minutes }} shell: bash env: - RUST_LOG: "metrics=debug" + GTEST_FILTER: ${{ inputs.integration_filter }} + RUST_LOG: ${{ inputs.rust_log != '' && inputs.rust_log || 'metrics=debug' }} LIVEKIT_CREATE_TOKEN_URL: ${{ steps.token_server.outputs.token-url }} run: | set -euo pipefail @@ -403,7 +419,7 @@ jobs: # TODO(BOT-477) Temporarily disable platform audio integration tests in CI until instability is resolved bash scripts/run-with-backtrace.sh \ ${{ env.BUILD_DIR }}/bin/livekit_integration_tests \ - --gtest_filter='-PlatformAudioIntegrationTest.*' \ + --gtest_filter="${GTEST_FILTER}" \ --gtest_repeat=${{ inputs.integration_repeat }} \ --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml @@ -613,4 +629,3 @@ jobs: name: coverage-report path: coverage-html/ retention-days: 14 - diff --git a/scripts/run-with-backtrace.sh b/scripts/run-with-backtrace.sh index bf35ac73..dfbab049 100644 --- a/scripts/run-with-backtrace.sh +++ b/scripts/run-with-backtrace.sh @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Runs a command normally, then prints a native backtrace from any core file it -# leaves after crashing. Linux cores are inspected with GDB and macOS cores -# with LLDB, avoiding debugger interference with normal process behavior. +# Runs a command normally, prints a live process stack when terminated, then +# prints a native backtrace from any core file left after crashing. Linux cores +# are inspected with GDB and macOS cores with LLDB, avoiding debugger +# interference with normal process behavior. set -euo pipefail if (($# == 0)); then @@ -40,10 +41,60 @@ mkdir -p "${LIVEKIT_CORE_DIR}" rm -f "${LIVEKIT_CORE_DIR}"/livekit-core.* ulimit -c unlimited +child_pid="" + +# Invoked indirectly by the signal traps below. +# shellcheck disable=SC2329 +dump_live_process() { + if [[ -z "${child_pid}" ]] || ! kill -0 "${child_pid}" 2>/dev/null; then + echo "::warning::Test process is no longer running; a live stack could not be captured." + return + fi + + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + echo "::group::Live test process stacks before termination" + fi + + if [[ "${platform}" == "Linux" ]]; then + timeout 30s gdb --batch --quiet \ + -ex "set pagination off" \ + -ex "info threads" \ + -ex "thread apply all backtrace" \ + -p "${child_pid}" || true + else + # `sample` is available on GitHub's macOS runners and does not require a + # crash/core file. A short sample provides all thread stacks while leaving + # enough of the runner's termination grace period to forward the signal. + /usr/bin/sample "${child_pid}" 3 10 || true + fi + + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + echo "::endgroup::" + fi +} + +# Invoked indirectly by the signal traps below. +# shellcheck disable=SC2329 +terminate_child() { + local signal_name="$1" + local exit_status="$2" + trap - TERM INT + dump_live_process + kill "-${signal_name}" "${child_pid}" 2>/dev/null || true + wait "${child_pid}" 2>/dev/null || true + exit "${exit_status}" +} + +trap 'terminate_child TERM 143' TERM +trap 'terminate_child INT 130' INT + set +e -"$@" +"$@" & +child_pid=$! +wait "${child_pid}" exit_code=$? set -e +trap - TERM INT shopt -s nullglob core_files=("${LIVEKIT_CORE_DIR}"/livekit-core.*) diff --git a/src/tests/integration/test_data_track.cpp b/src/tests/integration/test_data_track.cpp index 89690ab9..de1210e9 100644 --- a/src/tests/integration/test_data_track.cpp +++ b/src/tests/integration/test_data_track.cpp @@ -529,6 +529,7 @@ TEST_F(DataTrackE2ETest, SubscribeAfterUnpublishReportsTerminalError) { TEST_F(DataTrackE2ETest, PublishManyTracks) { auto rooms = testRooms(1); auto& room = rooms[0]; + std::cout << "[PublishManyTracks] room connected" << std::endl; std::vector> tracks; tracks.reserve(kPublishManyTrackCount); @@ -548,7 +549,7 @@ TEST_F(DataTrackE2ETest, PublishManyTracks) { } const auto elapsed = std::chrono::steady_clock::now() - start; - std::cout << "Publishing " << kPublishManyTrackCount << " tracks took " + std::cout << "[PublishManyTracks] publishing " << kPublishManyTrackCount << " tracks took " << std::chrono::duration_cast(elapsed).count() << " ms" << std::endl; EXPECT_LT(elapsed, kPublishManyTimeout); @@ -558,6 +559,7 @@ TEST_F(DataTrackE2ETest, PublishManyTracks) { // to make it onto the transport and "Failed to enqueue data track packet" // logs are expected. The purpose of this test is to verify publish/push // behavior and local track state, not end-to-end delivery of every packet. + std::cout << "[PublishManyTracks] pushing large frames" << std::endl; for (const auto& track : tracks) { auto push_result = track->tryPush(std::vector(kLargeFramePayloadBytes, 0xFA)); if (!push_result) { @@ -566,11 +568,23 @@ TEST_F(DataTrackE2ETest, PublishManyTracks) { } std::this_thread::sleep_for(50ms); } + std::cout << "[PublishManyTracks] large-frame pushes complete" << std::endl; + std::cout << "[PublishManyTracks] unpublishing tracks" << std::endl; for (const auto& track : tracks) { track->unpublishDataTrack(); EXPECT_FALSE(track->isPublished()); } + std::cout << "[PublishManyTracks] unpublish requests complete" << std::endl; + + // Preserve the natural destruction order (tracks before rooms), but make the + // teardown boundary visible so a timeout identifies the blocking phase. + tracks.clear(); + std::cout << "[PublishManyTracks] local track handles released" << std::endl; + + std::cout << "[PublishManyTracks] disconnecting room" << std::endl; + ASSERT_TRUE(room->disconnect()); + std::cout << "[PublishManyTracks] room disconnected" << std::endl; } TEST_F(DataTrackE2ETest, PublishDuplicateName) {