WIP: Integration test hang debugging

.github/workflows/ci.yml

@@ -64,6 +64,7 @@ jobs:
               - .build*
               - vcpkg.json
               - .github/workflows/ci.yml
+              - .github/workflows/data-track-publish-many.yml
               - .github/workflows/tests.yml
               - .github/workflows/nightly.yml
             docs:

.github/workflows/data-track-publish-many.yml

@@ -0,0 +1,30 @@
+name: Data Track Publish-Many Reproduction
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+    branches: ["main"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+  packages: read
+
+jobs:
+  macos-x64:
+    name: macOS x64 Reproduction
+    uses: ./.github/workflows/tests.yml
+    with:
+      build_type: debug
+      unit_repeat: 0
+      integration_repeat: 20
+      integration_filter: "DataTrackE2ETest.PublishManyTracks"
+      integration_timeout_minutes: 15
+      run_stress_tests: false
+      artifact_retention_days: 14
+      run_coverage: false
+      rust_log: "livekit_datatrack=debug,livekit=debug,livekit_ffi=debug,metrics=debug"
+      test_matrix: >-
+        {"include":[{"os":"macos-26-large","name":"macos-x64","macos_arch":"x86_64","e2e-testing":true}]}
+    secrets: inherit

.github/workflows/nightly.yml

@@ -26,4 +26,5 @@ jobs:
       stress_timeout_minutes: 120
       artifact_retention_days: 14
       run_coverage: false
+      rust_log: info
     secrets: inherit

.github/workflows/tests.yml

@@ -54,6 +54,22 @@ on:
         required: false
         type: boolean
         default: true
+      rust_log:
+        description: Optional RUST_LOG value for test jobs (e.g. info).
+        required: false
+        type: string
+        default: ""
+      integration_filter:
+        description: GoogleTest filter for integration tests.
+        required: false
+        type: string
+        default: "-PlatformAudioIntegrationTest.*"
+      test_matrix:
+        description: JSON test matrix; override to target a subset of platforms.
+        required: false
+        type: string
+        default: >-
+          {"include":[{"os":"ubuntu-latest","name":"linux-x64","e2e-testing":true},{"os":"ubuntu-24.04-arm","name":"linux-arm64","e2e-testing":true},{"os":"macos-26-xlarge","name":"macos-arm64","e2e-testing":true},{"os":"macos-26-large","name":"macos-x64","macos_arch":"x86_64","e2e-testing":true},{"os":"windows-2022","name":"windows-x64","e2e-testing":true}]}
   workflow_dispatch:
     inputs:
       build_type:
@@ -109,6 +125,22 @@ on:
         required: false
         type: boolean
         default: true
+      rust_log:
+        description: Optional RUST_LOG value for test jobs (e.g. info).
+        required: false
+        type: string
+        default: ""
+      integration_filter:
+        description: GoogleTest filter for integration tests.
+        required: false
+        type: string
+        default: "-PlatformAudioIntegrationTest.*"
+      test_matrix:
+        description: JSON test matrix; override to target a subset of platforms.
+        required: false
+        type: string
+        default: >-
+          {"include":[{"os":"ubuntu-latest","name":"linux-x64","e2e-testing":true},{"os":"ubuntu-24.04-arm","name":"linux-arm64","e2e-testing":true},{"os":"macos-26-xlarge","name":"macos-arm64","e2e-testing":true},{"os":"macos-26-large","name":"macos-x64","macos_arch":"x86_64","e2e-testing":true},{"os":"windows-2022","name":"windows-x64","e2e-testing":true}]}
 
 permissions:
   contents: read
@@ -118,6 +150,7 @@ permissions:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
+  RUST_LOG: ${{ inputs.rust_log }}
   # Disable Cargo incremental artifacts. The Rust FFI is built once per
   # submodule SHA and cached whole (see the cargo target cache below), so
   # incremental dirs add nothing but bloat the cached target/ directory and
@@ -137,25 +170,7 @@ jobs:
   test:
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            name: linux-x64
-            e2e-testing: true
-          - os: ubuntu-24.04-arm
-            name: linux-arm64
-            e2e-testing: true
-          - os: macos-26-xlarge
-            name: macos-arm64
-            e2e-testing: true
-          - os: macos-26-large
-            name: macos-x64
-            macos_arch: x86_64
-            e2e-testing: true
-          # Pinned to Windows 2022 for current VS 17 implementation
-          - os: windows-2022
-            name: windows-x64
-            e2e-testing: true
+      matrix: ${{ fromJSON(inputs.test_matrix) }}
 
     name: Test (${{ matrix.name }})
     runs-on: ${{ matrix.os }}
@@ -319,7 +334,7 @@ jobs:
 
       # ---------- Run unit tests ----------
       - name: Run unit tests (Unix)
-        if: runner.os != 'Windows'
+        if: runner.os != 'Windows' && inputs.unit_repeat > 0
         timeout-minutes: ${{ inputs.unit_timeout_minutes }}
         shell: bash
         run: |
@@ -330,7 +345,7 @@ jobs:
             --gtest_output=xml:${{ env.BUILD_DIR }}/unit-test-results.xml
 
       - name: Run unit tests (Windows)
-        if: runner.os == 'Windows'
+        if: runner.os == 'Windows' && inputs.unit_repeat > 0
         timeout-minutes: ${{ inputs.unit_timeout_minutes }}
         shell: pwsh
         run: |
@@ -395,15 +410,16 @@ jobs:
         timeout-minutes: ${{ inputs.integration_timeout_minutes }}
         shell: bash
         env:
-          RUST_LOG: "metrics=debug"
+          GTEST_FILTER: ${{ inputs.integration_filter }}
+          RUST_LOG: ${{ inputs.rust_log != '' && inputs.rust_log || 'metrics=debug' }}
           LIVEKIT_CREATE_TOKEN_URL: ${{ steps.token_server.outputs.token-url }}
         run: |
           set -euo pipefail
           source .token_helpers/set_data_track_test_tokens.bash
           # TODO(BOT-477) Temporarily disable platform audio integration tests in CI until instability is resolved
           bash scripts/run-with-backtrace.sh \
             ${{ env.BUILD_DIR }}/bin/livekit_integration_tests \
-            --gtest_filter='-PlatformAudioIntegrationTest.*' \
+            --gtest_filter="${GTEST_FILTER}" \
             --gtest_repeat=${{ inputs.integration_repeat }} \
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml
@@ -613,4 +629,3 @@ jobs:
           name: coverage-report
           path: coverage-html/
           retention-days: 14
-

scripts/run-with-backtrace.sh

@@ -14,9 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Runs a command normally, then prints a native backtrace from any core file it
-# leaves after crashing. Linux cores are inspected with GDB and macOS cores
-# with LLDB, avoiding debugger interference with normal process behavior.
+# Runs a command normally, prints a live process stack when terminated, then
+# prints a native backtrace from any core file left after crashing. Linux cores
+# are inspected with GDB and macOS cores with LLDB, avoiding debugger
+# interference with normal process behavior.
 set -euo pipefail
 
 if (($# == 0)); then
@@ -40,10 +41,60 @@ mkdir -p "${LIVEKIT_CORE_DIR}"
 rm -f "${LIVEKIT_CORE_DIR}"/livekit-core.*
 ulimit -c unlimited
 
+child_pid=""
+
+# Invoked indirectly by the signal traps below.
+# shellcheck disable=SC2329
+dump_live_process() {
+  if [[ -z "${child_pid}" ]] || ! kill -0 "${child_pid}" 2>/dev/null; then
+    echo "::warning::Test process is no longer running; a live stack could not be captured."
+    return
+  fi
+
+  if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+    echo "::group::Live test process stacks before termination"
+  fi
+
+  if [[ "${platform}" == "Linux" ]]; then
+    timeout 30s gdb --batch --quiet \
+      -ex "set pagination off" \
+      -ex "info threads" \
+      -ex "thread apply all backtrace" \
+      -p "${child_pid}" || true
+  else
+    # `sample` is available on GitHub's macOS runners and does not require a
+    # crash/core file. A short sample provides all thread stacks while leaving
+    # enough of the runner's termination grace period to forward the signal.
+    /usr/bin/sample "${child_pid}" 3 10 || true
+  fi
+
+  if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+    echo "::endgroup::"
+  fi
+}
+
+# Invoked indirectly by the signal traps below.
+# shellcheck disable=SC2329
+terminate_child() {
+  local signal_name="$1"
+  local exit_status="$2"
+  trap - TERM INT
+  dump_live_process
+  kill "-${signal_name}" "${child_pid}" 2>/dev/null || true
+  wait "${child_pid}" 2>/dev/null || true
+  exit "${exit_status}"
+}
+
+trap 'terminate_child TERM 143' TERM
+trap 'terminate_child INT 130' INT
+
 set +e
-"$@"
+"$@" &
+child_pid=$!
+wait "${child_pid}"
 exit_code=$?
 set -e
+trap - TERM INT
 
 shopt -s nullglob
 core_files=("${LIVEKIT_CORE_DIR}"/livekit-core.*)

src/tests/integration/test_data_track.cpp

@@ -529,6 +529,7 @@ TEST_F(DataTrackE2ETest, SubscribeAfterUnpublishReportsTerminalError) {
 TEST_F(DataTrackE2ETest, PublishManyTracks) {
   auto rooms = testRooms(1);
   auto& room = rooms[0];
+  std::cout << "[PublishManyTracks] room connected" << std::endl;
 
   std::vector<std::shared_ptr<LocalDataTrack>> tracks;
   tracks.reserve(kPublishManyTrackCount);
@@ -548,7 +549,7 @@ TEST_F(DataTrackE2ETest, PublishManyTracks) {
   }
   const auto elapsed = std::chrono::steady_clock::now() - start;
 
-  std::cout << "Publishing " << kPublishManyTrackCount << " tracks took "
+  std::cout << "[PublishManyTracks] publishing " << kPublishManyTrackCount << " tracks took "
             << std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() << " ms" << std::endl;
   EXPECT_LT(elapsed, kPublishManyTimeout);
 
@@ -558,6 +559,7 @@ TEST_F(DataTrackE2ETest, PublishManyTracks) {
   // to make it onto the transport and "Failed to enqueue data track packet"
   // logs are expected. The purpose of this test is to verify publish/push
   // behavior and local track state, not end-to-end delivery of every packet.
+  std::cout << "[PublishManyTracks] pushing large frames" << std::endl;
   for (const auto& track : tracks) {
     auto push_result = track->tryPush(std::vector<std::uint8_t>(kLargeFramePayloadBytes, 0xFA));
     if (!push_result) {
@@ -566,11 +568,23 @@ TEST_F(DataTrackE2ETest, PublishManyTracks) {
     }
     std::this_thread::sleep_for(50ms);
   }
+  std::cout << "[PublishManyTracks] large-frame pushes complete" << std::endl;
 
+  std::cout << "[PublishManyTracks] unpublishing tracks" << std::endl;
   for (const auto& track : tracks) {
     track->unpublishDataTrack();
     EXPECT_FALSE(track->isPublished());
   }
+  std::cout << "[PublishManyTracks] unpublish requests complete" << std::endl;
+
+  // Preserve the natural destruction order (tracks before rooms), but make the
+  // teardown boundary visible so a timeout identifies the blocking phase.
+  tracks.clear();
+  std::cout << "[PublishManyTracks] local track handles released" << std::endl;
+
+  std::cout << "[PublishManyTracks] disconnecting room" << std::endl;
+  ASSERT_TRUE(room->disconnect());
+  std::cout << "[PublishManyTracks] room disconnected" << std::endl;
 }
 
 TEST_F(DataTrackE2ETest, PublishDuplicateName) {