From e3de03829a17216be2f25fe053836134005e3f06 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Tue, 4 Aug 2026 15:26:03 -0700 Subject: [PATCH] ci: make the intermittent connect hang diagnosable and self-healing The suite intermittently freezes at its first SSH connection against mock-ssh-server (always a single ubuntu job, random Python version; observed across months of scheduled runs, e.g. Apr 26, Jul 25-28 and Aug 1-4 which were four red nightlies in a row). The 1-minute step timeout killed the job before anything could report, so every failure was an undiagnosable freeze. - pytest faulthandler_timeout=60 dumps all thread stacks when a test stalls, so the next hang shows where it is stuck - step timeout of 3 minutes leaves room for the dump and for asyncssh's login_timeout (120s) to raise a real error - pytest-rerunfailures retries the failed test once so a transient hang no longer reddens the whole run, while the dump stays in the log Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 8 ++++++-- pyproject.toml | 7 +++++++ requirements-dev.txt | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 77ba28f..2f3c45d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,8 +34,12 @@ jobs: python -m pip install -e . - name: Test - timeout-minutes: 1 - run: python -m pytest --cov-report=xml --cov=. + # Leave room for faulthandler_timeout (60s) to dump stacks and for + # asyncssh's login_timeout (120s) to raise before the step is killed. + timeout-minutes: 3 + run: >- + python -m pytest --cov-report=xml --cov=. + --reruns 1 --reruns-delay 2 - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/pyproject.toml b/pyproject.toml index 425bac0..ceb88ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,13 @@ namespaces = false [tool.setuptools_scm] +[tool.pytest.ini_options] +# The suite intermittently hangs on the first SSH connection against +# mock-ssh-server (Linux only). Dump all thread stacks when a test +# exceeds this, so a hang in CI is diagnosable instead of a mute +# step-timeout kill. +faulthandler_timeout = 60 + [tool.black] line-length = 79 target-version = ['py39'] diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b4add2..2433630 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ pytest pytest-asyncio pytest-cov +pytest-rerunfailures mock-ssh-server importlib-metadata >= 6.0.0