diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml index f68734209fa..a39b93567d3 100644 --- a/.github/actions/ccache-setup/action.yml +++ b/.github/actions/ccache-setup/action.yml @@ -42,9 +42,40 @@ runs: if command -v ccache >/dev/null 2>&1; then echo "ccache already installed: $(ccache --version | head -1)" elif [ "${{ runner.os }}" = "Linux" ]; then - sudo apt-get update -q - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - --no-install-recommends ccache + export DEBIAN_FRONTEND=noninteractive + # install-apt-deps stages the WHOLE ghcr bundle into + # /var/cache/apt/archives, and ccache is in the -minimal/-full + # lists, so in a job that ran it first the .deb is already on disk. + # Take it offline (--no-download): no apt-get update, nothing to + # stall on. Every other path here reaches the mirror, which is what + # used to hang these jobs for 10-40 min after the bundle had + # already installed cleanly. + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + --no-install-recommends --no-download ccache; then + echo "ccache installed offline from the staged .deb bundle" + else + # Same defence in depth as install-apt-deps: Acquire timeouts drop + # a stalled connection, `timeout` hard-kills a wedged apt-get, and + # only then does the retry loop get a non-zero exit to act on. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + ok="" + for i in 1 2; do + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \ + apt-get "${APT_OPTS[@]}" update -q && \ + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 180 \ + apt-get "${APT_OPTS[@]}" install -y \ + --no-install-recommends ccache; then + ok=1 + break + fi + echo "::warning::ccache apt install failed (attempt $i/2)" + sleep 5 + done + [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; } + fi elif [ "${{ runner.os }}" = "macOS" ]; then brew install ccache else diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml index b8cb68b4e4a..03417164832 100644 --- a/.github/actions/install-apt-deps/action.yml +++ b/.github/actions/install-apt-deps/action.yml @@ -5,9 +5,24 @@ inputs: description: 'Space-separated list of apt packages to install' required: true retries: - description: 'Number of retry attempts' + description: 'Number of retry attempts.' required: false - default: '3' + default: '2' + budget-seconds: + description: > + Wall-clock the whole retry loop may spend. Split across the attempts, + so a wedged mirror is reported by this action instead of the job being + cancelled around it. Must fit inside the caller's timeout-minutes + alongside pull-timeout; the default suits jobs of 15 minutes or more, + short jobs have to lower it. + required: false + default: '600' + pull-timeout: + description: > + Deadline for the ghcr bundle pull. Counts against the same + timeout-minutes as budget-seconds. + required: false + default: '300' retry-delay: description: 'Initial delay between retries (seconds, doubles each attempt)' required: false @@ -50,7 +65,7 @@ runs: # PRs read the public upstream image too rather than a nonexistent # ghcr.io//wolfssl-ci-debs. IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}" - if ! docker pull -q "$IMG" >/dev/null 2>&1; then + if ! timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" >/dev/null 2>&1; then echo "::notice::ghcr bundle $IMG unavailable; using apt" exit 0 fi @@ -77,21 +92,52 @@ runs: if: steps.ghcr.outputs.satisfied != 'true' shell: bash run: | - export DEBIAN_FRONTEND=noninteractive RETRIES=${{ inputs.retries }} DELAY=${{ inputs.retry-delay }} + BUDGET=${{ inputs.budget-seconds }} NO_REC="" if [ "${{ inputs.no-install-recommends }}" = "true" ]; then NO_REC="--no-install-recommends" fi + # A wedged mirror hangs apt rather than failing it, so the retry loop + # below never fired and the job burned its whole budget instead. + # Defend in depth: apt drops a stalled connection after 30s and retries + # it (Acquire timeouts - this is what actually detects a wedge, in + # ~90s), `timeout` hard-kills an apt-get that wedged outside its own + # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a + # retry can land on a different mirror. apt resumes from + # archives/partial/, so a killed transfer is not restarted from + # scratch. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + + # Spend budget-seconds over the attempts rather than a fixed + # per-attempt deadline: a caller with a short timeout-minutes would + # otherwise be cancelled mid-attempt, before the loop could report + # the failure. update gets a sixth of an attempt, install the rest, + # with floors so a small budget still leaves apt time to work. + PER=$((BUDGET / RETRIES)) + UPD=$((PER / 6)) + [ "$UPD" -ge 20 ] || UPD=20 + INS=$((PER - UPD)) + [ "$INS" -ge 40 ] || INS=40 + DEADLINE=$(($(date +%s) + BUDGET)) + + # sudo resets the environment, so DEBIAN_FRONTEND has to ride along + # on each privileged command rather than being exported once. for i in $(seq 1 $RETRIES); do - if sudo apt-get update -q && \ - sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then + # A previous attempt killed mid-unpack leaves dpkg needing this. + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \ + apt-get "${APT_OPTS[@]}" update -q && \ + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \ + apt-get "${APT_OPTS[@]}" install -y \ + $NO_REC ${{ inputs.packages }}; then exit 0 fi - if [ "$i" -eq "$RETRIES" ]; then - echo "::error::apt-get failed after $RETRIES attempts" + if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then + echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s" exit 1 fi echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..." diff --git a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt index c32e3ccb9cf..6fdbf8cfbba 100644 --- a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt +++ b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt @@ -4,6 +4,7 @@ autoconf automake build-essential +ccache crossbuild-essential-arm64 crossbuild-essential-armel crossbuild-essential-armhf diff --git a/.github/scripts/zephyr-4.x/zephyr-test.sh b/.github/scripts/zephyr-4.x/zephyr-test.sh index c225277cacc..66a2930d64c 100755 --- a/.github/scripts/zephyr-4.x/zephyr-test.sh +++ b/.github/scripts/zephyr-4.x/zephyr-test.sh @@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..." west zephyr-export echo "==> [container] Installing host packages (newlib, python3-venv)..." -sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true +# `|| true` keeps this best-effort, but without a timeout a wedged mirror +# stalls here silently until the job budget runs out. +APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) +sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \ + && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \ + python3-venv libnewlib-dev >/dev/null 2>&1 \ + || echo "==> [container] host package install skipped (apt unavailable)" python3 -m venv .venv source .venv/bin/activate pip3 install west diff --git a/.github/workflows/check-source-text.yml b/.github/workflows/check-source-text.yml index cf3efb95da2..7f9524563b8 100644 --- a/.github/workflows/check-source-text.yml +++ b/.github/workflows/check-source-text.yml @@ -45,6 +45,9 @@ jobs: - name: Install shellcheck uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: shellcheck python3-yaml ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml index d91ac4fb877..76f1a65489d 100644 --- a/.github/workflows/ci-deps-image.yml +++ b/.github/workflows/ci-deps-image.yml @@ -183,21 +183,27 @@ jobs: set -euo pipefail K="${{ steps.check.outputs.kernel }}" # linuxkm.yml installs only the headers; the membrowse linuxkm targets - # also need the build toolchain. Bundle the union - each consumer - # installs its own subset offline. - PKGS=(build-essential autoconf automake libtool "linux-headers-$K") + # also need the build toolchain, and ccache-setup installs ccache + # offline from whatever this bundle staged. Bundle the union - each + # consumer installs its own subset offline. + PKGS=(build-essential autoconf automake libtool ccache + "linux-headers-$K") echo "Packages: ${PKGS[*]}" export DEBIAN_FRONTEND=noninteractive rm -rf debs && mkdir -p debs sudo apt-get clean - retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; } - retry sudo apt-get update -q + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) + # 2 attempts, not 5: this job's timeout-minutes is 20, and an + # attempt cut off mid-flight reports nothing. + retry() { local i; for i in 1 2; do "$@" && return 0; sleep 5; done; "$@"; } + retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q # The whole set is required and this bundle is small, so resolve it as # one closure and let any download failure fail the job. We push only # on success, so a transient mirror error keeps the last good bundle # rather than publishing a partial one - which the kernel-label skip # would then pin in place until the kernel next changes (~monthly). - retry sudo apt-get install -y --download-only "${PKGS[@]}" + retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \ + --download-only "${PKGS[@]}" sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files" test -n "$(ls debs/*.deb 2>/dev/null)" # headers are never preinstalled diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml index af25dd52cf1..34bb4da8c41 100644 --- a/.github/workflows/cross-library.yml +++ b/.github/workflows/cross-library.yml @@ -64,10 +64,28 @@ jobs: run: | set -eux export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }} + # These containers are bare images with no bash, so this step runs + # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it + # word-splits. + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection (and are what actually detects a wedge), + # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, + # and the loop then retries - re-reading the mirror list. Two + # attempts at 60s+300s fit inside this job's timeout-minutes; apt + # resumes from archives/partial/, so a killed transfer is not lost. + APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" + for i in 1 2; do + if timeout -k 10 60 apt-get $APT_OPTS update -q && \ + timeout -k 10 300 apt-get $APT_OPTS install -y \ + --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }}; then + break + fi + test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done # Building only needs the commit under test, not history. The break check # that needs history runs in the compile job, not here. @@ -129,10 +147,28 @@ jobs: run: | set -eux export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }} + # These containers are bare images with no bash, so this step runs + # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it + # word-splits. + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection (and are what actually detects a wedge), + # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, + # and the loop then retries - re-reading the mirror list. Two + # attempts at 60s+300s fit inside this job's timeout-minutes; apt + # resumes from archives/partial/, so a killed transfer is not lost. + APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" + for i in 1 2; do + if timeout -k 10 60 apt-get $APT_OPTS update -q && \ + timeout -k 10 300 apt-get $APT_OPTS install -y \ + --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }}; then + break + fi + test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done # This job does not build wolfSSL, but the latest leg still checks out # wolfSSL history because check-break.sh scans commit messages here. The diff --git a/.github/workflows/cyrus-sasl.yml b/.github/workflows/cyrus-sasl.yml index 5c1ad746e0a..c4a5bc3c97e 100644 --- a/.github/workflows/cyrus-sasl.yml +++ b/.github/workflows/cyrus-sasl.yml @@ -66,6 +66,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: krb5-kdc krb5-otp libkrb5-dev libsocket-wrapper libnss-wrapper krb5-admin-server libdb5.3-dev ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml index ad6bc8f0137..1cf613e2ff4 100644 --- a/.github/workflows/falcon-interop.yml +++ b/.github/workflows/falcon-interop.yml @@ -91,8 +91,26 @@ jobs: steps: - name: Install build tools run: | - sudo apt-get update - sudo apt-get install -y ninja-build + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } + apt_retry ninja-build # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which # would delete an untracked oqs-install/ placed in the workspace by the @@ -161,8 +179,26 @@ jobs: steps: - name: Install build tools run: | - sudo apt-get update - sudo apt-get install -y autoconf automake libtool + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } + apt_retry autoconf automake libtool - name: Checkout wolfSSL uses: actions/checkout@v5 diff --git a/.github/workflows/ipmitool.yml b/.github/workflows/ipmitool.yml index b5df5fbe124..6db463a4e63 100644 --- a/.github/workflows/ipmitool.yml +++ b/.github/workflows/ipmitool.yml @@ -63,6 +63,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: libreadline-dev ghcr-debs-tag: ubuntu-24.04-full - name: Download lib diff --git a/.github/workflows/jwt-cpp.yml b/.github/workflows/jwt-cpp.yml index e3263f3f996..df9a56247f0 100644 --- a/.github/workflows/jwt-cpp.yml +++ b/.github/workflows/jwt-cpp.yml @@ -67,6 +67,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: libgtest-dev ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/linuxkm.yml b/.github/workflows/linuxkm.yml index 8c404a4b0c8..a7194f2065d 100644 --- a/.github/workflows/linuxkm.yml +++ b/.github/workflows/linuxkm.yml @@ -36,6 +36,9 @@ jobs: - name: Install linux-headers uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: linux-headers-$(uname -r) ghcr-debs-tag: ubuntu-24.04-linuxkm diff --git a/.github/workflows/mosquitto.yml b/.github/workflows/mosquitto.yml index 49b515d274b..dbea1074939 100644 --- a/.github/workflows/mosquitto.yml +++ b/.github/workflows/mosquitto.yml @@ -78,6 +78,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: build-essential libev-dev libssl-dev automake python3-docutils libcunit1 libcunit1-doc libcunit1-dev pkg-config make python3-psutil ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/pam-ipmi.yml b/.github/workflows/pam-ipmi.yml index de907d05aff..faa19001a5a 100644 --- a/.github/workflows/pam-ipmi.yml +++ b/.github/workflows/pam-ipmi.yml @@ -64,6 +64,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: libpam-dev ninja-build meson ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/rng-tools.yml b/.github/workflows/rng-tools.yml index 1027ab80fc6..f14877d0124 100644 --- a/.github/workflows/rng-tools.yml +++ b/.github/workflows/rng-tools.yml @@ -65,6 +65,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: libcurl4-openssl-dev libjansson-dev libp11-dev librtlsdr-dev libcap-dev ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 4e2b3fcd950..8a6389a6c91 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -822,7 +822,25 @@ jobs: - name: Install build deps + SBOM validators run: | - sudo apt-get update + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } # bison + autotools-dev are required by strace's ./bootstrap. # gcc-multilib + g++-multilib give strace's --enable-mpers=check # the 32-bit/x32 compilers it needs - without them mpers is @@ -830,7 +848,7 @@ jobs: # syscalls, diverging from what bomsh's devcontainer produces. # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3 # stage. - sudo apt-get install -y build-essential autoconf automake libtool \ + apt_retry build-essential autoconf automake libtool \ bison autotools-dev gcc-multilib g++-multilib \ python3 python3-pip git python3 -m pip install --user --upgrade pip