CI: bound apt calls with timeouts and install ccache from staged bundle

.github/actions/ccache-setup/action.yml

@@ -42,9 +42,40 @@ runs:
         if command -v ccache >/dev/null 2>&1; then
           echo "ccache already installed: $(ccache --version | head -1)"
         elif [ "${{ runner.os }}" = "Linux" ]; then
-          sudo apt-get update -q
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            --no-install-recommends ccache
+          export DEBIAN_FRONTEND=noninteractive
+          # install-apt-deps stages the WHOLE ghcr bundle into
+          # /var/cache/apt/archives, and ccache is in the -minimal/-full
+          # lists, so in a job that ran it first the .deb is already on disk.
+          # Take it offline (--no-download): no apt-get update, nothing to
+          # stall on. Every other path here reaches the mirror, which is what
+          # used to hang these jobs for 10-40 min after the bundle had
+          # already installed cleanly.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+               --no-install-recommends --no-download ccache; then
+            echo "ccache installed offline from the staged .deb bundle"
+          else
+            # Same defence in depth as install-apt-deps: Acquire timeouts drop
+            # a stalled connection, `timeout` hard-kills a wedged apt-get, and
+            # only then does the retry loop get a non-zero exit to act on.
+            APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                      -o Acquire::https::Timeout=30)
+            ok=""
+            for i in 1 2; do
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
+              if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \
+                     apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 180 \
+                     apt-get "${APT_OPTS[@]}" install -y \
+                     --no-install-recommends ccache; then
+                ok=1
+                break
+              fi
+              echo "::warning::ccache apt install failed (attempt $i/2)"
+              sleep 5
+            done
+            [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
+          fi
         elif [ "${{ runner.os }}" = "macOS" ]; then
           brew install ccache
         else

.github/actions/install-apt-deps/action.yml

@@ -5,9 +5,24 @@ inputs:
     description: 'Space-separated list of apt packages to install'
     required: true
   retries:
-    description: 'Number of retry attempts'
+    description: 'Number of retry attempts.'
     required: false
-    default: '3'
+    default: '2'
+  budget-seconds:
+    description: >
+      Wall-clock the whole retry loop may spend. Split across the attempts,
+      so a wedged mirror is reported by this action instead of the job being
+      cancelled around it. Must fit inside the caller's timeout-minutes
+      alongside pull-timeout; the default suits jobs of 15 minutes or more,
+      short jobs have to lower it.
+    required: false
+    default: '600'
+  pull-timeout:
+    description: >
+      Deadline for the ghcr bundle pull. Counts against the same
+      timeout-minutes as budget-seconds.
+    required: false
+    default: '300'
   retry-delay:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
@@ -50,7 +65,7 @@ runs:
         # PRs read the public upstream image too rather than a nonexistent
         # ghcr.io/<fork>/wolfssl-ci-debs.
         IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}"
-        if ! docker pull -q "$IMG" >/dev/null 2>&1; then
+        if ! timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" >/dev/null 2>&1; then
           echo "::notice::ghcr bundle $IMG unavailable; using apt"
           exit 0
         fi
@@ -77,21 +92,52 @@ runs:
       if: steps.ghcr.outputs.satisfied != 'true'
       shell: bash
       run: |
-        export DEBIAN_FRONTEND=noninteractive
         RETRIES=${{ inputs.retries }}
         DELAY=${{ inputs.retry-delay }}
+        BUDGET=${{ inputs.budget-seconds }}
         NO_REC=""
         if [ "${{ inputs.no-install-recommends }}" = "true" ]; then
           NO_REC="--no-install-recommends"
         fi
 
+        # A wedged mirror hangs apt rather than failing it, so the retry loop
+        # below never fired and the job burned its whole budget instead.
+        # Defend in depth: apt drops a stalled connection after 30s and retries
+        # it (Acquire timeouts - this is what actually detects a wedge, in
+        # ~90s), `timeout` hard-kills an apt-get that wedged outside its own
+        # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a
+        # retry can land on a different mirror. apt resumes from
+        # archives/partial/, so a killed transfer is not restarted from
+        # scratch.
+        APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                  -o Acquire::https::Timeout=30)
+
+        # Spend budget-seconds over the attempts rather than a fixed
+        # per-attempt deadline: a caller with a short timeout-minutes would
+        # otherwise be cancelled mid-attempt, before the loop could report
+        # the failure. update gets a sixth of an attempt, install the rest,
+        # with floors so a small budget still leaves apt time to work.
+        PER=$((BUDGET / RETRIES))
+        UPD=$((PER / 6))
+        [ "$UPD" -ge 20 ] || UPD=20
+        INS=$((PER - UPD))
+        [ "$INS" -ge 40 ] || INS=40
+        DEADLINE=$(($(date +%s) + BUDGET))
+
+        # sudo resets the environment, so DEBIAN_FRONTEND has to ride along
+        # on each privileged command rather than being exported once.
         for i in $(seq 1 $RETRIES); do
-          if sudo apt-get update -q && \
-             sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
+          # A previous attempt killed mid-unpack leaves dpkg needing this.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \
+                 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \
+                 apt-get "${APT_OPTS[@]}" install -y \
+                 $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
-          if [ "$i" -eq "$RETRIES" ]; then
-            echo "::error::apt-get failed after $RETRIES attempts"
+          if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then
+            echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s"
             exit 1
           fi
           echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."

.github/ci-deps/packages-ubuntu-22.04-minimal.txt

@@ -4,6 +4,7 @@
 autoconf
 automake
 build-essential
+ccache
 crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf

.github/scripts/zephyr-4.x/zephyr-test.sh

@@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..."
 west zephyr-export
 
 echo "==> [container] Installing host packages (newlib, python3-venv)..."
-sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true
+# `|| true` keeps this best-effort, but without a timeout a wedged mirror
+# stalls here silently until the job budget runs out.
+APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \
+  && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \
+       python3-venv libnewlib-dev >/dev/null 2>&1 \
+  || echo "==> [container] host package install skipped (apt unavailable)"
 python3 -m venv .venv
 source .venv/bin/activate
 pip3 install west

.github/workflows/check-source-text.yml

@@ -45,6 +45,9 @@ jobs:
       - name: Install shellcheck
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: shellcheck python3-yaml
           ghcr-debs-tag: ubuntu-24.04-full
 

.github/workflows/ci-deps-image.yml

@@ -183,21 +183,27 @@ jobs:
           set -euo pipefail
           K="${{ steps.check.outputs.kernel }}"
           # linuxkm.yml installs only the headers; the membrowse linuxkm targets
-          # also need the build toolchain. Bundle the union - each consumer
-          # installs its own subset offline.
-          PKGS=(build-essential autoconf automake libtool "linux-headers-$K")
+          # also need the build toolchain, and ccache-setup installs ccache
+          # offline from whatever this bundle staged. Bundle the union - each
+          # consumer installs its own subset offline.
+          PKGS=(build-essential autoconf automake libtool ccache
+                "linux-headers-$K")
           echo "Packages: ${PKGS[*]}"
           export DEBIAN_FRONTEND=noninteractive
           rm -rf debs && mkdir -p debs
           sudo apt-get clean
-          retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
-          retry sudo apt-get update -q
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+          # 2 attempts, not 5: this job's timeout-minutes is 20, and an
+          # attempt cut off mid-flight reports nothing.
+          retry() { local i; for i in 1 2; do "$@" && return 0; sleep 5; done; "$@"; }
+          retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only
           # on success, so a transient mirror error keeps the last good bundle
           # rather than publishing a partial one - which the kernel-label skip
           # would then pin in place until the kernel next changes (~monthly).
-          retry sudo apt-get install -y --download-only "${PKGS[@]}"
+          retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
+            --download-only "${PKGS[@]}"
           sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
           test -n "$(ls debs/*.deb 2>/dev/null)"  # headers are never preinstalled

.github/workflows/cross-library.yml

@@ -64,10 +64,28 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
+          done
 
       # Building only needs the commit under test, not history. The break check
       # that needs history runs in the compile job, not here.
@@ -129,10 +147,28 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
+          done
 
       # This job does not build wolfSSL, but the latest leg still checks out
       # wolfSSL history because check-break.sh scans commit messages here. The

.github/workflows/cyrus-sasl.yml

@@ -66,6 +66,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: krb5-kdc krb5-otp libkrb5-dev libsocket-wrapper libnss-wrapper krb5-admin-server libdb5.3-dev
           ghcr-debs-tag: ubuntu-24.04-full
 

.github/workflows/falcon-interop.yml

@@ -91,8 +91,26 @@ jobs:
     steps:
       - name: Install build tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y ninja-build
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
+            done
+            echo "::error::apt-get failed after 2 attempts"
+            return 1
+          }
+          apt_retry ninja-build
 
       # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which
       # would delete an untracked oqs-install/ placed in the workspace by the
@@ -161,8 +179,26 @@ jobs:
     steps:
       - name: Install build tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y autoconf automake libtool
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
+            done
+            echo "::error::apt-get failed after 2 attempts"
+            return 1
+          }
+          apt_retry autoconf automake libtool
 
       - name: Checkout wolfSSL
         uses: actions/checkout@v5

.github/workflows/ipmitool.yml

@@ -63,6 +63,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libreadline-dev
           ghcr-debs-tag: ubuntu-24.04-full
       - name: Download lib

.github/workflows/jwt-cpp.yml

@@ -67,6 +67,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libgtest-dev
           ghcr-debs-tag: ubuntu-24.04-full
 

.github/workflows/linuxkm.yml

@@ -36,6 +36,9 @@ jobs:
       - name: Install linux-headers
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: linux-headers-$(uname -r)
           ghcr-debs-tag: ubuntu-24.04-linuxkm
 

.github/workflows/mosquitto.yml

@@ -78,6 +78,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: build-essential libev-dev libssl-dev automake python3-docutils libcunit1 libcunit1-doc libcunit1-dev pkg-config make python3-psutil
           ghcr-debs-tag: ubuntu-24.04-full
 

.github/workflows/pam-ipmi.yml

@@ -64,6 +64,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libpam-dev ninja-build meson
           ghcr-debs-tag: ubuntu-24.04-full
 

.github/workflows/rng-tools.yml

@@ -65,6 +65,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libcurl4-openssl-dev libjansson-dev libp11-dev librtlsdr-dev libcap-dev
           ghcr-debs-tag: ubuntu-24.04-full