diff --git a/debian/changelog b/debian/changelog index 57ac518..27bbdd8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +puppet-code (0.1.0-1build323) noble; urgency=medium + + * commit event. see changes history in git log + + -- root Sat, 01 Aug 2026 03:21:10 +0000 + puppet-code (0.1.0-1build322) noble; urgency=medium * commit event. see changes history in git log diff --git a/modules/profile/files/github_runner/gha-boot-security-upgrade.sh b/modules/profile/files/github_runner/gha-boot-security-upgrade.sh new file mode 100644 index 0000000..77cea06 --- /dev/null +++ b/modules/profile/files/github_runner/gha-boot-security-upgrade.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# One-shot security patching during runner provisioning, with a hard cumulative +# time bound. +# +# Why a script instead of exec's tries/try_sleep: Puppet's `timeout` is +# PER-ATTEMPT, so `tries` multiplies the worst case to +# tries * timeout + (tries-1) * try_sleep with no cumulative cap. That matters +# here because an overrun is not merely a slow run -- ih-puppet applies with +# --detailed-exitcodes and exits 4/6 when a resource fails, which trips +# ih-bootstrap.sh's `trap _ih_signal_abandon ERR` and ABANDONs the instance. So a +# retry budget that can exceed the bootstrap lifecycle hook is a fleet-churn bug, +# not a latency bug. Bounding total wall clock here lets one legitimately long +# upgrade use the whole window while still capping the worst case. +# +# What actually needs retrying: both commands below can fail within seconds under +# lock contention. +# - `apt-get update` takes /var/lib/apt/lists/lock, which DPkg::Lock::Timeout +# does NOT cover (measured: fails in ~1s even with the option set). +# - `unattended-upgrade` refuses to run concurrently with itself. +# Contenders are routine: the Inspector and GuardDuty agents each dpkg-install +# about a minute into every boot, squarely inside the provisioning window. +# +# Note ih-puppet already runs the catalog twice and only checks the second exit +# code, so a transient failure gets one free retry above this script too. +# +# Usage: gha-boot-security-upgrade.sh [budget_seconds] [marker_path] + +# Deliberately no `set -e`: failures of the apt commands are expected and handled +# by the retry loop below. +set -uo pipefail + +BUDGET="${1:-480}" +MARKER="${2:-/run/gha-boot-upgrade.done}" + +deadline=$(( $(date +%s) + BUDGET )) +attempt=0 + +while :; do + attempt=$(( attempt + 1 )) + remaining=$(( deadline - $(date +%s) )) + + if [ "$remaining" -le 0 ]; then + echo "gha-boot-security-upgrade: ${BUDGET}s budget exhausted after ${attempt} attempt(s)" >&2 + exit 1 + fi + + echo "gha-boot-security-upgrade: attempt ${attempt}, ${remaining}s of budget left" + + # Each command is capped at the remaining budget so a single slow command + # cannot overshoot the deadline. + if timeout "$remaining" apt-get update -qq && timeout "$remaining" unattended-upgrade; then + # Written only on success, so a failed upgrade simply retries on the next + # Puppet apply. Lives on tmpfs so it clears on a real boot. + touch "$MARKER" + echo "gha-boot-security-upgrade: succeeded on attempt ${attempt}" + exit 0 + fi + + # Only sleep if there will still be budget to use afterwards. + if [ $(( deadline - $(date +%s) )) -gt 15 ]; then + sleep 15 + fi +done diff --git a/modules/profile/manifests/github_runner.pp b/modules/profile/manifests/github_runner.pp index 7f0e444..29f611d 100644 --- a/modules/profile/manifests/github_runner.pp +++ b/modules/profile/manifests/github_runner.pp @@ -65,12 +65,35 @@ # per boot rather than on every Puppet apply, and is written only on success, # so a failed upgrade simply retries on the next apply. This applies Ubuntu # security updates, which do not depend on the InfraHouse repos. + # + # The retry/bounding logic lives in the script rather than in exec's + # tries/try_sleep because exec's timeout is per-attempt, so tries would multiply + # the worst case with no cumulative cap. A resource failure here ABANDONs the + # instance (ih-puppet exits 4/6, ih-bootstrap's ERR trap signals ABANDON), so + # the total must stay inside the 1200s bootstrap hook budget -- nothing renews + # it, since gha-lifecycle-heartbeater.sh is a no-op outside Terminating:Wait. + $boot_upgrade_script = '/usr/local/bin/gha-boot-security-upgrade.sh' + $boot_upgrade_budget = 480 + + file { $boot_upgrade_script: + ensure => file, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/profile/github_runner/gha-boot-security-upgrade.sh', + } + exec { 'gha-boot-security-upgrade': - command => 'apt-get update -qq && unattended-upgrade && touch /run/gha-boot-upgrade.done', + command => "${boot_upgrade_script} ${boot_upgrade_budget}", path => '/usr/bin:/bin:/usr/sbin:/sbin', unless => 'test -f /run/gha-boot-upgrade.done', - timeout => 1200, - require => Class['profile::unattended_upgrades'], + # Slightly above the script's own budget so the script always gets to exit and + # log why it gave up, rather than being killed mid-report by Puppet. + timeout => $boot_upgrade_budget + 60, + require => [ + Class['profile::unattended_upgrades'], + File[$boot_upgrade_script], + ], } }