From c8977b0a95450083c6eded907f8ec6bb448f8914 Mon Sep 17 00:00:00 2001 From: Oleksandr Kuzminskyi Date: Fri, 31 Jul 2026 17:25:10 -0700 Subject: [PATCH] Promote apt lock timeout and bounded boot upgrade to sandbox Straight copy of the four items from environments/development, verified identical afterwards. No pre-existing divergence between the two overlays in these files, so nothing was clobbered. Validated on a development gha_runner (ip-10-0-1-142) before promoting: Notice: /Stage[init]/Profile::Apt_lock_timeout/File[/etc/apt/apt.conf.d/99-lock-timeout]/ensure: defined content The Stage[init] prefix confirms the drop-in lands before every Package resource in stage main, which was the part of the design most likely to surprise. That host's AMI predates infrahouse-ubuntu-pro#20, so the file did not exist and Puppet created it -- the exact case this exists for. The same run also confirmed the five unmask execs fire (so the units really are masked by cloud-init's bootcmd when Puppet runs), that the bounded upgrade script deploys and succeeds, that its /run marker stops it re-running on the second catalog pass, and that Package[postfix]/[mailutils]/[mutt] -- the three resources that failed in #289 -- now install cleanly. Bootstrap hook completed CONTINUE. Refs #289 Co-Authored-By: Claude Opus 5 (1M context) --- debian/changelog | 6 ++ .../gha-boot-security-upgrade.sh | 64 +++++++++++++++++++ .../profile/manifests/apt_lock_timeout.pp | 47 ++++++++++++++ .../profile/manifests/github_runner.pp | 29 ++++++++- .../modules/profile/manifests/repos.pp | 7 ++ 5 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 environments/sandbox/modules/profile/files/github_runner/gha-boot-security-upgrade.sh create mode 100644 environments/sandbox/modules/profile/manifests/apt_lock_timeout.pp diff --git a/debian/changelog b/debian/changelog index eabc465..57ac518 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +puppet-code (0.1.0-1build322) noble; urgency=medium + + * commit event. see changes history in git log + + -- root Sat, 01 Aug 2026 00:25:11 +0000 + puppet-code (0.1.0-1build321) noble; urgency=medium * commit event. see changes history in git log diff --git a/environments/sandbox/modules/profile/files/github_runner/gha-boot-security-upgrade.sh b/environments/sandbox/modules/profile/files/github_runner/gha-boot-security-upgrade.sh new file mode 100644 index 0000000..77cea06 --- /dev/null +++ b/environments/sandbox/modules/profile/files/github_runner/gha-boot-security-upgrade.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# One-shot security patching during runner provisioning, with a hard cumulative +# time bound. +# +# Why a script instead of exec's tries/try_sleep: Puppet's `timeout` is +# PER-ATTEMPT, so `tries` multiplies the worst case to +# tries * timeout + (tries-1) * try_sleep with no cumulative cap. That matters +# here because an overrun is not merely a slow run -- ih-puppet applies with +# --detailed-exitcodes and exits 4/6 when a resource fails, which trips +# ih-bootstrap.sh's `trap _ih_signal_abandon ERR` and ABANDONs the instance. So a +# retry budget that can exceed the bootstrap lifecycle hook is a fleet-churn bug, +# not a latency bug. Bounding total wall clock here lets one legitimately long +# upgrade use the whole window while still capping the worst case. +# +# What actually needs retrying: both commands below can fail within seconds under +# lock contention. +# - `apt-get update` takes /var/lib/apt/lists/lock, which DPkg::Lock::Timeout +# does NOT cover (measured: fails in ~1s even with the option set). +# - `unattended-upgrade` refuses to run concurrently with itself. +# Contenders are routine: the Inspector and GuardDuty agents each dpkg-install +# about a minute into every boot, squarely inside the provisioning window. +# +# Note ih-puppet already runs the catalog twice and only checks the second exit +# code, so a transient failure gets one free retry above this script too. +# +# Usage: gha-boot-security-upgrade.sh [budget_seconds] [marker_path] + +# Deliberately no `set -e`: failures of the apt commands are expected and handled +# by the retry loop below. +set -uo pipefail + +BUDGET="${1:-480}" +MARKER="${2:-/run/gha-boot-upgrade.done}" + +deadline=$(( $(date +%s) + BUDGET )) +attempt=0 + +while :; do + attempt=$(( attempt + 1 )) + remaining=$(( deadline - $(date +%s) )) + + if [ "$remaining" -le 0 ]; then + echo "gha-boot-security-upgrade: ${BUDGET}s budget exhausted after ${attempt} attempt(s)" >&2 + exit 1 + fi + + echo "gha-boot-security-upgrade: attempt ${attempt}, ${remaining}s of budget left" + + # Each command is capped at the remaining budget so a single slow command + # cannot overshoot the deadline. + if timeout "$remaining" apt-get update -qq && timeout "$remaining" unattended-upgrade; then + # Written only on success, so a failed upgrade simply retries on the next + # Puppet apply. Lives on tmpfs so it clears on a real boot. + touch "$MARKER" + echo "gha-boot-security-upgrade: succeeded on attempt ${attempt}" + exit 0 + fi + + # Only sleep if there will still be budget to use afterwards. + if [ $(( deadline - $(date +%s) )) -gt 15 ]; then + sleep 15 + fi +done diff --git a/environments/sandbox/modules/profile/manifests/apt_lock_timeout.pp b/environments/sandbox/modules/profile/manifests/apt_lock_timeout.pp new file mode 100644 index 0000000..454a519 --- /dev/null +++ b/environments/sandbox/modules/profile/manifests/apt_lock_timeout.pp @@ -0,0 +1,47 @@ +# @summary: Make apt-get wait for the dpkg lock instead of failing outright. +# +# Declared by profile::repos with `stage => init` so the drop-in exists before any +# Package resource in stage main. That ordering is the whole point: a drop-in +# applied halfway through the catalog does not help the Package resources Puppet +# already evaluated. +# +# Why this is needed at all: Ubuntu ships `binary::apt::DPkg::Lock::Timeout "120"`, +# and that scope applies ONLY to the `apt` command. Puppet's package provider, +# cloud-init and the AWS agents all shell out to `apt-get`, which inherits nothing +# and fails instantly on a held lock. Measured on noble/apt 2.8.3 against a held +# /var/lib/dpkg/lock-frontend: apt-get install exits 100 in 0s without the +# unscoped key, and 0 in 45s (waiting out a 40s lock) with it. +# +# Lock contention here is routine, not hypothetical: +# - the Inspector and GuardDuty agents each dpkg-install ~1 min into every boot +# - profile::unattended_upgrades deliberately unmasks and STARTS the apt-daily +# timers mid-catalog, so an unattended-upgrade can begin during the run +# +# NOTE: the path is deliberately the same file infrahouse-ubuntu-pro writes from +# its provision.sh. Two drop-ins both setting this key would resolve by lexical +# filename order, which is a silent trap -- so Puppet converges the AMI's file +# rather than racing a second one of its own. +# +# @param timeout +# Seconds apt-get waits for the dpkg lock. Sourced from the apt_lock_timeout +# custom fact (set via the cloud-init module's custom_facts), defaulting to the +# 300 that current AMIs ship. +# +# Interpolated as-is, so it makes no difference whether the fact arrives as an +# Integer or a String. +# +# Keep it well inside the gha_runner bootstrap lifecycle hook (1200s, +# default_result ABANDON): a genuinely wedged lock costs this many seconds per +# Package resource, and the hook is not renewed during bootstrap. +class profile::apt_lock_timeout ( + $timeout = pick_default($facts['apt_lock_timeout'], 300), +) { + + file { '/etc/apt/apt.conf.d/99-lock-timeout': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => "DPkg::Lock::Timeout \"${timeout}\";\n", + } +} diff --git a/environments/sandbox/modules/profile/manifests/github_runner.pp b/environments/sandbox/modules/profile/manifests/github_runner.pp index 7f0e444..29f611d 100644 --- a/environments/sandbox/modules/profile/manifests/github_runner.pp +++ b/environments/sandbox/modules/profile/manifests/github_runner.pp @@ -65,12 +65,35 @@ # per boot rather than on every Puppet apply, and is written only on success, # so a failed upgrade simply retries on the next apply. This applies Ubuntu # security updates, which do not depend on the InfraHouse repos. + # + # The retry/bounding logic lives in the script rather than in exec's + # tries/try_sleep because exec's timeout is per-attempt, so tries would multiply + # the worst case with no cumulative cap. A resource failure here ABANDONs the + # instance (ih-puppet exits 4/6, ih-bootstrap's ERR trap signals ABANDON), so + # the total must stay inside the 1200s bootstrap hook budget -- nothing renews + # it, since gha-lifecycle-heartbeater.sh is a no-op outside Terminating:Wait. + $boot_upgrade_script = '/usr/local/bin/gha-boot-security-upgrade.sh' + $boot_upgrade_budget = 480 + + file { $boot_upgrade_script: + ensure => file, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/profile/github_runner/gha-boot-security-upgrade.sh', + } + exec { 'gha-boot-security-upgrade': - command => 'apt-get update -qq && unattended-upgrade && touch /run/gha-boot-upgrade.done', + command => "${boot_upgrade_script} ${boot_upgrade_budget}", path => '/usr/bin:/bin:/usr/sbin:/sbin', unless => 'test -f /run/gha-boot-upgrade.done', - timeout => 1200, - require => Class['profile::unattended_upgrades'], + # Slightly above the script's own budget so the script always gets to exit and + # log why it gave up, rather than being killed mid-report by Puppet. + timeout => $boot_upgrade_budget + 60, + require => [ + Class['profile::unattended_upgrades'], + File[$boot_upgrade_script], + ], } } diff --git a/environments/sandbox/modules/profile/manifests/repos.pp b/environments/sandbox/modules/profile/manifests/repos.pp index 63752fe..218d252 100644 --- a/environments/sandbox/modules/profile/manifests/repos.pp +++ b/environments/sandbox/modules/profile/manifests/repos.pp @@ -7,4 +7,11 @@ tries => 5, }, } + + # Also in the init stage: the lock timeout has to be in place before anything in + # stage main starts installing packages. See the class for why apt-get needs an + # unscoped key of its own. + class { 'profile::apt_lock_timeout': + stage => init, + } }