From c2dbd4827305155a5fd9406ea5bf72795a54208b Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Tue, 25 Aug 2026 15:16:38 +0200 Subject: [PATCH] setup_docker: make LXC resource limits visible to nested containers Inside an LXC container, Docker gives every container it starts a fresh procfs in its own mount namespace. The lxcfs bind mounts that LXC placed on /proc/meminfo and friends live in the CT's namespace and never reach it, so each Docker container reads the physical host's totals instead of the CT's limits: CT sees: MemTotal: 8388608 kB container sees: MemTotal: 65648168 kB Enforcement was never affected, since the CT cgroup still caps every descendant. Visibility was, and anything that self-sizes from those numbers - JVM heap ergonomics, Node, Go, OpenMP thread pools - sizes against the wrong value and gets OOM-killed instead of throttling. Register a small runc wrapper as Docker's default runtime. On each container create it re-binds the lxcfs-backed files into the container and, when the container sets no limit of its own, applies the CT's memory limit to the container cgroup. Both halves are needed because runtimes read the limit two different ways: file readers use /proc, while musl and Node's uv_get_constrained_memory() consult the cgroup. CPU needs no handling - cpuset is inherited down the cgroup tree, so nested containers already observe the CT's core count. The one exception is /sys/devices/system/cpu/online, which glibc's sysconf(_SC_NPROCESSORS_ONLN) reads, so it is bound alongside the /proc files. The helper is a no-op outside an LXC container and wherever lxcfs is not mounted, so the five non-Docker installers that call setup_docker are unaffected. It fails open: no runc, no jq, or malformed JSON all fall through to the real runc unmodified, degrading to current behaviour rather than breaking the daemon. Opt out with DOCKER_LXCFS_VISIBILITY=false. Regenerate lib/API.txt for the new function. --- lib/API.txt | 1 + lib/runtime.func | 161 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/lib/API.txt b/lib/API.txt index 5a10bc0..45ad60d 100644 --- a/lib/API.txt +++ b/lib/API.txt @@ -9,6 +9,7 @@ _deploy_source_tarball _deploy_unpacked_archive _diagnose_deb_failure _docker_is_noninteractive +_docker_setup_lxcfs_visibility _download_source_tarball _forge_asset_urls _forge_descriptor diff --git a/lib/runtime.func b/lib/runtime.func index 6c2043e..528ff0b 100644 --- a/lib/runtime.func +++ b/lib/runtime.func @@ -140,6 +140,8 @@ setup_composer() { # DOCKER_PORTAINER - Install Portainer CE (optional, "true" to enable) # DOCKER_LOG_DRIVER - Log driver (optional, default: "journald") # DOCKER_SKIP_UPDATES - Skip container update check (optional, "true" to skip) +# DOCKER_LXCFS_VISIBILITY - Set to "false" to skip making LXC resource limits +# visible to nested containers (default: true) # # Features: # - Uses official Docker repository by default @@ -217,6 +219,161 @@ ensure_docker() { fi } +# ------------------------------------------------------------------------------ +# LXC resource visibility for nested Docker containers +# +# Description: +# Inside an LXC container, Docker gives every container it starts a fresh +# procfs in its own mount namespace. The lxcfs bind mounts that LXC placed on +# /proc/meminfo & friends live in the CT's namespace and never reach it, so +# each Docker container reads the *physical host's* totals rather than the +# CT's limits: +# +# CT sees: MemTotal: 8388608 kB +# container sees: MemTotal: 65648168 kB <- the whole hypervisor +# +# Enforcement is not affected (the CT cgroup still caps every descendant), +# but anything that self-sizes from those numbers - JVM heap ergonomics, +# Node, Go, OpenMP thread pools - sizes against the wrong value and gets +# OOM-killed instead of throttling. +# +# This registers a small runc wrapper as Docker's default runtime. On each +# container create it re-binds the lxcfs-backed files into the container and, +# when the container sets no limit of its own, applies the CT's memory limit +# to the container cgroup. Two mechanisms are needed because runtimes read +# the limit two different ways: file readers use /proc, while musl and +# Node's uv_get_constrained_memory() consult the cgroup. +# +# CPU needs no handling - cpuset is inherited down the cgroup tree, so +# nested containers already observe the CT's core count. +# +# Variables: +# DOCKER_LXCFS_VISIBILITY - Set to "false" to skip (default: true) +# +# Notes: +# - No-op outside an LXC container, and on any host where lxcfs is absent. +# - `docker run --runtime=runc` bypasses the wrapper for a single container. +# - An explicit --memory / mem_limit is never overridden. +# - The wrapper falls through to the real runc unmodified on any error, so a +# failure degrades to today's behaviour rather than breaking the daemon. +# - The limit is written at container creation. After changing the CT's +# memory with `pct set`, existing containers keep the old value until they +# are recreated, or re-synced with: +# docker update --memory= --memory-swap= +# - Monitoring containers are the one case that wants the opposite. lxcfs +# resolves *usage* by the reading process's own cgroup, so a container that +# graphs "system memory used" from /proc/meminfo will report only its own +# footprint once these binds are in place. Total is correct, used is not. +# Run those with `--runtime=runc` (compose: `runtime: runc`). Tools that +# read the Docker API instead - Portainer's dashboard, Arcane's Docker +# info panel - are unaffected, because the daemon reads the CT's values. +# CPU is not affected either way: /proc/stat is not cgroup-scoped. +# ------------------------------------------------------------------------------ +_docker_setup_lxcfs_visibility() { + local enabled="${DOCKER_LXCFS_VISIBILITY:-true}" + case "${enabled,,}" in + false | 0 | no) return 0 ;; + esac + + # Only meaningful inside an LXC container whose /proc is served by lxcfs + grep -qs 'fuse\.lxcfs' /proc/mounts || return 0 + + if ! command -v runc >/dev/null 2>&1; then + msg_warn "runc not found - skipping LXC resource visibility for Docker" + return 0 + fi + + if ! command -v jq >/dev/null 2>&1; then + $STD apt-get install -y jq || { + msg_warn "jq unavailable - skipping LXC resource visibility for Docker" + return 0 + } + fi + + msg_info "Configuring LXC resource visibility for Docker" + + cat <<'RUNC_LXCFS' >/usr/local/bin/runc-lxcfs +#!/bin/sh +# Managed by community-scripts/ProxmoxVE - see setup_docker() in misc/tools.func +# +# runc wrapper: let Docker containers running inside this LXC container observe +# the LXC container's resource limits instead of the physical host's. +# +# Docker mounts a fresh procfs per container, so the lxcfs binds LXC placed on +# /proc/meminfo & friends never reach it. Re-bind them into the OCI bundle, and +# apply the CT's memory limit to the container cgroup for runtimes that consult +# cgroups rather than /proc. +# +# Bypass for a single container with: docker run --runtime=runc ... +set -eu + +REAL_RUNC= +for candidate in /usr/bin/runc /usr/sbin/runc /usr/local/bin/runc; do + if [ -x "$candidate" ]; then + REAL_RUNC=$candidate + break + fi +done +[ -n "$REAL_RUNC" ] || exit 127 + +bundle="" +prev="" +for arg in "$@"; do + case "$prev" in + -b | --bundle) bundle="$arg" ;; + esac + case "$arg" in + --bundle=*) bundle="${arg#--bundle=}" ;; + esac + prev="$arg" +done + +# Only "create" carries a bundle; everything else passes straight through. +[ -n "$bundle" ] && [ -f "$bundle/config.json" ] || exec "$REAL_RUNC" "$@" + +mem=$(awk '/^MemTotal:/ { print $2 * 1024 }' /proc/meminfo) +[ -n "$mem" ] || exec "$REAL_RUNC" "$@" + +config="$bundle/config.json" +patched="$config.lxcfs" +if jq --argjson mem "$mem" ' + ["/proc/meminfo", "/proc/cpuinfo", "/proc/stat", "/proc/uptime", + "/proc/swaps", "/proc/loadavg", "/proc/diskstats", + "/sys/devices/system/cpu/online"] as $lxcfs + | [.mounts[].destination] as $taken + | .mounts += [ + $lxcfs[] + | select(. as $f | $taken | index($f) | not) + | {destination: ., type: "bind", source: ., options: ["rbind", "rprivate", "ro"]} + ] + | if (.linux.resources.memory.limit // 0) <= 0 + then .linux.resources.memory.limit = $mem + else . end +' "$config" >"$patched" 2>/dev/null && [ -s "$patched" ]; then + cat "$patched" >"$config" +fi +rm -f "$patched" + +exec "$REAL_RUNC" "$@" +RUNC_LXCFS + + chmod 755 /usr/local/bin/runc-lxcfs + + local tmpfile + tmpfile=$(mktemp) + if jq '. + { + "default-runtime": "lxcfs", + "runtimes": ((.runtimes // {}) + {"lxcfs": {"path": "/usr/local/bin/runc-lxcfs"}}) + }' /etc/docker/daemon.json >"$tmpfile" && [ -s "$tmpfile" ]; then + mv "$tmpfile" /etc/docker/daemon.json + msg_ok "Configured LXC resource visibility for Docker" + else + rm -f "$tmpfile" + rm -f /usr/local/bin/runc-lxcfs + msg_warn "Could not update /etc/docker/daemon.json - LXC resource visibility not enabled" + fi +} + setup_docker() { local docker_installed=false local portainer_installed=false @@ -273,6 +430,8 @@ setup_docker() { EOF fi + _docker_setup_lxcfs_visibility + # Enable and start Docker systemctl enable -q --now docker @@ -344,6 +503,8 @@ EOF EOF fi + _docker_setup_lxcfs_visibility + # Enable and start Docker systemctl enable -q --now docker fi