From 998a591a348e8d97a1631eff63d954615617135e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Mon, 27 Jul 2026 22:28:29 +0300 Subject: [PATCH 1/2] feat(nix): expose per-version extension packages, catalog, and instance tools Adds passthru.perVersion (version -> single-version derivation) to each multi-version extension (18 files, one line each) -- the attrset already existed inline as the buildEnv input, this surfaces it instead of discarding it. No change to any buildEnv output or extension identity. New flake-parts module nix/packages/extension-catalog.nix, exposing (packages + legacyPackages): - site-extensions-catalog- (15/17/orioledb-17): pg-extensions- catalog.json mapping { ext -> { extversion -> store path } }, plus bin/{resolve,update} preset with PG_EXTENSIONS_CATALOG. Assembled at build time from the wrappers' control files, so keys are real extversions (pgsql-http nix "1.5.0" -> "1.5") and include sub-extensions (postgis_topology, plcoffee, ...). unsafeDiscardReferences keeps the closure to just the JSON (6.8 KiB, not 2.4 GiB). - site-extensions-versions- (legacyPackages): the generic single-version packages the catalog points at (lean, no multi-version buildEnv), so the CI matrix builds and caches them. - Generic tools, one version each (not per-major): site-extensions-resolve (manifest json -> store paths), download-nix-store-paths (retry/timeout substitution, not extension-specific), site-extensions-update (chains both + nix-env --install --remove-all onto PROFILE, atomic). Part of PSQL-1530. --- nix/ext/hypopg.nix | 1 + nix/ext/index_advisor.nix | 1 + nix/ext/pg-safeupdate.nix | 1 + nix/ext/pg_cron/default.nix | 1 + nix/ext/pg_net.nix | 1 + nix/ext/pg_partman.nix | 1 + nix/ext/pg_repack.nix | 1 + nix/ext/pg_tle.nix | 1 + nix/ext/pgaudit.nix | 1 + nix/ext/pgmq/default.nix | 1 + nix/ext/pgroonga/default.nix | 1 + nix/ext/pgrouting/default.nix | 1 + nix/ext/pgsodium.nix | 1 + nix/ext/pgsql-http.nix | 1 + nix/ext/pgvector.nix | 1 + nix/ext/plv8/default.nix | 1 + nix/ext/postgis.nix | 1 + nix/ext/vault.nix | 1 + nix/packages/default.nix | 5 +- nix/packages/extension-catalog.nix | 183 +++++++++++++++++++++++++++++ 20 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 nix/packages/extension-catalog.nix diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index d95a1c7f71..9bd4927263 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -98,6 +98,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/index_advisor.nix b/nix/ext/index_advisor.nix index 3130d4bcc2..5892127142 100644 --- a/nix/ext/index_advisor.nix +++ b/nix/ext/index_advisor.nix @@ -83,6 +83,7 @@ pkgs.buildEnv { ]; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pg-safeupdate.nix b/nix/ext/pg-safeupdate.nix index 168772ce23..452e9c2c5e 100644 --- a/nix/ext/pg-safeupdate.nix +++ b/nix/ext/pg-safeupdate.nix @@ -83,6 +83,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pg_cron/default.nix b/nix/ext/pg_cron/default.nix index a823fa450b..4450f8e6bb 100644 --- a/nix/ext/pg_cron/default.nix +++ b/nix/ext/pg_cron/default.nix @@ -116,6 +116,7 @@ buildEnv { }; passthru = { + perVersion = lib.mapAttrs (name: value: build name value) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit switch-ext-version latestOnly; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index 87b994a5ef..f769bad314 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -145,6 +145,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pg_partman.nix b/nix/ext/pg_partman.nix index 4c36e765fc..0c8a4eee6e 100644 --- a/nix/ext/pg_partman.nix +++ b/nix/ext/pg_partman.nix @@ -100,6 +100,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit switch-ext-version libName latestOnly; diff --git a/nix/ext/pg_repack.nix b/nix/ext/pg_repack.nix index 1a7500ea27..78d9764063 100644 --- a/nix/ext/pg_repack.nix +++ b/nix/ext/pg_repack.nix @@ -140,6 +140,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index 45ba670385..9c7ef86755 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -111,6 +111,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix index 74429fa415..8fb727c666 100644 --- a/nix/ext/pgaudit.nix +++ b/nix/ext/pgaudit.nix @@ -240,6 +240,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgmq/default.nix b/nix/ext/pgmq/default.nix index 42613b6acd..59449c8edb 100644 --- a/nix/ext/pgmq/default.nix +++ b/nix/ext/pgmq/default.nix @@ -104,6 +104,7 @@ buildEnv { pathsToLink = [ "/share/postgresql/extension" ]; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgroonga/default.nix b/nix/ext/pgroonga/default.nix index 42345bd49a..b9c3829a0d 100644 --- a/nix/ext/pgroonga/default.nix +++ b/nix/ext/pgroonga/default.nix @@ -181,6 +181,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgrouting/default.nix b/nix/ext/pgrouting/default.nix index 87d190cdc7..cc00281e70 100644 --- a/nix/ext/pgrouting/default.nix +++ b/nix/ext/pgrouting/default.nix @@ -153,6 +153,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index 5a615a8722..b5a2dcb72c 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -112,6 +112,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index 5815f91607..885fcc472e 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -113,6 +113,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 5f613b3d52..2db9d0c123 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -96,6 +96,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/plv8/default.nix b/nix/ext/plv8/default.nix index 93a17434a1..731991937c 100644 --- a/nix/ext/plv8/default.nix +++ b/nix/ext/plv8/default.nix @@ -238,6 +238,7 @@ buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index 99afea6f5b..3f722d9ad9 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -256,6 +256,7 @@ in ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix index 18ffe0aa77..9ab5391b1d 100644 --- a/nix/ext/vault.nix +++ b/nix/ext/vault.nix @@ -99,6 +99,7 @@ pkgs.buildEnv { ''; passthru = { + perVersion = lib.mapAttrs (name: value: build name value.hash) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; inherit pname latestOnly; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index c3b1ae012e..6fa011f82d 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -1,6 +1,9 @@ { self, inputs, ... }: { - imports = [ ./postgres.nix ]; + imports = [ + ./postgres.nix + ./extension-catalog.nix + ]; perSystem = { inputs', diff --git a/nix/packages/extension-catalog.nix b/nix/packages/extension-catalog.nix new file mode 100644 index 0000000000..5f6d3ecf07 --- /dev/null +++ b/nix/packages/extension-catalog.nix @@ -0,0 +1,183 @@ +# Multi version extension catalog and update tools. +{ + perSystem = + { + pkgs, + lib, + self', + ... + }: + let + # Mirror a per-version package, adding unversioned control/library names + # only where missing — packages that already ship generic names (e.g. + # postgis and its `-3` convention) pass through untouched. + mkGenericVersion = + leanPkg: + pkgs.runCommand "${leanPkg.pname}-${leanPkg.version}-generic" { } '' + set -euo pipefail + shopt -s nullglob + mkdir -p "$out/lib" "$out/share/postgresql/extension" + ext="$out/share/postgresql/extension" + + for f in ${leanPkg}/lib/*; do ln -s "$f" "$out/lib/$(basename "$f")"; done + for f in ${leanPkg}/share/postgresql/extension/*; do ln -s "$f" "$ext/$(basename "$f")"; done + + # .control with default_version, from --.control + for cf in "$ext"/*--*.control; do + b=$(basename "$cf") + n="''${b%%--*}" + v="''${b#*--}"; v="''${v%.control}" + [ -e "$ext/$n.control" ] && continue + { echo "default_version = '$v'"; cat "$cf"; } > "$ext/$n.control" + done + + # .so -> -.so + for f in "$out"/lib/*-[0-9]*; do + b=$(basename "$f") + suffix=".''${b##*.}" + core="''${b%.*}" + base="''${core%-[0-9]*}" + [ "$base" != "$core" ] || continue + gen="$base$suffix" + [ -e "$out/lib/$gen" ] || ln -sfn "$b" "$out/lib/$gen" + done + + # Sanity check. No generic control is fine only for preload-only + # modules (e.g. pg-safeupdate), which still ship a library. + generic=0 + for c in "$ext"/*.control; do + case "$(basename "$c")" in + *--*) ;; + *) generic=$((generic + 1)) ;; + esac + done + libs=$(ls -A "$out/lib" 2>/dev/null | wc -l) + if [ "$generic" -eq 0 ] && [ "$libs" -eq 0 ]; then + echo "extension-catalog: empty wrapper for ${leanPkg.pname} (no control, no lib)" >&2 + exit 1 + fi + ''; + + wrappersFor = + extsSet: + lib.mapAttrs (_: drv: lib.mapAttrs (_: mkGenericVersion) drv.perVersion) ( + lib.filterAttrs (_: drv: lib.isDerivation drv && drv ? perVersion) extsSet + ); + + # Make catalog json from a set of wrappers: { : { : } } + # Keyed on each wrapper's actual default_version, not the nix attr name, + # which can differ (pgsql-http "1.5.0" builds extversion "1.5"). + mkCatalogFile = + wrappers: + pkgs.runCommand "pg-extensions-catalog" + { + paths = map toString (lib.concatMap lib.attrValues (lib.attrValues wrappers)); + nativeBuildInputs = [ pkgs.jq ]; + # Without this the literal wrapper paths in the JSON would pull + # every wrapper's closure into the catalog's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + } + '' + set -euo pipefail + obj='{}' + for path in "''${paths[@]}"; do + for ctrl in "$path"/share/postgresql/extension/*.control; do + base=$(basename "$ctrl") + case "$base" in *--*) continue ;; esac + name="''${base%.control}" + ver=$(sed -n "s/^default_version = '\(.*\)'.*/\1/p" "$ctrl" | head -1) + [ -n "$ver" ] || { echo "no default_version in $ctrl" >&2; exit 1; } + obj=$(printf '%s' "$obj" | jq --arg e "$name" --arg v "$ver" --arg p "$path" '.[$e][$v] = $p') + done + done + mkdir -p "$out/share" + printf '%s\n' "$obj" | jq -S . > "$out/share/pg-extensions-catalog.json" + ''; + + script = + name: text: + pkgs.writeShellApplication { + inherit name text; + runtimeInputs = [ + pkgs.coreutils + pkgs.jq + ]; + }; + + # Multi version extensions as attrsets: { = { = { = drv; }; }; } + perMajor = lib.genAttrs [ "15" "17" "orioledb-17" ] ( + major: wrappersFor self'.legacyPackages."psql_${major}".exts + ); + + # Catalog json plus bin/site-extensions-{resolve,update} that use it by default. + catalogs = lib.mapAttrs' ( + major: wrappers: + lib.nameValuePair "site-extensions-catalog-${major}" ( + pkgs.runCommand "site-extensions-catalog" { nativeBuildInputs = [ pkgs.makeWrapper ]; } '' + mkdir -p "$out/share" "$out/bin" + ln -s ${mkCatalogFile wrappers}/share/pg-extensions-catalog.json "$out/share/" + makeWrapper ${self'.packages.site-extensions-resolve}/bin/site-extensions-resolve \ + "$out/bin/site-extensions-resolve" \ + --set PG_EXTENSIONS_CATALOG "$out/share/pg-extensions-catalog.json" + makeWrapper ${self'.packages.site-extensions-update}/bin/site-extensions-update \ + "$out/bin/site-extensions-update" \ + --set PG_EXTENSIONS_CATALOG "$out/share/pg-extensions-catalog.json" + '' + ) + ) perMajor; + + versions = lib.mapAttrs' ( + major: wrappers: + lib.nameValuePair "site-extensions-versions-${major}" ( + lib.recurseIntoAttrs (lib.mapAttrs (_: lib.recurseIntoAttrs) wrappers) + ) + ) perMajor; + in + { + packages = catalogs // { + # Takes store paths as args and substitutes from binary cache, with retry. + download-nix-store-paths = script "download-nix-store-paths" '' + for path in "$@"; do + for attempt in 1 2 3; do + timeout -k 10s 120s nix-store -r "$path" >/dev/null && continue 2 + echo "WARNING: attempt $attempt failed for $path" >&2 + done + echo "ERROR: failed to realize $path" >&2 + exit 1 + done + ''; + + # Takes manifest json as argument. Format: {: }. + # Prints nix-store paths of the resolved extensions, one per line. + # Does not download or install. + site-extensions-resolve = script "site-extensions-resolve" '' + : "''${PG_EXTENSIONS_CATALOG:?PG_EXTENSIONS_CATALOG must point at a pg-extensions-catalog.json}" + manifest="''${1:?Usage: $0 path-to/pg-extensions.json}" + jq -r 'to_entries[] | "\(.key)=\(.value)"' "$manifest" | while IFS='=' read -r name version; do + [ -n "$name" ] || continue + jq -er --arg n "$name" --arg v "$version" \ + '.[$n][$v] // error("\($n)=\($v) not in catalog")' "$PG_EXTENSIONS_CATALOG" + done | sort -u + ''; + + # Takes manifest json as argument. + # Downloads and installs paths into the profile, replacing all existing ones. + site-extensions-update = pkgs.writeShellApplication { + name = "site-extensions-update"; + runtimeInputs = [ + self'.packages.site-extensions-resolve + self'.packages.download-nix-store-paths + ]; + text = '' + manifest="''${1:-/etc/adminapi/pg-extensions.json}" + profile="''${PROFILE:-/nix/var/nix/profiles/site-extensions}" + mapfile -t paths < <(site-extensions-resolve "$manifest") + download-nix-store-paths "''${paths[@]}" + nix-env --profile "$profile" --install "''${paths[@]}" --remove-all + ''; + }; + }; + legacyPackages = catalogs // versions; + }; +} From 3fe60a2fad059749ca66eaa4bb3d3cc78b4b1937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Tue, 28 Jul 2026 18:28:19 +0300 Subject: [PATCH 2/2] feat(ansible): delegate extension version handling to site-extensions-update postgres_prestart.sh's handle_extension_versions() now calls site-extensions-update directly instead of looping switch__version per extension -- that resolve/download/install pipeline now lives in the nix packages added by this branch. Manifest-missing guard preserved; failure is non-fatal (matches existing behavior, and ExecStartPre already ignores exit status via the -+ prefix). Also fixes drift in postgresql.service: it had fallen out of sync with salt's live copy of the unit (ExecStartPre missing the "-" ignore-failure prefix, ReadOnlyPaths missing /usr/local). Both now match what's actually deployed on the fleet. Depends on site-extensions-update being reachable on PATH (bundled into the site profile, tracked separately) before this can actually cut over. --- ansible/files/postgres_prestart.sh.j2 | 70 ++----------------- .../postgresql_config/postgresql.service | 4 +- 2 files changed, 7 insertions(+), 67 deletions(-) diff --git a/ansible/files/postgres_prestart.sh.j2 b/ansible/files/postgres_prestart.sh.j2 index df555c87e5..9622c4c015 100644 --- a/ansible/files/postgres_prestart.sh.j2 +++ b/ansible/files/postgres_prestart.sh.j2 @@ -32,78 +32,18 @@ update_orioledb_buffers() { fi } -check_extensions_file() { +handle_extension_versions() { local extensions_file="/etc/adminapi/pg-extensions.json" - if [ ! -f "$extensions_file" ]; then - log "extensions: No extensions file found, skipping extensions versions check" - return 0 #if file not found, skip - fi if [ ! -r "$extensions_file" ]; then - log "extensions: Cannot read extensions file" - return 1 #a true error, we should be able to read file - fi - return 0 -} - -switch_extension_version() { - local extension_name="$1" - local version="$2" - - # Use BIN_PATH environment variable or default to /var/lib/postgresql/.nix-profile - : ${BIN_PATH:="/var/lib/postgresql/.nix-profile"} - - local switch_script="$BIN_PATH/bin/switch_${extension_name}_version" - - if [ ! -x "$switch_script" ]; then - log "$extension_name: No version switch script available at $switch_script, skipping" + log "extensions: no manifest at $extensions_file, skipping" return 0 fi - log "$extension_name: Switching to version $version" - # Run directly as root since we're already running as root - "$switch_script" "$version" - local exit_code=$? - if [ $exit_code -eq 0 ]; then - log "$extension_name: Version switch completed successfully" + if site-extensions-update "$extensions_file"; then + log "extensions: profile updated" else - log "$extension_name: Version switch failed with exit code $exit_code" - fi - return $exit_code -} - -handle_extension_versions() { - if ! check_extensions_file; then - return - fi - - local extensions_file="/etc/adminapi/pg-extensions.json" - - # Get all extension names from the JSON file - local extensions - extensions=$(jq -r 'keys[]' "$extensions_file" 2>/dev/null) - - if [ -z "$extensions" ]; then - log "extensions: No extensions found in configuration" - return + log "extensions: update failed, continuing with existing profile" fi - - # Iterate through each extension - while IFS= read -r extension_name; do - # Get the version for this extension - local version - version=$(jq -r --arg ext "$extension_name" '.[$ext] // empty' "$extensions_file") - - if [ -z "$version" ]; then - log "$extension_name: No version specified, skipping" - continue - fi - - log "$extension_name: Found version $version in extensions file" - - # Don't fail if version switch fails - just log and continue - switch_extension_version "$extension_name" "$version" || log "$extension_name: Version switch failed but continuing" - - done <<< "$extensions" } main() { diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service index 68c37140bd..bad1422954 100644 --- a/ansible/files/postgresql_config/postgresql.service +++ b/ansible/files/postgresql_config/postgresql.service @@ -10,7 +10,7 @@ After=database-optimizations.service Type=notify User=postgres ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql -ExecStartPre=+/usr/local/bin/postgres_prestart.sh +ExecStartPre=-+/usr/local/bin/postgres_prestart.sh ExecReload=/bin/kill -HUP $MAINPID KillMode=mixed KillSignal=SIGINT @@ -22,7 +22,7 @@ OOMScoreAdjust=-1000 EnvironmentFile=-/etc/environment.d/postgresql.env LimitNOFILE=16384 {% if supabase_internal is defined %} -ReadOnlyPaths=/etc +ReadOnlyPaths=/etc /usr/local InaccessiblePaths=/root -/var/lib/supabase -/var/lib/supabase-admin-agent -/var/cache/supabase-admin-agent -/opt/saltstack -/etc/salt AppArmorProfile=-sbpostgres {% endif %}