From 956c29db39bc005ae61ff92f993e7adb1f22ee6e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 26 Jul 2026 13:52:01 +0700 Subject: [PATCH 1/6] docs: make release archives immutable Signed-off-by: Jeremi Joslin --- .github/scripts/ci_changes.py | 22 + .github/scripts/test_ci_changes.py | 33 ++ .github/workflows/ci.yml | 58 ++- .github/workflows/docs-pages.yml | 18 +- .github/workflows/release.yml | 80 +++- docs/site/.gitignore | 1 + docs/site/README.md | 30 +- docs/site/astro.config.mjs | 4 + docs/site/package-lock.json | 1 + docs/site/package.json | 9 +- docs/site/scripts/archive-bundle.mjs | 420 ++++++++++++++++++ docs/site/scripts/archive-bundle.test.mjs | 126 ++++++ docs/site/scripts/archive-lock.mjs | 238 ++++++++++ docs/site/scripts/archive-lock.test.mjs | 92 ++++ docs/site/scripts/assemble-archives.mjs | 239 ++++++++++ docs/site/scripts/assemble-archives.test.mjs | 142 ++++++ docs/site/scripts/astro-config.test.mjs | 4 + docs/site/scripts/build-archives.mjs | 28 +- docs/site/scripts/check-built-links.mjs | 19 +- docs/site/scripts/check-built-links.test.mjs | 14 +- .../scripts/check-evidence-links.test.mjs | 20 +- docs/site/scripts/check-seo.mjs | 32 +- docs/site/src/data/archive-lock.yaml | 47 ++ release/OPERATIONS.md | 19 +- release/VERIFY.md | 49 +- release/scripts/check-gates-inventory.py | 19 + release/scripts/registry-release | 79 +++- release/scripts/test_registry_release.py | 148 +++++- 28 files changed, 1944 insertions(+), 47 deletions(-) create mode 100644 docs/site/scripts/archive-bundle.mjs create mode 100644 docs/site/scripts/archive-bundle.test.mjs create mode 100644 docs/site/scripts/archive-lock.mjs create mode 100644 docs/site/scripts/archive-lock.test.mjs create mode 100644 docs/site/scripts/assemble-archives.mjs create mode 100644 docs/site/scripts/assemble-archives.test.mjs create mode 100644 docs/site/src/data/archive-lock.yaml diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 4a743c2d9..93efb108d 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -264,6 +264,27 @@ def classify( or path == ".github/workflows/docs-pages.yml" for path in paths ) + docs_archives = any( + path + in { + ".github/workflows/ci.yml", + ".github/workflows/docs-pages.yml", + ".github/workflows/release.yml", + "docs/site/astro.config.mjs", + "docs/site/package-lock.json", + "docs/site/package.json", + "docs/site/scripts/apply-archive-seo.mjs", + "docs/site/scripts/archive-bundle.mjs", + "docs/site/scripts/archive-lock.mjs", + "docs/site/scripts/assemble-archives.mjs", + "docs/site/scripts/build-archive.mjs", + "docs/site/scripts/build-archives.mjs", + "docs/site/src/data/archive-lock.yaml", + "docs/site/src/data/docsets.yaml", + "docs/site/src/data/repo-docs.yaml", + } + for path in paths + ) editors = complete or any(path.startswith("editors/") for path in paths) tutorial_infrastructure = any( @@ -313,6 +334,7 @@ def classify( "release_tool": release_tool, "release_source_proof": release_source_proof, "docs": docs, + "docs_archives": docs_archives, "editors": editors, "registryctl_tutorial": registryctl_tutorial, } diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index 61a7b5eef..f1e9c4c42 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -60,6 +60,7 @@ def test_ci_workflow_change_runs_the_complete_matrix(self) -> None: outputs = classify(self.workspace, (".github/workflows/ci.yml",)) self.assertCountEqual(outputs["rust_packages"], self.workspace.package_names) self.assertTrue(outputs["docs"]) + self.assertTrue(outputs["docs_archives"]) self.assertTrue(outputs["editors"]) def test_docs_only_change_skips_rust(self) -> None: @@ -70,6 +71,38 @@ def test_docs_only_change_skips_rust(self) -> None: self.assertFalse(outputs["rust"]) self.assertEqual(outputs["rust_matrix"], {"include": []}) self.assertTrue(outputs["docs"]) + self.assertFalse(outputs["docs_archives"]) + + def test_archive_content_is_immutable_during_routine_docs_changes(self) -> None: + current_content = classify( + self.workspace, + ("docs/site/src/content/docs/reference/glossary.mdx",), + ) + archive_lock = classify( + self.workspace, + ("docs/site/src/data/archive-lock.yaml",), + ) + archive_assembler = classify( + self.workspace, + ("docs/site/scripts/assemble-archives.mjs",), + ) + self.assertFalse(current_content["docs_archives"]) + self.assertTrue(archive_lock["docs_archives"]) + self.assertTrue(archive_assembler["docs_archives"]) + + def test_run_all_does_not_rebuild_immutable_archives_without_changed_paths(self) -> None: + outputs = classify(self.workspace, (), run_all=True) + self.assertTrue(outputs["docs"]) + self.assertFalse(outputs["docs_archives"]) + + def test_run_all_keeps_archive_sensitive_changed_paths(self) -> None: + outputs = classify( + self.workspace, + ("docs/site/scripts/archive-bundle.mjs",), + run_all=True, + ) + self.assertTrue(outputs["docs"]) + self.assertTrue(outputs["docs_archives"]) def test_other_workflow_changes_do_not_select_the_full_matrix(self) -> None: for workflow in sorted(RELEASE_SECURITY_WORKFLOWS): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37b97a18b..9495771c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: release_tool: ${{ steps.filter.outputs.release_tool }} release_source_proof: ${{ steps.filter.outputs.release_source_proof }} docs: ${{ steps.filter.outputs.docs }} + docs_archives: ${{ steps.filter.outputs.docs_archives }} editors: ${{ steps.filter.outputs.editors }} registryctl_tutorial: ${{ steps.filter.outputs.registryctl_tutorial }} steps: @@ -61,8 +62,8 @@ jobs: - name: Classify changed paths id: filter env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} shell: bash run: | set -euo pipefail @@ -82,8 +83,15 @@ jobs: else classifier+=(--all) fi + elif git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null && + git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then + # Keep push and merge-queue complete matrices while preserving + # paths for gates that intentionally do not run on every change. + git diff --name-only "${BASE_SHA}" "${HEAD_SHA}" > "${changed_files}" + classifier+=(--all --changed-files "${changed_files}") else - # Pushes and merge-queue candidates run the complete matrix. + # Runs without resolvable comparison commits use the complete + # matrix and leave opt-in immutable archive verification skipped. classifier+=(--all) fi "${classifier[@]}" @@ -903,6 +911,7 @@ jobs: needs: - changes - docs + - docs-archives if: ${{ always() }} runs-on: ubuntu-24.04 steps: @@ -911,6 +920,8 @@ jobs: CHANGES_RESULT: ${{ needs.changes.result }} REQUIRED: ${{ needs.changes.outputs.docs }} RESULT: ${{ needs.docs.result }} + ARCHIVES_REQUIRED: ${{ needs.changes.outputs.docs_archives }} + ARCHIVES_RESULT: ${{ needs.docs-archives.result }} shell: bash run: | set -euo pipefail @@ -922,6 +933,47 @@ jobs: exit 1 ;; esac + case "${CHANGES_RESULT}:${ARCHIVES_REQUIRED}:${ARCHIVES_RESULT}" in + success:true:success|success:false:skipped) + ;; + *) + echo "Docs archive path gate did not complete safely: changes=${CHANGES_RESULT} required=${ARCHIVES_REQUIRED} result=${ARCHIVES_RESULT}" >&2 + exit 1 + ;; + esac + + docs-archives: + name: Immutable docs archives + needs: changes + if: needs.changes.outputs.docs_archives == 'true' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + submodules: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 22.12.0 + cache: npm + cache-dependency-path: docs/site/package-lock.json + + - name: Install docs dependencies + working-directory: docs/site + run: npm ci + + - name: Reject changes to locked archive digests + working-directory: docs/site + env: + ARCHIVE_LOCK_BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || 'origin/main' }} + run: npm run check:archive-lock -- --base-ref "${ARCHIVE_LOCK_BASE_REF}" + + - name: Assemble and verify every archived release + working-directory: docs/site + run: npm run check:archives editor-extensions: name: Editor extensions diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml index 0d137307f..1f31181c7 100644 --- a/.github/workflows/docs-pages.yml +++ b/.github/workflows/docs-pages.yml @@ -53,13 +53,19 @@ jobs: PUBLIC_UMAMI_SCRIPT_SRC: ${{ vars.PUBLIC_UMAMI_SCRIPT_SRC }} PUBLIC_UMAMI_DOMAINS: ${{ vars.PUBLIC_UMAMI_DOMAINS }} - - name: Build archived docs + - name: Verify current machine-readable documentation working-directory: docs/site - run: npm run build:archives - env: - PUBLIC_UMAMI_WEBSITE_ID: ${{ vars.PUBLIC_UMAMI_WEBSITE_ID }} - PUBLIC_UMAMI_SCRIPT_SRC: ${{ vars.PUBLIC_UMAMI_SCRIPT_SRC }} - PUBLIC_UMAMI_DOMAINS: ${{ vars.PUBLIC_UMAMI_DOMAINS }} + run: npm run check:llms:built + + - name: Assemble immutable archived docs + working-directory: docs/site + run: npm run assemble:archives -- --bootstrap + + - name: Verify assembled documentation tree + working-directory: docs/site + run: | + npm run check:seo:built + npm run check:links:built - name: Configure Pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f29272d3..9fd798202 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,6 +138,7 @@ jobs: "${{ steps.release.outputs.tag_target }}" refs/remotes/origin/main release/scripts/registry-release validate \ "${{ steps.release.outputs.manifest }}" + release/scripts/registry-release validate-docsets release/scripts/registry-release validate-source \ "${{ steps.release.outputs.manifest }}" \ --tag "${{ steps.release.outputs.tag }}" \ @@ -150,6 +151,53 @@ jobs: REGISTRY_RELEASE_SOURCE_MODE=monorepo \ release/scripts/check-release-source-model.sh + docs-archive: + name: Build immutable release docs archive + needs: verify + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout exact tag target without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.2.2 + with: + ref: ${{ needs.verify.outputs.tag_target }} + fetch-depth: 0 + submodules: false + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v4.4.0 + with: + node-version: 22.12.0 + cache: npm + cache-dependency-path: docs/site/package-lock.json + + - name: Install locked docs dependencies + working-directory: docs/site + run: npm ci + + - name: Build and verify the tag-bound docs bundle + working-directory: docs/site + env: + DOCS_DOCSET: ${{ needs.verify.outputs.tag }} + DOCS_ARCHIVE_OUTPUT: ${{ runner.temp }}/registry-docs-${{ needs.verify.outputs.tag }}.tar.gz + run: | + set -euo pipefail + npm run build:archive + npm run archive:snapshot -- \ + "${DOCS_DOCSET}" \ + --output "${DOCS_ARCHIVE_OUTPUT}" \ + --verify-lock + + - name: Upload verified docs bundle + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: registry-docs-${{ needs.verify.outputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/registry-docs-${{ needs.verify.outputs.tag }}.tar.gz + if-no-files-found: error + retention-days: 7 + verify-candidate: name: Verify exact candidate before any public write needs: verify @@ -464,6 +512,7 @@ jobs: needs: - verify - verify-candidate + - docs-archive runs-on: ubuntu-24.04 permissions: actions: read @@ -603,6 +652,7 @@ jobs: - verify - verify-candidate - publish-images + - docs-archive runs-on: ubuntu-24.04 outputs: provenance_subjects: ${{ steps.provenance-subjects.outputs.base64_subjects }} @@ -627,6 +677,12 @@ jobs: name: ${{ needs.verify-candidate.outputs.bundle_name }} path: promotion + - name: Download unprivileged docs archive + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: registry-docs-${{ needs.verify.outputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }} + path: docs-archive + - name: Reverify candidate file inventory without executing product binaries env: EXPECTED_RECEIPT_SHA256: ${{ needs.verify.outputs.receipt_sha256 }} @@ -743,6 +799,27 @@ jobs: payload="promotion/artifacts/${payload_name}/dist" mkdir -p dist/{bin,image-evidence,binary-sbom,release-capsule} cp -R "${payload}/bin/." dist/bin/ + docs_archive="docs-archive/registry-docs-${{ needs.verify.outputs.tag }}.tar.gz" + test -f "${docs_archive}" + python3 - \ + "docs/site/src/data/archive-lock.yaml" \ + "${{ needs.verify.outputs.tag }}" \ + "${docs_archive}" <<'PY' + import hashlib + import pathlib + import sys + import yaml + + lock_path, docset_id, bundle_path = sys.argv[1:] + lock = yaml.safe_load(pathlib.Path(lock_path).read_text(encoding="utf-8")) + expected = lock["archives"][docset_id]["bundle_sha256"] + actual = hashlib.sha256(pathlib.Path(bundle_path).read_bytes()).hexdigest() + if actual != expected: + raise SystemExit( + f"docs archive digest {actual} does not match immutable lock {expected}" + ) + PY + cp "${docs_archive}" dist/bin/ cp -R "${payload}/images/." dist/image-evidence/ cp -R "${payload}/sbom/." dist/image-evidence/ cp -R "${payload}/grype/." dist/image-evidence/ @@ -801,7 +878,8 @@ jobs: --default-branch-protection unverified \ --repo . \ --default-branch origin/main \ - --require-registryctl-image-lock + --require-registryctl-image-lock \ + --require-registry-docs-archive - name: Install cosign uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.10.1 diff --git a/docs/site/.gitignore b/docs/site/.gitignore index c44b27a4b..e51d8d962 100644 --- a/docs/site/.gitignore +++ b/docs/site/.gitignore @@ -10,6 +10,7 @@ npm-debug.log* src/content/docs/products/ public/products/ .repo-docs-cache/ +.archive-bundles/ # OpenAPI specs fetched at the pinned ref by scripts/fetch-openapi.mjs # (build artifacts; redocly reads them but they are regenerated each build). diff --git a/docs/site/README.md b/docs/site/README.md index 20ee51e63..45afc29c8 100644 --- a/docs/site/README.md +++ b/docs/site/README.md @@ -25,7 +25,35 @@ npm run check The check command validates frontmatter, generated data, Markdown structure, prose style, OpenAPI snapshots, SVG accessibility, Astro types, the static -build, and generated Redoc API pages. +build, and generated Redoc API pages. It checks the current site only. Published +release archives are immutable bundles and are not rebuilt during routine docs +work. + +To verify the complete deployable tree, including every locked release archive: + +```sh +npm run check:archives +``` + +`src/data/archive-lock.yaml` is append-only. Each entry binds one archived +docset to the SHA-256 of its deterministic bundle and extracted site tree. +Existing entries must never be edited or removed. `npm run +check:archive-lock -- --base-ref origin/main` enforces that invariant. +Archived builds omit Pagefind because its generated index differs by host +platform; current documentation keeps search enabled. + +To publish a new archived docset, build only that docset and create its bundle: + +```sh +DOCS_DOCSET=vX.Y.Z npm run build:archive +npm run archive:snapshot -- vX.Y.Z --write-lock +npm run generate +npm run check:archive-lock -- --base-ref origin/main +``` + +The release workflow repeats those steps from the exact annotated tag and +publishes `registry-docs-vX.Y.Z.tar.gz` with the other signed, SBOM-covered, +SLSA-provenanced release files. ## Content Sources diff --git a/docs/site/astro.config.mjs b/docs/site/astro.config.mjs index 3df14bd1c..141103674 100644 --- a/docs/site/astro.config.mjs +++ b/docs/site/astro.config.mjs @@ -184,6 +184,10 @@ export default defineConfig({ starlight({ title: 'Registry stack docs', description: 'Documentation for Registry Stack: Registry Relay and Registry Notary, the runtime services that publish registry metadata, serve protected registry data, and issue evidence credentials.', + // Pagefind's generated index is platform-dependent. Archives are release + // artifacts, so disable their local search instead of allowing the same + // source release to produce different immutable bytes on macOS and Linux. + pagefind: !isArchivedBuild, plugins: [ // Generates /llms.txt, /llms-full.txt, and /llms-small.txt for // machine consumption. The `details` field carries the discovery diff --git a/docs/site/package-lock.json b/docs/site/package-lock.json index 27b5d0d28..06e37d1f6 100644 --- a/docs/site/package-lock.json +++ b/docs/site/package-lock.json @@ -26,6 +26,7 @@ "markdownlint-cli2": "^0.23.0", "remark-gfm": "^4.0.1", "starlight-llms-txt": "^0.11.0", + "tar": "^7.5.19", "typescript": "^6.0.3", "yaml": "^2.8.1" }, diff --git a/docs/site/package.json b/docs/site/package.json index 3565cb0c2..c120afd41 100644 --- a/docs/site/package.json +++ b/docs/site/package.json @@ -11,6 +11,8 @@ "build": "npm run generate && astro check && astro build", "build:archive": "node scripts/build-archive.mjs", "build:archives": "node scripts/build-archives.mjs", + "assemble:archives": "node scripts/assemble-archives.mjs", + "archive:snapshot": "node scripts/archive-lock.mjs snapshot", "preview": "astro preview", "generate": "node scripts/generate-data.mjs && node scripts/generate-project-starters.mjs && node scripts/fetch-openapi.mjs && node scripts/sync-repo-docs.mjs && node scripts/generate-sidebar.mjs", "test": "node --test \"scripts/**/*.test.mjs\"", @@ -31,8 +33,12 @@ "test:tutorial:registryctl": "node --test scripts/registryctl-tutorial.test.mjs", "check:tutorial:registryctl": "bash scripts/check-registryctl-tutorials.sh", "check:links": "npm run build && npm run check:links:built", - "check": "npm run generate && npm run check:evidence-links && npm run check:research-banners && npm run check:docset && npm run check:release-manifests && npm run check:content && npm run check:markdown && npm run check:style && npm run check:style:fixtures && npm run check:openapi && npm run check:config-vocabulary && npm run check:tutorial:dry-run && npm run check:svg && npm run build && npm run check:llms:built && npm run build:archives && npm run check:seo:built && npm run check:links:built", + "check": "npm run generate && npm run check:evidence-links && npm run check:research-banners && npm run check:docset && npm run check:release-manifests && npm run check:archive-lock && npm run check:content && npm run check:markdown && npm run check:style && npm run check:style:fixtures && npm run check:openapi && npm run check:config-vocabulary && npm run check:tutorial:dry-run && npm run check:svg && npm run build && npm run check:llms:built && npm run check:seo:current && npm run check:links:current", + "check:archives": "npm run build && npm run check:llms:built && npm run assemble:archives -- --bootstrap && npm run check:seo:built && npm run check:links:built", + "check:archive-lock": "node scripts/archive-lock.mjs check", + "check:seo:current": "node scripts/check-seo.mjs --scope current", "check:seo:built": "node scripts/check-seo.mjs", + "check:links:current": "node scripts/check-built-links.mjs --scope current", "check:links:built": "node scripts/check-built-links.mjs" }, "dependencies": { @@ -54,6 +60,7 @@ "markdownlint-cli2": "^0.23.0", "remark-gfm": "^4.0.1", "starlight-llms-txt": "^0.11.0", + "tar": "^7.5.19", "typescript": "^6.0.3", "yaml": "^2.8.1" }, diff --git a/docs/site/scripts/archive-bundle.mjs b/docs/site/scripts/archive-bundle.mjs new file mode 100644 index 000000000..82b130623 --- /dev/null +++ b/docs/site/scripts/archive-bundle.mjs @@ -0,0 +1,420 @@ +import { createHash } from 'node:crypto'; +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, posix, relative, resolve, sep } from 'node:path'; + +import * as tar from 'tar'; + +export const ARCHIVE_BUNDLE_SCHEMA = 'registry-docs.archive-bundle.v1'; +export const ARCHIVE_LOCK_SCHEMA = 'registry-docs.archive-lock.v1'; + +const sha256Pattern = /^[0-9a-f]{64}$/; +const archivePathPattern = /^\/v\/[a-z0-9][a-z0-9.-]*\/$/; +const archiveIdPattern = /^[a-z0-9][a-z0-9.-]*[a-z0-9]$/; +const maximumArchiveBundleBytes = 256 * 1024 * 1024; +const maximumExtractedBytes = 1024 * 1024 * 1024; +const maximumArchiveEntries = 100_000; + +export function canonicalValue(value) { + if (Array.isArray(value)) return value.map(canonicalValue); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalValue(value[key])]), + ); + } + return value; +} + +export function canonicalJson(value) { + return JSON.stringify(canonicalValue(value)); +} + +export function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +export async function fileDigest(path) { + return sha256(await readFile(path)); +} + +function isWithin(parent, child) { + const rel = relative(parent, child); + return rel === '' || (!rel.startsWith(`..${sep}`) && rel !== '..' && !rel.startsWith(sep)); +} + +async function existingKind(path) { + try { + return await lstat(path); + } catch (error) { + if (error?.code === 'ENOENT') return null; + throw error; + } +} + +async function requireRealDirectory(path, label) { + const info = await existingKind(path); + if (!info || info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`${label} must be a real directory: ${path}`); + } +} + +async function rejectUnsafeDirectory(path, label) { + const info = await existingKind(path); + if (info && (info.isSymbolicLink() || !info.isDirectory())) { + throw new Error(`${label} must be a real directory, not a symlink: ${path}`); + } +} + +export function archiveRelativePath(docset) { + if (docset?.status !== 'archived') { + throw new Error(`Docset "${docset?.id ?? ''}" is not archived`); + } + if (!archiveIdPattern.test(docset.id)) { + throw new Error(`Archived docset id is unsafe: ${docset.id}`); + } + if (typeof docset.path !== 'string' || !archivePathPattern.test(docset.path)) { + throw new Error( + `Archived docset "${docset.id}" path must be a safe path below /v/: ${docset.path}`, + ); + } + const rel = docset.path.slice(1, -1); + if (rel.split('/').some((part) => part === '.' || part === '..')) { + throw new Error(`Archived docset "${docset.id}" path contains traversal`); + } + return rel; +} + +export function archiveOutputDirectory(docsRoot, docset) { + const distRoot = resolve(docsRoot, 'dist'); + const output = resolve(distRoot, archiveRelativePath(docset)); + if (!isWithin(resolve(distRoot, 'v'), output)) { + throw new Error(`Archived docset "${docset.id}" resolves outside dist/v`); + } + return output; +} + +export async function validateArchiveOutputLocation(docsRoot, docset) { + const distRoot = resolve(docsRoot, 'dist'); + const versionRoot = resolve(distRoot, 'v'); + const output = archiveOutputDirectory(docsRoot, docset); + await rejectUnsafeDirectory(distRoot, 'archive dist root'); + await rejectUnsafeDirectory(versionRoot, 'archive version root'); + await rejectUnsafeDirectory(output, 'archive output'); + return output; +} + +export function archiveBundleName(docset) { + archiveRelativePath(docset); + return `registry-docs-${docset.id}.tar.gz`; +} + +export function publicArchiveBundlePath(docsRoot, docset) { + archiveRelativePath(docset); + return resolve(docsRoot, 'dist/_archive-bundles', `${docset.id}.tar.gz`); +} + +export function localArchiveBundlePath(docsRoot, docset) { + archiveRelativePath(docset); + return resolve(docsRoot, '.archive-bundles', `${docset.id}.tar.gz`); +} + +async function collectTree(root, current = root) { + const entries = await readdir(current, { withFileTypes: true }); + const directories = []; + const files = []; + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + const path = resolve(current, entry.name); + const info = await lstat(path); + if (info.isSymbolicLink()) { + throw new Error(`immutable archive trees cannot contain symlinks: ${path}`); + } + if (info.isDirectory()) { + directories.push(path); + const nested = await collectTree(root, path); + directories.push(...nested.directories); + files.push(...nested.files); + } else if (info.isFile()) { + files.push(path); + } else { + throw new Error(`archive tree contains an unsupported filesystem entry: ${path}`); + } + } + return { directories, files }; +} + +export async function treeDigest(root) { + await requireRealDirectory(root, 'archive tree'); + const { files } = await collectTree(root); + const hash = createHash('sha256'); + for (const path of files) { + const info = await lstat(path); + const rel = relative(root, path).replaceAll(sep, '/'); + hash.update(`${rel}\0${info.mode & 0o111 ? 'x' : '-'}\0`); + hash.update(await readFile(path)); + hash.update('\0'); + } + return hash.digest('hex'); +} + +export function archiveSourceRefs(docset) { + return Object.entries(docset.products ?? {}) + .map(([name, product]) => ({ + name, + version: product?.version, + ref: product?.ref, + })) + .sort((left, right) => left.name.localeCompare(right.name)); +} + +export function archiveMetadata(docset, outputDigest) { + if (!sha256Pattern.test(outputDigest)) { + throw new Error('archive output digest must be 64 lowercase hexadecimal characters'); + } + archiveRelativePath(docset); + return { + schema_version: ARCHIVE_BUNDLE_SCHEMA, + docset_id: docset.id, + archive_path: docset.path, + source: docset.source, + published_at: docset.published_at, + source_refs: archiveSourceRefs(docset), + tree_sha256: outputDigest, + }; +} + +async function stagedTarEntries(stagingRoot) { + const { directories, files } = await collectTree(stagingRoot); + return [...directories, ...files] + .map((path) => relative(stagingRoot, path).replaceAll(sep, '/')) + .sort(); +} + +export async function createArchiveBundle({ + docsRoot = process.cwd(), + docset, + bundlePath = localArchiveBundlePath(docsRoot, docset), +} = {}) { + const output = await validateArchiveOutputLocation(docsRoot, docset); + await requireRealDirectory(output, `archive output for ${docset.id}`); + const outputDigest = await treeDigest(output); + const metadata = archiveMetadata(docset, outputDigest); + const staging = await mkdtemp(resolve(tmpdir(), 'registry-docs-archive-bundle-')); + try { + await writeFile( + resolve(staging, 'metadata.json'), + `${JSON.stringify(metadata, null, 2)}\n`, + { mode: 0o644 }, + ); + await cp(output, resolve(staging, 'site'), { + recursive: true, + force: false, + errorOnExist: true, + preserveTimestamps: false, + }); + const entries = await stagedTarEntries(staging); + await mkdir(dirname(bundlePath), { recursive: true }); + await rm(bundlePath, { force: true }); + await Promise.resolve(tar.create( + { + cwd: staging, + file: bundlePath, + filter(_path, info) { + const permissions = info.isDirectory() + ? 0o755 + : info.mode & 0o111 + ? 0o755 + : 0o644; + info.mode = (info.mode & ~0o7777) | permissions; + return true; + }, + gzip: { level: 9, mtime: 0 }, + mtime: new Date(0), + noDirRecurse: true, + noPax: true, + portable: true, + strict: true, + }, + entries, + )); + } finally { + await rm(staging, { recursive: true, force: true }); + } + return { + bundle_path: bundlePath, + bundle_sha256: await fileDigest(bundlePath), + tree_sha256: outputDigest, + metadata, + }; +} + +function validateTarEntry(path, entry) { + if (path.includes('\\') || path.startsWith('/') || path.includes('\0')) { + throw new Error(`archive bundle contains an unsafe path: ${path}`); + } + const normalized = posix.normalize(path); + if ( + normalized !== path || + normalized === '..' || + normalized.startsWith('../') || + !( + normalized === 'metadata.json' || + normalized === 'site' || + normalized.startsWith('site/') + ) + ) { + throw new Error(`archive bundle contains an unsafe path: ${path}`); + } + if (!['Directory', 'File'].includes(entry.type)) { + throw new Error(`archive bundle contains unsupported ${entry.type} entry: ${path}`); + } + return true; +} + +function assertMetadataMatchesDocset(metadata, docset) { + const expected = { + schema_version: ARCHIVE_BUNDLE_SCHEMA, + docset_id: docset.id, + archive_path: docset.path, + source: docset.source, + published_at: docset.published_at, + source_refs: archiveSourceRefs(docset), + tree_sha256: metadata.tree_sha256, + }; + if (canonicalJson(metadata) !== canonicalJson(expected)) { + throw new Error(`archive bundle metadata does not match docset ${docset.id}`); + } + if (!sha256Pattern.test(metadata.tree_sha256)) { + throw new Error(`archive bundle ${docset.id} has an invalid tree digest`); + } +} + +export async function inspectArchiveBundle({ + bundlePath, + docset, + expectedBundleSha256, + expectedTreeSha256, +} = {}) { + if (!sha256Pattern.test(expectedBundleSha256)) { + throw new Error(`archive bundle ${docset.id} has no valid locked bundle digest`); + } + if (!sha256Pattern.test(expectedTreeSha256)) { + throw new Error(`archive bundle ${docset.id} has no valid locked tree digest`); + } + const bundleInfo = await lstat(bundlePath); + if ( + bundleInfo.isSymbolicLink() || + !bundleInfo.isFile() || + bundleInfo.size > maximumArchiveBundleBytes + ) { + throw new Error( + `archive bundle ${docset.id} must be a regular file no larger than ${maximumArchiveBundleBytes} bytes`, + ); + } + const actualBundleDigest = await fileDigest(bundlePath); + if (actualBundleDigest !== expectedBundleSha256) { + throw new Error( + `archive bundle ${docset.id} digest ${actualBundleDigest} does not match lock ${expectedBundleSha256}`, + ); + } + + const extraction = await mkdtemp(resolve(tmpdir(), 'registry-docs-archive-extract-')); + try { + let entryCount = 0; + let extractedBytes = 0; + await Promise.resolve(tar.extract({ + cwd: extraction, + file: bundlePath, + filter(path, entry) { + entryCount += 1; + extractedBytes += Number(entry.size ?? 0); + if (entryCount > maximumArchiveEntries) { + throw new Error( + `archive bundle ${docset.id} exceeds ${maximumArchiveEntries} entries`, + ); + } + if (extractedBytes > maximumExtractedBytes) { + throw new Error( + `archive bundle ${docset.id} exceeds ${maximumExtractedBytes} extracted bytes`, + ); + } + return validateTarEntry(path, entry); + }, + preservePaths: false, + strict: true, + })); + const rootEntries = (await readdir(extraction)).sort(); + if (canonicalJson(rootEntries) !== canonicalJson(['metadata.json', 'site'])) { + throw new Error(`archive bundle ${docset.id} must contain only metadata.json and site/`); + } + const metadataInfo = await lstat(resolve(extraction, 'metadata.json')); + if (metadataInfo.isSymbolicLink() || !metadataInfo.isFile()) { + throw new Error(`archive bundle ${docset.id} metadata must be a regular file`); + } + await requireRealDirectory(resolve(extraction, 'site'), `archive bundle ${docset.id} site`); + const metadata = JSON.parse(await readFile(resolve(extraction, 'metadata.json'), 'utf8')); + assertMetadataMatchesDocset(metadata, docset); + const actualTreeDigest = await treeDigest(resolve(extraction, 'site')); + if ( + actualTreeDigest !== metadata.tree_sha256 || + actualTreeDigest !== expectedTreeSha256 + ) { + throw new Error( + `archive bundle ${docset.id} tree digest ${actualTreeDigest} does not match its lock`, + ); + } + return { extraction, metadata, bundle_sha256: actualBundleDigest, tree_sha256: actualTreeDigest }; + } catch (error) { + await rm(extraction, { recursive: true, force: true }); + throw error; + } +} + +export async function restoreArchiveBundle({ + docsRoot = process.cwd(), + bundlePath, + docset, + expectedBundleSha256, + expectedTreeSha256, + publishBundle = true, +} = {}) { + const inspected = await inspectArchiveBundle({ + bundlePath, + docset, + expectedBundleSha256, + expectedTreeSha256, + }); + try { + const output = await validateArchiveOutputLocation(docsRoot, docset); + await rm(output, { recursive: true, force: true }); + await mkdir(dirname(output), { recursive: true }); + await cp(resolve(inspected.extraction, 'site'), output, { + recursive: true, + force: false, + errorOnExist: true, + preserveTimestamps: false, + }); + const publicBundle = publicArchiveBundlePath(docsRoot, docset); + if (publishBundle && resolve(bundlePath) !== resolve(publicBundle)) { + await mkdir(dirname(publicBundle), { recursive: true }); + await cp(bundlePath, publicBundle, { force: true }); + } + return { + ...inspected, + output, + public_bundle: publishBundle ? publicBundle : null, + }; + } finally { + await rm(inspected.extraction, { recursive: true, force: true }); + } +} diff --git a/docs/site/scripts/archive-bundle.test.mjs b/docs/site/scripts/archive-bundle.test.mjs new file mode 100644 index 000000000..a7a2fac91 --- /dev/null +++ b/docs/site/scripts/archive-bundle.test.mjs @@ -0,0 +1,126 @@ +import assert from 'node:assert/strict'; +import { + chmod, + mkdir, + mkdtemp, + open, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import { + createArchiveBundle, + inspectArchiveBundle, + restoreArchiveBundle, + treeDigest, +} from './archive-bundle.mjs'; + +const docset = { + id: 'v1.2.3', + path: '/v/1.2.3/', + status: 'archived', + source: 'registry-stack-v1.2.3', + published_at: '2026-07-26', + products: { + 'registry-stack': { + version: 'v1.2.3', + ref: 'a'.repeat(40), + }, + }, +}; + +async function fixture(t) { + const root = await mkdtemp(resolve(tmpdir(), 'registry-docs-bundle-test-')); + t.after(() => rm(root, { recursive: true, force: true })); + const output = resolve(root, 'dist/v/1.2.3'); + await mkdir(resolve(output, 'guide'), { recursive: true }); + await writeFile(resolve(output, 'index.html'), '

Archive

\n'); + await writeFile(resolve(output, 'guide/index.html'), '

Guide

\n'); + return { root, output }; +} + +test('creates byte-for-byte deterministic archive bundles and restores them', async (t) => { + const { root, output } = await fixture(t); + const firstPath = resolve(root, 'first.tar.gz'); + const secondPath = resolve(root, 'second.tar.gz'); + const first = await createArchiveBundle({ docsRoot: root, docset, bundlePath: firstPath }); + await chmod(resolve(output, 'index.html'), 0o600); + const second = await createArchiveBundle({ docsRoot: root, docset, bundlePath: secondPath }); + + assert.equal(first.bundle_sha256, second.bundle_sha256); + assert.equal(first.tree_sha256, second.tree_sha256); + assert.deepEqual(await readFile(firstPath), await readFile(secondPath)); + + await rm(output, { recursive: true }); + const restored = await restoreArchiveBundle({ + docsRoot: root, + bundlePath: secondPath, + docset, + expectedBundleSha256: second.bundle_sha256, + expectedTreeSha256: second.tree_sha256, + }); + assert.equal(await readFile(resolve(restored.output, 'guide/index.html'), 'utf8'), '

Guide

\n'); + assert.deepEqual(await readFile(restored.public_bundle), await readFile(secondPath)); +}); + +test('rejects a bundle that does not match the immutable digest', async (t) => { + const { root } = await fixture(t); + const bundlePath = resolve(root, 'archive.tar.gz'); + const result = await createArchiveBundle({ docsRoot: root, docset, bundlePath }); + await writeFile(bundlePath, Buffer.concat([await readFile(bundlePath), Buffer.from('changed')])); + + await assert.rejects( + inspectArchiveBundle({ + bundlePath, + docset, + expectedBundleSha256: result.bundle_sha256, + expectedTreeSha256: result.tree_sha256, + }), + /does not match lock/, + ); +}); + +test('rejects oversized bundles before reading or extracting them', async (t) => { + const { root } = await fixture(t); + const bundlePath = resolve(root, 'oversized.tar.gz'); + const handle = await open(bundlePath, 'w'); + await handle.truncate(256 * 1024 * 1024 + 1); + await handle.close(); + + await assert.rejects( + inspectArchiveBundle({ + bundlePath, + docset, + expectedBundleSha256: 'a'.repeat(64), + expectedTreeSha256: 'b'.repeat(64), + }), + /no larger than/, + ); +}); + +test('rejects symlinks in immutable archive trees', async (t) => { + const { output } = await fixture(t); + await symlink('index.html', resolve(output, 'linked.html')); + await assert.rejects(treeDigest(output), /cannot contain symlinks/); +}); + +test('rejects docset metadata drift even when given a valid bundle digest', async (t) => { + const { root } = await fixture(t); + const bundlePath = resolve(root, 'archive.tar.gz'); + const result = await createArchiveBundle({ docsRoot: root, docset, bundlePath }); + + await assert.rejects( + inspectArchiveBundle({ + bundlePath, + docset: { ...docset, source: 'different-source' }, + expectedBundleSha256: result.bundle_sha256, + expectedTreeSha256: result.tree_sha256, + }), + /metadata does not match/, + ); +}); diff --git a/docs/site/scripts/archive-lock.mjs b/docs/site/scripts/archive-lock.mjs new file mode 100644 index 000000000..5b7893fd6 --- /dev/null +++ b/docs/site/scripts/archive-lock.mjs @@ -0,0 +1,238 @@ +import { execFile } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +import YAML from 'yaml'; + +import { + ARCHIVE_LOCK_SCHEMA, + canonicalJson, + createArchiveBundle, + localArchiveBundlePath, +} from './archive-bundle.mjs'; +import { getDocset, loadDocsets } from './docsets.mjs'; + +const run = promisify(execFile); +const sha256Pattern = /^[0-9a-f]{64}$/; + +export async function loadArchiveLock({ + lockPath = resolve(process.cwd(), 'src/data/archive-lock.yaml'), +} = {}) { + return YAML.parse(await readFile(lockPath, 'utf8')); +} + +export function validateArchiveLock(lock, docsets) { + const errors = []; + if (!lock || typeof lock !== 'object' || Array.isArray(lock)) { + return ['archive-lock.yaml must contain a top-level object']; + } + if (lock.schema_version !== ARCHIVE_LOCK_SCHEMA) { + errors.push(`archive-lock.yaml schema_version must be ${ARCHIVE_LOCK_SCHEMA}`); + } + if (!lock.archives || typeof lock.archives !== 'object' || Array.isArray(lock.archives)) { + errors.push('archive-lock.yaml archives must be a map'); + return errors; + } + const expectedIds = docsets.docsets + .filter((docset) => docset.status === 'archived') + .map((docset) => docset.id) + .sort(); + const actualIds = Object.keys(lock.archives).sort(); + for (const missing of expectedIds.filter((id) => !actualIds.includes(id))) { + errors.push(`archive-lock.yaml is missing archived docset ${missing}`); + } + for (const unexpected of actualIds.filter((id) => !expectedIds.includes(id))) { + errors.push(`archive-lock.yaml contains non-archived docset ${unexpected}`); + } + for (const id of actualIds) { + const entry = lock.archives[id]; + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + errors.push(`archive-lock.yaml ${id} entry must be a map`); + continue; + } + const unknown = Object.keys(entry).filter( + (key) => !['bundle_sha256', 'tree_sha256'].includes(key), + ); + if (unknown.length > 0) { + errors.push(`archive-lock.yaml ${id} has unknown field ${unknown[0]}`); + } + for (const field of ['bundle_sha256', 'tree_sha256']) { + if (!sha256Pattern.test(entry[field] ?? '')) { + errors.push(`archive-lock.yaml ${id}.${field} must be 64 lowercase hex characters`); + } + } + } + return errors; +} + +export function assertArchiveLockImmutable(baseLock, currentLock) { + const errors = []; + for (const [id, entry] of Object.entries(baseLock?.archives ?? {})) { + if (!currentLock?.archives || !(id in currentLock.archives)) { + errors.push(`immutable archive lock entry ${id} was removed`); + continue; + } + if (canonicalJson(entry) !== canonicalJson(currentLock.archives[id])) { + errors.push(`immutable archive lock entry ${id} was changed`); + } + } + return errors; +} + +export function addArchiveLockEntry(lock, docsetId, result) { + if (lock?.schema_version !== ARCHIVE_LOCK_SCHEMA || !lock.archives) { + throw new Error('archive-lock.yaml must be valid before adding an archive'); + } + if (Object.hasOwn(lock.archives, docsetId)) { + throw new Error(`immutable archive lock entry ${docsetId} already exists`); + } + lock.archives[docsetId] = { + bundle_sha256: result.bundle_sha256, + tree_sha256: result.tree_sha256, + }; + return lock; +} + +async function archiveLockAtGitRef(baseRef, repoRoot) { + try { + const { stdout } = await run( + 'git', + ['show', `${baseRef}:docs/site/src/data/archive-lock.yaml`], + { cwd: repoRoot, maxBuffer: 4 * 1024 * 1024 }, + ); + return YAML.parse(stdout); + } catch (error) { + if ( + error?.code === 128 && + /does not exist|exists on disk, but not in/.test(error.stderr ?? '') + ) { + return null; + } + throw error; + } +} + +export async function checkArchiveLock({ + docsRoot = process.cwd(), + baseRef = null, +} = {}) { + const docsets = await loadDocsets({ dataDir: resolve(docsRoot, 'src/data') }); + const lock = await loadArchiveLock({ + lockPath: resolve(docsRoot, 'src/data/archive-lock.yaml'), + }); + const errors = validateArchiveLock(lock, docsets); + if (baseRef) { + const baseLock = await archiveLockAtGitRef(baseRef, resolve(docsRoot, '../..')); + if (baseLock) errors.push(...assertArchiveLockImmutable(baseLock, lock)); + } + if (errors.length > 0) throw new Error(errors.join('\n')); + return { docsets, lock }; +} + +export async function snapshotArchive({ + docsRoot = process.cwd(), + docsetId, + bundlePath = null, + verifyLock = false, + writeLock = false, +} = {}) { + if (verifyLock && writeLock) { + throw new Error('--verify-lock and --write-lock cannot be used together'); + } + const docsets = await loadDocsets({ dataDir: resolve(docsRoot, 'src/data') }); + const docset = getDocset(docsets, docsetId); + const result = await createArchiveBundle({ + docsRoot, + docset, + bundlePath: bundlePath ?? localArchiveBundlePath(docsRoot, docset), + }); + if (verifyLock) { + const lock = await loadArchiveLock({ + lockPath: resolve(docsRoot, 'src/data/archive-lock.yaml'), + }); + const expected = lock.archives?.[docset.id]; + if ( + expected?.bundle_sha256 !== result.bundle_sha256 || + expected?.tree_sha256 !== result.tree_sha256 + ) { + throw new Error( + `archive bundle ${docset.id} does not match its immutable lock entry`, + ); + } + } + if (writeLock) { + const lockPath = resolve(docsRoot, 'src/data/archive-lock.yaml'); + const lock = await loadArchiveLock({ lockPath }); + addArchiveLockEntry(lock, docset.id, result); + await writeFile(lockPath, YAML.stringify(lock), 'utf8'); + } + return { docset, result }; +} + +function parseArgs(args) { + const command = args.shift(); + if (command === 'check') { + let baseRef = null; + while (args.length > 0) { + const option = args.shift(); + if (option === '--base-ref' && args[0]) { + baseRef = args.shift(); + } else { + throw new Error('usage: archive-lock.mjs check [--base-ref ]'); + } + } + return { command, baseRef }; + } + if (command === 'snapshot' && args.length >= 1) { + const docsetId = args.shift(); + let bundlePath = null; + let verifyLock = false; + let writeLock = false; + while (args.length > 0) { + const option = args.shift(); + if (option === '--output' && args[0]) { + bundlePath = resolve(args.shift()); + } else if (option === '--verify-lock') { + verifyLock = true; + } else if (option === '--write-lock') { + writeLock = true; + } else { + throw new Error( + 'usage: archive-lock.mjs snapshot [--output ] [--verify-lock|--write-lock]', + ); + } + } + return { command, docsetId, bundlePath, verifyLock, writeLock }; + } + throw new Error( + 'usage: archive-lock.mjs check [--base-ref ] | snapshot [--output ] [--verify-lock|--write-lock]', + ); +} + +async function main(args) { + const parsed = parseArgs([...args]); + if (parsed.command === 'check') { + const { lock } = await checkArchiveLock({ baseRef: parsed.baseRef }); + console.log(`Archive lock check passed for ${Object.keys(lock.archives).length} archive(s).`); + return; + } + const { docset, result } = await snapshotArchive(parsed); + const entry = { + bundle_sha256: result.bundle_sha256, + tree_sha256: result.tree_sha256, + }; + process.stdout.write( + `${docset.id}:\n${YAML.stringify(entry).replace(/^/gm, ' ').trimEnd()}\n`, + ); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + await main(process.argv.slice(2)); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/docs/site/scripts/archive-lock.test.mjs b/docs/site/scripts/archive-lock.test.mjs new file mode 100644 index 000000000..8d39a06f0 --- /dev/null +++ b/docs/site/scripts/archive-lock.test.mjs @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + addArchiveLockEntry, + assertArchiveLockImmutable, + validateArchiveLock, +} from './archive-lock.mjs'; + +const digest = 'a'.repeat(64); +const docsets = { + current: 'latest', + docsets: [ + { id: 'latest', status: 'current' }, + { id: 'v1.0.0', status: 'archived' }, + ], +}; + +function lock(overrides = {}) { + return { + schema_version: 'registry-docs.archive-lock.v1', + archives: { + 'v1.0.0': { + bundle_sha256: digest, + tree_sha256: 'b'.repeat(64), + }, + }, + ...overrides, + }; +} + +test('validates exact archived docset coverage and digest shapes', () => { + assert.deepEqual(validateArchiveLock(lock(), docsets), []); + assert.match( + validateArchiveLock(lock({ archives: {} }), docsets).join('\n'), + /missing archived docset v1.0.0/, + ); + assert.match( + validateArchiveLock( + lock({ + archives: { + ...lock().archives, + latest: { bundle_sha256: digest, tree_sha256: digest }, + }, + }), + docsets, + ).join('\n'), + /contains non-archived docset latest/, + ); +}); + +test('immutable lock entries can be added but not changed or removed', () => { + const base = lock(); + const added = lock({ + archives: { + ...base.archives, + 'v1.1.0': { bundle_sha256: 'c'.repeat(64), tree_sha256: 'd'.repeat(64) }, + }, + }); + assert.deepEqual(assertArchiveLockImmutable(base, added), []); + assert.match( + assertArchiveLockImmutable(base, lock({ archives: {} })).join('\n'), + /was removed/, + ); + assert.match( + assertArchiveLockImmutable( + base, + lock({ + archives: { + 'v1.0.0': { ...base.archives['v1.0.0'], tree_sha256: 'e'.repeat(64) }, + }, + }), + ).join('\n'), + /was changed/, + ); +}); + +test('adds a new lock entry but refuses to overwrite immutable bytes', () => { + const current = lock(); + addArchiveLockEntry(current, 'v1.1.0', { + bundle_sha256: 'c'.repeat(64), + tree_sha256: 'd'.repeat(64), + }); + assert.equal(current.archives['v1.1.0'].bundle_sha256, 'c'.repeat(64)); + assert.throws( + () => addArchiveLockEntry(current, 'v1.0.0', { + bundle_sha256: 'e'.repeat(64), + tree_sha256: 'f'.repeat(64), + }), + /already exists/, + ); +}); diff --git a/docs/site/scripts/assemble-archives.mjs b/docs/site/scripts/assemble-archives.mjs new file mode 100644 index 000000000..e983e1b6c --- /dev/null +++ b/docs/site/scripts/assemble-archives.mjs @@ -0,0 +1,239 @@ +import { createWriteStream } from 'node:fs'; +import { lstat, mkdir, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { Readable, Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { fileURLToPath } from 'node:url'; + +import { + createArchiveBundle, + localArchiveBundlePath, + restoreArchiveBundle, +} from './archive-bundle.mjs'; +import { loadArchiveLock, validateArchiveLock } from './archive-lock.mjs'; +import { buildDocsetArchive } from './build-archives.mjs'; +import { loadDocsets } from './docsets.mjs'; + +const defaultArchiveBaseUrl = 'https://docs.registrystack.org/_archive-bundles'; +const defaultReleaseBaseUrl = + 'https://github.com/registrystack/registry-stack/releases/download'; +const maximumBundleBytes = 256 * 1024 * 1024; + +function trimTrailingSlash(value) { + return value.replace(/\/+$/, ''); +} + +export function archiveBundleUrls(docset, { + archiveBaseUrl = process.env.DOCS_ARCHIVE_BASE_URL || defaultArchiveBaseUrl, + releaseBaseUrl = process.env.DOCS_RELEASE_BASE_URL || defaultReleaseBaseUrl, +} = {}) { + return [ + `${trimTrailingSlash(releaseBaseUrl)}/${docset.id}/registry-docs-${docset.id}.tar.gz`, + `${trimTrailingSlash(archiveBaseUrl)}/${docset.id}.tar.gz`, + ]; +} + +async function downloadBundle(url, output, { + fetchImpl = fetch, + maximumBytes = maximumBundleBytes, +} = {}) { + const response = await fetchImpl(url, { + headers: { 'user-agent': 'registry-docs-archive-assembler/1' }, + redirect: 'follow', + }); + if (response.status === 404) return false; + if (!response.ok) { + throw new Error(`archive bundle download ${url} returned HTTP ${response.status}`); + } + const declaredLength = Number(response.headers.get('content-length')); + if (Number.isFinite(declaredLength) && declaredLength > maximumBytes) { + throw new Error(`archive bundle download ${url} exceeds ${maximumBytes} bytes`); + } + if (!response.body) { + throw new Error(`archive bundle download ${url} returned no body`); + } + await mkdir(dirname(output), { recursive: true }); + let received = 0; + const limit = new Transform({ + transform(chunk, _encoding, callback) { + received += chunk.length; + if (received > maximumBytes) { + callback(new Error(`archive bundle download ${url} exceeds ${maximumBytes} bytes`)); + } else { + callback(null, chunk); + } + }, + }); + await pipeline(Readable.fromWeb(response.body), limit, createWriteStream(output, { mode: 0o600 })); + return true; +} + +async function restoreDownloadedBundle({ + docsRoot, + docset, + lockEntry, + urls, + fetchImpl, +}) { + const temporary = await mkdtemp(resolve(tmpdir(), 'registry-docs-archive-download-')); + try { + const bundlePath = resolve(temporary, `${docset.id}.tar.gz`); + for (const [index, url] of urls.entries()) { + if (!await downloadBundle(url, bundlePath, { fetchImpl })) continue; + await restoreArchiveBundle({ + docsRoot, + bundlePath, + docset, + expectedBundleSha256: lockEntry.bundle_sha256, + expectedTreeSha256: lockEntry.tree_sha256, + // Release assets are canonical and do not need to be duplicated in + // Pages. The fallback URL is republished so pre-contract releases stay + // self-hosting. + publishBundle: index > 0, + }); + return url; + } + return null; + } finally { + await rm(temporary, { recursive: true, force: true }); + } +} + +async function restoreLocalBundle({ docsRoot, docset, lockEntry }) { + const bundlePath = localArchiveBundlePath(docsRoot, docset); + try { + const info = await lstat(bundlePath); + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error(`local archive bundle must be a regular file: ${bundlePath}`); + } + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } + await restoreArchiveBundle({ + docsRoot, + bundlePath, + docset, + expectedBundleSha256: lockEntry.bundle_sha256, + expectedTreeSha256: lockEntry.tree_sha256, + }); + return true; +} + +async function bootstrapArchive({ + docsRoot, + docset, + lockEntry, + buildArchive, +}) { + await buildArchive(docset, { docsRoot }); + const bundlePath = localArchiveBundlePath(docsRoot, docset); + const result = await createArchiveBundle({ docsRoot, docset, bundlePath }); + if ( + result.bundle_sha256 !== lockEntry.bundle_sha256 || + result.tree_sha256 !== lockEntry.tree_sha256 + ) { + throw new Error( + `bootstrapped archive ${docset.id} does not match its immutable lock entry`, + ); + } + await restoreArchiveBundle({ + docsRoot, + bundlePath, + docset, + expectedBundleSha256: lockEntry.bundle_sha256, + expectedTreeSha256: lockEntry.tree_sha256, + }); +} + +async function restoreCurrentGeneratedData(docsRoot, currentDocsetId) { + const { spawn } = await import('node:child_process'); + await new Promise((resolveRun, rejectRun) => { + const child = spawn('npm', ['run', 'generate'], { + cwd: docsRoot, + env: { ...process.env, DOCS_DOCSET: currentDocsetId, DOCS_BASE: '/' }, + shell: process.platform === 'win32', + stdio: 'inherit', + }); + child.on('exit', (code) => { + if (code === 0) resolveRun(); + else rejectRun(new Error(`npm run generate exited ${code}`)); + }); + child.on('error', rejectRun); + }); +} + +export async function assembleArchives({ + docsRoot = process.cwd(), + bootstrap = false, + fetchImpl = fetch, + buildArchive = buildDocsetArchive, + restoreGeneratedData = restoreCurrentGeneratedData, + urlResolver = archiveBundleUrls, +} = {}) { + const docsets = await loadDocsets({ dataDir: resolve(docsRoot, 'src/data') }); + const lock = await loadArchiveLock({ + lockPath: resolve(docsRoot, 'src/data/archive-lock.yaml'), + }); + const errors = validateArchiveLock(lock, docsets); + if (errors.length > 0) throw new Error(errors.join('\n')); + + let bootstrapped = 0; + let restored = 0; + const sources = {}; + for (const docset of docsets.docsets.filter((entry) => entry.status === 'archived')) { + const lockEntry = lock.archives[docset.id]; + if (await restoreLocalBundle({ docsRoot, docset, lockEntry })) { + restored += 1; + sources[docset.id] = 'local'; + continue; + } + const source = await restoreDownloadedBundle({ + docsRoot, + docset, + lockEntry, + urls: urlResolver(docset), + fetchImpl, + }); + if (source) { + restored += 1; + sources[docset.id] = source; + continue; + } + if (!bootstrap) { + throw new Error( + `no immutable bundle is published for ${docset.id}; rerun with --bootstrap only in archive publication verification`, + ); + } + await bootstrapArchive({ docsRoot, docset, lockEntry, buildArchive }); + bootstrapped += 1; + sources[docset.id] = 'bootstrap'; + } + if (bootstrapped > 0) { + await restoreGeneratedData(docsRoot, docsets.current); + } + return { bootstrapped, restored, sources }; +} + +function parseArgs(args) { + let bootstrap = false; + for (const arg of args) { + if (arg === '--bootstrap') bootstrap = true; + else throw new Error('usage: assemble-archives.mjs [--bootstrap]'); + } + return { bootstrap }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const result = await assembleArchives(parseArgs(process.argv.slice(2))); + console.log( + `Assembled ${result.restored + result.bootstrapped} immutable archive(s): ` + + `${result.restored} restored, ${result.bootstrapped} bootstrapped.`, + ); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/docs/site/scripts/assemble-archives.test.mjs b/docs/site/scripts/assemble-archives.test.mjs new file mode 100644 index 000000000..637990fd8 --- /dev/null +++ b/docs/site/scripts/assemble-archives.test.mjs @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import YAML from 'yaml'; + +import { assembleArchives } from './assemble-archives.mjs'; +import { createArchiveBundle } from './archive-bundle.mjs'; + +const docset = { + id: 'v1.2.3', + label: 'v1.2.3', + path: '/v/1.2.3/', + status: 'archived', + source: 'registry-stack-v1.2.3', + published_at: '2026-07-26', + description: 'Test archive.', + products: { + 'registry-stack': { + version: 'v1.2.3', + ref: 'a'.repeat(40), + }, + }, +}; + +async function fixture(t) { + const sourceRoot = await mkdtemp(resolve(tmpdir(), 'registry-docs-assemble-source-')); + const targetRoot = await mkdtemp(resolve(tmpdir(), 'registry-docs-assemble-target-')); + t.after(() => Promise.all([ + rm(sourceRoot, { recursive: true, force: true }), + rm(targetRoot, { recursive: true, force: true }), + ])); + + const sourceOutput = resolve(sourceRoot, 'dist/v/1.2.3'); + await mkdir(sourceOutput, { recursive: true }); + await writeFile(resolve(sourceOutput, 'index.html'), '

Frozen

\n'); + const bundlePath = resolve(sourceRoot, 'bundle.tar.gz'); + const bundle = await createArchiveBundle({ docsRoot: sourceRoot, docset, bundlePath }); + + await mkdir(resolve(targetRoot, 'src/data'), { recursive: true }); + await writeFile( + resolve(targetRoot, 'src/data/docsets.yaml'), + YAML.stringify({ + current: 'latest', + docsets: [ + { + ...docset, + id: 'latest', + label: 'Latest', + path: '/', + status: 'current', + source: 'registry-stack-main', + products: { + 'registry-stack': { version: 'v1.2.3', ref: 'HEAD' }, + }, + }, + docset, + ], + }), + ); + await writeFile( + resolve(targetRoot, 'src/data/archive-lock.yaml'), + YAML.stringify({ + schema_version: 'registry-docs.archive-lock.v1', + archives: { + [docset.id]: { + bundle_sha256: bundle.bundle_sha256, + tree_sha256: bundle.tree_sha256, + }, + }, + }), + ); + return { bundlePath, sourceRoot, targetRoot }; +} + +test('restores a locked release bundle without rebuilding', async (t) => { + const { bundlePath, targetRoot } = await fixture(t); + const body = await readFile(bundlePath); + let buildCalls = 0; + const result = await assembleArchives({ + docsRoot: targetRoot, + fetchImpl: async () => new Response(body, { status: 200 }), + buildArchive: async () => { buildCalls += 1; }, + restoreGeneratedData: async () => {}, + }); + + assert.equal(buildCalls, 0); + assert.equal(result.restored, 1); + assert.equal(await readFile(resolve(targetRoot, 'dist/v/1.2.3/index.html'), 'utf8'), '

Frozen

\n'); +}); + +test('only bootstraps missing bundles when explicitly allowed', async (t) => { + const { targetRoot } = await fixture(t); + const fetchImpl = async () => new Response(null, { status: 404 }); + await assert.rejects( + assembleArchives({ + docsRoot: targetRoot, + fetchImpl, + buildArchive: async () => {}, + restoreGeneratedData: async () => {}, + }), + /rerun with --bootstrap/, + ); + + let generatedDataRestores = 0; + const result = await assembleArchives({ + docsRoot: targetRoot, + bootstrap: true, + fetchImpl, + buildArchive: async (_docset, { docsRoot }) => { + const output = resolve(docsRoot, 'dist/v/1.2.3'); + await mkdir(output, { recursive: true }); + await writeFile(resolve(output, 'index.html'), '

Frozen

\n'); + }, + restoreGeneratedData: async () => { generatedDataRestores += 1; }, + }); + assert.equal(result.bootstrapped, 1); + assert.equal(generatedDataRestores, 1); + assert.deepEqual( + await readFile(resolve(targetRoot, 'dist/_archive-bundles/v1.2.3.tar.gz')), + await readFile(resolve(targetRoot, '.archive-bundles/v1.2.3.tar.gz')), + ); +}); + +test('does not fall back after a published bundle fails digest verification', async (t) => { + const { targetRoot } = await fixture(t); + let requests = 0; + await assert.rejects( + assembleArchives({ + docsRoot: targetRoot, + fetchImpl: async () => { + requests += 1; + return new Response('untrusted', { status: 200 }); + }, + restoreGeneratedData: async () => {}, + }), + /does not match lock/, + ); + assert.equal(requests, 1); +}); diff --git a/docs/site/scripts/astro-config.test.mjs b/docs/site/scripts/astro-config.test.mjs index 1b2e43254..7853ad66a 100644 --- a/docs/site/scripts/astro-config.test.mjs +++ b/docs/site/scripts/astro-config.test.mjs @@ -67,3 +67,7 @@ test('unsupported archive flags cannot make current components and config disagr assert.equal(context.isArchivedBuild, false); assert.equal(context.currentDocsetRedirect(currentOnlyPath), `/snapshot${currentOnlyPath}`); }); + +test('archived builds disable platform-dependent Pagefind output', () => { + assert.match(configSource, /pagefind:\s*!isArchivedBuild/); +}); diff --git a/docs/site/scripts/build-archives.mjs b/docs/site/scripts/build-archives.mjs index 99ba8d09a..ca323bab3 100644 --- a/docs/site/scripts/build-archives.mjs +++ b/docs/site/scripts/build-archives.mjs @@ -3,12 +3,12 @@ import { rm } from 'node:fs/promises'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { applyArchiveSeo } from './apply-archive-seo.mjs'; +import { + archiveOutputDirectory, + validateArchiveOutputLocation, +} from './archive-bundle.mjs'; import { loadDocsets } from './docsets.mjs'; -function outDirForDocset(docset) { - return `dist${docset.path.replace(/\/$/, '')}`; -} - async function run(command, args, env) { await new Promise((resolveRun, rejectRun) => { const child = spawn(command, args, { @@ -24,17 +24,29 @@ async function run(command, args, env) { }); } -export async function buildDocsetArchive(docset) { +export async function buildDocsetArchive(docset, { + docsRoot = process.cwd(), +} = {}) { if (docset.status !== 'archived') { throw new Error(`Docset "${docset.id}" is not archived`); } - const env = { ...process.env, DOCS_DOCSET: docset.id, DOCS_BASE: docset.path }; - const outDir = outDirForDocset(docset); + const env = { + ...process.env, + DOCS_DOCSET: docset.id, + DOCS_BASE: docset.path, + TZ: 'UTC', + // Archives are immutable release files, so their bytes cannot depend on + // mutable deployment analytics configuration. + PUBLIC_UMAMI_WEBSITE_ID: '', + PUBLIC_UMAMI_SCRIPT_SRC: '', + PUBLIC_UMAMI_DOMAINS: '', + }; + const outDir = await validateArchiveOutputLocation(docsRoot, docset); await rm(outDir, { recursive: true, force: true }); await run('npm', ['run', 'generate'], env); await run('npx', ['astro', 'check'], env); - await run('npx', ['astro', 'build', '--outDir', outDir], env); + await run('npx', ['astro', 'build', '--outDir', archiveOutputDirectory(docsRoot, docset)], env); await applyArchiveSeo(outDir); console.log(`Built archived docset ${docset.id} at ${outDir}.`); } diff --git a/docs/site/scripts/check-built-links.mjs b/docs/site/scripts/check-built-links.mjs index 9a003263d..ab8b08f92 100644 --- a/docs/site/scripts/check-built-links.mjs +++ b/docs/site/scripts/check-built-links.mjs @@ -7,6 +7,14 @@ const distDir = 'dist'; const attrPattern = /\s(?:href|src)=["']([^"']+)["']/g; const idPattern = /\sid=["']([^"']+)["']/g; +function scopeFromArgs(args) { + if (args.length === 0) return 'all'; + if (args.length === 2 && args[0] === '--scope' && ['all', 'current'].includes(args[1])) { + return args[1]; + } + throw new Error('usage: check-built-links.mjs [--scope current|all]'); +} + async function exists(path) { try { await stat(path); @@ -93,15 +101,21 @@ const errors = []; let checked = 0; const idsByFile = new Map(); const evidencePaths = await currentEvidencePaths(); +const scope = scopeFromArgs(process.argv.slice(2)); +const archivedRootPattern = /^\/v\/[^/]+\//; + +const files = (await htmlFiles(distDir)).filter( + (file) => scope === 'all' || archiveRoot(file) === null, +); -for (const file of await htmlFiles(distDir)) { +for (const file of files) { const html = await readFile(file, 'utf8'); const ids = new Set(); for (const match of html.matchAll(idPattern)) ids.add(match[1]); idsByFile.set(file, ids); } -for (const file of await htmlFiles(distDir)) { +for (const file of files) { const html = await readFile(file, 'utf8'); for (const match of html.matchAll(attrPattern)) { const raw = match[1]; @@ -120,6 +134,7 @@ for (const file of await htmlFiles(distDir)) { const url = resolveInternal(raw, file); if (!url) continue; + if (scope === 'current' && archivedRootPattern.test(splitUrl(url)[0])) continue; checked += 1; const [path, fragment] = splitUrl(url); diff --git a/docs/site/scripts/check-built-links.test.mjs b/docs/site/scripts/check-built-links.test.mjs index b23751cb5..f3a7e4940 100644 --- a/docs/site/scripts/check-built-links.test.mjs +++ b/docs/site/scripts/check-built-links.test.mjs @@ -39,8 +39,8 @@ function fixture(t, archivedHref) { return root; } -function run(root) { - return spawnSync(process.execPath, [checker], { cwd: root, encoding: 'utf8' }); +function run(root, args = []) { + return spawnSync(process.execPath, [checker, ...args], { cwd: root, encoding: 'utf8' }); } test('allows an archived standards page to cite root-relative current evidence', (t) => { @@ -54,3 +54,13 @@ test('keeps rejecting unrelated links that escape an archive', (t) => { assert.equal(result.status, 1); assert.match(result.stderr, /links outside its archive/); }); + +test('current scope defers immutable archive targets to the archive gate', (t) => { + const root = fixture(t, '/explanation/current/'); + write(root, 'dist/index.html', 'Release'); + const current = run(root, ['--scope', 'current']); + const all = run(root); + assert.equal(current.status, 0, current.stderr); + assert.equal(all.status, 1); + assert.match(all.stderr, /links to missing/); +}); diff --git a/docs/site/scripts/check-evidence-links.test.mjs b/docs/site/scripts/check-evidence-links.test.mjs index 099badfe5..21c201f15 100644 --- a/docs/site/scripts/check-evidence-links.test.mjs +++ b/docs/site/scripts/check-evidence-links.test.mjs @@ -227,9 +227,25 @@ test('extracts only policy-owned YAML fields', () => { test('release verification uses the resolved tag target without installing the docs tree', () => { const workflow = readFileSync(resolve(repositoryRoot, '.github/workflows/release.yml'), 'utf8'); + const verifyJob = workflow.slice( + workflow.indexOf(' verify:\n'), + workflow.indexOf(' docs-archive:\n'), + ); assert.match( - workflow, + verifyJob, /npm run check:evidence-links --\s+--source-ref "\$\{\{ steps\.release\.outputs\.tag_target \}\}"/, ); - assert.doesNotMatch(workflow, /npm ci/); + assert.doesNotMatch(verifyJob, /npm ci/); +}); + +test('Pages checks the current machine-readable corpus before inserting archives', () => { + const workflow = readFileSync( + resolve(repositoryRoot, '.github/workflows/docs-pages.yml'), + 'utf8', + ); + const llmsCheck = workflow.indexOf('npm run check:llms:built'); + const archiveAssembly = workflow.indexOf('npm run assemble:archives'); + assert.ok(llmsCheck >= 0); + assert.ok(archiveAssembly > llmsCheck); + assert.equal(workflow.indexOf('npm run check:llms:built', llmsCheck + 1), -1); }); diff --git a/docs/site/scripts/check-seo.mjs b/docs/site/scripts/check-seo.mjs index 269b9df3d..975f8f3a9 100644 --- a/docs/site/scripts/check-seo.mjs +++ b/docs/site/scripts/check-seo.mjs @@ -4,6 +4,14 @@ import { loadDocsets } from './docsets.mjs'; const distDir = 'dist'; +function scopeFromArgs(args) { + if (args.length === 0) return 'all'; + if (args.length === 2 && args[0] === '--scope' && ['all', 'current'].includes(args[1])) { + return args[1]; + } + throw new Error('usage: check-seo.mjs [--scope current|all]'); +} + async function exists(path) { try { await stat(path); @@ -31,6 +39,7 @@ function archiveRootForFile(file, archivedDocsets) { const manifest = await loadDocsets(); const archivedDocsets = manifest.docsets.filter((docset) => docset.status === 'archived'); +const scope = scopeFromArgs(process.argv.slice(2)); const errors = []; let latestChecked = 0; let archivedChecked = 0; @@ -39,21 +48,24 @@ if (!await exists(join(distDir, 'sitemap-index.xml'))) { errors.push('Latest sitemap is missing: dist/sitemap-index.xml'); } -for (const docset of archivedDocsets) { - const archiveDir = join(distDir, docset.path); - const archiveSitemap = join(archiveDir, 'sitemap-index.xml'); - const archiveSitemapPage = join(archiveDir, 'sitemap-0.xml'); - if (await exists(archiveSitemap)) { - errors.push(`Archived docset ${docset.id} must not publish sitemap-index.xml`); - } - if (await exists(archiveSitemapPage)) { - errors.push(`Archived docset ${docset.id} must not publish sitemap-0.xml`); +if (scope === 'all') { + for (const docset of archivedDocsets) { + const archiveDir = join(distDir, docset.path); + const archiveSitemap = join(archiveDir, 'sitemap-index.xml'); + const archiveSitemapPage = join(archiveDir, 'sitemap-0.xml'); + if (await exists(archiveSitemap)) { + errors.push(`Archived docset ${docset.id} must not publish sitemap-index.xml`); + } + if (await exists(archiveSitemapPage)) { + errors.push(`Archived docset ${docset.id} must not publish sitemap-0.xml`); + } } } for (const file of await htmlFiles(distDir)) { const html = await readFile(file, 'utf8'); const isArchived = Boolean(archiveRootForFile(file, archivedDocsets)); + if (scope === 'current' && isArchived) continue; const hasNoindex = //.test(html); const hasSitemapLink = /]*\brel=["']sitemap["'])[^>]*>/i.test(html); @@ -73,7 +85,7 @@ for (const file of await htmlFiles(distDir)) { } } -if (archivedDocsets.length > 0 && archivedChecked === 0) { +if (scope === 'all' && archivedDocsets.length > 0 && archivedChecked === 0) { errors.push('No archived HTML files were checked.'); } diff --git a/docs/site/src/data/archive-lock.yaml b/docs/site/src/data/archive-lock.yaml new file mode 100644 index 000000000..4cbb0118f --- /dev/null +++ b/docs/site/src/data/archive-lock.yaml @@ -0,0 +1,47 @@ +schema_version: registry-docs.archive-lock.v1 +archives: + v0.13.0: + bundle_sha256: acd75c7b5ced5d2d14c21a04312ad21dac654a09fdfff692612553dbdfb8dd59 + tree_sha256: fcf63200a9d4e0d5aa42d819e7107e1d2934ed8c9a973e08c9eb8f3bf14886ae + v0.12.2: + bundle_sha256: 159f46ee4534408992976e2b8e4d3b611674cf8598dfae742adabcbe5ec12253 + tree_sha256: e31135cfc735df46f38c9a79d7f80ce484bac538d88ffc7709c3825745f4362a + v0.12.1: + bundle_sha256: 79d5386b9a93c1789af9469447dd4268a50ba001f8a1d7005096f291a4e4cb4f + tree_sha256: f42bae577b388779498931f49bcf031db5dc5b1dad065e30ccaa278e1f1c8c42 + v0.12.0: + bundle_sha256: cbf67e3b7f1e9b10f9f6226ce4942418270815e85d852ad8a3a040ee4d061e5c + tree_sha256: 50cb42e2bb426d99db511326fa8d68bd10b562fdf50219378b2aeecf39d5b38c + v0.11.0: + bundle_sha256: d260947162aa565c28fb2aa96fa0ab17f3fc09ce4211cb1532ac819bf74b8dc5 + tree_sha256: b4321dcc2e18b7968847a3f7e532d7129ed00d14806f23219ac8f9ebeaef3d78 + v0.10.0: + bundle_sha256: 1da99fd82a7778ade7aa2995b42d8644e179610c5af118b550d0d3e7f1ab833a + tree_sha256: 0002c9c8e8c28fc69a54f6f18039a303e3c50c0f49d149636b279a867ffd4011 + v0.9.0: + bundle_sha256: 5df98897696644385c53372f528a2f967218014cd965968a0bb3c44498742425 + tree_sha256: 67a5e24fa90e55c7111bdc53f9aab8814b89286fee57df7a6460111892f9ea80 + v0.8.4: + bundle_sha256: 36b15c6a11bfd7fffa3fba4700f497310b04fe35a7e47cb73ec453f4e719bee0 + tree_sha256: e41940dd9641330153d10568d487170b1f13290e576b98c0adc66c051f10e86a + v0.8.3: + bundle_sha256: 7ab235ea0b6be4659c35b8393de18d4693172664fbf9ddc8a400980131b6255a + tree_sha256: ccb18537e23b80266e8b3eaaf33621671fd42d97988ebcb35413ab70b89892f4 + v0.8.2: + bundle_sha256: 47eefcd0246df0ed13d94f563033e4e176f3965575b56a510018827d063307f3 + tree_sha256: 88d6e2d2d4bb88afde9e22adc4f2c82274af9f4edd5f8a29d07c170a39bfda86 + v0.8.1: + bundle_sha256: 6a155a7eb2d7e04120c8c1c0e5615934de318a217150a866cd2a906298503b10 + tree_sha256: b7714c5ded3d441ae4706d30e7fa940e6bac591af9a2013216fd29b757544244 + beta-5: + bundle_sha256: e691e39d19379877ce0dc549497a00f892bcc58f8d7d5fbcb82f95779781d6ca + tree_sha256: 010739e3b0f04babdcc1853b5f021debce953ba0dd612fe398b7153867cfc5bc + beta-4: + bundle_sha256: a692527fcea94ef544302a688c93b33be1bf69a9816013750dc6a397b403d69e + tree_sha256: 43423bbfa14dfcf7363b47c43e155151e32d24411768d7f1cefa25b36f050ed7 + beta-3: + bundle_sha256: ca21bb0cf0b40a29a377f54435f10ff548f7e1efd6af18ceeb5981ab48ca886f + tree_sha256: ac44d33367fa1da6ba4392e28ff74e39a593e733b5b4c5a4c7c30135d6e211e5 + beta-2026-06-12: + bundle_sha256: fe79d51e32d0a21eb7e2a36bd8ce3ba689556b95bde795e38457c592f7a3ce8a + tree_sha256: 1efec40da020a14a6b07ddf5c475ea122539f691cc2f2ac13296ef257ed29ec6 diff --git a/release/OPERATIONS.md b/release/OPERATIONS.md index af76f18fa..92506f002 100644 --- a/release/OPERATIONS.md +++ b/release/OPERATIONS.md @@ -70,7 +70,24 @@ The gate is preventive; no prior hosted candidate disk exhaustion is recorded. ## Request A Candidate -Start from a clean local checkout with current `origin/main`. +Before requesting the candidate, build the one new archived docset and append +its bundle and tree digests to the immutable lock: + +```sh +cd docs/site +npm ci +DOCS_DOCSET=vX.Y.Z npm run build:archive +npm run archive:snapshot -- vX.Y.Z --write-lock +npm run generate +npm run check:archive-lock -- --base-ref origin/main +cd ../.. +``` + +Commit the new lock entry with the rest of the release preparation. Existing +entries cannot be changed or removed. The candidate, tag workflow, and Pages +deployment all verify the same bytes. + +Then start from a clean local checkout with current `origin/main`. Resolve the exact protected-main commit and choose an unused release ID. ```sh diff --git a/release/VERIFY.md b/release/VERIFY.md index 24dc63076..599b5adb5 100644 --- a/release/VERIFY.md +++ b/release/VERIFY.md @@ -311,6 +311,47 @@ The provenance subject set covers release artifacts before their generated exact Relay and Notary digest evidence are all independent signed and provenance-covered subjects. +## Verify An Immutable Documentation Archive + +Releases created after the immutable archive contract publish +`registry-docs-${tag}.tar.gz`. The archive digest must match the append-only +`docs/site/src/data/archive-lock.yaml` entry committed at that tag. + +```bash +docs_archive="registry-docs-${tag}.tar.gz" +docs_archive_sbom="${docs_archive}.spdx.json" + +gh release download "${tag}" \ + --repo registrystack/registry-stack \ + --pattern "${docs_archive}" \ + --pattern "${docs_archive}.sig" \ + --pattern "${docs_archive}.pem" \ + --pattern "${docs_archive_sbom}" \ + --pattern "${provenance}" + +expected="$( + gh api "repos/registrystack/registry-stack/contents/docs/site/src/data/archive-lock.yaml?ref=${tag}" \ + --jq .content | base64 --decode | + python3 -c 'import sys,yaml; tag=sys.argv[1]; print(yaml.safe_load(sys.stdin)["archives"][tag]["bundle_sha256"])' "${tag}" +)" +test "$(sha256sum "${docs_archive}" | awk '{print $1}')" = "${expected}" + +cosign verify-blob "${docs_archive}" \ + --signature "${docs_archive}.sig" \ + --certificate "${docs_archive}.pem" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/registrystack/registry-stack/.github/workflows/release.yml@refs/tags/${tag}" + +slsa-verifier verify-artifact "${docs_archive}" \ + --provenance-path "${provenance}" \ + --source-uri github.com/registrystack/registry-stack \ + --source-tag "${tag}" +``` + +The file-subject SBOM and release capsule classify this payload as +`registry-docs-archive`. Pages verifies the same locked bundle and extracted +tree digests before serving it. + ## Install Or Move Registryctl For `v0.9.0` and later, the matching release installer checksum-verifies and @@ -338,7 +379,8 @@ installation. The candidate workflow compiles the release payload before the tag exists. The tag-bound workflow verifies the exact candidate run, attempt, receipt, and bytes, then promotes those bytes without rebuilding product binaries. -It signs binaries, the registryctl image lock, checksums, release file SBOMs, +It signs binaries, the registryctl image lock, the immutable documentation +archive, checksums, release file SBOMs, image-input binary SBOMs, image evidence files, image SBOMs, Grype reports, release capsules, and the candidate receipt before upload. The separate SLSA generator publishes tag-bound provenance for those @@ -363,8 +405,9 @@ For post-tag verification, require the pushed image's Linux amd64 image config and ordered rootfs layers to match the retained candidate proof, then verify the published index's BuildKit provenance separately. -The release capsule summarizes binary asset hashes, the image lock under -`release_files`, file SBOM asset names, image digests, image SBOMs, Grype +The release capsule summarizes binary asset hashes, the image lock and +documentation archive under `release_files`, file SBOM asset names, image +digests, image SBOMs, Grype reports, workflow lineage, and release warnings. It does not carry per-artifact signature or provenance status fields. Verify cosign signatures from sibling `.sig` and `.pem` files, and verify the separately uploaded diff --git a/release/scripts/check-gates-inventory.py b/release/scripts/check-gates-inventory.py index 873a755f0..424d9f014 100644 --- a/release/scripts/check-gates-inventory.py +++ b/release/scripts/check-gates-inventory.py @@ -353,6 +353,19 @@ '--source-ref "${{ steps.release.outputs.tag_target }}"', ), ), + ( + "Immutable documentation archive publication", + ".github/workflows/release.yml", + ( + "docs-archive:\n name: Build immutable release docs archive", + "name: Build and verify the tag-bound docs bundle", + "npm run build:archive", + "--verify-lock", + "name: Download unprivileged docs archive", + "does not match immutable lock", + "--require-registry-docs-archive", + ), + ), ( "Exact promotion run attempt and receipt binding", ".github/workflows/release.yml", @@ -693,6 +706,12 @@ ) ORDERED_RELEASE_SECURITY_GATES: tuple[tuple[str, str, str, str], ...] = ( + ( + "Immutable docs archive before first public image write", + ".github/workflows/release.yml", + "name: Build and verify the tag-bound docs bundle", + "name: Promote provenance-bearing candidate indexes without rewriting", + ), ( "Promotion binding parsed before candidate verification", ".github/workflows/release.yml", diff --git a/release/scripts/registry-release b/release/scripts/registry-release index 4b99469d7..dc97810c1 100755 --- a/release/scripts/registry-release +++ b/release/scripts/registry-release @@ -25,6 +25,7 @@ import release_candidate ROOT = Path(__file__).resolve().parents[2] RELEASE_MANIFEST_DIR = ROOT / "release" / "manifests" DOCSETS_PATH = ROOT / "docs" / "site" / "src" / "data" / "docsets.yaml" +ARCHIVE_LOCK_SCHEMA = "registry-docs.archive-lock.v1" HEX40 = re.compile(r"^[0-9a-f]{40}$") SHA256_HEX = re.compile(r"^[0-9a-f]{64}$") SEMVER_TAG = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") @@ -445,10 +446,17 @@ def validate(manifest_path: Path) -> int: return 0 -def validate_docsets(manifest_dir: Path, docsets_path: Path) -> int: +def validate_docsets( + manifest_dir: Path, + docsets_path: Path, + archive_lock_path: Path | None = None, +) -> int: errors: list[str] = [] + if archive_lock_path is None: + archive_lock_path = docsets_path.with_name("archive-lock.yaml") try: docsets_data = load_yaml(docsets_path) + archive_lock = load_yaml(archive_lock_path) manifests = [ load_yaml(path) for path in sorted(manifest_dir.glob("registry-stack-*.yaml")) @@ -462,6 +470,45 @@ def validate_docsets(manifest_dir: Path, docsets_path: Path) -> int: print("error: docsets.yaml docsets must be a list", file=sys.stderr) return 1 + archived_ids = sorted( + str(docset.get("id", "")) + for docset in docsets + if isinstance(docset, dict) and docset.get("status") == "archived" + ) + if not isinstance(archive_lock, dict): + errors.append("archive-lock.yaml must contain an object") + archive_entries: dict = {} + else: + if archive_lock.get("schema_version") != ARCHIVE_LOCK_SCHEMA: + errors.append( + f"archive-lock.yaml schema_version must be {ARCHIVE_LOCK_SCHEMA}" + ) + candidate_entries = archive_lock.get("archives") + if not isinstance(candidate_entries, dict): + errors.append("archive-lock.yaml archives must be an object") + archive_entries = {} + else: + archive_entries = candidate_entries + locked_ids = sorted(str(docset_id) for docset_id in archive_entries) + for docset_id in sorted(set(archived_ids) - set(locked_ids)): + errors.append(f"archive-lock.yaml is missing archived docset {docset_id}") + for docset_id in sorted(set(locked_ids) - set(archived_ids)): + errors.append(f"archive-lock.yaml contains non-archived docset {docset_id}") + for docset_id, entry in sorted(archive_entries.items()): + if not isinstance(entry, dict): + errors.append(f"archive-lock.yaml {docset_id} entry must be an object") + continue + unknown = sorted(set(entry) - {"bundle_sha256", "tree_sha256"}) + if unknown: + errors.append( + f"archive-lock.yaml {docset_id} has unknown field {unknown[0]}" + ) + for field in ("bundle_sha256", "tree_sha256"): + if SHA256_HEX.fullmatch(str(entry.get(field, ""))) is None: + errors.append( + f"archive-lock.yaml {docset_id}.{field} must be 64 lowercase hex characters" + ) + manifests_by_tag: dict[str, dict] = {} for manifest in manifests: if not isinstance(manifest, dict) or not isinstance(manifest.get("stack"), dict): @@ -890,6 +937,7 @@ def render_capsule( repo: Path, default_branch: str, require_registryctl_image_lock: bool = False, + require_registry_docs_archive: bool = False, ) -> int: manifest = load_yaml(manifest_path) if not isinstance(manifest, dict): @@ -904,6 +952,7 @@ def render_capsule( binaries = [] release_files = [] expected_image_lock_name = f"registryctl-v{version}-image-lock.json" + expected_docs_archive_name = f"registry-docs-v{version}.tar.gz" for path in sorted(binary_dir.iterdir()) if binary_dir.exists() else []: if not path.is_file() or path.name == "SHA256SUMS": continue @@ -911,6 +960,10 @@ def render_capsule( raise ValueError( f"unexpected registryctl image lock filename {path.name}; expected {expected_image_lock_name}" ) + if path.name.startswith("registry-docs-") and path.name != expected_docs_archive_name: + raise ValueError( + f"unexpected registry docs archive filename {path.name}; expected {expected_docs_archive_name}" + ) file_sha256 = verified_binary_sha256(path, checksums) sbom = binary_sbom_dir / f"{path.name}.spdx.json" if not sbom.is_file(): @@ -932,13 +985,28 @@ def render_capsule( if path.name == expected_image_lock_name: file_evidence["kind"] = "registryctl-release-image-lock" release_files.append(file_evidence) + elif path.name == expected_docs_archive_name: + file_evidence["kind"] = "registry-docs-archive" + release_files.append(file_evidence) else: binaries.append(file_evidence) - if require_registryctl_image_lock and len(release_files) != 1: + image_locks = [ + entry for entry in release_files + if entry["kind"] == "registryctl-release-image-lock" + ] + docs_archives = [ + entry for entry in release_files + if entry["kind"] == "registry-docs-archive" + ] + if require_registryctl_image_lock and len(image_locks) != 1: raise ValueError( f"capsule requires exactly one registryctl release image lock named {expected_image_lock_name}" ) + if require_registry_docs_archive and len(docs_archives) != 1: + raise ValueError( + f"capsule requires exactly one registry docs archive named {expected_docs_archive_name}" + ) images = [] for digest_path in sorted(image_evidence_dir.rglob("*.digest")): @@ -993,6 +1061,8 @@ def render_capsule( if errors: raise ValueError("; ".join(errors)) for release_file in release_files: + if release_file["kind"] != "registryctl-release-image-lock": + continue validate_registryctl_image_lock_document( load_json(Path(release_file["path"])), version=version, @@ -2719,6 +2789,7 @@ def main() -> int: "--manifest-dir", type=Path, default=RELEASE_MANIFEST_DIR ) docsets_parser.add_argument("--docsets", type=Path, default=DOCSETS_PATH) + docsets_parser.add_argument("--archive-lock", type=Path) source_parser = subparsers.add_parser("validate-source") source_parser.add_argument("manifest", type=Path) source_parser.add_argument("--tag", required=True) @@ -2768,6 +2839,7 @@ def main() -> int: capsule_parser.add_argument("--repo", type=Path, default=ROOT) capsule_parser.add_argument("--default-branch", default="origin/main") capsule_parser.add_argument("--require-registryctl-image-lock", action="store_true") + capsule_parser.add_argument("--require-registry-docs-archive", action="store_true") prepare_parser = subparsers.add_parser("prepare") prepare_parser.add_argument("--version", required=True) prepare_parser.add_argument("--release-id", required=True) @@ -2806,7 +2878,7 @@ def main() -> int: if args.command == "validate": return validate(args.manifest) if args.command == "validate-docsets": - return validate_docsets(args.manifest_dir, args.docsets) + return validate_docsets(args.manifest_dir, args.docsets, args.archive_lock) if args.command == "validate-source": return validate_source(args.manifest, args.tag, args.repo, args.default_branch) if args.command == "audit": @@ -2846,6 +2918,7 @@ def main() -> int: args.repo, args.default_branch, args.require_registryctl_image_lock, + args.require_registry_docs_archive, ) if args.command == "prepare": return run_release_plan( diff --git a/release/scripts/test_registry_release.py b/release/scripts/test_registry_release.py index b0dc73dc5..ea5092f2e 100755 --- a/release/scripts/test_registry_release.py +++ b/release/scripts/test_registry_release.py @@ -68,10 +68,10 @@ def test_required_ci_contexts_wrap_path_gated_jobs(self) -> None: required_job = jobs[required_job_id] self.assertEqual(context_name, required_job["name"]) - self.assertEqual( - {"changes", check_job_id}, - set(required_job["needs"]), - ) + expected_needs = {"changes", check_job_id} + if check_job_id == "docs": + expected_needs.add("docs-archives") + self.assertEqual(expected_needs, set(required_job["needs"])) self.assertEqual("${{ always() }}", required_job["if"]) self.assertEqual(1, len(required_job["steps"])) step = required_job["steps"][0] @@ -784,6 +784,37 @@ def test_release_workflow_never_replaces_published_assets(self) -> None: self.assertIn("--verify-tag", step) self.assertIn("GitHub Release is no longer absent", step) + def test_release_workflow_builds_docs_without_publish_permissions(self) -> None: + workflow = yaml.safe_load( + (ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") + ) + docs_job = workflow["jobs"]["docs-archive"] + publisher = workflow["jobs"]["github-release"] + self.assertEqual(["verify"], [docs_job["needs"]]) + self.assertEqual({"contents": "read"}, docs_job["permissions"]) + checkout = next( + step + for step in docs_job["steps"] + if step.get("name", "").startswith("Checkout exact tag target") + ) + self.assertFalse(checkout["with"]["persist-credentials"]) + docs_script = "\n".join( + step.get("run", "") + for step in docs_job["steps"] + if isinstance(step, dict) + ) + self.assertIn("npm run build:archive", docs_script) + self.assertIn("--verify-lock", docs_script) + self.assertIn("docs-archive", publisher["needs"]) + self.assertIn("docs-archive", workflow["jobs"]["publish-images"]["needs"]) + publish_script = "\n".join( + step.get("run", "") + for step in publisher["steps"] + if isinstance(step, dict) + ) + self.assertIn("does not match immutable lock", publish_script) + self.assertIn("--require-registry-docs-archive", publish_script) + def test_candidate_receipt_checks_its_in_progress_run_identity(self) -> None: workflow = (ROOT / ".github/workflows/release-candidate.yml").read_text( encoding="utf-8" @@ -1041,6 +1072,34 @@ def test_validate_docsets_rejects_missing_release_manifest(self) -> None: self.assertNotEqual(0, result.returncode) self.assertIn("has no release manifest", result.stderr) + def test_validate_docsets_rejects_missing_archive_lock_entry(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest_dir, docsets = write_docset_fixture(root) + archive_lock = root / "archive-lock.yaml" + archive_lock.write_text( + yaml.safe_dump( + { + "schema_version": "registry-docs.archive-lock.v1", + "archives": {}, + } + ), + encoding="utf-8", + ) + + result = run_tool( + "validate-docsets", + "--manifest-dir", + str(manifest_dir), + "--docsets", + str(docsets), + "--archive-lock", + str(archive_lock), + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("missing archived docset v0.8.0", result.stderr) + def test_audit_import_map(self) -> None: result = run_tool("audit", "release/manifests/import-map-2026-06-24.yaml") self.assertEqual(0, result.returncode, result.stderr) @@ -1482,6 +1541,69 @@ def test_render_capsule_required_image_lock_fails_when_omitted(self) -> None: "requires exactly one registryctl release image lock", result.stderr ) + def test_render_capsule_classifies_required_docs_archive_as_release_file( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_ref = init_release_repo(root) + manifest = write_manifest(root, source_ref=source_ref) + binary_dir = write_binary_fixture(root) + docs_archive = binary_dir / "registry-docs-v0.8.0.tar.gz" + docs_archive.write_bytes(b"immutable docs bundle\n") + checksums = "".join( + subprocess.check_output( + ["sha256sum", path.name], + cwd=binary_dir, + text=True, + ) + for path in sorted(binary_dir.iterdir()) + if path.is_file() and path.name != "SHA256SUMS" + ) + (binary_dir / "SHA256SUMS").write_text(checksums, encoding="utf-8") + binary_sbom_dir = write_binary_sbom_fixture(root, binary_dir) + image_dir = write_image_fixture(root) + output_json = root / "capsule.json" + + result = render_capsule( + manifest, + binary_dir, + image_dir, + output_json, + root / "capsule.md", + root, + binary_sbom_dir=binary_sbom_dir, + require_registry_docs_archive=True, + ) + evidence = json.loads(output_json.read_text(encoding="utf-8")) + + self.assertEqual(0, result.returncode, result.stderr) + docs_files = [ + item + for item in evidence["release_files"] + if item["kind"] == "registry-docs-archive" + ] + self.assertEqual(1, len(docs_files)) + self.assertEqual("registry-docs-v0.8.0.tar.gz", docs_files[0]["name"]) + + def test_render_capsule_required_docs_archive_fails_when_omitted(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_ref = init_release_repo(root) + manifest = write_manifest(root, source_ref=source_ref) + result = render_capsule( + manifest, + write_binary_fixture(root), + write_image_fixture(root), + root / "capsule.json", + root / "capsule.md", + root, + require_registry_docs_archive=True, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("requires exactly one registry docs archive", result.stderr) + def test_render_capsule_includes_cross_platform_binaries(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -2214,6 +2336,7 @@ def write_docset_fixture(root: Path) -> tuple[Path, Path]: "docsets": [ { "id": "v0.8.0", + "status": "archived", "source": "registry-stack-v0.8.0", "products": { "registry-stack": { @@ -2231,6 +2354,20 @@ def write_docset_fixture(root: Path) -> tuple[Path, Path]: ), encoding="utf-8", ) + (root / "archive-lock.yaml").write_text( + yaml.safe_dump( + { + "schema_version": "registry-docs.archive-lock.v1", + "archives": { + "v0.8.0": { + "bundle_sha256": "a" * 64, + "tree_sha256": "b" * 64, + } + }, + } + ), + encoding="utf-8", + ) return manifest_dir, docsets @@ -2446,6 +2583,7 @@ def render_capsule( *, binary_sbom_dir: Path | None = None, require_registryctl_image_lock: bool = False, + require_registry_docs_archive: bool = False, ) -> subprocess.CompletedProcess[str]: if binary_sbom_dir is None: binary_sbom_dir = write_binary_sbom_fixture(repo, binary_dir) @@ -2477,6 +2615,8 @@ def render_capsule( ] if require_registryctl_image_lock: args.append("--require-registryctl-image-lock") + if require_registry_docs_archive: + args.append("--require-registry-docs-archive") return run_tool(*args) From a8749c1e36abdfb297d2cdc84aebc05c4717380e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 26 Jul 2026 13:58:03 +0700 Subject: [PATCH 2/6] docs: snapshot archive files safely Signed-off-by: Jeremi Joslin --- docs/site/scripts/archive-bundle.mjs | 117 +++++++++++++++++++-------- 1 file changed, 83 insertions(+), 34 deletions(-) diff --git a/docs/site/scripts/archive-bundle.mjs b/docs/site/scripts/archive-bundle.mjs index 82b130623..fc46a4a8a 100644 --- a/docs/site/scripts/archive-bundle.mjs +++ b/docs/site/scripts/archive-bundle.mjs @@ -1,10 +1,11 @@ import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; import { cp, lstat, mkdir, mkdtemp, - readFile, + open, readdir, rm, writeFile, @@ -44,8 +45,46 @@ export function sha256(value) { return createHash('sha256').update(value).digest('hex'); } +async function readRegularFile(path, label, { + maximumBytes = Number.POSITIVE_INFINITY, +} = {}) { + let handle; + try { + handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === 'ELOOP') { + throw new Error(`${label} must be a regular file, not a symlink: ${path}`); + } + throw error; + } + try { + const before = await handle.stat(); + if (!before.isFile()) { + throw new Error(`${label} must be a regular file: ${path}`); + } + if (before.size > maximumBytes) { + throw new Error(`${label} must be no larger than ${maximumBytes} bytes`); + } + const contents = await handle.readFile(); + const after = await handle.stat(); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) { + throw new Error(`${label} changed while it was being read: ${path}`); + } + return { contents, info: after }; + } finally { + await handle.close(); + } +} + export async function fileDigest(path) { - return sha256(await readFile(path)); + const { contents } = await readRegularFile(path, 'digest input'); + return sha256(contents); } function isWithin(parent, child) { @@ -158,10 +197,10 @@ export async function treeDigest(root) { const { files } = await collectTree(root); const hash = createHash('sha256'); for (const path of files) { - const info = await lstat(path); + const { contents, info } = await readRegularFile(path, 'archive tree entry'); const rel = relative(root, path).replaceAll(sep, '/'); hash.update(`${rel}\0${info.mode & 0o111 ? 'x' : '-'}\0`); - hash.update(await readFile(path)); + hash.update(contents); hash.update('\0'); } return hash.digest('hex'); @@ -207,21 +246,25 @@ export async function createArchiveBundle({ } = {}) { const output = await validateArchiveOutputLocation(docsRoot, docset); await requireRealDirectory(output, `archive output for ${docset.id}`); - const outputDigest = await treeDigest(output); - const metadata = archiveMetadata(docset, outputDigest); const staging = await mkdtemp(resolve(tmpdir(), 'registry-docs-archive-bundle-')); + let outputDigest; + let metadata; try { - await writeFile( - resolve(staging, 'metadata.json'), - `${JSON.stringify(metadata, null, 2)}\n`, - { mode: 0o644 }, - ); await cp(output, resolve(staging, 'site'), { recursive: true, + dereference: false, force: false, errorOnExist: true, preserveTimestamps: false, + verbatimSymlinks: true, }); + outputDigest = await treeDigest(resolve(staging, 'site')); + metadata = archiveMetadata(docset, outputDigest); + await writeFile( + resolve(staging, 'metadata.json'), + `${JSON.stringify(metadata, null, 2)}\n`, + { mode: 0o644 }, + ); const entries = await stagedTarEntries(staging); await mkdir(dirname(bundlePath), { recursive: true }); await rm(bundlePath, { force: true }); @@ -311,30 +354,29 @@ export async function inspectArchiveBundle({ if (!sha256Pattern.test(expectedTreeSha256)) { throw new Error(`archive bundle ${docset.id} has no valid locked tree digest`); } - const bundleInfo = await lstat(bundlePath); - if ( - bundleInfo.isSymbolicLink() || - !bundleInfo.isFile() || - bundleInfo.size > maximumArchiveBundleBytes - ) { - throw new Error( - `archive bundle ${docset.id} must be a regular file no larger than ${maximumArchiveBundleBytes} bytes`, - ); - } - const actualBundleDigest = await fileDigest(bundlePath); + const { contents: bundleContents } = await readRegularFile( + bundlePath, + `archive bundle ${docset.id}`, + { maximumBytes: maximumArchiveBundleBytes }, + ); + const actualBundleDigest = sha256(bundleContents); if (actualBundleDigest !== expectedBundleSha256) { throw new Error( `archive bundle ${docset.id} digest ${actualBundleDigest} does not match lock ${expectedBundleSha256}`, ); } - const extraction = await mkdtemp(resolve(tmpdir(), 'registry-docs-archive-extract-')); + const temporary = await mkdtemp(resolve(tmpdir(), 'registry-docs-archive-extract-')); + const extraction = resolve(temporary, 'extracted'); + const bundleSnapshot = resolve(temporary, 'bundle.tar.gz'); try { + await mkdir(extraction); + await writeFile(bundleSnapshot, bundleContents, { mode: 0o600 }); let entryCount = 0; let extractedBytes = 0; await Promise.resolve(tar.extract({ cwd: extraction, - file: bundlePath, + file: bundleSnapshot, filter(path, entry) { entryCount += 1; extractedBytes += Number(entry.size ?? 0); @@ -357,12 +399,12 @@ export async function inspectArchiveBundle({ if (canonicalJson(rootEntries) !== canonicalJson(['metadata.json', 'site'])) { throw new Error(`archive bundle ${docset.id} must contain only metadata.json and site/`); } - const metadataInfo = await lstat(resolve(extraction, 'metadata.json')); - if (metadataInfo.isSymbolicLink() || !metadataInfo.isFile()) { - throw new Error(`archive bundle ${docset.id} metadata must be a regular file`); - } + const { contents: metadataContents } = await readRegularFile( + resolve(extraction, 'metadata.json'), + `archive bundle ${docset.id} metadata`, + ); await requireRealDirectory(resolve(extraction, 'site'), `archive bundle ${docset.id} site`); - const metadata = JSON.parse(await readFile(resolve(extraction, 'metadata.json'), 'utf8')); + const metadata = JSON.parse(metadataContents.toString('utf8')); assertMetadataMatchesDocset(metadata, docset); const actualTreeDigest = await treeDigest(resolve(extraction, 'site')); if ( @@ -373,9 +415,16 @@ export async function inspectArchiveBundle({ `archive bundle ${docset.id} tree digest ${actualTreeDigest} does not match its lock`, ); } - return { extraction, metadata, bundle_sha256: actualBundleDigest, tree_sha256: actualTreeDigest }; + return { + bundle_sha256: actualBundleDigest, + bundle_snapshot: bundleSnapshot, + extraction, + metadata, + temporary, + tree_sha256: actualTreeDigest, + }; } catch (error) { - await rm(extraction, { recursive: true, force: true }); + await rm(temporary, { recursive: true, force: true }); throw error; } } @@ -405,9 +454,9 @@ export async function restoreArchiveBundle({ preserveTimestamps: false, }); const publicBundle = publicArchiveBundlePath(docsRoot, docset); - if (publishBundle && resolve(bundlePath) !== resolve(publicBundle)) { + if (publishBundle) { await mkdir(dirname(publicBundle), { recursive: true }); - await cp(bundlePath, publicBundle, { force: true }); + await cp(inspected.bundle_snapshot, publicBundle, { force: true }); } return { ...inspected, @@ -415,6 +464,6 @@ export async function restoreArchiveBundle({ public_bundle: publishBundle ? publicBundle : null, }; } finally { - await rm(inspected.extraction, { recursive: true, force: true }); + await rm(inspected.temporary, { recursive: true, force: true }); } } From bf01a117cf500da49dedb0d430eafd9348738017 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 26 Jul 2026 18:14:38 +0700 Subject: [PATCH 3/6] ci: verify archive-dependent scripts Signed-off-by: Jeremi Joslin --- .github/scripts/ci_changes.py | 3 +++ .github/scripts/test_ci_changes.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 93efb108d..74fb9c65b 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -279,6 +279,9 @@ def classify( "docs/site/scripts/assemble-archives.mjs", "docs/site/scripts/build-archive.mjs", "docs/site/scripts/build-archives.mjs", + "docs/site/scripts/check-built-links.mjs", + "docs/site/scripts/check-seo.mjs", + "docs/site/scripts/docsets.mjs", "docs/site/src/data/archive-lock.yaml", "docs/site/src/data/docsets.yaml", "docs/site/src/data/repo-docs.yaml", diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index f1e9c4c42..06a8a2339 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -90,6 +90,16 @@ def test_archive_content_is_immutable_during_routine_docs_changes(self) -> None: self.assertTrue(archive_lock["docs_archives"]) self.assertTrue(archive_assembler["docs_archives"]) + def test_archive_dependent_scripts_select_archive_verification(self) -> None: + for path in ( + "docs/site/scripts/check-built-links.mjs", + "docs/site/scripts/check-seo.mjs", + "docs/site/scripts/docsets.mjs", + ): + with self.subTest(path=path): + outputs = classify(self.workspace, (path,)) + self.assertTrue(outputs["docs_archives"]) + def test_run_all_does_not_rebuild_immutable_archives_without_changed_paths(self) -> None: outputs = classify(self.workspace, (), run_all=True) self.assertTrue(outputs["docs"]) From 425aee58dd5c0532740239bfe8df82ec3d43b8cd Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 26 Jul 2026 18:39:21 +0700 Subject: [PATCH 4/6] docs: refresh archive cutover baseline Signed-off-by: Jeremi Joslin --- docs/site/src/data/archive-lock.yaml | 60 ++++++++++++++-------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/docs/site/src/data/archive-lock.yaml b/docs/site/src/data/archive-lock.yaml index 4cbb0118f..7b58dca9e 100644 --- a/docs/site/src/data/archive-lock.yaml +++ b/docs/site/src/data/archive-lock.yaml @@ -1,47 +1,47 @@ schema_version: registry-docs.archive-lock.v1 archives: v0.13.0: - bundle_sha256: acd75c7b5ced5d2d14c21a04312ad21dac654a09fdfff692612553dbdfb8dd59 - tree_sha256: fcf63200a9d4e0d5aa42d819e7107e1d2934ed8c9a973e08c9eb8f3bf14886ae + bundle_sha256: 6d676d1b1323aea9cdc4562df7a791bee40518dc5ad3a93b6c872c4768db24eb + tree_sha256: 3ab749ac997d832d29f53c7f0705c12b29062607c396c340eb82f39416781ce2 v0.12.2: - bundle_sha256: 159f46ee4534408992976e2b8e4d3b611674cf8598dfae742adabcbe5ec12253 - tree_sha256: e31135cfc735df46f38c9a79d7f80ce484bac538d88ffc7709c3825745f4362a + bundle_sha256: 0a1ae3a7bcdc4d679e04b2562230a0f2c7437b7914c4610e0bb4a4f67192ab40 + tree_sha256: 6b3066d2c76fc862d46c458ff863c6e5c50e3fb9a3f9e0b1b3d96f90d8b031d2 v0.12.1: - bundle_sha256: 79d5386b9a93c1789af9469447dd4268a50ba001f8a1d7005096f291a4e4cb4f - tree_sha256: f42bae577b388779498931f49bcf031db5dc5b1dad065e30ccaa278e1f1c8c42 + bundle_sha256: 156ced805a2fd9f677e802e827709af6457f2efa04852d12a23f34a7b8e5eac6 + tree_sha256: 908e4fd65478a27ea45b4bd43374bb65d38fa672a14389cf73f05bfe403c3713 v0.12.0: - bundle_sha256: cbf67e3b7f1e9b10f9f6226ce4942418270815e85d852ad8a3a040ee4d061e5c - tree_sha256: 50cb42e2bb426d99db511326fa8d68bd10b562fdf50219378b2aeecf39d5b38c + bundle_sha256: 192599e03bf2357546feade0d465d4d3bfe1c9b061b336c4fb55d18a93f3c266 + tree_sha256: a40825e7dc0bb491022acbf1b65244c3980370b94d3c6c307d6d05d3aa667f1a v0.11.0: - bundle_sha256: d260947162aa565c28fb2aa96fa0ab17f3fc09ce4211cb1532ac819bf74b8dc5 - tree_sha256: b4321dcc2e18b7968847a3f7e532d7129ed00d14806f23219ac8f9ebeaef3d78 + bundle_sha256: 04c065709cd9ddc3bf141e35fe432ba3277cbc3fc7abbbae83fc83891ca9fee0 + tree_sha256: b373612d3c1cc0e60073c285cec0fb28ac4a53d13605ba97450bed6b330c1908 v0.10.0: - bundle_sha256: 1da99fd82a7778ade7aa2995b42d8644e179610c5af118b550d0d3e7f1ab833a - tree_sha256: 0002c9c8e8c28fc69a54f6f18039a303e3c50c0f49d149636b279a867ffd4011 + bundle_sha256: ed42be854624b84afdb2c292708135316f7e051b5b363dc9d3c0840c0d727075 + tree_sha256: 0a5ed55179d20669de596a1c344ff4091458f0417ab4e643b8cceaa817dfe4ff v0.9.0: - bundle_sha256: 5df98897696644385c53372f528a2f967218014cd965968a0bb3c44498742425 - tree_sha256: 67a5e24fa90e55c7111bdc53f9aab8814b89286fee57df7a6460111892f9ea80 + bundle_sha256: 8faf937fcbb4a2063881df0d5ffe120c9e0e496a7d967df3c54210eb831e629e + tree_sha256: 9d5950128bc48ef41d5a4fb93c3d4f3496a54ff97fd8b5c69ff3927ed5c0becd v0.8.4: - bundle_sha256: 36b15c6a11bfd7fffa3fba4700f497310b04fe35a7e47cb73ec453f4e719bee0 - tree_sha256: e41940dd9641330153d10568d487170b1f13290e576b98c0adc66c051f10e86a + bundle_sha256: 58c1d41bab071c949b4f93b41890a7d7eb6bd570b3934a95fb791753511f450c + tree_sha256: 18817c1ccf2cf313aac8143c53938522cfe27621724e6beaa0e43eaa663fc2e9 v0.8.3: - bundle_sha256: 7ab235ea0b6be4659c35b8393de18d4693172664fbf9ddc8a400980131b6255a - tree_sha256: ccb18537e23b80266e8b3eaaf33621671fd42d97988ebcb35413ab70b89892f4 + bundle_sha256: b2eccf8fb49ac08ad97f8451bee0300dfeada2ceba2c920cea4650706860fd04 + tree_sha256: 527ea79ac9084ebea2dfe614e0f8481505a733e512ea1d8e890b624f47bc50c6 v0.8.2: - bundle_sha256: 47eefcd0246df0ed13d94f563033e4e176f3965575b56a510018827d063307f3 - tree_sha256: 88d6e2d2d4bb88afde9e22adc4f2c82274af9f4edd5f8a29d07c170a39bfda86 + bundle_sha256: d23ea041163515d618e53abe0696571d49e1bf01ff0c93d6238a5a5b2f7f3eff + tree_sha256: 784ec14b486359307bf26ed280649d23130c21391e4c5bf457802d407f6e7dfc v0.8.1: - bundle_sha256: 6a155a7eb2d7e04120c8c1c0e5615934de318a217150a866cd2a906298503b10 - tree_sha256: b7714c5ded3d441ae4706d30e7fa940e6bac591af9a2013216fd29b757544244 + bundle_sha256: 291efa082d49ae597bdbd981886b952b8770f8919b5f3afab52f990271550a6b + tree_sha256: 49cd144f9a622bb1522bc784ae8ae0d302516e301e79d143a5cec7b66a14b029 beta-5: - bundle_sha256: e691e39d19379877ce0dc549497a00f892bcc58f8d7d5fbcb82f95779781d6ca - tree_sha256: 010739e3b0f04babdcc1853b5f021debce953ba0dd612fe398b7153867cfc5bc + bundle_sha256: 2cf54f1224d8b13e670ad639edb6671edbae0e19e6f9a04fe6f2bb85c90c6612 + tree_sha256: 42ee3865b12a25dae5b539b4308d65ff0b934b4c9e937b37b76ea20360f204d0 beta-4: - bundle_sha256: a692527fcea94ef544302a688c93b33be1bf69a9816013750dc6a397b403d69e - tree_sha256: 43423bbfa14dfcf7363b47c43e155151e32d24411768d7f1cefa25b36f050ed7 + bundle_sha256: 1d4de05448d48556f64cede658944e93f451c02dfd3d4ebd7a05ac9b9dbbe91e + tree_sha256: 13a869bdae2bfef35e90dcc69671bc811c48ec1ea93ca72730d725f27b3667fe beta-3: - bundle_sha256: ca21bb0cf0b40a29a377f54435f10ff548f7e1efd6af18ceeb5981ab48ca886f - tree_sha256: ac44d33367fa1da6ba4392e28ff74e39a593e733b5b4c5a4c7c30135d6e211e5 + bundle_sha256: 6dfd12046eb3e2820f08bfa84c6834306678199e539451d0b0d27c662a7a7ba0 + tree_sha256: a8b1c59c047c696c2aa58334dea55fc3d65df8daefbed7ef238bc7a730d1b5f4 beta-2026-06-12: - bundle_sha256: fe79d51e32d0a21eb7e2a36bd8ce3ba689556b95bde795e38457c592f7a3ce8a - tree_sha256: 1efec40da020a14a6b07ddf5c475ea122539f691cc2f2ac13296ef257ed29ec6 + bundle_sha256: feedc7da32364f484be472c2d9741de03395e17057a3ff8416a0aaf967440c67 + tree_sha256: bcd573eab9c2ab801fde6637b8c8fb33cedd79941a1e47d4296df3b5fae76854 From 4ca2921b0a2ef94542bdc061b332d2db510acfcc Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 26 Jul 2026 18:40:41 +0700 Subject: [PATCH 5/6] ci: verify archive classifier changes Signed-off-by: Jeremi Joslin --- .github/scripts/ci_changes.py | 1 + .github/scripts/test_ci_changes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 74fb9c65b..3ede1d963 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -267,6 +267,7 @@ def classify( docs_archives = any( path in { + ".github/scripts/ci_changes.py", ".github/workflows/ci.yml", ".github/workflows/docs-pages.yml", ".github/workflows/release.yml", diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index 06a8a2339..8831f1ced 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -92,6 +92,7 @@ def test_archive_content_is_immutable_during_routine_docs_changes(self) -> None: def test_archive_dependent_scripts_select_archive_verification(self) -> None: for path in ( + ".github/scripts/ci_changes.py", "docs/site/scripts/check-built-links.mjs", "docs/site/scripts/check-seo.mjs", "docs/site/scripts/docsets.mjs", From de10c1b01bc1ba8916d779847146eba793039d08 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 26 Jul 2026 18:54:48 +0700 Subject: [PATCH 6/6] docs: validate archive link targets Signed-off-by: Jeremi Joslin --- docs/site/scripts/check-built-links.mjs | 17 +++++++++- docs/site/scripts/check-built-links.test.mjs | 35 ++++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/site/scripts/check-built-links.mjs b/docs/site/scripts/check-built-links.mjs index ab8b08f92..82f038676 100644 --- a/docs/site/scripts/check-built-links.mjs +++ b/docs/site/scripts/check-built-links.mjs @@ -2,6 +2,7 @@ import { readdir, readFile, stat } from 'node:fs/promises'; import { dirname, join, normalize, relative } from 'node:path'; import { extractEvidenceUrlsFromYaml } from './check-evidence-links.mjs'; +import { loadDocsets } from './docsets.mjs'; const distDir = 'dist'; const attrPattern = /\s(?:href|src)=["']([^"']+)["']/g; @@ -103,6 +104,13 @@ const idsByFile = new Map(); const evidencePaths = await currentEvidencePaths(); const scope = scopeFromArgs(process.argv.slice(2)); const archivedRootPattern = /^\/v\/[^/]+\//; +const archivedRoots = scope === 'current' + ? new Set( + (await loadDocsets()).docsets + .filter((docset) => docset.status === 'archived') + .map((docset) => docset.path), + ) + : new Set(); const files = (await htmlFiles(distDir)).filter( (file) => scope === 'all' || archiveRoot(file) === null, @@ -134,7 +142,14 @@ for (const file of files) { const url = resolveInternal(raw, file); if (!url) continue; - if (scope === 'current' && archivedRootPattern.test(splitUrl(url)[0])) continue; + const archivedRoot = splitUrl(url)[0].match(archivedRootPattern)?.[0]; + if (scope === 'current' && archivedRoot) { + checked += 1; + if (!archivedRoots.has(archivedRoot)) { + errors.push(`${relative('.', file)} links to unknown archive ${raw}`); + } + continue; + } checked += 1; const [path, fragment] = splitUrl(url); diff --git a/docs/site/scripts/check-built-links.test.mjs b/docs/site/scripts/check-built-links.test.mjs index f3a7e4940..b265f746f 100644 --- a/docs/site/scripts/check-built-links.test.mjs +++ b/docs/site/scripts/check-built-links.test.mjs @@ -26,6 +26,29 @@ function fixture(t, archivedHref) { `Evidence`, ); write(root, 'src/data/contracts.yaml', '[]\n'); + write( + root, + 'src/data/docsets.yaml', + `current: latest +docsets: + - id: latest + label: Latest + path: / + status: current + source: main + published_at: 2026-07-26 + description: Current docs. + products: {} + - id: v1 + label: Version 1 + path: /v/v1/ + status: archived + source: v1 + published_at: 2026-07-25 + description: Archived docs. + products: {} +`, + ); write( root, 'src/data/standards.yaml', @@ -55,12 +78,20 @@ test('keeps rejecting unrelated links that escape an archive', (t) => { assert.match(result.stderr, /links outside its archive/); }); -test('current scope defers immutable archive targets to the archive gate', (t) => { +test('current scope accepts a declared archive without requiring its target tree', (t) => { const root = fixture(t, '/explanation/current/'); - write(root, 'dist/index.html', 'Release'); + write(root, 'dist/index.html', 'Release'); const current = run(root, ['--scope', 'current']); const all = run(root); assert.equal(current.status, 0, current.stderr); assert.equal(all.status, 1); assert.match(all.stderr, /links to missing/); }); + +test('current scope rejects links to undeclared archives', (t) => { + const root = fixture(t, '/explanation/current/'); + write(root, 'dist/index.html', 'Release'); + const current = run(root, ['--scope', 'current']); + assert.equal(current.status, 1); + assert.match(current.stderr, /links to unknown archive/); +});