Pack eng/docker-tools content in ImageBuilder

AGENTS.md

@@ -54,6 +54,13 @@ Breaking changes to `eng/docker-tools/` must be documented in `eng/docker-tools/
 
 For comprehensive documentation on the docker-tools infrastructure, pipeline architecture, image building workflows, and troubleshooting, see [eng/docker-tools/DEV-GUIDE.md](eng/docker-tools/DEV-GUIDE.md).
 
+### Templates Bundled In ImageBuilder
+
+ImageBuilder ships a copy of the entire `eng/docker-tools/` directory so that source-code changes and pipeline-template changes can be made together in the same commit. See [issue #2130](https://github.com/dotnet/docker-tools/issues/2130) for background.
+
+- `src/Infrastructure/` (`Microsoft.DotNet.DockerTools.Infrastructure`) embeds a copy of `eng/docker-tools/` under `src/Infrastructure/Content/` as assembly resources, which ImageBuilder's `update` command writes back to disk. The copy must live under `src/` because the Docker build context is `src/`, so `eng/docker-tools/` is not reachable from the container build.
+- **The two copies are not kept identical.** Make infrastructure changes in `src/Infrastructure/Content/` — that copy ships in ImageBuilder. This repo's own `eng/docker-tools/` is regenerated by the `update` command when a newer ImageBuilder arrives via automatic dependency updates, so the two legitimately differ in between: `eng/docker-tools/` tracks the ImageBuilder version this repo currently consumes, while `Content/` tracks what the next one will ship. See [src/Infrastructure/README.md](src/Infrastructure/README.md).
+
 ### Service Connections and Authentication
 
 `publishConfig` is the source of truth for registry authentication. Registry service connections live in `publishConfig.RegistryAuthentication`. Non-registry connections (e.g., kusto, marStatus, cleanServiceConnection) are separate fields or passed via `additionalServiceConnections`.

Microsoft.DotNet.DockerTools.slnx

@@ -9,4 +9,5 @@
   <Project Path="src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj" />
   <Project Path="src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj" />
   <Project Path="src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj" />
+  <Project Path="src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj" />
 </Solution>

eng/docker-tools/Update-ImageBuilder.ps1

@@ -0,0 +1,89 @@
+#!/usr/bin/env pwsh
+
+<#
+.SYNOPSIS
+Example script that updates the bundled docker-tools infrastructure in the current repo to a specific
+ImageBuilder image.
+
+.DESCRIPTION
+ImageBuilder ships a copy of the eng/docker-tools infrastructure and writes it back to disk via its
+'update' command. The reference that 'update' records into
+eng/docker-tools/templates/variables/docker-images.yml is supplied as an argument rather than being
+baked into the build, so the caller decides exactly which image the repo should pin to.
+
+This example resolves the multi-platform (manifest list / image index) digest of an ImageBuilder image
+(the published 'latest' tag by default) and passes that digest reference to the 'update' command, which
+runs inside the same image with the repository mounted so it can rewrite eng/docker-tools on disk.
+
+.PARAMETER ImageBuilderImage
+The ImageBuilder image to resolve and run. Defaults to the published 'latest' tag.
+
+.PARAMETER RepoRoot
+The root of the git repository to update. Defaults to the current directory.
+
+.NOTES
+To exercise an unpublished ImageBuilder (for example, the 'update' command before it is released),
+build the image, push it to a registry it can be pulled from, and pass its reference via
+-ImageBuilderImage. The digest is read from the registry, so the image must be pushed first.
+#>
+[CmdletBinding()]
+param(
+    [string]
+    $ImageBuilderImage = "mcr.microsoft.com/dotnet-buildtools/image-builder:latest",
+
+    [string]
+    $RepoRoot = (Get-Location).Path
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+function Exec {
+    param ([string] $Cmd)
+
+    Write-Output "Executing: '$Cmd'"
+    Invoke-Expression $Cmd
+    if ($LASTEXITCODE -ne 0) {
+        throw "Failed: '$Cmd'"
+    }
+}
+
+# Strip any existing tag or digest so the resolved digest can be appended to the bare repository name.
+# A tag is a ':' within the final path segment; a registry host's ':port' precedes the last '/', so it
+# must not be mistaken for a tag.
+function Get-RepositoryName {
+    param ([string] $Reference)
+
+    $withoutDigest = $Reference.Split('@')[0]
+    $lastSlash = $withoutDigest.LastIndexOf('/')
+    $lastColon = $withoutDigest.LastIndexOf(':')
+    if ($lastColon -gt $lastSlash) {
+        return $withoutDigest.Substring(0, $lastColon)
+    }
+
+    return $withoutDigest
+}
+
+# Resolve the multi-platform digest. 'docker buildx imagetools inspect' reads the top-level manifest
+# straight from the registry, so for a multi-arch image this is the manifest list (image index) digest
+# rather than a single platform's digest. Pinning the index keeps the reference valid on every
+# platform the pipeline runs on.
+$digest = (docker buildx imagetools inspect $ImageBuilderImage --format '{{.Manifest.Digest}}')
+if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($digest)) {
+    throw "Unable to resolve a multi-platform digest for '$ImageBuilderImage'."
+}
+
+$repository = Get-RepositoryName $ImageBuilderImage
+$imageBuilderRef = "$repository@$($digest.Trim())"
+
+Write-Output "Resolved ImageBuilder reference: $imageBuilderRef"
+
+# Run 'update' from the resolved digest, mounting the repository so it can write eng/docker-tools to
+# disk. The command must run from the repository root, which is why $RepoRoot is the mounted working
+# directory. Running by the same digest that gets recorded keeps the writer and the pinned reference
+# identical.
+Exec ("docker run --rm " `
+    + "-v `"${RepoRoot}:/repo`" " `
+    + "-w /repo " `
+    + "$imageBuilderRef " `
+    + "update $imageBuilderRef")

src/Dockerfile.linux

@@ -20,6 +20,7 @@ WORKDIR /image-builder
 COPY NuGet.config ./
 COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/
 COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/
+COPY Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj ./Infrastructure/
 RUN dotnet restore -r linux-$TARGETARCH ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj
 
 # copy everything else and publish

src/Dockerfile.windows

@@ -15,6 +15,7 @@ ENV NUGET_PACKAGES=/nuget
 COPY NuGet.config ./
 COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/
 COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/
+COPY Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj ./Infrastructure/
 RUN dotnet restore -r win-x64 ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj
 
 # copy everything else and publish

src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs

@@ -4,6 +4,7 @@
 
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -20,7 +21,12 @@ internal sealed class InMemoryFileSystem : IFileSystem
     private readonly HashSet<string> _directories = [];
 
     /// <summary>
-    /// Paths written via <see cref="WriteAllText"/> or <see cref="WriteAllTextAsync"/>.
+    /// The path returned by <see cref="GetCurrentDirectory"/>. Defaults to the platform root.
+    /// </summary>
+    public string CurrentDirectory { get; set; } = Path.DirectorySeparatorChar.ToString();
+
+    /// <summary>
+    /// Paths written via <see cref="WriteAllText"/>, <see cref="WriteAllTextAsync"/>, or <see cref="CreateFile"/>.
     /// </summary>
     public List<string> FilesWritten { get; } = [];
 
@@ -39,31 +45,44 @@ internal sealed class InMemoryFileSystem : IFileSystem
     /// </summary>
     public List<string> DirectoriesCreated { get; } = [];
 
+    /// <summary>
+    /// Paths deleted via <see cref="DeleteDirectory"/>.
+    /// </summary>
+    public List<string> DirectoriesDeleted { get; } = [];
+
     /// <summary>
     /// Seeds a file with text content before a test runs.
     /// </summary>
     public void AddFile(string path, string contents) =>
-        _files[path] = Encoding.UTF8.GetBytes(contents);
+        SetFile(path, Encoding.UTF8.GetBytes(contents));
 
     /// <summary>
     /// Seeds a file with binary content before a test runs.
     /// </summary>
     public void AddFile(string path, byte[] contents) =>
-        _files[path] = contents;
+        SetFile(path, contents);
+
+    /// <summary>
+    /// Seeds an empty directory before a test runs.
+    /// </summary>
+    public void AddDirectory(string path) =>
+        _directories.Add(path);
 
     public void WriteAllText(string path, string contents)
     {
-        _files[path] = Encoding.UTF8.GetBytes(contents);
+        SetFile(path, Encoding.UTF8.GetBytes(contents));
         FilesWritten.Add(path);
     }
 
     public Task WriteAllTextAsync(string path, string? contents, CancellationToken cancellationToken = default)
     {
-        _files[path] = Encoding.UTF8.GetBytes(contents ?? string.Empty);
+        SetFile(path, Encoding.UTF8.GetBytes(contents ?? string.Empty));
         FilesWritten.Add(path);
         return Task.CompletedTask;
     }
 
+    public Stream CreateFile(string path) => new CommitOnDisposeStream(this, path);
+
     public byte[] ReadAllBytes(string path)
     {
         FilesRead.Add(path);
@@ -93,22 +112,99 @@ public Task<string> ReadAllTextAsync(string path, CancellationToken cancellation
 
     public bool FileExists(string path) => _files.ContainsKey(path);
 
+    public bool DirectoryExists(string path) =>
+        _directories.Contains(path)
+        || _files.Keys.Any(filePath => IsUnder(path, filePath))
+        || _directories.Any(directory => IsUnder(path, directory));
+
     public void DeleteFile(string path)
     {
         _files.Remove(path);
         FilesDeleted.Add(path);
     }
 
+    public void DeleteDirectory(string path, bool recursive)
+    {
+        if (!DirectoryExists(path))
+        {
+            // Mirror Directory.Delete, which throws when the target directory does not exist.
+            throw new DirectoryNotFoundException($"Could not find a part of the path '{path}'.");
+        }
+
+        if (recursive)
+        {
+            foreach (string file in _files.Keys.Where(filePath => IsUnder(path, filePath)).ToList())
+            {
+                _files.Remove(file);
+                FilesDeleted.Add(file);
+            }
+
+            foreach (string directory in _directories.Where(dir => dir == path || IsUnder(path, dir)).ToList())
+            {
+                _directories.Remove(directory);
+                DirectoriesDeleted.Add(directory);
+            }
+
+            return;
+        }
+
+        _directories.Remove(path);
+        DirectoriesDeleted.Add(path);
+    }
+
     public DirectoryInfo CreateDirectory(string path)
     {
         _directories.Add(path);
         DirectoriesCreated.Add(path);
         return new DirectoryInfo(path);
     }
 
+    public string GetCurrentDirectory() => CurrentDirectory;
+
+    private static bool IsUnder(string directory, string candidate) =>
+        candidate.StartsWith(directory.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar);
+
+    private void SetFile(string path, byte[] bytes)
+    {
+        _files[path] = bytes;
+
+        // Materialize ancestor directories so they persist independently of the file,
+        // mirroring a real filesystem where deleting a file leaves its directory behind.
+        string? directory = Path.GetDirectoryName(path);
+        while (!string.IsNullOrEmpty(directory))
+        {
+            _directories.Add(directory);
+            directory = Path.GetDirectoryName(directory);
+        }
+    }
+
     /// <summary>
     /// Gets the text content of a file, for test assertions.
     /// </summary>
     public string GetFileText(string path) =>
         Encoding.UTF8.GetString(_files[path]);
+
+    /// <summary>
+    /// Gets the binary content of a file, for test assertions.
+    /// </summary>
+    public byte[] GetFileBytes(string path) => _files[path];
+
+    /// <summary>
+    /// Writable stream returned by <see cref="CreateFile"/> that commits its contents to the
+    /// in-memory store when disposed, mirroring the <see cref="FileStream"/> returned by
+    /// <see cref="File.Create(string)"/>.
+    /// </summary>
+    private sealed class CommitOnDisposeStream(InMemoryFileSystem fileSystem, string path) : MemoryStream
+    {
+        protected override void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                fileSystem.SetFile(path, ToArray());
+                fileSystem.FilesWritten.Add(path);
+            }
+
+            base.Dispose(disposing);
+        }
+    }
 }

src/ImageBuilder.Tests/PathHelperTests.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using Shouldly;
+
+namespace Microsoft.DotNet.ImageBuilder.Tests;
+
+[TestClass]
+public class PathHelperTests
+{
+    [TestMethod]
+    public void SafeCombine_RelativeSegments_CombinesUnderBasePath()
+    {
+        string basePath = $"{Path.DirectorySeparatorChar}repo";
+
+        string result = PathHelper.SafeCombine(basePath, "eng", "docker-tools");
+
+        result.ShouldBe(Path.Combine(basePath, "eng", "docker-tools"));
+    }
+
+    [TestMethod]
+    public void SafeCombine_ForwardSlashSegment_ConvertsToPlatformSeparator()
+    {
+        string basePath = $"{Path.DirectorySeparatorChar}repo";
+
+        string result = PathHelper.SafeCombine(basePath, "templates/variables/docker-images.yml");
+
+        result.ShouldBe(Path.Combine(basePath, "templates", "variables", "docker-images.yml"));
+    }
+
+    [TestMethod]
+    public void SafeCombine_RootedSegment_Throws()
+    {
+        string basePath = $"{Path.DirectorySeparatorChar}repo";
+        string rootedSegment = $"{Path.DirectorySeparatorChar}etc{Path.DirectorySeparatorChar}passwd";
+
+        ArgumentException exception =
+            Should.Throw<ArgumentException>(() => PathHelper.SafeCombine(basePath, rootedSegment));
+        exception.Message.ShouldContain("rooted");
+    }
+
+    [TestMethod]
+    [DataRow("..")]
+    [DataRow("../escape")]
+    [DataRow("nested/../../escape")]
+    public void SafeCombine_TraversalSegment_Throws(string traversalSegment)
+    {
+        string basePath = $"{Path.DirectorySeparatorChar}repo";
+
+        ArgumentException exception =
+            Should.Throw<ArgumentException>(() => PathHelper.SafeCombine(basePath, traversalSegment));
+        exception.Message.ShouldContain("traversal");
+    }
+
+    [TestMethod]
+    [DataRow("")]
+    [DataRow("   ")]
+    [DataRow(null)]
+    public void SafeCombine_NullOrWhitespaceSegment_Throws(string? segment)
+    {
+        string basePath = $"{Path.DirectorySeparatorChar}repo";
+
+        Should.Throw<ArgumentException>(() => PathHelper.SafeCombine(basePath, segment!));
+    }
+}