From 22a30932c3a4fc2ab28e756e7272b4fab320e7c4 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 22 Jun 2026 11:07:02 -0700 Subject: [PATCH 1/7] Extract shared Cottle document configuration into a helper GenerateArtifactsCommand built its Cottle DocumentConfiguration inline. Move that configuration into a CottleDocumentConfiguration.Create() helper in a new Templating namespace so the same template delimiters and whitespace handling can be reused by other commands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Commands/GenerateArtifactsCommand.cs | 10 ++----- .../Templating/CottleDocumentConfiguration.cs | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 src/ImageBuilder/Templating/CottleDocumentConfiguration.cs diff --git a/src/ImageBuilder/Commands/GenerateArtifactsCommand.cs b/src/ImageBuilder/Commands/GenerateArtifactsCommand.cs index 4563b4254..2285d87a8 100644 --- a/src/ImageBuilder/Commands/GenerateArtifactsCommand.cs +++ b/src/ImageBuilder/Commands/GenerateArtifactsCommand.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Cottle; using Cottle.Exceptions; +using Microsoft.DotNet.ImageBuilder.Templating; namespace Microsoft.DotNet.ImageBuilder.Commands { @@ -20,14 +21,7 @@ public delegate (IReadOnlyDictionary Symbols, string Indent) GetTe public abstract class GenerateArtifactsCommand : ManifestCommand where TOptions : GenerateArtifactsOptions, new() { - private readonly DocumentConfiguration _config = new DocumentConfiguration - { - BlockBegin = "{{", - BlockContinue = "^", - BlockEnd = "}}", - Escape = '@', - Trimmer = DocumentConfiguration.TrimNothing - }; + private readonly DocumentConfiguration _config = CottleDocumentConfiguration.Create(); private readonly IEnvironmentService _environmentService; private readonly List _invalidTemplates = new List(); diff --git a/src/ImageBuilder/Templating/CottleDocumentConfiguration.cs b/src/ImageBuilder/Templating/CottleDocumentConfiguration.cs new file mode 100644 index 000000000..930cb622e --- /dev/null +++ b/src/ImageBuilder/Templating/CottleDocumentConfiguration.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Cottle; + +namespace Microsoft.DotNet.ImageBuilder.Templating; + +/// +/// Creates the shared Cottle document configuration used by ImageBuilder template rendering. +/// +internal static class CottleDocumentConfiguration +{ + /// + /// Creates a Cottle configuration that uses ImageBuilder's template delimiters and preserves + /// whitespace exactly. + /// + public static DocumentConfiguration Create() => + new() + { + BlockBegin = "{{", + BlockContinue = "^", + BlockEnd = "}}", + Escape = '@', + Trimmer = DocumentConfiguration.TrimNothing + }; +} From cc1dedab64c666330ccee170f2866094c65397f8 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 22 Jun 2026 11:07:11 -0700 Subject: [PATCH 2/7] Extend IFileSystem with directory and stream operations Add CreateFile, DirectoryExists, DeleteDirectory, and GetCurrentDirectory to IFileSystem (and the FileSystem pass-through implementation) so callers can perform directory-aware, streaming filesystem work behind the existing abstraction. Update the InMemoryFileSystem test double to implement the new members, tracking created/deleted directories and committing CreateFile streams on dispose. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Helpers/InMemoryFileSystem.cs | 106 +++++++++++++++++- src/ImageBuilder/FileSystem.cs | 20 ++++ src/ImageBuilder/IFileSystem.cs | 24 ++++ 3 files changed, 145 insertions(+), 5 deletions(-) diff --git a/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs b/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs index 272d452b2..aebd610a2 100644 --- a/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs +++ b/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -20,7 +21,12 @@ internal sealed class InMemoryFileSystem : IFileSystem private readonly HashSet _directories = []; /// - /// Paths written via or . + /// The path returned by . Defaults to the platform root. + /// + public string CurrentDirectory { get; set; } = Path.DirectorySeparatorChar.ToString(); + + /// + /// Paths written via , , or . /// public List FilesWritten { get; } = []; @@ -39,31 +45,44 @@ internal sealed class InMemoryFileSystem : IFileSystem /// public List DirectoriesCreated { get; } = []; + /// + /// Paths deleted via . + /// + public List DirectoriesDeleted { get; } = []; + /// /// Seeds a file with text content before a test runs. /// public void AddFile(string path, string contents) => - _files[path] = Encoding.UTF8.GetBytes(contents); + SetFile(path, Encoding.UTF8.GetBytes(contents)); /// /// Seeds a file with binary content before a test runs. /// public void AddFile(string path, byte[] contents) => - _files[path] = contents; + SetFile(path, contents); + + /// + /// Seeds an empty directory before a test runs. + /// + public void AddDirectory(string path) => + _directories.Add(path); public void WriteAllText(string path, string contents) { - _files[path] = Encoding.UTF8.GetBytes(contents); + SetFile(path, Encoding.UTF8.GetBytes(contents)); FilesWritten.Add(path); } public Task WriteAllTextAsync(string path, string? contents, CancellationToken cancellationToken = default) { - _files[path] = Encoding.UTF8.GetBytes(contents ?? string.Empty); + SetFile(path, Encoding.UTF8.GetBytes(contents ?? string.Empty)); FilesWritten.Add(path); return Task.CompletedTask; } + public Stream CreateFile(string path) => new CommitOnDisposeStream(this, path); + public byte[] ReadAllBytes(string path) { FilesRead.Add(path); @@ -93,12 +112,46 @@ public Task ReadAllTextAsync(string path, CancellationToken cancellation public bool FileExists(string path) => _files.ContainsKey(path); + public bool DirectoryExists(string path) => + _directories.Contains(path) + || _files.Keys.Any(filePath => IsUnder(path, filePath)) + || _directories.Any(directory => IsUnder(path, directory)); + public void DeleteFile(string path) { _files.Remove(path); FilesDeleted.Add(path); } + public void DeleteDirectory(string path, bool recursive) + { + if (!DirectoryExists(path)) + { + // Mirror Directory.Delete, which throws when the target directory does not exist. + throw new DirectoryNotFoundException($"Could not find a part of the path '{path}'."); + } + + if (recursive) + { + foreach (string file in _files.Keys.Where(filePath => IsUnder(path, filePath)).ToList()) + { + _files.Remove(file); + FilesDeleted.Add(file); + } + + foreach (string directory in _directories.Where(dir => dir == path || IsUnder(path, dir)).ToList()) + { + _directories.Remove(directory); + DirectoriesDeleted.Add(directory); + } + + return; + } + + _directories.Remove(path); + DirectoriesDeleted.Add(path); + } + public DirectoryInfo CreateDirectory(string path) { _directories.Add(path); @@ -106,9 +159,52 @@ public DirectoryInfo CreateDirectory(string path) return new DirectoryInfo(path); } + public string GetCurrentDirectory() => CurrentDirectory; + + private static bool IsUnder(string directory, string candidate) => + candidate.StartsWith(directory.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar); + + private void SetFile(string path, byte[] bytes) + { + _files[path] = bytes; + + // Materialize ancestor directories so they persist independently of the file, + // mirroring a real filesystem where deleting a file leaves its directory behind. + string? directory = Path.GetDirectoryName(path); + while (!string.IsNullOrEmpty(directory)) + { + _directories.Add(directory); + directory = Path.GetDirectoryName(directory); + } + } + /// /// Gets the text content of a file, for test assertions. /// public string GetFileText(string path) => Encoding.UTF8.GetString(_files[path]); + + /// + /// Gets the binary content of a file, for test assertions. + /// + public byte[] GetFileBytes(string path) => _files[path]; + + /// + /// Writable stream returned by that commits its contents to the + /// in-memory store when disposed, mirroring the returned by + /// . + /// + private sealed class CommitOnDisposeStream(InMemoryFileSystem fileSystem, string path) : MemoryStream + { + protected override void Dispose(bool disposing) + { + if (disposing) + { + fileSystem.SetFile(path, ToArray()); + fileSystem.FilesWritten.Add(path); + } + + base.Dispose(disposing); + } + } } diff --git a/src/ImageBuilder/FileSystem.cs b/src/ImageBuilder/FileSystem.cs index 39a4d0e4b..c807b9839 100644 --- a/src/ImageBuilder/FileSystem.cs +++ b/src/ImageBuilder/FileSystem.cs @@ -11,6 +11,10 @@ namespace Microsoft.DotNet.ImageBuilder; /// /// Default filesystem implementation that delegates to and . /// +/// +/// Every member must be a thin, one-to-one pass-through to the system API with no added behavior; +/// any higher-level logic lives in the caller. +/// public sealed class FileSystem : IFileSystem { /// @@ -21,6 +25,10 @@ public void WriteAllText(string path, string contents) => public Task WriteAllTextAsync(string path, string? contents, CancellationToken cancellationToken = default) => File.WriteAllTextAsync(path, contents, cancellationToken); + /// + public Stream CreateFile(string path) => + File.Create(path); + /// public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path); @@ -41,11 +49,23 @@ public Task ReadAllTextAsync(string path, CancellationToken cancellation public bool FileExists(string path) => File.Exists(path); + /// + public bool DirectoryExists(string path) => + Directory.Exists(path); + /// public void DeleteFile(string path) => File.Delete(path); + /// + public void DeleteDirectory(string path, bool recursive) => + Directory.Delete(path, recursive); + /// public DirectoryInfo CreateDirectory(string path) => Directory.CreateDirectory(path); + + /// + public string GetCurrentDirectory() => + Directory.GetCurrentDirectory(); } diff --git a/src/ImageBuilder/IFileSystem.cs b/src/ImageBuilder/IFileSystem.cs index a21980b50..7757d8cdc 100644 --- a/src/ImageBuilder/IFileSystem.cs +++ b/src/ImageBuilder/IFileSystem.cs @@ -23,6 +23,12 @@ public interface IFileSystem /// Task WriteAllTextAsync(string path, string? contents, CancellationToken cancellationToken = default); + /// + /// Creates or overwrites a file in the specified path and returns a writable stream + /// (mirrors ). + /// + Stream CreateFile(string path); + /// /// Opens a binary file, reads the contents into a byte array, and then closes the file. /// @@ -48,13 +54,31 @@ public interface IFileSystem /// bool FileExists(string path); + /// + /// Determines whether the specified directory exists. + /// + bool DirectoryExists(string path); + /// /// Deletes the specified file. /// void DeleteFile(string path); + /// + /// Deletes the specified directory, throwing if it does not exist (mirrors ). + /// + /// + /// true to delete the directory, its subdirectories, and all files; otherwise false. + /// + void DeleteDirectory(string path, bool recursive); + /// /// Creates all directories and subdirectories in the specified path unless they already exist. /// DirectoryInfo CreateDirectory(string path); + + /// + /// Gets the current working directory of the application. + /// + string GetCurrentDirectory(); } From 5790e7ce1404a63be0eb45439037110bbbdbd9c7 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 22 Jun 2026 11:07:32 -0700 Subject: [PATCH 3/7] Add Infrastructure project embedding eng/docker-tools content Add the Microsoft.DotNet.DockerTools.Infrastructure library, which embeds a copy of the entire eng/docker-tools directory (pipeline templates, PowerShell scripts, and docs) under src/Infrastructure/Content/ as assembly resources. InfrastructureContent exposes the embedded files by relative path via a trim/AOT-safe manifest-resource index. The copy lives under src/ so it is reachable from the container build context, and register the project in the solution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Microsoft.DotNet.DockerTools.slnx | 1 + src/Infrastructure/Content/CHANGELOG.md | 157 ++++++ src/Infrastructure/Content/DEV-GUIDE.md | 508 ++++++++++++++++++ .../Content/Dockerfile.WithRepo | 6 + src/Infrastructure/Content/Dockerfile.syft | 16 + .../Content/Get-BaseImageStatus.ps1 | 33 ++ .../Content/Get-ImageBuilder.ps1 | 13 + .../Content/Get-ImageNameVars.ps1 | 12 + .../Content/Install-DotNetSdk.ps1 | 64 +++ .../Content/Invoke-CleanupDocker.ps1 | 20 + .../Content/Invoke-ImageBuilder.ps1 | 105 ++++ .../Content/Invoke-WithRetry.ps1 | 41 ++ src/Infrastructure/Content/Pull-Image.ps1 | 18 + src/Infrastructure/Content/Retain-Build.ps1 | 43 ++ src/Infrastructure/Content/build.ps1 | 76 +++ src/Infrastructure/Content/readme.md | 30 ++ .../Content/skill-helpers/AzureDevOps.ps1 | 85 +++ .../Content/skill-helpers/Get-BuildLog.ps1 | 21 + .../skill-helpers/Get-FailingPipelines.ps1 | 63 +++ .../skill-helpers/Get-RecentBuilds.ps1 | 58 ++ .../skill-helpers/Show-BuildTimeline.ps1 | 77 +++ .../skill-helpers/Show-PullRequestBuilds.ps1 | 99 ++++ .../Show-PullRequestComments.ps1 | 117 ++++ .../Content/templates/1es-official.yml | 62 +++ .../Content/templates/1es-unofficial.yml | 70 +++ src/Infrastructure/Content/templates/1es.yml | 86 +++ .../Content/templates/jobs/build-images.yml | 143 +++++ .../templates/jobs/cg-build-projects.yml | 70 +++ .../jobs/copy-base-images-staging.yml | 40 ++ .../templates/jobs/copy-base-images.yml | 57 ++ .../templates/jobs/generate-matrix.yml | 82 +++ .../Content/templates/jobs/post-build.yml | 114 ++++ .../Content/templates/jobs/publish.yml | 292 ++++++++++ .../Content/templates/jobs/sign-images.yml | 66 +++ .../jobs/test-images-linux-client.yml | 30 ++ .../jobs/test-images-windows-client.yml | 25 + .../templates/stages/build-and-test.yml | 351 ++++++++++++ .../stages/dotnet/build-and-test.yml | 141 +++++ .../stages/dotnet/build-test-publish-repo.yml | 84 +++ .../stages/dotnet/publish-config-nonprod.yml | 119 ++++ .../stages/dotnet/publish-config-prod.yml | 117 ++++ .../templates/stages/dotnet/publish.yml | 64 +++ .../Content/templates/stages/publish.yml | 88 +++ .../templates/steps/annotate-eol-digests.yml | 43 ++ .../templates/steps/clean-acr-images.yml | 26 + .../templates/steps/cleanup-docker-linux.yml | 15 + .../steps/cleanup-docker-windows.yml | 18 + .../templates/steps/copy-base-images.yml | 37 ++ .../steps/download-build-artifact.yml | 40 ++ .../templates/steps/generate-appsettings.yml | 34 ++ .../Content/templates/steps/init-common.yml | 247 +++++++++ .../templates/steps/init-imagebuilder.yml | 146 +++++ .../templates/steps/init-signing-linux.yml | 124 +++++ .../templates/steps/init-testrunner.yml | 21 + .../templates/steps/parse-test-arg-arrays.yml | 15 + .../templates/steps/publish-artifact.yml | 28 + .../templates/steps/publish-readmes.yml | 29 + .../steps/reference-service-connections.yml | 72 +++ .../Content/templates/steps/retain-build.yml | 9 + .../templates/steps/run-imagebuilder.yml | 81 +++ .../templates/steps/run-pwsh-with-auth.yml | 39 ++ .../Content/templates/steps/set-dry-run.yml | 30 ++ .../steps/set-image-info-path-var.yml | 19 + .../steps/test-images-linux-client.yml | 119 ++++ .../steps/test-images-windows-client.yml | 69 +++ .../templates/steps/validate-branch.yml | 52 ++ .../steps/wait-for-mcr-doc-ingestion.yml | 21 + .../steps/wait-for-mcr-image-ingestion.yml | 37 ++ .../templates/task-prefix-decorator.yml | 63 +++ .../templates/variables/common-paths.yml | 6 + .../Content/templates/variables/common.yml | 89 +++ .../variables/dnceng-build-pools.yml | 58 ++ .../variables/dnceng-project-names.yml | 8 + .../templates/variables/dnceng-signing.yml | 10 + .../templates/variables/docker-images.yml | 7 + .../variables/dotnet/build-test-publish.yml | 48 ++ .../templates/variables/dotnet/common.yml | 14 + .../variables/dotnet/secrets-unofficial.yml | 5 + .../templates/variables/dotnet/secrets.yml | 17 + .../Content/templates/variables/sdl-pool.yml | 9 + src/Infrastructure/InfrastructureContent.cs | 86 +++ ...t.DotNet.DockerTools.Infrastructure.csproj | 33 ++ src/Infrastructure/README.md | 34 ++ 83 files changed, 5622 insertions(+) create mode 100644 src/Infrastructure/Content/CHANGELOG.md create mode 100644 src/Infrastructure/Content/DEV-GUIDE.md create mode 100644 src/Infrastructure/Content/Dockerfile.WithRepo create mode 100644 src/Infrastructure/Content/Dockerfile.syft create mode 100644 src/Infrastructure/Content/Get-BaseImageStatus.ps1 create mode 100644 src/Infrastructure/Content/Get-ImageBuilder.ps1 create mode 100644 src/Infrastructure/Content/Get-ImageNameVars.ps1 create mode 100644 src/Infrastructure/Content/Install-DotNetSdk.ps1 create mode 100644 src/Infrastructure/Content/Invoke-CleanupDocker.ps1 create mode 100644 src/Infrastructure/Content/Invoke-ImageBuilder.ps1 create mode 100644 src/Infrastructure/Content/Invoke-WithRetry.ps1 create mode 100644 src/Infrastructure/Content/Pull-Image.ps1 create mode 100644 src/Infrastructure/Content/Retain-Build.ps1 create mode 100644 src/Infrastructure/Content/build.ps1 create mode 100644 src/Infrastructure/Content/readme.md create mode 100644 src/Infrastructure/Content/skill-helpers/AzureDevOps.ps1 create mode 100755 src/Infrastructure/Content/skill-helpers/Get-BuildLog.ps1 create mode 100755 src/Infrastructure/Content/skill-helpers/Get-FailingPipelines.ps1 create mode 100644 src/Infrastructure/Content/skill-helpers/Get-RecentBuilds.ps1 create mode 100755 src/Infrastructure/Content/skill-helpers/Show-BuildTimeline.ps1 create mode 100644 src/Infrastructure/Content/skill-helpers/Show-PullRequestBuilds.ps1 create mode 100755 src/Infrastructure/Content/skill-helpers/Show-PullRequestComments.ps1 create mode 100644 src/Infrastructure/Content/templates/1es-official.yml create mode 100644 src/Infrastructure/Content/templates/1es-unofficial.yml create mode 100644 src/Infrastructure/Content/templates/1es.yml create mode 100644 src/Infrastructure/Content/templates/jobs/build-images.yml create mode 100644 src/Infrastructure/Content/templates/jobs/cg-build-projects.yml create mode 100644 src/Infrastructure/Content/templates/jobs/copy-base-images-staging.yml create mode 100644 src/Infrastructure/Content/templates/jobs/copy-base-images.yml create mode 100644 src/Infrastructure/Content/templates/jobs/generate-matrix.yml create mode 100644 src/Infrastructure/Content/templates/jobs/post-build.yml create mode 100644 src/Infrastructure/Content/templates/jobs/publish.yml create mode 100644 src/Infrastructure/Content/templates/jobs/sign-images.yml create mode 100644 src/Infrastructure/Content/templates/jobs/test-images-linux-client.yml create mode 100644 src/Infrastructure/Content/templates/jobs/test-images-windows-client.yml create mode 100644 src/Infrastructure/Content/templates/stages/build-and-test.yml create mode 100644 src/Infrastructure/Content/templates/stages/dotnet/build-and-test.yml create mode 100644 src/Infrastructure/Content/templates/stages/dotnet/build-test-publish-repo.yml create mode 100644 src/Infrastructure/Content/templates/stages/dotnet/publish-config-nonprod.yml create mode 100644 src/Infrastructure/Content/templates/stages/dotnet/publish-config-prod.yml create mode 100644 src/Infrastructure/Content/templates/stages/dotnet/publish.yml create mode 100644 src/Infrastructure/Content/templates/stages/publish.yml create mode 100644 src/Infrastructure/Content/templates/steps/annotate-eol-digests.yml create mode 100644 src/Infrastructure/Content/templates/steps/clean-acr-images.yml create mode 100644 src/Infrastructure/Content/templates/steps/cleanup-docker-linux.yml create mode 100644 src/Infrastructure/Content/templates/steps/cleanup-docker-windows.yml create mode 100644 src/Infrastructure/Content/templates/steps/copy-base-images.yml create mode 100644 src/Infrastructure/Content/templates/steps/download-build-artifact.yml create mode 100644 src/Infrastructure/Content/templates/steps/generate-appsettings.yml create mode 100644 src/Infrastructure/Content/templates/steps/init-common.yml create mode 100644 src/Infrastructure/Content/templates/steps/init-imagebuilder.yml create mode 100644 src/Infrastructure/Content/templates/steps/init-signing-linux.yml create mode 100644 src/Infrastructure/Content/templates/steps/init-testrunner.yml create mode 100644 src/Infrastructure/Content/templates/steps/parse-test-arg-arrays.yml create mode 100644 src/Infrastructure/Content/templates/steps/publish-artifact.yml create mode 100644 src/Infrastructure/Content/templates/steps/publish-readmes.yml create mode 100644 src/Infrastructure/Content/templates/steps/reference-service-connections.yml create mode 100644 src/Infrastructure/Content/templates/steps/retain-build.yml create mode 100644 src/Infrastructure/Content/templates/steps/run-imagebuilder.yml create mode 100644 src/Infrastructure/Content/templates/steps/run-pwsh-with-auth.yml create mode 100644 src/Infrastructure/Content/templates/steps/set-dry-run.yml create mode 100644 src/Infrastructure/Content/templates/steps/set-image-info-path-var.yml create mode 100644 src/Infrastructure/Content/templates/steps/test-images-linux-client.yml create mode 100644 src/Infrastructure/Content/templates/steps/test-images-windows-client.yml create mode 100644 src/Infrastructure/Content/templates/steps/validate-branch.yml create mode 100644 src/Infrastructure/Content/templates/steps/wait-for-mcr-doc-ingestion.yml create mode 100644 src/Infrastructure/Content/templates/steps/wait-for-mcr-image-ingestion.yml create mode 100644 src/Infrastructure/Content/templates/task-prefix-decorator.yml create mode 100644 src/Infrastructure/Content/templates/variables/common-paths.yml create mode 100644 src/Infrastructure/Content/templates/variables/common.yml create mode 100644 src/Infrastructure/Content/templates/variables/dnceng-build-pools.yml create mode 100644 src/Infrastructure/Content/templates/variables/dnceng-project-names.yml create mode 100644 src/Infrastructure/Content/templates/variables/dnceng-signing.yml create mode 100644 src/Infrastructure/Content/templates/variables/docker-images.yml create mode 100644 src/Infrastructure/Content/templates/variables/dotnet/build-test-publish.yml create mode 100644 src/Infrastructure/Content/templates/variables/dotnet/common.yml create mode 100644 src/Infrastructure/Content/templates/variables/dotnet/secrets-unofficial.yml create mode 100644 src/Infrastructure/Content/templates/variables/dotnet/secrets.yml create mode 100644 src/Infrastructure/Content/templates/variables/sdl-pool.yml create mode 100644 src/Infrastructure/InfrastructureContent.cs create mode 100644 src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj create mode 100644 src/Infrastructure/README.md diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index 0c8d96696..21c095d09 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -9,4 +9,5 @@ + diff --git a/src/Infrastructure/Content/CHANGELOG.md b/src/Infrastructure/Content/CHANGELOG.md new file mode 100644 index 000000000..99dbceb69 --- /dev/null +++ b/src/Infrastructure/Content/CHANGELOG.md @@ -0,0 +1,157 @@ +# Docker Tools / ImageBuilder Changelog + +All breaking changes and new features in `eng/docker-tools` will be documented in this file. + +--- + +## 2026-04-02: Extra Docker build options can be passed through ImageBuilder + +- Pull request: [#2063](https://github.com/dotnet/docker-tools/pull/2063) + +ImageBuilder's `build` command now accepts repeated `--build-option` arguments and forwards them directly to +`docker build`. This allows repos to pass options such as `--ulimit nofile=65536:65536` or `--network host` +through `imageBuilderBuildArgs`, in addition to standard Dockerfile `--build-arg` values. + +**How to use:** + +```yaml +customBuildInitSteps: +- powershell: | + $args = '--build-option "--ulimit nofile=65536:65536"' + echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$args" +``` + +Repeat `--build-option` for multiple Docker arguments, and quote values that contain spaces. + +--- + +## 2026-03-25: Manifest list creation moved to Post_Build + +- Issue: [#2002](https://github.com/dotnet/docker-tools/issues/2002) + +Manifest lists are now created during `Post_Build` instead of during `Publish`. They are copied to the publish registry via ACR import along with the platform images, rather than being recreated from scratch during publish. + +--- + +## 2026-03-18: CG build template supports skipping .NET SDK installation + +- Issue: [#2029](https://github.com/dotnet/docker-tools/issues/2029) + +`cg-build-projects.yml` now accepts `skipDotNetInstall` (boolean, default `false`) and +`initSteps` (stepList, default `[]`) parameters. Setting `skipDotNetInstall: true` skips +the built-in SDK installation. Setting `initSteps` will execute those custom steps +at the beginning of the job. + +--- + +## 2026-03-12: Service connection OIDC changes + +- Pull request: [#2013](https://github.com/dotnet/docker-tools/pull/2013) +- Issue: [#2012](https://github.com/dotnet/docker-tools/issues/2012) + +`setup-service-connections.yml` has been removed. Azure DevOps no longer +issues OIDC tokens for service connections referenced in a separate stage. +Service connections are now referenced per-job via +`reference-service-connections.yml`. + +**How to update:** + +- Remove any `serviceConnections` parameters passed to `1es-official.yml` or + `1es-unofficial.yml` - they are no longer accepted. +- Remove any calls to `setup-service-connections.yml` from stage templates. +- Non-registry service connections (e.g., kusto, marStatus) should be passed + via `additionalServiceConnections` to the job templates that need them. + +--- + +## 2026-03-04: Pre-build validation gated by `preBuildTestScriptPath` variable + +- Pull request: [#1997](https://github.com/dotnet/docker-tools/pull/1997) + +The `PreBuildValidation` job condition now checks the new `preBuildTestScriptPath` variable instead of `testScriptPath`. +This allows repos to independently control whether pre-build validation runs, without affecting functional tests. + +The new variable defaults to `$(testScriptPath)`, so existing repos that have pre-build tests are not affected. +Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely. + +### Update (2026-03-11): Use `preBuildTestScriptPath` for test execution + +- Pull request: [#2011](https://github.com/dotnet/docker-tools/pull/2011) + +The `PreBuildValidation` job now uses `preBuildTestScriptPath` for test execution instead of `testScriptPath`. +Previously, the job condition was gated on `preBuildTestScriptPath` but the test execution step still used `testScriptPath`, +which meant PreBuildValidation could not be enabled independently when `testScriptPath` was empty. +Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely. + +--- + +## 2026-02-19: Separate Registry Endpoints from Authentication + +- Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945) +- Issue: [#1914](https://github.com/dotnet/docker-tools/issues/1914) + +Authentication details (`serviceConnection`, `resourceGroup`, `subscription`) have been moved from individual registry endpoints into a centralized `RegistryAuthentication` list. +This fixes an issue where ACR authentication could fail when multiple service connections existed for the same registry. + +**Before:** Each registry endpoint embedded its own authentication: + +```yaml +publishConfig: + BuildRegistry: + server: $(acr.server) + repoPrefix: "my-prefix/" + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(serviceConnectionName) + id: $(serviceConnection.id) + clientId: $(serviceConnection.clientId) + tenantId: $(tenant) + PublishRegistry: + server: $(acr.server) + repoPrefix: "publish/" + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(publishServiceConnectionName) + id: $(publishServiceConnection.id) + clientId: $(publishServiceConnection.clientId) + tenantId: $(tenant) +``` + +**After:** Registry endpoints only contain `server` and `repoPrefix`. Authentication is centralized: + +```yaml +publishConfig: + BuildRegistry: + server: $(acr.server) + repoPrefix: "my-prefix/" + PublishRegistry: + server: $(acr.server) + repoPrefix: "publish/" + RegistryAuthentication: + - server: $(acr.server) + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(serviceConnectionName) + id: $(serviceConnection.id) + clientId: $(serviceConnection.clientId) + tenantId: $(tenant) +``` + +How to update: +- Update any publishConfig parameters to match the new structure. + - Multiple registries can share authentication. If two registries use the same ACR server, only one entry is needed in `RegistryAuthentication`. + - The new structure should match [ImageBuilder's Configuration Model](https://github.com/dotnet/docker-tools/tree/a82572386854f15af441c50c6efa698a627e9f2b/src/ImageBuilder/Configuration). +- Update service connection setup (if using `setup-service-connections.yml`): + - The template now supports looking up service connections from `publishConfig.RegistryAuthentication` + - Use the new `usesRegistries` parameter to specify which registries need auth setup: + ```yaml + - template: eng/docker-tools/templates/stages/setup-service-connections.yml + parameters: + publishConfig: ${{ variables.publishConfig }} + usesRegistries: + - $(buildRegistry.server) + - $(publishRegistry.server) + ``` diff --git a/src/Infrastructure/Content/DEV-GUIDE.md b/src/Infrastructure/Content/DEV-GUIDE.md new file mode 100644 index 000000000..471b0d39e --- /dev/null +++ b/src/Infrastructure/Content/DEV-GUIDE.md @@ -0,0 +1,508 @@ +# Developer Guide: Using the docker-tools Infrastructure + +This guide walks you through the practical scenarios and workflows for using the docker-tools infrastructure. The `eng/docker-tools` directory is a **shared infrastructure layer** used across all .NET Docker repositories (dotnet-docker, dotnet-buildtools-prereqs-docker, dotnet-framework-docker). It solves a fundamental challenge: building, testing, and publishing Docker images across multiple operating systems (Alpine, Ubuntu, Azure Linux, Windows Server variants), multiple CPU architectures (amd64, arm64, arm32), and multiple .NET versions—all while maintaining consistency and reliability. + +At its core, the infrastructure provides: + +- **PowerShell scripts** for local image building and Docker operations—so you can test Dockerfile changes on your machine before committing +- **Azure Pipelines templates** for CI/CD (build, test, publish)—a composable template system that orchestrates builds across dozens of OS/architecture combinations in parallel +- **ImageBuilder orchestration**—a specialized .NET tool that understands manifest files, manages image dependencies, handles multi-arch manifest creation, and coordinates the entire build process +- **Caching and optimization**—intelligent systems that skip unchanged images and minimize redundant work +- **SBOM generation**—automatic Software Bill of Materials creation for supply chain security + +The infrastructure handles complexity that would otherwise be overwhelming: a single commit to a repo can trigger builds of hundreds of image variants across Linux and Windows agents, each requiring proper build sequencing, testing, and eventual publication to Microsoft Artifact Registry (MAR). + +**Important:** Files in `eng/docker-tools/` are synchronized across repositories by automation in the [dotnet/docker-tools](https://github.com/dotnet/docker-tools) repository. If you need to make changes to this infrastructure, submit them there—changes made directly in consuming repos will be overwritten. + +--- + +## Local Development Scenarios + +### Scenario: Building Docker Images Locally + +The most common local task is building images to test Dockerfile changes before pushing. + +**Quick Build - All Images:** +```powershell +./eng/docker-tools/build.ps1 +``` + +**Filter by OS:** +```powershell +# Build only Alpine images +./eng/docker-tools/build.ps1 -OS "alpine" + +# Build Ubuntu 24.04 images +./eng/docker-tools/build.ps1 -OS "noble" +``` + +**Filter by Architecture:** +```powershell +# Build arm64 images only +./eng/docker-tools/build.ps1 -Architecture "arm64" +``` + +**Filter by Path:** +```powershell +# Build images from a specific directory +./eng/docker-tools/build.ps1 -Paths "src/runtime/8.0/alpine3.20" + +# Build all 8.0 runtime images using glob pattern +./eng/docker-tools/build.ps1 -Paths "*runtime*8.0*" +``` + +**Combine Filters:** +```powershell +# Build .NET 8.0 Alpine arm64 images +./eng/docker-tools/build.ps1 -Version "8.0" -OS "alpine" -Architecture "arm64" +``` + +**Filter by Product Version (if applicable):** +```powershell +# Build only .NET 8.0 images +./eng/docker-tools/build.ps1 -Version "8.0" + +# Build .NET 6.0 and 8.0 images +./eng/docker-tools/build.ps1 -Version "6.0","8.0" +``` + +### Understanding What Happens Under the Hood + +When you run [`build.ps1`](build.ps1), here's the chain of execution: + +``` +build.ps1 + │ + ├── Translates your filter parameters into ImageBuilder CLI args + │ + └── Calls Invoke-ImageBuilder.ps1 "build --version X --os-version Y ..." + │ + ├── On Linux: Runs ImageBuilder in a Docker container + │ └── Builds image: microsoft-dotnet-imagebuilder-withrepo + │ └── Mounts Docker socket and repo contents + │ + └── On Windows: Extracts ImageBuilder locally (due to Docker-in-Docker limitations) + └── Runs Microsoft.DotNet.ImageBuilder.exe directly +``` + +### Scenario: Running ImageBuilder Directly + +For advanced scenarios, you may want to invoke ImageBuilder with specific commands: + +```powershell +# Run any ImageBuilder command +./eng/docker-tools/Invoke-ImageBuilder.ps1 "build --help" + +# Generate the build matrix (useful for debugging pipeline behavior) +./eng/docker-tools/Invoke-ImageBuilder.ps1 "generateBuildMatrix --manifest manifest.json --type platformDependencyGraph" + +# Validate manifest syntax +./eng/docker-tools/Invoke-ImageBuilder.ps1 "validateManifest --manifest manifest.json" +``` + +--- + +## Understanding the Pipeline Architecture + +### The Build Flow + +The pipeline behaves differently depending on the build context: + +**Public PR Builds**: +``` +Build Stage + ├── PreBuildValidation + ├── GenerateBuildMatrix + └── Build Jobs (dry-run, no push) + └── Inline tests after each build + │ + ▼ + Post_Build Stage + └── Merge artifacts + │ + ▼ + Publish Stage (dry-run) + └── All publish operations run but skip actual pushes + │ + ▼ + (end) +``` +- Images are built but **not pushed** to any registry +- Tests run inline within each build job +- Publish stage runs in dry-run mode (validates publish logic without pushing) +- Validates that Dockerfiles build successfully + +**Internal Official Builds**: +``` +Build Stage + ├── PreBuildValidation + ├── CopyBaseImages → staging ACR + ├── GenerateBuildMatrix + └── Build Jobs (push to staging ACR) + │ + ▼ + Post_Build Stage + ├── Merge image info files + └── Consolidate SBOMs + │ + ▼ + Test Stage + ├── GenerateTestMatrix + └── Test Jobs + │ + ▼ + Publish Stage + ├── Copy images to production ACR + ├── Create multi-arch manifests + ├── Wait for MAR ingestion + ├── Update READMEs + ├── Publish image info to versions repo + └── Apply EOL annotations +``` +- Full pipeline with all stages +- Images flow: `BuildRegistry` → `PublishRegistry` → MAR (see [`publish-config-prod.yml`](templates/stages/dotnet/publish-config-prod.yml) for ACR definitions) +- Tests run against staged images +- Only successful builds get published + +### Build Matrix Generation + +The `generateBuildMatrix` command is key to understanding how builds are parallelized. It: + +1. **Reads the manifest.json** - Understands which images exist +2. **Builds a dependency graph** - Knows that `runtime-deps` must build before `runtime` +3. **Groups by platform** - Creates jobs for each OS/Architecture combo +4. **Optimizes with caching** - Can detect and exclude unchanged images (see [Image Caching](#image-caching) below) + +### Controlling Which Build Stages Run + +The `stages` variable is a comma-separated string that controls which pipeline stages execute: + +```yaml +variables: +- name: stages + value: "build,test,sign,publish" # Run all stages +``` + +Common patterns: +- `"build"` - Build only, no tests, signing, or publishing +- `"build,test"` - Build and test, but don't sign or publish +- `"build,test,sign"` - Build, test, and sign, but don't publish +- `"sign"` - Sign only (when re-running failed signing from a previous build) +- `"publish"` - Publish only (when re-running a failed publish from a previous build) +- `"build,test,sign,publish"` - Full pipeline + +**Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files and consolidate SBOMs. + +The stages variable is useful for: +- Re-running just the publish stage after fixing a transient failure +- Skipping tests during initial development +- Running isolated stages for debugging + +### Decoupling build OS from the base image OS + +By default, a platform's `osVersion` represents the base image OS version, but also determines what +build leg an image is built in. This can cause problems when build image and base image don't match +up. For example, building a .NET app on Windows Server 2025 and copying the artifacts into a +Windows Server 2019 base image won't work, because the build matrix generation will attempt to +build the image on the Server 2019 build leg (which can't run Server 2025 images). + +To fix this, set the optional `buildOsVersion` field in order to override only the OS used in the +build matrix generation. Here is an example of building a Windows Server 2019 image using Windows +Server 2025: + +```jsonc +{ + // ... + "os": "windows", + "osVersion": "windowsservercore-ltsc2019", + "buildOsVersion": "windowsservercore-ltsc2025" + // ... +} +``` + +### Image Info Files: The Build's Memory + +Image info files (defined by [`ImageArtifactDetails`](https://github.com/dotnet/docker-tools/blob/main/src/ImageBuilder/Models/Image/ImageArtifactDetails.cs)) are the mechanism that tracks what was built: + +```json +{ + "repos": [{ + "repo": "dotnet/runtime", + "images": [{ + "platforms": [{ + "dockerfile": "src/runtime/8.0/alpine3.20/amd64/Dockerfile", + "digest": "sha256:abc123...", + "created": "2024-01-15T10:30:00Z", + "commitUrl": "https://github.com/dotnet/dotnet-docker/commit/..." + }] + }] + }] +} +``` + +**How they flow through the pipeline:** +1. **Build stage**: Each build job produces an image-info fragment +2. **Post_Build stage**: Fragments are merged into a single `image-info.json` +3. **Test stage**: Uses merged info to know which images to test +4. **Publish stage**: Uses info to know which images to copy/publish +5. **Versions repo**: Final info is committed to the versions repo + +The [versions repo](https://github.com/dotnet/versions) stores the "source of truth" image info. Future builds compare against this to determine what's changed and skip unchanged images. + +**Using Image Info for Investigations** + +Image info files are invaluable when you need to track down information about a specific image, particularly when starting from a digest reported by a customer or security scan. + +*Scenario: "Which commit produced this image?"* + +Given a digest like `sha256:abc123...`, you can trace it back to its source: + +1. **Check the versions repo history** - The `dotnet/versions` repo contains historical image info committed after each publish. Use `git log -p --all -S 'sha256:abc123'` to find the commit that introduced this digest. + +2. **From the image info entry**, you'll find: + - `commitUrl` - The exact source commit that built this image + - `dockerfile` - Which Dockerfile produced it + - `created` - When it was built + - `simpleTags` - The tags applied to this image + +*Scenario: "What was in the last successful build?"* + +Download the `image-info` artifact from a pipeline run in Azure DevOps: +1. Navigate to the pipeline run +2. Go to the "Published" artifacts section +3. Download `image-info` (merged) or individual `*-image-info-*` fragments + +*Scenario: "When did we last publish updates to a specific image?"* + +Use the versions repo git history: +```bash +# In the dotnet/versions repo +git log --oneline -- build-info/docker/image-info.dotnet-dotnet-docker-main.json +``` + +Each commit corresponds to a publish operation and includes the full image info at that point in time. + +*Scenario: "Compare what changed between two publishes"* + +```bash +git diff -- build-info/docker/image-info.dotnet-dotnet-docker-main.json +``` + +This shows which images were added, removed, or rebuilt (new digests) between the two publishes. + +### The Publish Flow in Detail + +The publish stage does more than just push images. Here's the sequence: + +1. **Copy Images** — `copyAcrImages` copies from build ACR to publish ACR +2. **Publish Manifest** — `publishManifest` creates multi-arch manifest lists +3. **Wait for MAR Ingestion** — Polls MAR until images are available (timeout configurable) +4. **Publish READMEs** — Updates documentation in the registry +5. **Wait for Doc Ingestion** — Ensures README changes are live +6. **Merge & Publish Image Info** — Updates the versions repo with new image metadata +7. **Ingest Kusto Image Info** — Sends telemetry to Kusto for analytics +8. **Generate & Apply EOL Annotations** — Marks images with end-of-life dates +9. **Post Publish Notification** — Creates GitHub issues/notifications about the publish + +### Dry-Run Mode + +For testing pipeline changes without actually publishing: + +```yaml +# In pipeline variables or at runtime +variables: +- name: dryRunArg + value: "--dry-run" +``` + +Or the infrastructure automatically enables dry-run for: +- Pull request builds +- Builds from non-official branches +- Public project builds + +The [`set-dry-run.yml`](templates/steps/set-dry-run.yml) step template determines this automatically based on context. + +--- + +## Automatic Image Rebuilds + +The infrastructure includes automation that monitors for base image updates and triggers rebuilds when dependencies change. + +### How It Works + +A scheduled pipeline ([`check-base-image-updates.yml`](https://github.com/dotnet/docker-tools/blob/main/eng/pipelines/check-base-image-updates.yml)) runs every 4 hours and: + +1. **Checks for stale images** — Compares the base image digests used in our published images against the current digests in upstream registries +2. **Identifies affected images** — Determines which of our images need rebuilding because their base image changed +3. **Queues targeted builds** — Automatically triggers builds for only the affected images, not the entire repo + +This ensures that security patches and updates in base images (like `alpine`, `ubuntu`, `mcr.microsoft.com/windows/nanoserver`) flow through to images without manual intervention. + +### Failure Handling and Recovery + +The system has built-in retry logic but requires manual intervention after repeated failures: + +**Automatic retry behavior:** +- If a triggered build fails, the system will attempt to rebuild every 4 hours +- After **3 unsuccessful attempts**, the system stops queuing new builds for that image +- This prevents endless rebuild loops when there's a genuine issue requiring human attention + +**After fixing the issue:** + +Once you've fixed the underlying problem (Dockerfile change, test fix, etc.) and have a successful build: + +1. Navigate to the successful pipeline run in Azure DevOps +2. Add the `autobuilder` label to that run +3. This signals to the infrastructure that a successful build has occurred +4. The system will resume automatic rebuilds for that image as needed + +The `autobuilder` label is how the infrastructure tracks that the failure cycle has been broken and normal operations can resume. + +--- + +### Image Caching + +The infrastructure includes caching to avoid rebuilding images that haven't changed. Caching operates at two levels: + +**1. Matrix Trimming (job-level caching)** + +When `trimCachedImagesForMatrix` is enabled, the `generateBuildMatrix` command excludes platforms from the build matrix if they would result in cache hits. This means no build job is even created for those platforms—they're completely skipped. + +**2. Build-time Caching** + +Even if a platform isn't trimmed from the matrix, the `build` command checks each image against the cache before building. If the image is cached, it outputs `CACHE HIT`, pulls the previously-built image from the registry, and skips the actual Docker build. + +#### Cache Conditions + +An image is considered cached when **both** of the following conditions are true: + +1. **Base image digest is unchanged** — The digest of the base image (FROM image) matches the digest recorded in the image info file from the last successful publish. If the upstream base image has been updated, this condition fails and the image will be rebuilt. + +2. **Dockerfile commit is unchanged** — The git commit URL for the Dockerfile matches the commit URL recorded in the image info file. If you've modified the Dockerfile, this condition fails and the image will be rebuilt. + +Caching compares against the published image info stored in the [versions repo](https://github.com/dotnet/versions). This means caching compares against what's been officially published, not what's in your current branch. + +#### Disabling Caching + +To force a rebuild regardless of cache state, set the `noCache` parameter to `true` when queuing the build. This disables both matrix trimming and build-time caching. + +--- + +## Common Customization Patterns + +### Pattern: Adding Build Arguments + +Pass Dockerfile `ARG` values via ImageBuilder: + +```yaml +customBuildInitSteps: +- powershell: | + $args = "--build-arg MY_VAR=value" + echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$args" +``` + +To pass raw options directly to `docker build`, use `--build-option`. Quote values that contain spaces: + +```yaml +customBuildInitSteps: +- powershell: | + $args = '--build-option "--ulimit nofile=65536:65536"' + echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$args" +``` + +### Pattern: Re-running Stages with `stages` and `sourceBuildPipelineRunId` + +A powerful pattern is combining the `stages` variable with the `sourceBuildPipelineRunId` pipeline parameter to run specific stages using artifacts from a previous build. This is useful for: +1. Skipping stages you don't need to run +2. Avoiding unnecessary re-builds after test/publishing infrastructure fixes + +Note: For simple retries of failed jobs, use the Azure Pipelines UI "Re-run failed jobs" feature instead. + +**Scenario: Test failed, need to run publish anyway** + +* Set `sourceBuildPipelineRunId` to the build which built the images +* Set `stages` to `publish` + +**How it works:** + +1. `sourceBuildPipelineRunId` tells the pipeline which previous run to pull artifacts from +2. The [`download-build-artifact.yml`](templates/steps/download-build-artifact.yml) step uses this ID to fetch `image-info.json` from that run +3. Specified stage(s) use the downloaded image info to know which images exist + +**Common recovery patterns:** + +| Scenario | stages | sourceBuildPipelineRunId | +|----------|--------|--------------------------| +| Normal full build | `"build,test,sign,publish"` | `$(Build.BuildId)` (default) | +| Re-run publish after infra fix | `"publish"` | ID of the successful build run | +| Re-test after infra fix | `"test"` | ID of the build run to test | +| Re-sign after infra fix | `"sign"` | ID of the build run to sign | +| Build only (no publish) | `"build"` | `$(Build.BuildId)` (default) | +| Test + publish (skip build) | `"test,publish"` | ID of the build run | +| Sign + publish (skip build/test) | `"sign,publish"` | ID of the build run | + +**In the Azure DevOps UI:** + +When you queue a new run, you can override these as runtime parameters: +1. Set `stages` to the stage(s) you want to run +2. Set `sourceBuildPipelineRunId` to the run ID containing the artifacts you need (find the build ID in the URL when viewing a pipeline run, e.g., `buildId=123456`) + +This avoids the multi-hour rebuild cycle when you just need to retry a failed operation. + +When signing is enabled, use `"publish"` by itself only if the images from `sourceBuildPipelineRunId` were already signed and the current run is not building new images. Use `"sign,publish"` when the current run still needs to sign them before publishing. + +--- + +## Troubleshooting + +### Why isn't my Dockerfile being built? + +When you trigger a pipeline run, you might find that your Dockerfile isn't being built. + +#### Symptom 1: The Dockerfile isn't included in any build job + +If your Dockerfile doesn't appear in any build job, first verify the Dockerfile is included in the manifest file. + +**How to verify:** Check `manifest.json` to ensure your Dockerfile path is defined under the appropriate repo and image. You can also run `generateBuildMatrix` locally to see which Dockerfiles are included: + +```powershell +./eng/docker-tools/Invoke-ImageBuilder.ps1 "generateBuildMatrix --manifest manifest.json --type platformDependencyGraph" +``` + +**How to fix:** Add the Dockerfile to `manifest.json` under the correct repo, image, and platform configuration. + +#### Symptom 2: The pipeline job isn't running at all + +If the Dockerfile is in the manifest but you don't see a build job for it, the build matrix was likely trimmed due to [matrix trimming](#image-caching). + +**How to verify:** Look at the "Generate platformDependencyGraph Matrix" step output in the `GenerateBuildMatrix` job. This is an example of what the output in that step looks like: + +```yaml +windowsLtsc2025Amd64: + src-windowsservercore-ltsc2025-helix-graph: + imageBuilderPaths: --path src/windowsservercore/ltsc2025/helix/amd64 --path src/windowsservercore/ltsc2025/helix/webassembly-net8/amd64 --path src/windowsservercore/ltsc2025/helix/webassembly/amd64 + legName: windows-ltsc2025amd64src-windowsservercore-ltsc2025-helix-graph + osType: windows + architecture: amd64 + osVersions: --os-version windowsservercore-ltsc2025 +``` + +If your Dockerfile path doesn't appear in any of the matrix legs, it was trimmed. + +**How to fix:** Set the `noCache` parameter to `true` when queuing the build. + +#### Symptom 3: The build output shows `CACHE HIT` + +If your build job runs but you see `CACHE HIT` in the output of the `Build Images` step and the Dockerfile isn't actually built, the [build-time caching](#image-caching) determined that the image doesn't need to be rebuilt. This is an example of what the output in that step looks like: + +``` +Image info's Dockerfile commit: https://github.com/dotnet/dotnet-buildtools-prereqs-docker/blob/aa85f0dcc3b3d6757c80dc8c2a6f38c290b372cc/src/windowsservercore/ltsc2025/helix/amd64/Dockerfile +Latest Dockerfile commit: https://github.com/dotnet/dotnet-buildtools-prereqs-docker/blob/aa85f0dcc3b3d6757c80dc8c2a6f38c290b372cc/src/windowsservercore/ltsc2025/helix/amd64/Dockerfile +Dockerfile commits match: True + +CACHE HIT + +-- EXECUTING: docker pull mcr.microsoft.com/dotnet-buildtools/prereqs@sha256:40d36a0aab610f4d513ed7c7300a5d962968a547ffe8a859a0e599691b74b77f +``` + +**How to fix:** Set the `noCache` parameter to `true` when queuing the build. diff --git a/src/Infrastructure/Content/Dockerfile.WithRepo b/src/Infrastructure/Content/Dockerfile.WithRepo new file mode 100644 index 000000000..d1126b3d1 --- /dev/null +++ b/src/Infrastructure/Content/Dockerfile.WithRepo @@ -0,0 +1,6 @@ +# Use this Dockerfile to create an ImageBuilder image +ARG IMAGE +FROM $IMAGE + +WORKDIR /repo +COPY . . diff --git a/src/Infrastructure/Content/Dockerfile.syft b/src/Infrastructure/Content/Dockerfile.syft new file mode 100644 index 000000000..2e564e2ae --- /dev/null +++ b/src/Infrastructure/Content/Dockerfile.syft @@ -0,0 +1,16 @@ +ARG SYFT_IMAGE_NAME +ARG TARGET_IMAGE_NAME + +FROM ${SYFT_IMAGE_NAME} AS syft +FROM ${TARGET_IMAGE_NAME} AS scan-image + +FROM syft AS run-scan +ARG TARGET_IMAGE_NAME +ENV SYFT_CHECK_FOR_APP_UPDATE=0 \ + SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME} +USER root +RUN --mount=from=scan-image,source=/,target=/rootfs \ + ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"] + +FROM scratch AS output +COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json diff --git a/src/Infrastructure/Content/Get-BaseImageStatus.ps1 b/src/Infrastructure/Content/Get-BaseImageStatus.ps1 new file mode 100644 index 000000000..6b2546265 --- /dev/null +++ b/src/Infrastructure/Content/Get-BaseImageStatus.ps1 @@ -0,0 +1,33 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Outputs the status of external base images referenced in the Dockerfiles. +#> +[cmdletbinding()] +param( + # Path to the manifest file to use + [string] + $Manifest = "manifest.json", + + # Architecture to filter Dockerfiles to + [string] + $Architecture = "*", + + # A value indicating whether to run the script continously + [switch] + $Continuous, + + # Number of seconds to wait between each iteration + [int] + $ContinuousDelay = 10 +) + +Set-StrictMode -Version Latest + +$imageBuilderArgs = "getBaseImageStatus --manifest $Manifest --architecture $Architecture" +if ($Continuous) { + $imageBuilderArgs += " --continuous --continuous-delay $ContinuousDelay" +} + +& "$PSScriptRoot/Invoke-ImageBuilder.ps1" -ImageBuilderArgs $imageBuilderArgs diff --git a/src/Infrastructure/Content/Get-ImageBuilder.ps1 b/src/Infrastructure/Content/Get-ImageBuilder.ps1 new file mode 100644 index 000000000..3eb35240b --- /dev/null +++ b/src/Infrastructure/Content/Get-ImageBuilder.ps1 @@ -0,0 +1,13 @@ +#!/usr/bin/env pwsh + +# Load common image names +$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 +foreach ($varName in $imageNameVars.Keys) { + Set-Variable -Name $varName -Value $imageNameVars[$varName] -Scope Global +} + +& docker inspect ${imageNames.imagebuilderName} | Out-Null +if (-not $?) { + Write-Output "Pulling" + & $PSScriptRoot/Invoke-WithRetry.ps1 "docker pull ${imageNames.imagebuilderName}" +} diff --git a/src/Infrastructure/Content/Get-ImageNameVars.ps1 b/src/Infrastructure/Content/Get-ImageNameVars.ps1 new file mode 100644 index 000000000..0b255c925 --- /dev/null +++ b/src/Infrastructure/Content/Get-ImageNameVars.ps1 @@ -0,0 +1,12 @@ +# Returns a hashtable of variable name-to-value mapping representing the image name variables +# used by the common build infrastructure. + +$vars = @{} +Get-Content $PSScriptRoot/templates/variables/docker-images.yml | + Where-Object { $_.Trim().Length -gt 0 -and $_.Trim() -notlike 'variables:' -and $_.Trim() -notlike '# *' } | + ForEach-Object { + $parts = $_.Split(':', 2) + $vars[$parts[0].Trim()] = $parts[1].Trim() + } + +return $vars diff --git a/src/Infrastructure/Content/Install-DotNetSdk.ps1 b/src/Infrastructure/Content/Install-DotNetSdk.ps1 new file mode 100644 index 000000000..35f6516e4 --- /dev/null +++ b/src/Infrastructure/Content/Install-DotNetSdk.ps1 @@ -0,0 +1,64 @@ +#!/usr/bin/env pwsh +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +<# +.SYNOPSIS +Install the .NET Core SDK at the specified path. + +.PARAMETER InstallPath +The path where the .NET Core SDK is to be installed. + +.PARAMETER Channel +The version of the .NET Core SDK to be installed. + +#> +[cmdletbinding()] +param( + [string] + $InstallPath, + [string] + $Channel = "9.0" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (!(Test-Path "$InstallPath")) { + New-Item -ItemType Directory -Path "$InstallPath" -Force | Out-Null +} + +$IsRunningOnUnix = $PSVersionTable.contains("Platform") -and $PSVersionTable.Platform -eq "Unix" +if ($IsRunningOnUnix) { + $DotnetInstallScript = "dotnet-install.sh" +} +else { + $DotnetInstallScript = "dotnet-install.ps1" +} + +$DotnetInstallScriptPath = Join-Path -Path $InstallPath -ChildPath $DotnetInstallScript + +if (!(Test-Path $DotnetInstallScriptPath)) { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + & "$PSScriptRoot/Invoke-WithRetry.ps1" "Invoke-WebRequest 'https://builds.dotnet.microsoft.com/dotnet/scripts/v1/$DotnetInstallScript' -OutFile $DotnetInstallScriptPath" +} + +$DotnetChannel = $Channel + +$InstallFailed = $false +if ($IsRunningOnUnix) { + & chmod +x $DotnetInstallScriptPath + & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath --channel $DotnetChannel --install-dir $InstallPath" -Retries 5 + $InstallFailed = ($LASTEXITCODE -ne 0) +} +else { + & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath -Channel $DotnetChannel -InstallDir $InstallPath" -Retries 5 + $InstallFailed = (-not $?) +} + +# See https://github.com/NuGet/NuGet.Client/pull/4259 +$Env:NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY = "6,1500" + +if ($InstallFailed) { throw "Failed to install the .NET Core SDK" } diff --git a/src/Infrastructure/Content/Invoke-CleanupDocker.ps1 b/src/Infrastructure/Content/Invoke-CleanupDocker.ps1 new file mode 100644 index 000000000..ad637872c --- /dev/null +++ b/src/Infrastructure/Content/Invoke-CleanupDocker.ps1 @@ -0,0 +1,20 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +docker ps -a -q | ForEach-Object { docker rm -f $_ } + +docker volume prune -f + +# Preserve the tagged Windows base images and the common eng infra images (e.g. ImageBuilder) +# to avoid the expense of having to repull continuously. +$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 + +docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | + Where-Object { + $localImage = $_ + $localImage.Contains(": ")` + -Or -Not ($localImage.StartsWith("mcr.microsoft.com/windows")` + -Or ($imageNameVars.Values.Where({ $localImage.StartsWith($_) }, 'First').Count -gt 0)) } | + ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] } | + Select-Object -Unique | + ForEach-Object { docker rmi -f $_ } diff --git a/src/Infrastructure/Content/Invoke-ImageBuilder.ps1 b/src/Infrastructure/Content/Invoke-ImageBuilder.ps1 new file mode 100644 index 000000000..4cec5ba2b --- /dev/null +++ b/src/Infrastructure/Content/Invoke-ImageBuilder.ps1 @@ -0,0 +1,105 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Executes ImageBuilder with the specified args. + +.PARAMETER ImageBuilderArgs +The args to pass to ImageBuilder. + +.PARAMETER ReuseImageBuilderImage +Indicates that a previously built ImageBuilder image is presumed to exist locally and that +it should be used for this execution of the script. This allows some optimization when +multiple calls are being made to this script that don't require a fresh image (i.e. the +repo contents in the image don't need to be or should not be updated with each call to +this script). + +.PARAMETER OnCommandExecuted +A ScriptBlock that will be invoked after the ImageBuilder command has been executed. +This allows the caller to execute extra logic in the context of the ImageBuilder while +its container is still running. +The ScriptBlock is passed the following argument values: + 1. Container name +#> +[cmdletbinding()] +param( + [string] + $ImageBuilderArgs, + + [switch] + $ReuseImageBuilderImage, + + [scriptblock] + $OnCommandExecuted +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Log { + param ([string] $Message) + + Write-Output $Message +} + +function Exec { + param ([string] $Cmd) + + Log "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + $host.SetShouldExit($LASTEXITCODE) + exit $LASTEXITCODE + throw "Failed: '$Cmd'" + } +} + +$imageBuilderContainerName = "ImageBuilder-$(Get-Date -Format yyyyMMddhhmmss)" +$containerCreated = $false + +pushd $PSScriptRoot/../../ +try { + $activeOS = docker version -f "{{ .Server.Os }}" + if ($activeOS -eq "linux") { + # On Linux, ImageBuilder is run within a container. + $imageBuilderImageName = "microsoft-dotnet-imagebuilder-withrepo" + if ($ReuseImageBuilderImage -ne $True) { + & ./eng/docker-tools/Get-ImageBuilder.ps1 + Exec ("docker build -t $imageBuilderImageName --build-arg " ` + + "IMAGE=${imageNames.imageBuilderName} -f eng/docker-tools/Dockerfile.WithRepo .") + } + + $imageBuilderCmd = "docker run --name $imageBuilderContainerName -v /var/run/docker.sock:/var/run/docker.sock $imageBuilderImageName" + $containerCreated = $true + } + else { + # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container. + # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved + $imageBuilderFolder = ".Microsoft.DotNet.ImageBuilder" + $imageBuilderCmd = [System.IO.Path]::Combine($imageBuilderFolder, "Microsoft.DotNet.ImageBuilder.exe") + if (-not (Test-Path -Path "$imageBuilderCmd" -PathType Leaf)) { + & ./eng/docker-tools/Get-ImageBuilder.ps1 + Exec "docker create --name $imageBuilderContainerName ${imageNames.imageBuilderName}" + $containerCreated = $true + if (Test-Path -Path $imageBuilderFolder) + { + Remove-Item -Recurse -Force -Path $imageBuilderFolder + } + + Exec "docker cp ${imageBuilderContainerName}:/image-builder $imageBuilderFolder" + } + } + + Exec "$imageBuilderCmd $ImageBuilderArgs" + + if ($OnCommandExecuted) { + Invoke-Command $OnCommandExecuted -ArgumentList $imageBuilderContainerName + } +} +finally { + if ($containerCreated) { + Exec "docker container rm -f $imageBuilderContainerName" + } + + popd +} diff --git a/src/Infrastructure/Content/Invoke-WithRetry.ps1 b/src/Infrastructure/Content/Invoke-WithRetry.ps1 new file mode 100644 index 000000000..0aa2c2737 --- /dev/null +++ b/src/Infrastructure/Content/Invoke-WithRetry.ps1 @@ -0,0 +1,41 @@ +#!/usr/bin/env pwsh + +# Executes a command and retries if it fails. +[cmdletbinding()] +param ( + [Parameter(Mandatory = $true)][string]$Cmd, + [int]$Retries = 2, + [int]$WaitFactor = 6 + ) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$count = 0 +$completed = $false + +Write-Output "Executing '$Cmd'" + +while (-not $completed) { + try { + Invoke-Expression $Cmd + if (-not $(Test-Path variable:LASTEXITCODE) -or $LASTEXITCODE -eq 0) { + $completed = $true + continue + } + } + catch { + } + + $count++ + + if ($count -lt $Retries) { + $wait = [Math]::Pow($WaitFactor, $count - 1) + Write-Output "Retry $count/$Retries, retrying in $wait seconds..." + Start-Sleep $wait + } + else { + Write-Output "Retry $count/$Retries, no more retries left." + throw "Failed to execute '$Cmd'" + } +} diff --git a/src/Infrastructure/Content/Pull-Image.ps1 b/src/Infrastructure/Content/Pull-Image.ps1 new file mode 100644 index 000000000..2d0a82cf5 --- /dev/null +++ b/src/Infrastructure/Content/Pull-Image.ps1 @@ -0,0 +1,18 @@ +#!/usr/bin/env pwsh + +[cmdletbinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Image, + + [Parameter(Mandatory = $false)] + [int]$Retries = 2, + + [Parameter(Mandatory = $false)] + [int]$WaitFactor = 6 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor diff --git a/src/Infrastructure/Content/Retain-Build.ps1 b/src/Infrastructure/Content/Retain-Build.ps1 new file mode 100644 index 000000000..7f836e121 --- /dev/null +++ b/src/Infrastructure/Content/Retain-Build.ps1 @@ -0,0 +1,43 @@ +# Adapted from https://github.com/dotnet/arcade/blob/main/eng/docker-tools/retain-build.ps1 +Param( + [Parameter(Mandatory = $true)][int] $BuildId, + [Parameter(Mandatory = $true)][string] $AzdoOrgUri, + [Parameter(Mandatory = $true)][string] $AzdoProject, + [Parameter(Mandatory = $true)][string] $Token +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +function Get-AzDOHeaders( + [string] $Token) { + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${Token}")) + $headers = @{"Authorization" = "Basic $base64AuthInfo" } + return $headers +} + +function Update-BuildRetention( + [string] $AzdoOrgUri, + [string] $AzdoProject, + [int] $BuildId, + [string] $Token) { + $headers = Get-AzDOHeaders -Token $Token + $requestBody = "{ + `"keepForever`": `"true`" + }" + + $requestUri = "${AzdoOrgUri}/${AzdoProject}/_apis/build/builds/${BuildId}?api-version=6.0" + Write-Host "Attempting to retain build using the following URI: ${requestUri} ..." + + try { + Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" + Write-Host "Updated retention settings for build ${BuildId}." + } + catch { + Write-Host "##[error] Failed to update retention settings for build: $($_.Exception.Response.StatusDescription)" + exit 1 + } +} + +Update-BuildRetention -AzdoOrgUri $AzdoOrgUri -AzdoProject $AzdoProject -BuildId $BuildId -Token $Token +exit 0 diff --git a/src/Infrastructure/Content/build.ps1 b/src/Infrastructure/Content/build.ps1 new file mode 100644 index 000000000..13c0806fb --- /dev/null +++ b/src/Infrastructure/Content/build.ps1 @@ -0,0 +1,76 @@ +#!/usr/bin/env pwsh + +<# + .SYNOPSIS + Builds the Dockerfiles +#> + +[cmdletbinding()] +param( + # Product versions to filter by + [string[]]$Version = "*", + + # Names of OS to filter by + [string[]]$OS, + + # Type of architecture to filter by + [string]$Architecture, + + # Additional custom path filters + [string[]]$Paths, + + # Path to manifest file + [string]$Manifest = "manifest.json", + + # Additional args to pass to ImageBuilder + [string]$OptionalImageBuilderArgs +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Log { + param ([string] $Message) + + Write-Output $Message +} + +function Exec { + param ([string] $Cmd) + + Log "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + throw "Failed: '$Cmd'" + } +} + +pushd $PSScriptRoot/../.. +try { + $args = $OptionalImageBuilderArgs + + if ($Version) { + $args += ($Version | foreach { ' --version "{0}"' -f $_ }) + } + + if ($OS) { + $args += ($OS | foreach { ' --os-version "{0}"' -f $_ }) + } + + if ($Architecture) { + $args += ' --architecture "{0}"' -f $Architecture + } + + if ($Paths) { + $args += ($Paths | foreach { ' --path "{0}"' -f $_ }) + } + + if ($Manifest) { + $args += ' --manifest "{0}"' -f $Manifest + } + + ./eng/docker-tools/Invoke-ImageBuilder.ps1 "build $args" +} +finally { + popd +} diff --git a/src/Infrastructure/Content/readme.md b/src/Infrastructure/Content/readme.md new file mode 100644 index 000000000..11d6717ec --- /dev/null +++ b/src/Infrastructure/Content/readme.md @@ -0,0 +1,30 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all .NET Docker repos. If you need to make changes to these files, open an issue or submit a pull request in https://github.com/dotnet/docker-tools. + +For guidance on using this infrastructure, see the [Developer Guide](DEV-GUIDE.md). \ No newline at end of file diff --git a/src/Infrastructure/Content/skill-helpers/AzureDevOps.ps1 b/src/Infrastructure/Content/skill-helpers/AzureDevOps.ps1 new file mode 100644 index 000000000..fd30c92ef --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/AzureDevOps.ps1 @@ -0,0 +1,85 @@ +#!/usr/bin/env pwsh +# Lightweight wrapper for authenticated Azure DevOps REST API calls. +# Uses `az account get-access-token` for bearer token auth. +# +# Usage: +# . ./AzureDevOps.ps1 +# $response = Invoke-AzDORestMethod -Organization myorg -Project myproject ` +# -Endpoint "pipelines/42/runs" -Method POST -Body @{ resources = @{} } + +$ErrorActionPreference = "Stop" + +function Get-AzDOAccessToken { + <# + .SYNOPSIS + Returns a bearer token for Azure DevOps. + #> + + # Well-known Entra ID application ID for Azure DevOps + $tokenJson = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Failed to get access token. Run 'az login' first. Output: $tokenJson" + } + + $parsed = $tokenJson | ConvertFrom-Json + return $parsed.accessToken +} + +function Invoke-AzDORestMethod { + <# + .SYNOPSIS + Calls an Azure DevOps REST API endpoint with automatic authentication. + .PARAMETER Organization + Azure DevOps organization name (not the full URL). + .PARAMETER Project + Azure DevOps project name. + .PARAMETER Endpoint + API path after _apis/ (e.g. "pipelines/42/runs", "build/builds"). + .PARAMETER Method + HTTP method. Defaults to GET. + .PARAMETER Body + Request body as a hashtable. Automatically converted to JSON. + .PARAMETER ApiVersion + API version. Defaults to 7.1. + .PARAMETER QueryParams + Optional hashtable of additional query string parameters. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string] $Organization, + [Parameter(Mandatory)][string] $Project, + [Parameter(Mandatory)][string] $Endpoint, + [string] $Method = "GET", + [hashtable] $Body, + [string] $ApiVersion = "7.1", + [hashtable] $QueryParams + ) + + $token = Get-AzDOAccessToken + $headers = @{ + Authorization = "Bearer $token" + "Content-Type" = "application/json" + } + + $query = "api-version=$ApiVersion" + if ($QueryParams) { + foreach ($key in $QueryParams.Keys) { + $value = [System.Uri]::EscapeDataString([string]$QueryParams[$key]) + $query += "&$key=$value" + } + } + + $uri = "https://dev.azure.com/$Organization/$Project/_apis/$($Endpoint)?$query" + + $params = @{ + Uri = $uri + Headers = $headers + Method = $Method + } + + if ($Body) { + $params.Body = $Body | ConvertTo-Json -Depth 10 + } + + return Invoke-RestMethod @params +} diff --git a/src/Infrastructure/Content/skill-helpers/Get-BuildLog.ps1 b/src/Infrastructure/Content/skill-helpers/Get-BuildLog.ps1 new file mode 100755 index 000000000..1c430ad18 --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/Get-BuildLog.ps1 @@ -0,0 +1,21 @@ +#!/usr/bin/env pwsh +# Retrieves a build log by log ID. +# Usage: +# ./Get-BuildLog.ps1 -Organization dnceng -Project internal -BuildId 12345 -LogId 47 + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $Organization, + [Parameter(Mandatory)][string] $Project, + [Parameter(Mandatory)][int] $BuildId, + [Parameter(Mandatory)][int] $LogId +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot/AzureDevOps.ps1" + +Invoke-AzDORestMethod ` + -Organization $Organization ` + -Project $Project ` + -Endpoint "build/builds/$BuildId/logs/$LogId" diff --git a/src/Infrastructure/Content/skill-helpers/Get-FailingPipelines.ps1 b/src/Infrastructure/Content/skill-helpers/Get-FailingPipelines.ps1 new file mode 100755 index 000000000..cfc3750c9 --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/Get-FailingPipelines.ps1 @@ -0,0 +1,63 @@ +#!/usr/bin/env pwsh +# Lists pipeline definitions in a folder whose most recent completed build did not succeed. +# Usage: +# ./Get-FailingPipelines.ps1 -Organization dnceng -Project internal -Folder dotnet/docker-tools + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $Organization, + [Parameter(Mandatory)][string] $Project, + [Parameter(Mandatory)][string] $Folder, + [switch] $IncludeWarnings +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot/AzureDevOps.ps1" + +# Normalize folder: accept "dotnet/docker-tools" or "\dotnet\docker-tools". +$normalizedFolder = "\" + ($Folder.Trim('\', '/') -replace '/', '\') + +$failingResults = @("failed", "canceled") +if ($IncludeWarnings) { + $failingResults += "partiallySucceeded" +} + +$definitions = Invoke-AzDORestMethod ` + -Organization $Organization ` + -Project $Project ` + -Endpoint "build/definitions" ` + -QueryParams @{ + path = $normalizedFolder + includeLatestBuilds = "true" + } + +$failing = @() +foreach ($def in $definitions.value) { + $latest = $def.latestCompletedBuild + if (-not $latest) { continue } + if ($failingResults -notcontains $latest.result) { continue } + + $failing += [pscustomobject]@{ + Definition = $def.name + Result = $latest.result + BuildId = $latest.id + BuildNumber = $latest.buildNumber + Branch = $latest.sourceBranch + FinishTime = $latest.finishTime + Url = "https://dev.azure.com/$Organization/$Project/_build/results?buildId=$($latest.id)" + } +} + +Write-Host "## Failing pipelines in $normalizedFolder" +Write-Host "" +Write-Host "Found $($failing.Count) of $($definitions.value.Count) pipeline(s) with a failing latest run." +Write-Host "" + +if ($failing.Count -gt 0) { + Write-Host "Pipeline | Result | Build | Branch | Finished | Link" + Write-Host "--- | --- | --- | --- | --- | ---" + foreach ($item in $failing | Sort-Object Definition) { + Write-Host "$($item.Definition) | $($item.Result) | $($item.BuildId) | $($item.Branch) | $($item.FinishTime) | $($item.Url)" + } +} diff --git a/src/Infrastructure/Content/skill-helpers/Get-RecentBuilds.ps1 b/src/Infrastructure/Content/skill-helpers/Get-RecentBuilds.ps1 new file mode 100644 index 000000000..24d14ccb9 --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/Get-RecentBuilds.ps1 @@ -0,0 +1,58 @@ +#!/usr/bin/env pwsh +# Lists all build runs in the last N hours for pipelines under a given folder. +# Usage: +# ./Get-RecentBuilds.ps1 -Organization dnceng -Project internal -Folder dotnet/docker-tools +# ./Get-RecentBuilds.ps1 -Organization dnceng -Project internal -Folder dotnet/docker-tools -Hours 48 + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $Organization, + [Parameter(Mandatory)][string] $Project, + [Parameter(Mandatory)][string] $Folder, + [int] $Hours = 24 +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot/AzureDevOps.ps1" + +$normalizedFolder = "\" + ($Folder.Trim('\', '/') -replace '/', '\') +$minTime = [DateTime]::UtcNow.AddHours(-$Hours).ToString("o") + +$definitions = Invoke-AzDORestMethod ` + -Organization $Organization ` + -Project $Project ` + -Endpoint "build/definitions" ` + -QueryParams @{ path = $normalizedFolder } + +if (-not $definitions.value -or $definitions.value.Count -eq 0) { + Write-Host "## No pipelines found in $normalizedFolder" + return +} + +$definitionIds = ($definitions.value | ForEach-Object { $_.id }) -join "," + +$builds = Invoke-AzDORestMethod ` + -Organization $Organization ` + -Project $Project ` + -Endpoint "build/builds" ` + -QueryParams @{ + definitions = $definitionIds + minTime = $minTime + queryOrder = "finishTimeDescending" + } + +Write-Host "## Builds in $normalizedFolder (last $Hours hours)" +Write-Host "" +Write-Host "Found $($builds.value.Count) build(s) across $($definitions.value.Count) pipeline(s)." +Write-Host "" + +if ($builds.value.Count -gt 0) { + Write-Host "Pipeline | State | Build | Branch | Finished | Link" + Write-Host "--- | --- | --- | --- | --- | ---" + foreach ($build in $builds.value) { + $state = if ($build.status -eq "completed") { $build.result } else { $build.status } + $url = "https://dev.azure.com/$Organization/$Project/_build/results?buildId=$($build.id)" + Write-Host "$($build.definition.name) | $state | $($build.id) | $($build.sourceBranch) | $($build.finishTime) | $url" + } +} diff --git a/src/Infrastructure/Content/skill-helpers/Show-BuildTimeline.ps1 b/src/Infrastructure/Content/skill-helpers/Show-BuildTimeline.ps1 new file mode 100755 index 000000000..7de331e59 --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/Show-BuildTimeline.ps1 @@ -0,0 +1,77 @@ +#!/usr/bin/env pwsh +# Prints the build timeline as an indented tree with result indicators. +# Usage: +# ./Show-BuildTimeline.ps1 -Organization dnceng -Project internal -BuildId 12345 +# ./Show-BuildTimeline.ps1 -Organization dnceng -Project internal -BuildId 12345 -ShowAllTasks + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $Organization, + [Parameter(Mandatory)][string] $Project, + [Parameter(Mandatory)][int] $BuildId, + [switch] $ShowAllTasks +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot/AzureDevOps.ps1" + +$build = Invoke-AzDORestMethod ` + -Organization $Organization ` + -Project $Project ` + -Endpoint "build/builds/$BuildId" + +Write-Host "## Build $BuildId - $($build.definition.name)" +Write-Host "" +Write-Host "- Status: $($build.status) $(if ($build.result) { "($($build.result))" })" +Write-Host "- Branch: $($build.sourceBranch)" +Write-Host "- Queued: $($build.queueTime)" +Write-Host "- URL: $($build._links.web.href)" +Write-Host "" + +$timeline = Invoke-AzDORestMethod ` + -Organization $Organization ` + -Project $Project ` + -Endpoint "build/builds/$BuildId/timeline" + +$records = $timeline.records + +# Build a lookup of children grouped by parentId +$childrenOf = @{} +foreach ($record in $records) { + $parentId = $record.parentId + if (-not $parentId) { $parentId = "" } + if (-not $childrenOf.ContainsKey($parentId)) { + $childrenOf[$parentId] = [System.Collections.Generic.List[object]]::new() + } + $childrenOf[$parentId].Add($record) +} + +# Sort children by order within each group +foreach ($key in @($childrenOf.Keys)) { + $childrenOf[$key] = $childrenOf[$key] | Sort-Object { $_.order } +} + +function Write-TimelineNode([string] $nodeId, [int] $depth) { + $children = $childrenOf[$nodeId] + if (-not $children) { return } + + foreach ($child in $children) { + $isTask = $child.type -eq "Task" + $isFailing = $child.result -in @("failed", "canceled", "abandoned") -or $child.state -eq "inProgress" + if ($isTask -and -not $ShowAllTasks -and -not $isFailing) { continue } + + $indent = " " * $depth + $status = if ($child.result) { $child.result } else { $child.state } + + $logId = $child.log.id + $logLabel = if ($logId) { " #$logId" } else { "" } + Write-Host "${indent}- $($child.type)$logLabel | $($child.name) | $status" + Write-TimelineNode $child.id ($depth + 1) + } +} + +Write-Host "### Build Timeline" +Write-Host "" +Write-TimelineNode "" 0 +Write-Host "" diff --git a/src/Infrastructure/Content/skill-helpers/Show-PullRequestBuilds.ps1 b/src/Infrastructure/Content/skill-helpers/Show-PullRequestBuilds.ps1 new file mode 100644 index 000000000..d49224e0d --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/Show-PullRequestBuilds.ps1 @@ -0,0 +1,99 @@ +#!/usr/bin/env pwsh +# Shows all PR checks as a summary table, then expands AzDO build timelines for any +# checks that point at Azure Pipelines (https://dev.azure.com/...). +# Requires `gh` CLI authenticated against the target repo. +# +# Usage: +# ./Show-PullRequestBuilds.ps1 -PullRequest 2100 +# ./Show-PullRequestBuilds.ps1 -PullRequest 2100 -Repo dotnet/docker-tools +# ./Show-PullRequestBuilds.ps1 -PullRequest 2100 -ShowAllTasks + +[CmdletBinding()] +param( + [Parameter(Mandatory)][int] $PullRequest, + [string] $Repo, + [switch] $ShowAllTasks +) + +$ErrorActionPreference = "Stop" + +$ghArgs = @("pr", "view", $PullRequest, "--json", "statusCheckRollup") +if ($Repo) { $ghArgs += @("--repo", $Repo) } + +$checksJson = & gh @ghArgs 2>&1 +if ($LASTEXITCODE -ne 0) { + throw "gh pr view failed: $checksJson" +} + +$checks = ($checksJson | ConvertFrom-Json).statusCheckRollup + +# statusCheckRollup mixes two shapes: +# CheckRun: { name, status, conclusion, detailsUrl, workflowName } +# StatusContext: { context, state, targetUrl, description } +# Normalize them. +$normalized = foreach ($check in $checks) { + if ($check.PSObject.Properties.Name -contains "context") { + [pscustomobject]@{ + Name = $check.context + State = $check.state + Url = $check.targetUrl + } + } + else { + $state = if ($check.conclusion) { $check.conclusion } else { $check.status } + [pscustomobject]@{ + Name = $check.name + State = $state + Url = $check.detailsUrl + } + } +} + +# AzDO build results URLs look like: +# https://dev.azure.com///_build/results?buildId=... +$pattern = '^https?://dev\.azure\.com/(?[^/]+)/(?[^/]+)/_build/results\?.*buildId=(?\d+)' + +$builds = @() +foreach ($check in $normalized) { + if (-not $check.Url) { continue } + $match = [regex]::Match($check.Url, $pattern) + if (-not $match.Success) { continue } + + $builds += [pscustomobject]@{ + Org = $match.Groups["org"].Value + Project = $match.Groups["project"].Value + BuildId = [int]$match.Groups["buildId"].Value + } +} + +# Deduplicate by buildId (a single build can produce multiple check-run rows). +$builds = $builds | Sort-Object BuildId -Unique + +$title = if ($Repo) { "$Repo#$PullRequest" } else { "PR #$PullRequest" } +Write-Host "## Checks for $title" +Write-Host "" +Write-Host "$($normalized.Count) check(s); $($builds.Count) Azure Pipelines build(s)." +Write-Host "" + +if ($normalized.Count -gt 0) { + Write-Host "Check | State | URL" + Write-Host "--- | --- | ---" + foreach ($check in $normalized | Sort-Object Name) { + Write-Host "$($check.Name) | $($check.State) | $($check.Url)" + } + Write-Host "" +} + +if ($builds.Count -eq 0) { return } + +$timelineScript = "$PSScriptRoot/Show-BuildTimeline.ps1" + +foreach ($build in $builds) { + Write-Host "---" + Write-Host "" + & $timelineScript ` + -Organization $build.Org ` + -Project $build.Project ` + -BuildId $build.BuildId ` + -ShowAllTasks:$ShowAllTasks +} diff --git a/src/Infrastructure/Content/skill-helpers/Show-PullRequestComments.ps1 b/src/Infrastructure/Content/skill-helpers/Show-PullRequestComments.ps1 new file mode 100755 index 000000000..acb1d59bd --- /dev/null +++ b/src/Infrastructure/Content/skill-helpers/Show-PullRequestComments.ps1 @@ -0,0 +1,117 @@ +#!/usr/bin/env pwsh +# Shows a focused summary of a pull request: metadata, reviews, issue-level comments, +# and inline review comments (which `gh pr view --json` does not expose). +# Requires `gh` CLI authenticated against the target repo. +# +# Usage: +# ./Show-PullRequestComments.ps1 2100 +# ./Show-PullRequestComments.ps1 2100 -Repo dotnet/docker-tools + +[CmdletBinding()] +param( + [Parameter(Mandatory)][int] $PullRequest, + [string] $Repo +) + +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true + +function Write-BlockComment { + param([string] $Text) + if (-not $Text) { return } + $Text.TrimEnd() -split "`n" | ForEach-Object { Write-Host "> $_" } + Write-Host "" +} + +# Fetch PR overview. +$viewArgs = @( + "pr", "view", $PullRequest, + "--json", "number,title,state,author,baseRefName,headRefName,isDraft,url,additions,deletions,changedFiles,reviewDecision,labels,reviews,comments" +) +if ($Repo) { $viewArgs += @("--repo", $Repo) } + +$prJson = & gh @viewArgs +$pr = $prJson | ConvertFrom-Json + +# Fetch inline review comments via REST API. `gh pr view --json` exposes review bodies +# but drops the inline diff comments attached to specific files/lines. +$apiPath = if ($Repo) { + "repos/$Repo/pulls/$PullRequest/comments" +} else { + "repos/{owner}/{repo}/pulls/$PullRequest/comments" +} + +$inlineJson = & gh api --paginate $apiPath +$inline = $inlineJson | ConvertFrom-Json + +# Render. +$title = if ($Repo) { "$Repo#$PullRequest" } else { "PR #$PullRequest" } +Write-Host "## $title - $($pr.title)" +Write-Host "" +Write-Host "- State: $($pr.state)$(if ($pr.isDraft) { ' (draft)' })" +Write-Host "- Author: $($pr.author.login)" +Write-Host "- Branch: $($pr.headRefName) -> $($pr.baseRefName)" +Write-Host "- Changes: +$($pr.additions)/-$($pr.deletions) ($($pr.changedFiles) files)" +Write-Host "- Review decision: $($pr.reviewDecision)" +if ($pr.labels) { + Write-Host "- Labels: $(($pr.labels | ForEach-Object { $_.name }) -join ', ')" +} +Write-Host "- URL: $($pr.url)" +Write-Host "" + +# Conversation: top-level issue comments and review submissions, merged in +# chronological order. +$conversation = @() +foreach ($comment in $pr.comments) { + $conversation += [pscustomobject]@{ + Timestamp = [datetime]$comment.createdAt + Header = "**$($comment.author.login)** commented at $($comment.createdAt):" + Body = $comment.body + } +} +foreach ($review in $pr.reviews) { + $header = if ($review.state -eq "COMMENTED") { + "**$($review.author.login)** left a comment at $($review.submittedAt):" + } else { + "**$($review.author.login)** reviewed ($($review.state)) at $($review.submittedAt):" + } + $conversation += [pscustomobject]@{ + Timestamp = [datetime]$review.submittedAt + Header = $header + Body = $review.body + } +} +$conversation = $conversation | Sort-Object Timestamp + +Write-Host "### Conversation ($($conversation.Count))" +Write-Host "" +if ($conversation.Count -eq 0) { + Write-Host "_None_" +} else { + foreach ($entry in $conversation) { + Write-Host $entry.Header + Write-BlockComment $entry.Body + } +} +Write-Host "" + +# Inline review comments grouped by file and line. +Write-Host "### Code review comments ($($inline.Count))" +Write-Host "" +if ($inline.Count -eq 0) { + Write-Host "_None_" +} else { + $grouped = $inline | + Group-Object -Property { "$($_.path):$(if ($_.line) { $_.line } else { $_.original_line })" } | + Sort-Object Name + foreach ($group in $grouped) { + $first = $group.Group[0] + $line = if ($first.line) { $first.line } else { $first.original_line } + Write-Host "#### $($first.path) (line $line)" + Write-Host "" + foreach ($comment in $group.Group | Sort-Object created_at) { + Write-Host "**$($comment.user.login)** commented at $($comment.created_at):" + Write-BlockComment $comment.body + } + } +} diff --git a/src/Infrastructure/Content/templates/1es-official.yml b/src/Infrastructure/Content/templates/1es-official.yml new file mode 100644 index 000000000..8eb33d656 --- /dev/null +++ b/src/Infrastructure/Content/templates/1es-official.yml @@ -0,0 +1,62 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. +# +# The `cgDryRun` parameter will run CG but not submit the results, for testing purposes. + +parameters: +- name: cgDryRun + type: boolean + default: false +- name: stages + type: stageList + default: [] +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + whatIf: ${{ parameters.cgDryRun }} + showAlertLink: true + policheck: + enabled: true + sourceRepositoriesToScan: + exclude: + - repository: VersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/src/Infrastructure/Content/templates/1es-unofficial.yml b/src/Infrastructure/Content/templates/1es-unofficial.yml new file mode 100644 index 000000000..c4a849c53 --- /dev/null +++ b/src/Infrastructure/Content/templates/1es-unofficial.yml @@ -0,0 +1,70 @@ +# This unofficial template will always run CG in "what if" mode, which will not submit results to the CG. SDL tools may +# also be disabled for testing purposes. +# +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: disableSDL + type: boolean + default: false + displayName: Disable SDL +- name: stages + type: stageList + default: [] +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + # Use a unique task prefix for unofficial pipelines + taskPrefix: "🟦" + baseTemplate: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + whatIf: true + showAlertLink: true + enableAllTools: ${{ not(parameters.disableSDL) }} + policheck: + enabled: true + sbom: + enabled: true + sourceRepositoriesToScan: + exclude: + - repository: InternalVersionsRepo + - repository: PublicVersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/src/Infrastructure/Content/templates/1es.yml b/src/Infrastructure/Content/templates/1es.yml new file mode 100644 index 000000000..4056610e4 --- /dev/null +++ b/src/Infrastructure/Content/templates/1es.yml @@ -0,0 +1,86 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: stages + type: stageList + default: [] +# List of repositories that will be excluded from SDL scanning. This should +# only be used when including other repos without building their source code. +# E.g. for the dotnet/versions repo. +- name: reposToExcludeFromScanning + type: object + default: [] +# The pool that will be used for initializing service connections. +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# The pool that will be used for SDL jobs. +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows +# Container image SBOMs are generated manually during the build job. 1ESPT's +# automatic SBOM generation only adds unnecessary steps and artifacts to +# builds. SBOM is not needed for JSON outputs. If a pipeline outputs binary +# artifacts that ship to customers, then set this parameter to true. +- name: enableSbom + type: boolean + default: false +# Network isolation policy that will be enabled for jobs. The default policy +# allows all outbound connections except for public package feeds and known +# malicious endpoints. If this policy breaks the build, then it can be set to +# "Permissive" temporarily until external dependencies are resolved. +# See the network isolation documentation for more details: +# https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-build/cloudbuild/security/1espt-network-isolation +- name: networkIsolationPolicy + type: string + default: Permissive,CFSClean + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + settings: + networkIsolationPolicy: ${{ parameters.networkIsolationPolicy }} + sdl: + sbom: + enabled: ${{ parameters.enableSbom }} + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + policheck: + enabled: true + ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}: + sourceRepositoriesToScan: + exclude: + - ${{ each repo in parameters.reposToExcludeFromScanning }}: + - repository: ${{ repo }} + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/src/Infrastructure/Content/templates/jobs/build-images.yml b/src/Infrastructure/Content/templates/jobs/build-images.yml new file mode 100644 index 000000000..7327b6d69 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/build-images.yml @@ -0,0 +1,143 @@ +parameters: + name: null + pool: {} + matrix: {} + dockerClientOS: null + buildJobTimeout: 60 + # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). + # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. + customInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before the build starts. + # Use for build-specific initialization (e.g., setting variables, additional setup). + customBuildInitSteps: [] + publishConfig: null + versionsRepoRef: "" + noCache: false + internalProjectName: null + publicProjectName: null + storageAccountServiceConnection: null + +jobs: +- job: ${{ parameters.name }} + condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + dependsOn: + - PreBuildValidation + - CopyBaseImages + - GenerateBuildMatrix + pool: ${{ parameters.pool }} + strategy: + matrix: $[ ${{ parameters.matrix }} ] + timeoutInMinutes: ${{ parameters.buildJobTimeout }} + variables: + imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions) + sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom + imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo + imageInfoContainerDir: $(artifactsPath)/imageInfo + steps: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: ${{ parameters.dockerClientOS }} + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + cleanupDocker: true + customInitSteps: ${{ parameters.customInitSteps }} + - ${{ parameters.customBuildInitSteps }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + dockerClientOS: ${{ parameters.dockerClientOS }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + # Check .name instead of the whole object - null parameters can become + # empty objects through template layers, making ${{ if }} truthy. + ${{ if parameters.storageAccountServiceConnection.name }}: + serviceConnections: + - name: ${{ parameters.storageAccountServiceConnection.name }} + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self + parameters: + publicSourceBranch: $(publicSourceBranch) + - powershell: echo "##vso[task.setvariable variable=imageBuilderBuildArgs]" + condition: eq(variables.imageBuilderBuildArgs, '') + displayName: Initialize Image Builder Build Args + - powershell: | + New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force + + # Reference the existing imageBuilderBuildArgs variable as an environment variable rather than injecting it directly + # with the $(imageBuilderBuildArgs) syntax. This is to avoid issues where the string may contain single quotes $ chars + # which really mess up assigning to a variable. It would require assigning the string with single quotes but also needing + # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since + # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates + # the environment variable for us. + $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" + if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} --push" + } + + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)" + } + + echo "imageBuilderBuildArgs: $imageBuilderBuildArgs" + echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs" + displayName: Set Image Builder Build Args + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + name: BuildImages + displayName: Build Images + ${{ if parameters.storageAccountServiceConnection }}: + serviceConnections: + - name: storage + id: ${{ parameters.storageAccountServiceConnection.id }} + tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }} + clientId: ${{ parameters.storageAccountServiceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + dockerClientOS: ${{ parameters.dockerClientOS }} + args: >- + build + --manifest $(manifest) + $(imageBuilderPaths) + $(osVersions) + --os-type $(osType) + --architecture $(architecture) + --retry + --digests-out-var 'builtImages' + $(manifestVariables) + $(imageBuilderBuildArgs) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(imageInfoHostDir) + artifactName: $(legName)-image-info-$(System.JobAttempt) + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: + - powershell: | + $images = "$(BuildImages.builtImages)" + if (-not $images) { return 0 } + $syftImageName = "${{ parameters.publishConfig.PublicMirrorRegistry.server }}/$(imageNames.syft)" + & $(engDockerToolsPath)/Pull-Image.ps1 $syftImageName + $images -Split ',' | ForEach-Object { + echo "Generating SBOM for $_"; + $targetImageName = "$_"; + $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.BuildRegistry.server }}/${{ parameters.publishConfig.BuildRegistry.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); + $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; + New-Item -Type Directory -Path $sbomChildDir > $null; + docker build --output=$sbomChildDir -f $(engDockerToolsPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engDockerToolsPath); + } + displayName: Generate SBOMs + condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/docker-tools/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self + parameters: + condition: ne(variables.testScriptPath, '') + skipCommonInit: true + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(sbomDirectory) + artifactName: $(legName)-sboms + displayName: Publish SBOM + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + condition: ne(variables['BuildImages.builtImages'], '') diff --git a/src/Infrastructure/Content/templates/jobs/cg-build-projects.yml b/src/Infrastructure/Content/templates/jobs/cg-build-projects.yml new file mode 100644 index 000000000..10d50689f --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/cg-build-projects.yml @@ -0,0 +1,70 @@ +# This job builds all projects in the repository. It is intended to be used for CG purposes. +# The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they +# need to be built outside of Dockerfiles. +parameters: +# Setting cgDryRun will run CG but not submit the results +- name: cgDryRun + type: boolean + default: false + displayName: CG Dry Run +# When true, the job skips the .NET SDK installation. +- name: skipDotNetInstall + type: boolean + default: false + displayName: Skip .NET SDK Installation +# See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values +- name: dotnetVersionChannel + type: string + default: '9.0' + displayName: .NET Version +# Additional steps to run before building projects (e.g. custom SDK installation). +- name: initSteps + type: stepList + default: [] + +jobs: +- job: BuildProjects + displayName: Build Projects + pool: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux + steps: + - ${{ each step in parameters.initSteps }}: + - ${{ step }} + - ${{ if eq(parameters.skipDotNetInstall, false) }}: + - powershell: > + ./eng/docker-tools/Install-DotNetSdk.ps1 -Channel ${{ parameters.dotnetVersionChannel }} -InstallPath "$(Build.SourcesDirectory)/.dotnet" + displayName: Install .NET SDK + - script: > + find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 $(Build.SourcesDirectory)/.dotnet/dotnet build + displayName: Build Projects + + # Component Detection is only automatically run on production branches. + # To run Component Detection on non-production branches, the task must be manually injected. + - ${{ if eq(parameters.cgDryRun, true) }}: + - powershell: | + Write-Host "##vso[build.updatebuildnumber]$env:BUILD_BUILDNUMBER (Dry run)" + Write-Host "##vso[build.addbuildtag]dry-run" + + if ("$(officialBranches)".Split(',').Contains("$(Build.SourceBranch)")) + { + Write-Host "##vso[task.logissue type=error]Cannot run a CG dry-run build from an official branch ($(officialBranches))." + Write-Host "##vso[task.logissue type=error]Run the pipeline again from a different branch to avoid registering scan results." + exit 1 + } + displayName: Update Build Number + - task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (manually injected) + inputs: + # Running CG with `whatIf: true` or `scanType: LogOnly` outputs a list of detected components, but doesn't show + # which components would trigger an alert on a production build. + # As long as the build isn't ran on a tracked branch (typically main or nightly), it's OK to submit components + # to CG for analysis. Only the results for tracked branches matter. + scanType: Register + whatIf: false + alertWarningLevel: Low + failOnAlert: false + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + timeoutInMinutes: 10 diff --git a/src/Infrastructure/Content/templates/jobs/copy-base-images-staging.yml b/src/Infrastructure/Content/templates/jobs/copy-base-images-staging.yml new file mode 100644 index 000000000..02c6cc84e --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/copy-base-images-staging.yml @@ -0,0 +1,40 @@ +parameters: +- name: name + type: string + default: null +- name: pool + type: object + default: {} +- name: publishConfig + type: object + default: null +# Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). +# Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. +- name: customInitSteps + type: stepList + default: [] +# Custom steps that run after ImageBuilder is set up but before copy-base-images runs. +- name: customCopyBaseImagesInitSteps + type: stepList + default: [] +- name: additionalOptions + type: string + default: '' +- name: continueOnError + type: string + default: false +- name: versionsRepoRef + type: string + default: "" + +jobs: +- template: /eng/docker-tools/templates/jobs/copy-base-images.yml@self + parameters: + name: ${{ parameters.name }} + pool: ${{ parameters.pool }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + additionalOptions: ${{ parameters.additionalOptions }} + acr: ${{ parameters.publishConfig.InternalMirrorRegistry }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/src/Infrastructure/Content/templates/jobs/copy-base-images.yml b/src/Infrastructure/Content/templates/jobs/copy-base-images.yml new file mode 100644 index 000000000..4a5ed5e88 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/copy-base-images.yml @@ -0,0 +1,57 @@ +parameters: +- name: name + type: string + default: null +- name: pool + type: object + default: {} +- name: publishConfig + type: object + default: null +- name: acr + type: object + default: null +# Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). +# Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. +- name: customInitSteps + type: stepList + default: [] +# Custom steps that run after ImageBuilder is set up but before copy-base-images runs. +- name: customCopyBaseImagesInitSteps + type: stepList + default: [] +- name: additionalOptions + type: string + default: '' +- name: continueOnError + type: string + default: false +- name: forceDryRun + type: boolean + default: false +- name: versionsRepoRef + type: string + default: "" + +jobs: +- job: ${{ parameters.name }} + pool: ${{ parameters.pool }} + steps: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.acr.server }} + - ${{ parameters.customCopyBaseImagesInitSteps }} + - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self + parameters: + acr: ${{ parameters.acr }} + additionalOptions: ${{ parameters.additionalOptions }} + continueOnError: ${{ parameters.continueOnError }} + forceDryRun: ${{ parameters.forceDryRun }} diff --git a/src/Infrastructure/Content/templates/jobs/generate-matrix.yml b/src/Infrastructure/Content/templates/jobs/generate-matrix.yml new file mode 100644 index 000000000..fe668c65e --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/generate-matrix.yml @@ -0,0 +1,82 @@ +parameters: + matrixType: null + name: null + pool: {} + customBuildLegGroupArgs: "" + isTestStage: false + internalProjectName: null + noCache: false + publishConfig: null + # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). + # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. + customInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before matrix generation runs. + customGenerateMatrixInitSteps: [] + versionsRepoRef: "" + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + pool: ${{ parameters.pool }} + steps: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + # When --trim-cached-images is active, ImageBuilder checks base image digests + # in the ACR mirror registry, which requires OIDC auth via this service connection. + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + - ${{ parameters.customGenerateMatrixInitSteps }} + - template: /eng/docker-tools/templates/steps/retain-build.yml@self + - template: /eng/docker-tools/templates/steps/validate-branch.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml + parameters: + publicSourceBranch: $(publicSourceBranch) + - ${{ if eq(parameters.isTestStage, true) }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - powershell: | + $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)" + + if ("${{ parameters.isTestStage}}" -eq "true") { + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" + } + elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" + } + + echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions" + displayName: Set GenerateBuildMatrix Variables + - script: > + echo "##vso[task.setvariable variable=generateBuildMatrixCommand] + generateBuildMatrix + --manifest $(manifest) + --type ${{ parameters.matrixType }} + --os-type '*' + --architecture '*' + --product-version-components $(productVersionComponents) + ${{ parameters.customBuildLegGroupArgs }} + $(imageBuilder.pathArgs) + $(manifestVariables) + $(commonMatrixAndBuildOptions) + $(additionalGenerateBuildMatrixOptions)" + displayName: Set GenerateBuildMatrix Command + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + name: matrix + displayName: Generate ${{ parameters.matrixType }} Matrix + internalProjectName: internal + args: $(generateBuildMatrixCommand) diff --git a/src/Infrastructure/Content/templates/jobs/post-build.yml b/src/Infrastructure/Content/templates/jobs/post-build.yml new file mode 100644 index 000000000..32b9c7999 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/post-build.yml @@ -0,0 +1,114 @@ +parameters: + pool: {} + internalProjectName: null + publicProjectName: null + customInitSteps: [] + publishConfig: null + +jobs: +- job: Build + pool: ${{ parameters.pool }} + variables: + imageInfosSubDir: "/image-infos" + imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)" + imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)" + imageInfosOutputSubDir: "/output" + sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom" + steps: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + - ${{ if parameters.publishConfig }}: + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + # This can fail if no build jobs ran to produce any artifacts + continueOnError: true + - powershell: | + # Move all image-info artifacts to their own directory + New-Item -ItemType Directory -Path $(imageInfosHostDir) + Get-ChildItem -Directory -Filter "*-image-info-*" $(Build.ArtifactStagingDirectory) | + Move-Item -Verbose -Destination $(imageInfosHostDir) + displayName: Collect Image Info Files + - powershell: | + # Move the contents of all the SBOM artifact directories to a single location + New-Item -ItemType Directory -Path $(sbomOutputDir) + Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) | + ForEach-Object { + Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(sbomOutputDir) + } + displayName: Consolidate SBOMs to Single Directory + - powershell: | + # Deletes the artifacts from all the unsuccessful jobs + Get-ChildItem $(imageInfosHostDir) -Directory | + ForEach-Object { + [pscustomobject]@{ + # Parse the artifact name to separate the base of the name from the job attempt number + BaseName = $_.Name.Substring(0, $_.Name.LastIndexOf('-')); + JobAttempt = $_.Name.Substring($_.Name.LastIndexOf('-') + 1) + FullName = $_.FullName + } + } | + Group-Object BaseName | + # Delete all but the last artifact from each base name + ForEach-Object { + $_.Group | + Sort-Object JobAttempt | + Select-Object -ExpandProperty FullName -SkipLast 1 | + Remove-Item -Recurse -Force + } + displayName: Prune Publish Artifacts + - powershell: | + $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)" + if ($imageInfoFiles.Count -eq 0) { + echo "No image info files found." + echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true" + exit 0 + } + + New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force + $(runImageBuilderCmd) mergeImageInfo ` + --manifest $(manifest) ` + $(imageInfosContainerDir) ` + $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` + $(manifestVariables) + name: MergeImageInfoFiles + displayName: Merge Image Info Files + - ${{ if parameters.publishConfig }}: + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Create Manifest Lists + internalProjectName: ${{ parameters.internalProjectName }} + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'), ne(variables['Build.Reason'], 'PullRequest')) + args: >- + createManifestList + '$(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json' + --repo-prefix '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}' + --os-type '*' + --architecture '*' + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.BuildRegistry.server }}' + $(manifestVariables) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) + path: $(sbomOutputDir) + artifactName: sboms + displayName: Publish SBOM Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) + path: $(imageInfosHostDir)$(imageInfosOutputSubDir) + artifactName: image-info + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} diff --git a/src/Infrastructure/Content/templates/jobs/publish.yml b/src/Infrastructure/Content/templates/jobs/publish.yml new file mode 100644 index 000000000..5839be2d7 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/publish.yml @@ -0,0 +1,292 @@ +parameters: + pool: {} + internalProjectName: null + publishConfig: null + customInitSteps: [] + customPostInitSteps: [] + customPublishVariables: [] + sourceBuildPipelineDefinitionId: "" + sourceBuildPipelineRunId: "" + versionsRepoRef: null + versionsRepoPath: "" + # When true, overrides the commit SHA in merged image info files to use the current repository commit. + # This ensures that updated images reference the correct commit in their commitUrl properties. + overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + +jobs: +- job: Publish + pool: ${{ parameters.pool }} + timeoutInMinutes: 90 + + variables: + - name: imageBuilder.commonCmdArgs + value: >- + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.PublishRegistry.server }}' + $(manifestVariables) + $(imageBuilder.queueArgs) + - name: publishNotificationRepoName + value: $(Build.Repository.Name) + - name: branchName + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}: + value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: + value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] + - name: imageInfoHostDir + value: $(Build.ArtifactStagingDirectory)/imageInfo + - name: imageInfoContainerDir + value: $(artifactsPath)/imageInfo + - name: sourceBuildIdOutputDir + value: $(Build.ArtifactStagingDirectory)/sourceBuildId + - name: commitOverrideArg + ${{ if eq(parameters.overrideImageInfoCommit, true) }}: + value: --commit-override $(Build.SourceVersion) + ${{ else }}: + value: '' + - ${{ parameters.customPublishVariables }} + + steps: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + - ${{ parameters.publishConfig.PublishRegistry.server }} + serviceConnections: ${{ parameters.additionalServiceConnections }} + + - template: /eng/docker-tools/templates/steps/retain-build.yml@self + + - pwsh: | + $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI + echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName" + displayName: Set Publish Variables + + - ${{ parameters.customPostInitSteps }} + + - template: /eng/docker-tools/templates/steps/validate-branch.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(imageInfoHostDir) + artifactName: image-info + piplineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + # This can fail in scenarios where no build jobs have run to produce any artifacts + continueOnError: true + + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self + parameters: + publicSourceBranch: $(publicSourceBranch) + + - template: /eng/docker-tools/templates/steps/set-dry-run.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + + - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)" + displayName: Set Publish Variables + + - script: > + $(runImageBuilderCmd) trimUnchangedPlatforms + '$(imageInfoContainerDir)/image-info.json' + displayName: Trim Unchanged Images + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Copy Images + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + copyAcrImages + '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}' + '${{ parameters.publishConfig.BuildRegistry.server }}' + --os-type '*' + --architecture '*' + --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' + --image-info '$(imageInfoContainerDir)/image-info.json' + $(dryRunArg) + $(imageBuilder.pathArgs) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(imageInfoHostDir) + artifactName: image-info-final-$(System.JobAttempt) + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + + - template: /eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + imageInfoPath: '$(imageinfoContainerDir)/image-info.json' + minQueueTime: $(imageQueueTime) + dryRunArg: $(dryRunArg) + condition: succeeded() + + - template: /eng/docker-tools/templates/steps/publish-readmes.yml@self + parameters: + dryRunArg: $(dryRunArg) + condition: and(succeeded(), eq(variables['publishReadme'], 'true')) + + - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data + displayName: Create EOL Annotation Data Directory + + - script: |- + cd $(versionsRepoRoot) + git pull origin $(gitHubVersionsRepoInfo.branch) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Pull Latest Changes from Versions Repo + + - script: >- + cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Copy Latest Image Info from Versions Repo + + - script: > + $(runImageBuilderCmd) mergeImageInfo + $(imageInfoContainerDir) + $(imageInfoContainerDir)/full-image-info-new.json + $(manifestVariables) + $(dryRunArg) + --manifest $(manifest) + --publish + --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json + $(commitOverrideArg) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Merge Image Info + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Ingest Kusto Image Info + serviceConnections: + - name: kusto + id: $(kusto.serviceConnection.id) + tenantId: $(kusto.serviceConnection.tenantId) + clientId: $(kusto.serviceConnection.clientId) + internalProjectName: ${{ parameters.internalProjectName }} + condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) + args: >- + ingestKustoImageInfo + '$(imageInfoContainerDir)/image-info.json' + '$(kusto.cluster)' + '$(kusto.database)' + '$(kusto.imageTable)' + '$(kusto.layerTable)' + --os-type '*' + --architecture '*' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Generate EOL Annotation Data + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: >- + generateEolAnnotationDataForPublish + '${{ parameters.publishConfig.PublishRegistry.server }}' + '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' + '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' + '$(imageInfoContainerDir)/full-image-info-orig.json' + '$(imageInfoContainerDir)/full-image-info-new.json' + $(generateEolAnnotationDataExtraOptions) + $(dryRunArg) + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/eol-annotation-data + artifactName: eol-annotation-data-$(System.JobAttempt) + displayName: Publish EOL Annotation Data Artifact + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + + - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self + parameters: + acr: ${{ parameters.publishConfig.PublishRegistry }} + dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json + + - script: > + $(runImageBuilderCmd) publishImageInfo + '$(imageInfoContainerDir)/full-image-info-new.json' + '$(gitHubVersionsRepoInfo.userName)' + '$(gitHubVersionsRepoInfo.email)' + $(gitHubVersionsRepoInfo.authArgs) + --git-owner '$(gitHubVersionsRepoInfo.org)' + --git-repo '$(gitHubVersionsRepoInfo.repo)' + --git-branch '$(gitHubVersionsRepoInfo.branch)' + --git-path '$(gitHubImageInfoVersionsPath)' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info + + # Task displayNames names are hardcoded to reference the task prefix used by 1ES official + # pipelines in eng/docker-tools/templates/1es-official.yml. + # + # These will fail if they are dependend on by an unofficial pipeline since they use a unique task + # prefix compared to official pipelines (see eng/docker-tools/templates/1es-unofficial.yml). This is + # acceptable because unofficial pipelines should not publish images. + # + # https://github.com/dotnet/docker-tools/issues/1698 tracks making this command no longer depend + # on individual step displayNames. + # + # Skipped for PR builds because manifest lists are not created in PR builds (see post-build.yml). + # Without manifest lists present in image-info.json, the postPublishNotification fails with a NRE. + - script: > + $(runImageBuilderCmd) postPublishNotification + '$(publishNotificationRepoName)' + '$(branchName)' + '$(imageInfoContainerDir)/image-info.json' + $(Build.BuildId) + '$(System.AccessToken)' + '$(azdoOrgName)' + '$(System.TeamProject)' + $(gitHubNotificationsRepoInfo.authArgs) + '$(gitHubNotificationsRepoInfo.org)' + '$(gitHubNotificationsRepoInfo.repo)' + --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' + --task "🟪 Copy Images" + --task "🟪 Wait for Image Ingestion" + --task "🟪 Publish Readmes" + --task "🟪 Wait for MCR Doc Ingestion" + --task "🟪 Publish Image Info" + --task "🟪 Ingest Kusto Image Info" + --task "🟪 Generate EOL Annotation Data" + --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.PublishRegistry.server }})" + --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.PublishRegistry.server }})" + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + displayName: Post Publish Notification + condition: and(always(), eq(variables['publishNotificationsEnabled'], 'true'), ne(variables['Build.Reason'], 'PullRequest')) + + - powershell: | + # Default to current build number if parameter was not overridden + $buildId = "${{ parameters.sourceBuildPipelineRunId }}" + if ($buildId -eq "") { + $buildId = "$(Build.BuildNumber)" + } + + New-Item -ItemType Directory -Path $(sourceBuildIdOutputDir) + Set-Content -Path $(sourceBuildIdOutputDir)/source-build-id.txt -Value "$buildId" + condition: succeeded() + displayName: Write Source Build ID to File + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(sourceBuildIdOutputDir) + artifactName: source-build-id + displayName: Publish Source Build ID Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} diff --git a/src/Infrastructure/Content/templates/jobs/sign-images.yml b/src/Infrastructure/Content/templates/jobs/sign-images.yml new file mode 100644 index 000000000..e9bf51ec1 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/sign-images.yml @@ -0,0 +1,66 @@ +# Signs container images using ESRP/Notary v2. +# This job downloads the merged image-info artifact and signs all images listed in it. +parameters: + pool: {} + internalProjectName: null + publicProjectName: null + customInitSteps: [] + publishConfig: null + sourceBuildPipelineRunId: "" + +jobs: +- job: Sign + pool: ${{ parameters.pool }} + variables: + imageInfoDir: $(Build.ArtifactStagingDirectory)/image-info + steps: + + # Setup docker and ImageBuilder (checkout must happen before init-signing + # so that the Install-DotNetSdk.ps1 script is available) + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + setupImageBuilder: true + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + + # Install MicroBuild signing plugin for ESRP container image signing + - template: /eng/docker-tools/templates/steps/init-signing-linux.yml@self + parameters: + signType: ${{ parameters.publishConfig.Signing.SignType }} + dockerRunOptionsVariableName: signingDockerRunOptions + + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + + # Download merged image-info artifact from Post_Build stage (or from a previous pipeline run) + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(imageInfoDir) + artifactName: image-info + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: 🔏 Sign Container Images + internalProjectName: ${{ parameters.internalProjectName }} + linuxOnlyExtraDockerRunArgs: $(signingDockerRunOptions) + args: >- + signImages + $(artifactsPath)/image-info/image-info.json + --registry-override ${{ parameters.publishConfig.BuildRegistry.server }} + --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: ✅ Verify Container Image Signatures + internalProjectName: ${{ parameters.internalProjectName }} + linuxOnlyExtraDockerRunArgs: $(signingDockerRunOptions) + args: >- + verifySignatures + $(artifactsPath)/image-info/image-info.json + --registry-override ${{ parameters.publishConfig.BuildRegistry.server }} + --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} diff --git a/src/Infrastructure/Content/templates/jobs/test-images-linux-client.yml b/src/Infrastructure/Content/templates/jobs/test-images-linux-client.yml new file mode 100644 index 000000000..450e067f5 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/test-images-linux-client.yml @@ -0,0 +1,30 @@ +parameters: + name: null + pool: {} + matrix: {} + testJobTimeout: 60 + preBuildValidation: false + internalProjectName: null + publishConfig: null + customInitSteps: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + ${{ if eq(parameters.preBuildValidation, 'false') }}: + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + strategy: + matrix: $[ ${{ parameters.matrix }} ] + ${{ if eq(parameters.preBuildValidation, 'true') }}: + condition: and(succeeded(), ne(variables.preBuildTestScriptPath, '')) + pool: ${{ parameters.pool }} + timeoutInMinutes: ${{ parameters.testJobTimeout }} + steps: + - template: /eng/docker-tools/templates/steps/test-images-linux-client.yml@self + parameters: + preBuildValidation: ${{ parameters.preBuildValidation }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/src/Infrastructure/Content/templates/jobs/test-images-windows-client.yml b/src/Infrastructure/Content/templates/jobs/test-images-windows-client.yml new file mode 100644 index 000000000..390f01da6 --- /dev/null +++ b/src/Infrastructure/Content/templates/jobs/test-images-windows-client.yml @@ -0,0 +1,25 @@ +parameters: + name: null + pool: {} + matrix: {} + testJobTimeout: 60 + internalProjectName: null + publishConfig: null + customInitSteps: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + pool: ${{ parameters.pool }} + strategy: + matrix: $[ ${{ parameters.matrix }} ] + timeoutInMinutes: ${{ parameters.testJobTimeout }} + steps: + - template: /eng/docker-tools/templates/steps/test-images-windows-client.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/src/Infrastructure/Content/templates/stages/build-and-test.yml b/src/Infrastructure/Content/templates/stages/build-and-test.yml new file mode 100644 index 000000000..d21e8de90 --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/build-and-test.yml @@ -0,0 +1,351 @@ +parameters: + buildMatrixType: platformDependencyGraph + testMatrixType: platformVersionedOs + buildMatrixCustomBuildLegGroupArgs: "" + testMatrixCustomBuildLegGroupArgs: "" + # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). + # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. + customInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before copy-base-images runs. + customCopyBaseImagesInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before matrix generation runs. + customGenerateMatrixInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before the build starts. + # Use for build-specific initialization (e.g., setting variables, additional setup). + customBuildInitSteps: [] + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + noCache: false + publishConfig: null + + internalProjectName: null + publicProjectName: null + + versionsRepoRef: "" + + storageAccountServiceConnection: null + + linuxAmd64Pool: + vmImage: $(defaultLinuxAmd64PoolImage) + linuxArm32Pool: + vmImage: $(defaultLinuxArm32PoolImage) + linuxArm64Pool: + vmImage: $(defaultLinuxArm64PoolImage) + windows2016Pool: + vmImage: $(defaultWindows2016PoolImage) + windows1809Pool: + vmImage: $(defaultWindows1809PoolImage) + windows2022Pool: + vmImage: $(defaultWindows2022PoolImage) + windows2025Pool: + vmImage: $(defaultWindows2025PoolImage) + + +################################################################################ +# Build Images +################################################################################ +stages: +- stage: Build + condition: and(succeeded(), contains(variables['stages'], 'build')) + dependsOn: [] + jobs: + + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables + + - template: /eng/docker-tools/templates/jobs/copy-base-images-staging.yml@self + parameters: + name: CopyBaseImages + publishConfig: ${{ parameters.publishConfig }} + pool: ${{ parameters.linuxAmd64Pool }} + additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" + customInitSteps: ${{ parameters.customInitSteps }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self + parameters: + matrixType: ${{ parameters.buildMatrixType }} + name: GenerateBuildMatrix + pool: ${{ parameters.linuxAmd64Pool }} + customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + internalProjectName: ${{ parameters.internalProjectName }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_amd64 + pool: ${{ parameters.linuxAmd64Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_arm64 + pool: ${{ parameters.linuxArm64Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_arm32 + pool: ${{ parameters.linuxArm32Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows1809_amd64 + pool: ${{ parameters.windows1809Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows2022_amd64 + pool: ${{ parameters.windows2022Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: WindowsLtsc2016_amd64 + pool: ${{ parameters.windows2016Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customInitSteps }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + +################################################################################ +# Post-Build +################################################################################ +- stage: Post_Build + dependsOn: Build + condition: and(succeeded(), contains(variables['stages'], 'build')) + jobs: + - template: /eng/docker-tools/templates/jobs/post-build.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + +################################################################################ +# Sign Images +################################################################################ +- ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}: + - stage: Sign + dependsOn: Post_Build + condition: " + and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + and( + contains(variables['stages'], 'sign'), + or( + and( + succeeded(), + contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'build')))))" + jobs: + - template: /eng/docker-tools/templates/jobs/sign-images.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + +################################################################################ +# Test Images +################################################################################ +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - stage: Test + dependsOn: Post_Build + condition: " + and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + ne(variables['testScriptPath'], ''), + and( + contains(variables['stages'], 'test'), + or( + and( + succeeded(), + contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'build')))))" + jobs: + - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self + parameters: + matrixType: ${{ parameters.testMatrixType }} + name: GenerateTestMatrix + pool: ${{ parameters.linuxAmd64Pool }} + customBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + isTestStage: true + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customInitSteps }} + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_amd64 + pool: ${{ parameters.linuxAmd64Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_arm64 + pool: ${{ parameters.linuxArm64Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] + testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_arm32 + pool: ${{ parameters.linuxArm32Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] + testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows1809_amd64 + pool: ${{ parameters.windows1809Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2022_amd64 + pool: ${{ parameters.windows2022Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: WindowsLtsc2016_amd64 + pool: ${{ parameters.windows2016Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/src/Infrastructure/Content/templates/stages/dotnet/build-and-test.yml b/src/Infrastructure/Content/templates/stages/dotnet/build-and-test.yml new file mode 100644 index 000000000..a9e8f05fd --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/dotnet/build-and-test.yml @@ -0,0 +1,141 @@ +# A wrapper template around the common build-test-publish-repo template with settings +# specific to the .NET team's infrastructure. + +parameters: + linuxAmd64Pool: "" + + # Custom init steps for all jobs + customInitSteps: [] + + # (Optional) This service connection should be an Azure Resource Manager + # service connection to a storage account that's needed during image builds. + # It can be used to build images with access to private/internal bits. + # If specified, this service connection will be used to pass a storage + # account access token as `--build-arg ACCESSTOKEN=***` to all image builds. + storageAccountServiceConnection: null + + # Parameters for pre-build jobs + customGenerateMatrixInitSteps: [] + customCopyBaseImagesInitSteps: [] + + # Build parameters + noCache: false + publishConfig: null + buildMatrixType: platformDependencyGraph + buildMatrixCustomBuildLegGroupArgs: "" + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + customBuildInitSteps: [] + + # Test parameters + testMatrixType: platformVersionedOs + testMatrixCustomBuildLegGroupArgs: "" + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + internalProjectName: null + publicProjectName: null + + versionsRepoRef: null + +stages: +- template: /eng/docker-tools/templates/stages/build-and-test.yml@self + parameters: + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customInitSteps: ${{ parameters.customInitSteps }} + customTestInitSteps: ${{ parameters.customTestInitSteps }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + buildMatrixType: ${{ parameters.buildMatrixType }} + testMatrixType: ${{ parameters.testMatrixType }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + + # Only clone versions repo if we need to reference it during the build in order to cache images. + ${{ if eq(parameters.noCache, false) }}: + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + # Linux AMD64 + linuxAmd64Pool: + ${{ if ne(parameters.linuxAmd64Pool, '') }}: + ${{ parameters.linuxAmd64Pool }} + ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxAmd64PublicPoolName) + demands: ImageOverride -equals $(linuxAmd64PublicPoolImage) + os: linux + ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) + os: linux + + # Linux Arm64 + linuxArm64Pool: + os: linux + hostArchitecture: Arm64 + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxArm64PublicPoolName) + demands: ImageOverride -equals $(linuxArm64PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxArm64InternalPoolName) + image: $(linuxArm64InternalPoolImage) + + # Linux Arm32 + linuxArm32Pool: + os: linux + hostArchitecture: Arm64 + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxArm32PublicPoolName) + demands: ImageOverride -equals $(linuxArm32PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxArm32InternalPoolName) + image: $(linuxArm32InternalPoolImage) + + # Windows Server 2016 + windows2016Pool: + os: windows + name: $(windowsServer2016PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2016PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2016InternalPoolImage) + + # Windows Server 2019 (1809) + windows1809Pool: + os: windows + name: $(windowsServer2019PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2019PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2019InternalPoolImage) + + # Windows Server 2022 + windows2022Pool: + os: windows + name: $(windowsServer2022PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2022PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2022InternalPoolImage) + + # Windows Server 2025 + windows2025Pool: + os: windows + name: $(windowsServer2025PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2025PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2025InternalPoolImage) diff --git a/src/Infrastructure/Content/templates/stages/dotnet/build-test-publish-repo.yml b/src/Infrastructure/Content/templates/stages/dotnet/build-test-publish-repo.yml new file mode 100644 index 000000000..c590a5287 --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/dotnet/build-test-publish-repo.yml @@ -0,0 +1,84 @@ +# This template wraps the .NET-specific build-and-test and publish templates + +parameters: + linuxAmd64Pool: "" + + # Custom init steps for all jobs + customInitSteps: [] + + # Parameters for pre-build jobs + customGenerateMatrixInitSteps: [] + customCopyBaseImagesInitSteps: [] + + # Build parameters + noCache: false + publishConfig: null + buildMatrixType: platformDependencyGraph + buildMatrixCustomBuildLegGroupArgs: "" + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + customBuildInitSteps: [] + + # Test parameters + testMatrixType: platformVersionedOs + testMatrixCustomBuildLegGroupArgs: "" + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + # Publish parameters + customPublishInitSteps: [] + + # Additional service connections not in publishConfig.RegistryAuthentication + # that need OIDC token access (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + + # Other common parameters + internalProjectName: null + publicProjectName: null + versionsRepoRef: "" + +stages: +- template: /eng/docker-tools/templates/stages/dotnet/build-and-test.yml@self + parameters: + linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }} + # Pre-build + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + # Build + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + buildMatrixType: ${{ parameters.buildMatrixType }} + buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customInitSteps: ${{ parameters.customInitSteps }} + # Test + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + testMatrixType: ${{ parameters.testMatrixType }} + testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + linuxAmdTestJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + linuxArmTestJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + customTestInitSteps: ${{ parameters.customTestInitSteps }} + # Other + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + +- template: /eng/docker-tools/templates/stages/dotnet/publish.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + customPublishInitSteps: ${{ parameters.customPublishInitSteps }} + customInitSteps: ${{ parameters.customInitSteps }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/src/Infrastructure/Content/templates/stages/dotnet/publish-config-nonprod.yml b/src/Infrastructure/Content/templates/stages/dotnet/publish-config-nonprod.yml new file mode 100644 index 000000000..0ce87d4d2 --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/dotnet/publish-config-nonprod.yml @@ -0,0 +1,119 @@ +# This pipeline template injects the publish config for the dotnet-docker +# non-production (unofficial) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-prod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + +# Enable container image signing +- name: enableSigning + type: boolean + default: false + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + # publishConfig schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs. + # This will get converted to JSON and placed in appsettings.json to be loaded by ImageBuilder at runtime. + publishConfig: + InternalMirrorRegistry: + server: $(acr-staging-test.server) + repoPrefix: $(internalMirrorRepoPrefix) + + PublicMirrorRegistry: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) + + BuildRegistry: + server: $(acr-staging-test.server) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + + PublishRegistry: + server: $(acr-test.server) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + + RegistryAuthentication: + - server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + - server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + - server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) + + cleanServiceConnection: + name: $(clean-test.serviceConnectionName) + id: $(clean-test.serviceConnection.id) + clientId: $(clean-test.serviceConnection.clientId) + tenantId: $(testTenant) + + testServiceConnection: + name: $(test-nonprod.serviceConnectionName) + id: $(test-nonprod.serviceConnection.id) + clientId: $(test-nonprod.serviceConnection.clientId) + tenantId: $(testTenant) + + Signing: + Enabled: ${{ parameters.enableSigning }} + ImageSigningKeyCode: $(microBuildSigningKeyCode.testing) + ReferrerSigningKeyCode: $(microBuildSigningKeyCode.testing) + # Use signType 'real' even for non-prod to actually sign with the test certificate. + # The 'test' signType skips signing entirely on linux; the test keycode provides a non-production certificate. + SignType: real + TrustStoreName: test diff --git a/src/Infrastructure/Content/templates/stages/dotnet/publish-config-prod.yml b/src/Infrastructure/Content/templates/stages/dotnet/publish-config-prod.yml new file mode 100644 index 000000000..f90e5e806 --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/dotnet/publish-config-prod.yml @@ -0,0 +1,117 @@ +# This pipeline template injects the publish config for the dotnet-docker +# production (official) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-nonprod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + +# Enable container image signing +- name: enableSigning + type: boolean + default: false + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + # publishConfig schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs. + # This will get converted to JSON and placed in appsettings.json to be loaded by ImageBuilder at runtime. + publishConfig: + InternalMirrorRegistry: + server: $(acr-staging.server) + repoPrefix: $(internalMirrorRepoPrefix) + + PublicMirrorRegistry: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) + + BuildRegistry: + server: $(acr-staging.server) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + + PublishRegistry: + server: $(acr.server) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + + RegistryAuthentication: + - server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + - server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + - server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) + + cleanServiceConnection: + name: $(clean.serviceConnectionName) + id: $(clean.serviceConnection.id) + clientId: $(clean.serviceConnection.clientId) + tenantId: $(clean.serviceConnection.tenantId) + + testServiceConnection: + name: $(test.serviceConnectionName) + id: $(test.serviceConnection.id) + clientId: $(test.serviceConnection.clientId) + tenantId: $(test.serviceConnection.tenantId) + + Signing: + Enabled: ${{ parameters.enableSigning }} + ImageSigningKeyCode: $(microBuildSigningKeyCode.containers) + ReferrerSigningKeyCode: $(microBuildSigningKeyCode.attestations) + SignType: real + TrustStoreName: supplychain diff --git a/src/Infrastructure/Content/templates/stages/dotnet/publish.yml b/src/Infrastructure/Content/templates/stages/dotnet/publish.yml new file mode 100644 index 000000000..5cac237b9 --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/dotnet/publish.yml @@ -0,0 +1,64 @@ +# This template wraps the common publish stage template with settings specific +# to the .NET team's infrastructure. + +parameters: + internalProjectName: null + publicProjectName: null + publishConfig: null + pool: "" + isStandalonePublish: false + customPublishInitSteps: [] + customInitSteps: [] + sourceBuildPipelineDefinitionId: '' + sourceBuildPipelineRunId: '' + versionsRepoRef: null + overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + +stages: +- template: /eng/docker-tools/templates/stages/publish.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} + isStandalonePublish: ${{ parameters.isStandalonePublish }} + customInitSteps: ${{ parameters.customInitSteps }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + + customPublishInitSteps: + - pwsh: | + # When reporting the repo name in the publish notification, we don't want to include + # the org part of the repo name (e.g. we want "dotnet-docker", not "dotnet-dotnet-docker"). + # This also accounts for the different separators between AzDO and GitHub repo names. + + $repoName = "$(Build.Repository.Name)" + + $orgSeparatorIndex = $repoName.IndexOf("/") + if ($orgSeparatorIndex -eq -1) { + $orgSeparatorIndex = $repoName.IndexOf("-") + } + + if ($orgSeparatorIndex -ge 0) { + $repoName = $repoName.Substring($orgSeparatorIndex + 1) + } + echo "##vso[task.setvariable variable=publishNotificationRepoName]$repoName" + displayName: "Set Custom Repo Name Var" + - ${{ parameters.customPublishInitSteps }} + + pool: + ${{ if ne(parameters.pool, '') }}: + ${{ parameters.pool }} + ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxAmd64PublicPoolName) + demands: ImageOverride -equals $(linuxAmd64PublicPoolImage) + os: linux + ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) + os: linux diff --git a/src/Infrastructure/Content/templates/stages/publish.yml b/src/Infrastructure/Content/templates/stages/publish.yml new file mode 100644 index 000000000..2c17283ec --- /dev/null +++ b/src/Infrastructure/Content/templates/stages/publish.yml @@ -0,0 +1,88 @@ +parameters: + customPublishInitSteps: [] + customPublishVariables: [] + customInitSteps: [] + + internalProjectName: null + publicProjectName: null + + publishConfig: null + + isStandalonePublish: false + + pool: + vmImage: $(defaultLinuxAmd64PoolImage) + + sourceBuildPipelineDefinitionId: '' + sourceBuildPipelineRunId: '' + + versionsRepoRef: null + versionsRepoPath: "versions" + + # When true, any updated images will have the SHA in their commit URL updated + # to the commit that this pipeline is running on, instead of the commit they + # were built from. Use in combination with isStandalonePublish to ensure that + # internally built images still reference public Dockerfiles. + overrideImageInfoCommit: false + + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + +################################################################################ +# Publish Images +################################################################################ +stages: +- stage: Publish + ${{ if eq(parameters.isStandalonePublish, true) }}: + dependsOn: [] + ${{ else }}: + dependsOn: + - ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}: + - Sign + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - Test + - ${{ else }}: + - Post_Build + # Run when all of the following are true: + # 1. The pipeline has not been canceled. + # 2. The stages variable includes 'publish'. + # 3. Either signing is not enabled, this run is reusing previously signed images, or the Sign stage succeeded. + # 4. Either the stages variable does not include 'build', or Post_Build succeeded. + # 5. Either the stages variable does not include 'test', or Test succeeded/was skipped. + condition: " + and( + not(canceled()), + contains(variables['stages'], 'publish'), + or( + ne(lower('${{ parameters.publishConfig.Signing.Enabled }}'), 'true'), + and( + not(contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'sign')) + ), + in(dependencies.Sign.result, 'Succeeded', 'SucceededWithIssues') + ), + or( + not(contains(variables['stages'], 'build')), + succeeded('Post_Build') + ), + or( + not(contains(variables['stages'], 'test')), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped') + ) + )" + jobs: + - template: /eng/docker-tools/templates/jobs/publish.yml@self + parameters: + pool: ${{ parameters.pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customPublishVariables: ${{ parameters.customPublishVariables }} + customInitSteps: ${{ parameters.customInitSteps }} + customPostInitSteps: ${{ parameters.customPublishInitSteps }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + versionsRepoPath: ${{ parameters.versionsRepoPath }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} diff --git a/src/Infrastructure/Content/templates/steps/annotate-eol-digests.yml b/src/Infrastructure/Content/templates/steps/annotate-eol-digests.yml new file mode 100644 index 000000000..8e2f7571b --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/annotate-eol-digests.yml @@ -0,0 +1,43 @@ +parameters: +- name: acr + type: object +# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command +- name: dataFile + type: string + +steps: + - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests + displayName: Create Annotation Digests Directory + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Annotate EOL Images (${{ parameters.acr.server }}) + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: >- + annotateEolDigests + "${{ parameters.dataFile }}" + "${{ parameters.acr.server }}" + "${{ parameters.acr.repoPrefix }}" + $(artifactsPath)/annotation-digests/annotation-digests.txt + $(dryRunArg) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/annotation-digests + artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) + displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Annotation Ingestion (${{ parameters.acr.server }}) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'), eq(variables['waitForIngestionEnabled'], 'true')) + args: >- + waitForMarAnnotationIngestion + $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/src/Infrastructure/Content/templates/steps/clean-acr-images.yml b/src/Infrastructure/Content/templates/steps/clean-acr-images.yml new file mode 100644 index 000000000..7a44560bb --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/clean-acr-images.yml @@ -0,0 +1,26 @@ +parameters: + repo: null + acr: null + action: null + age: null + customArgs: '' + internalProjectName: null +steps: + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + # Options are documented in CleanAcrImagesOptions.cs + ${{ if eq(parameters.action, 'delete') }}: + displayName: "Delete ${{ parameters.repo }}" + ${{ elseif parameters.age }}: + displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }} > ${{ parameters.age }}d)" + ${{ else }}: + displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }})" + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + cleanAcrImages + ${{ parameters.repo }} + ${{ parameters.acr.server }} + --action ${{ parameters.action }} + --age ${{ parameters.age }} + ${{ parameters.customArgs }} + $(dryRunArg) diff --git a/src/Infrastructure/Content/templates/steps/cleanup-docker-linux.yml b/src/Infrastructure/Content/templates/steps/cleanup-docker-linux.yml new file mode 100644 index 000000000..ddf0f9b09 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/cleanup-docker-linux.yml @@ -0,0 +1,15 @@ +parameters: + condition: true + +steps: + ################################################################################ + # Cleanup local Docker server + ################################################################################ +- script: docker stop $(docker ps -q) || true + displayName: Stop Running Containers + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- script: docker system prune -a -f --volumes + displayName: Cleanup Docker + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/src/Infrastructure/Content/templates/steps/cleanup-docker-windows.yml b/src/Infrastructure/Content/templates/steps/cleanup-docker-windows.yml new file mode 100644 index 000000000..a2c41c20b --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/cleanup-docker-windows.yml @@ -0,0 +1,18 @@ +parameters: + condition: true + +steps: + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- powershell: $(engDockerToolsPath)/Invoke-CleanupDocker.ps1 + displayName: Cleanup Docker Images + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- powershell: | + if (Test-Path $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder) { + Remove-Item $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder -Force -Recurse; + } + displayName: Cleanup Image Builder + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/src/Infrastructure/Content/templates/steps/copy-base-images.yml b/src/Infrastructure/Content/templates/steps/copy-base-images.yml new file mode 100644 index 000000000..6664c8f9a --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/copy-base-images.yml @@ -0,0 +1,37 @@ +parameters: +- name: acr + type: object + default: + server: "" + repoPrefix: "" +- name: additionalOptions + type: string + default: "" +- name: continueOnError + type: string + default: false +- name: forceDryRun + type: boolean + default: false + +steps: +- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run" +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Copy Base Images + continueOnError: ${{ parameters.continueOnError }} + internalProjectName: 'internal' + # Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined, + # PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an + # error + args: >- + copyBaseImages + $(dockerHubRegistryCreds) + $(customCopyBaseImagesArgs) + --repo-prefix '${{ parameters.acr.repoPrefix }}' + --registry-override '${{ parameters.acr.server }}' + --os-type 'linux' + --architecture '*' + $env:DRYRUNARG + ${{ parameters.additionalOptions }} diff --git a/src/Infrastructure/Content/templates/steps/download-build-artifact.yml b/src/Infrastructure/Content/templates/steps/download-build-artifact.yml new file mode 100644 index 000000000..965f49320 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/download-build-artifact.yml @@ -0,0 +1,40 @@ +parameters: + # Destination directory on the pipeline agent's filesystem, relative or absolute. + targetPath: "" + # The build/pipeline artifact to download. If the value is left empty, + # the task downloads all artifacts associated with the pipeline run. + artifactName: "" + # AKA pipeline/definition - optional. + # If this is left empty, use the current pipeline's definition ID. + # You can get this from the URL of the pipeline's overview page on Azure DevOps. + # Example: https://dev.azure.com/$org/$project/_build?definitionId=373 + pipelineDefinitionId: "" + # AKA runId/buildId/pipelineId - optional. + # The identifier of the pipeline run from which to download the artifacts. + # If this is left empty, then always download from the current pipeline + # You can get this from the URL of the specific pipeline run, for example: + # https://dev.azure.com/$org/$project/_build/results?buildId=2709155&view=results + pipelineRunId: "" + condition: true + continueOnError: false + +steps: +# https://learn.microsoft.com/azure/devops/pipelines/tasks/reference/download-pipeline-artifact-v2 +- task: DownloadPipelineArtifact@2 + inputs: + ${{ if ne(parameters.pipelineRunId, '') }}: + buildType: specific + project: $(System.TeamProject) + ${{ if ne(parameters.pipelineDefinitionId, '') }}: + definition: ${{ parameters.pipelineDefinitionId }} + ${{ else }}: + definition: $(System.DefinitionId) + buildId: ${{ parameters.pipelineRunId }} + buildVersionToDownload: specific + ${{ else }}: + buildType: current + targetPath: ${{ parameters.targetPath }} + artifactName: ${{ parameters.artifactName }} + displayName: Download Build Artifact(s) + condition: and(succeeded(), ${{ parameters.condition }}) + continueOnError: ${{ parameters.continueOnError }} diff --git a/src/Infrastructure/Content/templates/steps/generate-appsettings.yml b/src/Infrastructure/Content/templates/steps/generate-appsettings.yml new file mode 100644 index 000000000..b1243e703 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/generate-appsettings.yml @@ -0,0 +1,34 @@ +# .NET Microsoft.Extensions.Configuration reads appsettings.json from the working directory +# where ImageBuilder is run. Place it in the repo root so it will be available at runtime. +parameters: +# See: +# - publish-config-prod.yml +# - publish-config-nonprod.yml +# - PublishConfiguration.cs +- name: publishConfig + type: object +# This should be the path to $(Build.ArtifactStagingDirectory). It is parameterized +# here since it is mounted into the ImageBuilder container at runtime. +- name: artifactStagingDirectory + type: string + default: "" +- name: condition + type: string + default: "true" + +steps: +- powershell: |- + # Escape backslashes for JSON compatibility (Windows paths like D:\a\_work become D:\\a\\_work) + $artifactStagingDirectory = "${{ parameters.artifactStagingDirectory }}" -replace '\\', '\\' + $appsettingsJsonContent = @" + { + "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }}, + "BuildConfiguration": { + "ArtifactStagingDirectory": "$artifactStagingDirectory" + } + } + "@ + Set-Content -Path "appsettings.json" -Value $appsettingsJsonContent + Get-Content -Path "appsettings.json" + displayName: Output publish configuration + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), ${{ parameters.condition }}) diff --git a/src/Infrastructure/Content/templates/steps/init-common.yml b/src/Infrastructure/Content/templates/steps/init-common.yml new file mode 100644 index 000000000..eda572618 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/init-common.yml @@ -0,0 +1,247 @@ +# Common initialization steps for all Docker Tools pipeline jobs. It handles: +# - Repository checkout (single or multi-repo with versions repo) +# - Path resolution that adapts to checkout configuration +# - ImageBuilder setup (containerized on Linux, native on Windows) +# - Test runner setup +# - Docker environment cleanup +parameters: +- name: dockerClientOS + type: string + values: + - linux + - windows + +# Whether to set up ImageBuilder +- name: setupImageBuilder + type: boolean + default: true + +# Whether to delete existing Docker images +- name: cleanupDocker + type: boolean + default: false + +# Whether or not to run the steps in this template +- name: condition + type: string + default: "true" + +# Custom steps to set up ImageBuilder instead of pulling it. +# When provided, these steps run instead of the default pull-based setup. +# The steps should result in $(imageNames.imageBuilder) being available locally. +# Used by bootstrap pipelines that build ImageBuilder from source. +- name: customInitSteps + type: stepList + default: [] + +# Registry and authentication configuration for publishing images. +# Contains server URLs, repo prefixes, subscriptions, and resource groups. +# When null, build/publish steps that require registry access will be skipped. +- name: publishConfig + type: object + default: null + +# Reference to a versions repository (e.g., "dotnet-versions") for multi-repo checkout. +# When set, enables image caching by providing access to historical image-info files. +# When empty, single-repo checkout is used and caching is disabled. +- name: versionsRepoRef + type: string + default: "" +# Local path where the versions repository will be checked out. +# Only used when versionsRepoRef is set. +- name: versionsRepoPath + type: string + default: "versions" + +steps: +# Repository Checkout +# Multi-repo checkout is used when a versions repository is needed for caching. +# The versions repo contains historical image-info files used to determine which +# images need rebuilding. Credentials are persisted for later git operations. +- checkout: self +- ${{ if ne(parameters.versionsRepoRef, '') }}: + - checkout: ${{ parameters.versionsRepoRef }} + path: s/${{ parameters.versionsRepoPath }} + persistCredentials: true + fetchDepth: 1 + condition: succeeded() + +# Base Variable Initialization +# Sets foundational variables that subsequent steps build upon: +# - sourceBranch: Cleaned branch name without refs/ prefixes, used for filtering +# - commonMatrixAndBuildOptions: Base CLI args shared by matrix generation and +# build commands. Starts with just --source-repo; registry-specific options +# are added conditionally below. +- powershell: | + # Source branch + $sourceBranch = $Env:BUILD_SOURCEBRANCH -replace "refs/heads/","" -replace "refs/tags/","" -replace "refs/pull/","" + Write-Host "Setting sourceBranch to '$sourceBranch'" + echo "##vso[task.setvariable variable=sourceBranch]$sourceBranch" + + # Common matrix and build options (base) + $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" + Write-Host "Setting commonMatrixAndBuildOptions to '$commonMatrixAndBuildOptions'" + echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" + displayName: Set Source Branch and Base Options + condition: and(succeeded(), ${{ parameters.condition }}) + +# Build Registry Configuration +# Extends commonMatrixAndBuildOptions with registry-specific settings: +# - Internal builds: Use internal mirror registry prefix and build registry +# server to pull/push from private ACR instead of public MCR +# - Public builds: Override non-MCR base images to use public mirror, reducing +# external dependencies and improving build reliability +- ${{ if parameters.publishConfig }}: + - powershell: | + $commonMatrixAndBuildOptions = "$(commonMatrixAndBuildOptions)" + if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.InternalMirrorRegistry.repoPrefix }} --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}" + } + + if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + } + + Write-Host "Setting commonMatrixAndBuildOptions to '$commonMatrixAndBuildOptions'" + echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" + displayName: Set Build Registry Options + condition: and(succeeded(), ${{ parameters.condition }}) + +# Repository Path Resolution +# Resolves paths differently based on checkout mode: +# +# Multi-repo checkout (versionsRepoRef is set): +# - Both main repo and versions repo are checked out side-by-side +# - Paths must be prefixed with repo name (e.g., "dotnet-docker/src/manifest.json") +# - Caching is ENABLED because versions repo provides historical image-info files +# - Build.Repository.Name may include org prefix (e.g., "dotnet/dotnet-docker") +# which must be stripped to get the actual checkout directory name +# +# Single-repo checkout (versionsRepoRef is empty): +# - Only the main repo is checked out at Build.Repository.LocalPath +# - Paths are relative to repo root without prefix +# - Caching is disabled because no versions repo means no historical image-info +# +# Key outputs: +# - versionsBasePath: Prefix for paths in versions repo (empty or "versions/") +# - pipelineDisabledCache: "true" disables caching, "false" enables it +# - repoRoot, engDockerToolsPath: Resolved absolute paths for scripts +- powershell: | + function Set-PipelineVariable($name, $value) { + Write-Host "Setting $name to '$value'" + echo "##vso[task.setvariable variable=$name]$value" + } + + # Repository paths - differ based on single vs multi-repo checkout + if ("${{ parameters.versionsRepoRef }}" -ne "") { + # Multi-repo checkout + $versionsBasePath = "${{ parameters.versionsRepoPath }}/" + $pipelineDisabledCache = "false" + + $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") + if ($pathSeparatorIndex -ge 0) { + $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) + } + else { + $buildRepoName = "$(Build.Repository.Name)" + } + + $repoRoot = "$(Build.Repository.LocalPath)/$buildRepoName" + $versionsRepoRoot = "$(Build.Repository.LocalPath)/${{ parameters.versionsRepoPath }}" + $engDockerToolsPath = "$repoRoot/$(engDockerToolsRelativePath)" + + $engPath = "$repoRoot/eng" + $manifest = "$buildRepoName/$(manifest)" + $testResultsDirectory = "$buildRepoName/$testResultsDirectory" + + if ("$(testScriptPath)") { + $testScriptPath = "$buildRepoName/$(testScriptPath)" + } + + Set-PipelineVariable "buildRepoName" $buildRepoName + Set-PipelineVariable "repoRoot" $repoRoot + Set-PipelineVariable "versionsRepoRoot" $versionsRepoRoot + Set-PipelineVariable "engDockerToolsPath" $engDockerToolsPath + Set-PipelineVariable "manifest" $manifest + Set-PipelineVariable "engPath" $engPath + Set-PipelineVariable "testScriptPath" $testScriptPath + Set-PipelineVariable "testResultsDirectory" $testResultsDirectory + } + else { + # Single-repo checkout + $versionsBasePath = "" + $pipelineDisabledCache = "true" + $repoRoot = "$(Build.Repository.LocalPath)" + $engDockerToolsPath = "$repoRoot/$(engDockerToolsRelativePath)" + + Set-PipelineVariable "repoRoot" $repoRoot + Set-PipelineVariable "engDockerToolsPath" $engDockerToolsPath + } + + Set-PipelineVariable "versionsBasePath" $versionsBasePath + Set-PipelineVariable "pipelineDisabledCache" $pipelineDisabledCache + displayName: Set Repository Path Variables + condition: and(succeeded(), ${{ parameters.condition }}) + +# TSA Configuration - 1ES Pipeline Templates require tsaoptions.json at +# Build.SourcesDirectory. In a multi-repo checkout scenario, +# Build.SourcesDirectory differs from the main repo's checkout location, so we +# must copy the config file to the expected location. +- ${{ if ne(parameters.versionsRepoRef, '') }}: + - task: CopyFiles@2 + displayName: Copy TSA Config + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' + Contents: '.config/tsaoptions.json' + TargetFolder: '$(Build.SourcesDirectory)' + +# Artifacts Path Configuration +# Linux: Uses Build.ArtifactStagingDirectory directly so container and host paths match +# through an identity volume mount +# Windows: Uses Build.ArtifactStagingDirectory directly since ImageBuilder runs +# as an extracted executable, not in a container +- ${{ if eq(parameters.dockerClientOS, 'linux') }}: + - script: | + echo "Setting artifactsPath to '$(Build.ArtifactStagingDirectory)'" + echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" + displayName: Define Artifacts Path Variable + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if eq(parameters.dockerClientOS, 'windows') }}: + - powershell: | + Write-Host "Setting artifactsPath to '$(Build.ArtifactStagingDirectory)'" + echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" + displayName: Define Artifacts Path Variable + condition: and(succeeded(), ${{ parameters.condition }}) + +# Docker Cleanup - Removes existing Docker images and containers to ensure a clean build state. +# Linux: Only runs when explicitly requested via cleanupDocker parameter +# Windows: Always runs because Windows agents have limited disk space and +# accumulated images cause build failures +- ${{ if and(eq(parameters.dockerClientOS, 'linux'), eq(parameters.cleanupDocker, true)) }}: + - template: /eng/docker-tools/templates/steps/cleanup-docker-linux.yml@self + parameters: + condition: ${{ parameters.condition }} +- ${{ if eq(parameters.dockerClientOS, 'windows') }}: + - template: /eng/docker-tools/templates/steps/cleanup-docker-windows.yml@self + parameters: + condition: ${{ parameters.condition }} + +# ImageBuilder Setup +# Linux: Runs containerized +# - Pull the ImageBuilder image from MCR +# - Build a "withrepo" image that layers the source code into the container +# - Generate docker run commands that mount Docker socket and artifact paths +# Windows: Runs as extracted executable +# - Pull the ImageBuilder image, create a temporary container, and copy out +# the executable to Build.BinariesDirectory +# - Run directly as .exe (no container) because Windows containers have +# limitations with Docker-in-Docker and volume mounts +# Custom setup (customInitSteps) overrides both Linux and Windows setup steps. +- ${{ if eq(parameters.setupImageBuilder, true) }}: + - template: /eng/docker-tools/templates/steps/init-imagebuilder.yml@self + parameters: + dockerClientOS: ${{ parameters.dockerClientOS }} + publishConfig: ${{ parameters.publishConfig }} + condition: ${{ parameters.condition }} + customInitSteps: ${{ parameters.customInitSteps }} diff --git a/src/Infrastructure/Content/templates/steps/init-imagebuilder.yml b/src/Infrastructure/Content/templates/steps/init-imagebuilder.yml new file mode 100644 index 000000000..b85c62b6c --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/init-imagebuilder.yml @@ -0,0 +1,146 @@ +# ImageBuilder setup steps for Docker Tools pipelines. Handles: +# - Custom init steps (when provided by caller) +# - Default pull-based setup (Linux containerized, Windows native executable) +# - appsettings.json generation for publish configuration +parameters: +- name: dockerClientOS + type: string + values: + - linux + - windows + +- name: publishConfig + type: object + default: null + +- name: condition + type: string + default: "true" + +- name: customInitSteps + type: stepList + default: [] + +steps: +# Custom ImageBuilder setup (e.g., bootstrap from source) +- ${{ if gt(length(parameters.customInitSteps), 0) }}: + # Set dockerClientOS variable so custom setup steps can use it + - script: echo "##vso[task.setvariable variable=dockerClientOS]${{ parameters.dockerClientOS }}" + displayName: Set dockerClientOS variable + condition: and(succeeded(), ${{ parameters.condition }}) + - ${{ parameters.customInitSteps }} +# Default: Pull pre-built ImageBuilder image +- ${{ else }}: + - ${{ if eq(parameters.dockerClientOS, 'linux') }}: + - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.imageBuilder) + displayName: Pull Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - ${{ if eq(parameters.dockerClientOS, 'windows') }}: + # Windows: Extract ImageBuilder executable from container image + # Windows containers don't support Docker socket mounting, so we run + # ImageBuilder as a native executable rather than in a container + - powershell: $(engDockerToolsPath)/Invoke-WithRetry.ps1 "docker pull $(imageNames.imageBuilder)" + displayName: Pull Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - script: docker create --name setupImageBuilder-$(Build.BuildId)-$(System.JobId) $(imageNames.imageBuilder) + displayName: Create Setup Container + condition: and(succeeded(), ${{ parameters.condition }}) + - script: > + docker cp + setupImageBuilder-$(Build.BuildId)-$(System.JobId):/image-builder + $(Build.BinariesDirectory)/.Microsoft.DotNet.ImageBuilder + displayName: Copy Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - script: docker rm -f setupImageBuilder-$(Build.BuildId)-$(System.JobId) + displayName: Cleanup Setup Container + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + +# Generate appsettings.json with publish configuration for ImageBuilder to read +- template: /eng/docker-tools/templates/steps/generate-appsettings.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + artifactStagingDirectory: $(artifactsPath) + condition: ${{ parameters.condition }} + +# On Linux, build the "withrepo" image that includes the repo's source code. +# The withrepo image layers the checked-out repository into the ImageBuilder +# container at /repo, so ImageBuilder can access manifests and Dockerfiles +- ${{ if eq(parameters.dockerClientOS, 'linux') }}: + - script: >- + docker build + -t $(imageNames.imageBuilder.withrepo) + --build-arg IMAGE=$(imageNames.imageBuilder) + -f $(engDockerToolsPath)/Dockerfile.WithRepo . + displayName: Build Image for Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + # Define runImageBuilderCmd and runAuthedImageBuilderCmd variables + # These are the primary interface for downstream ImageBuilder invocations: + # - runImageBuilderCmd: For operations that don't need Azure DevOps auth + # - runAuthedImageBuilderCmd: Passes OIDC tokens for ACR/Azure operations + # The commands mount Docker socket (for building images) and artifact directory + - task: PowerShell@2 + displayName: Define ImageBuilder Command Variables + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + $imageBuilderImageName = "$(imageNames.imageBuilder.withrepo)" + Write-Host "##vso[task.setvariable variable=imageBuilderImageName]$imageBuilderImageName" + + $dockerRunBaseCmd = @( + "docker run --rm" + ) + + $dockerRunArgs = @( + "-v /var/run/docker.sock:/var/run/docker.sock" + "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" + "-w /repo" + "$(imageBuilderDockerRunExtraOptions)" + ) + + $authedDockerRunArgs = @( + "-e", 'SYSTEM_ACCESSTOKEN' + "-e", 'SYSTEM_OIDCREQUESTURI' + ) + + $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs + $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + + # Base commands without image name for templates that need to insert + # extra docker run args before the image name (e.g. signing) + $runImageBuilderBaseCmd = $($dockerRunCmd -join ' ') + $runAuthedImageBuilderBaseCmd = $($authedDockerRunCmd -join ' ') + + Write-Host "##vso[task.setvariable variable=runImageBuilderBaseCmd]$runImageBuilderBaseCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderBaseCmd]$runAuthedImageBuilderBaseCmd" + + # Full commands with image name for direct invocation by other templates + $runImageBuilderCmd = "$runImageBuilderBaseCmd $imageBuilderImageName" + $runAuthedImageBuilderCmd = "$runAuthedImageBuilderBaseCmd $imageBuilderImageName" + + Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd" + +# On Windows, point to the extracted executable path +# Both runImageBuilderCmd and runAuthedImageBuilderCmd are the same because +# Windows runs natively and inherits environment variables automatically +- ${{ if eq(parameters.dockerClientOS, 'windows') }}: + - task: PowerShell@2 + displayName: Define runImageBuilderCmd Variables + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + $runImageBuilderCmd = "$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe" + Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runImageBuilderCmd" + # On Windows the base commands are the same as the full commands since + # there is no container image name to append + Write-Host "##vso[task.setvariable variable=runImageBuilderBaseCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderBaseCmd]$runImageBuilderCmd" + # Set imageBuilderImageName to empty - on Windows there is no container image + # since ImageBuilder runs as a native exe. run-imagebuilder.yml appends this + # variable to the command line, so it must be defined (but empty) to avoid + # leaving an unexpanded $(imageBuilderImageName) literal in the script. + Write-Host "##vso[task.setvariable variable=imageBuilderImageName]" diff --git a/src/Infrastructure/Content/templates/steps/init-signing-linux.yml b/src/Infrastructure/Content/templates/steps/init-signing-linux.yml new file mode 100644 index 000000000..d6c3d4681 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/init-signing-linux.yml @@ -0,0 +1,124 @@ +# Installs the MicroBuild signing plugin for ESRP container image signing. +# After installation, MBSIGN_APPFOLDER environment variable points to DDSignFiles.dll location. +parameters: +- name: signType + type: string + default: test + values: + - test + - real + +- name: condition + type: string + default: "true" + +- name: microBuildOutputFolder + type: string + default: $(Agent.TempDirectory)/MicroBuild + +# Name of the pipeline variable to set with the signing docker run options. +# The variable will contain both the MicroBuild plugin volume mount and +# the --env-file flag, ready to pass as extraDockerRunOptions to run-imagebuilder. +- name: dockerRunOptionsVariableName + type: string + +steps: +# Install .NET 8.0 SDK for MicroBuild plugin installation using dotnet-install.sh. +# We avoid UseDotNet@2 because it sets DOTNET_ROOT globally, which breaks PowerShell +# (pwsh) on Azure Linux 3 where pwsh requires the .NET 9.0 runtime from the system +# .NET installation. Instead, we install to an isolated directory and only expose it +# to the MicroBuild task via its env block. +- powershell: > + $(engDockerToolsPath)/Install-DotNetSdk.ps1 + -InstallPath "${{ parameters.microBuildOutputFolder }}/.dotnet" + -Channel "8.0" + displayName: Install .NET SDK for MicroBuild Plugin + condition: and(succeeded(), ${{ parameters.condition }}) + +# Create a global.json in the MicroBuild folder that pins to the installed SDK. +# This prevents the repo's global.json from causing SDK resolution failures +# when MicroBuild runs dotnet restore from this directory. +- script: | + mkdir -p ${{ parameters.microBuildOutputFolder }} + version=$(${{ parameters.microBuildOutputFolder }}/.dotnet/dotnet --version) + cat > ${{ parameters.microBuildOutputFolder }}/global.json << EOF + { + "sdk": { + "version": "$version" + } + } + EOF + displayName: Create global.json for MicroBuild + condition: and(succeeded(), ${{ parameters.condition }}) + +- task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild Signing Plugin + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + version: $(MicroBuildPluginVersion) + ${{ if eq(parameters.signType, 'test') }}: + signType: test + ${{ else }}: + signType: real + zipSources: false + feedSource: $(MicroBuildFeedSource) + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + workingDirectory: ${{ parameters.microBuildOutputFolder }} + env: + TeamName: $(TeamName) + MicroBuildOutputFolderOverride: $(Agent.TempDirectory)/MicroBuild + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + PATH: ${{ parameters.microBuildOutputFolder }}/.dotnet:$(PATH) + +# Configure docker run options for signing. +# Writes an env file with signing variables and sets $(signingDockerRunOptions) +# with both the MicroBuild plugin volume mount and the --env-file flag. +- task: PowerShell@2 + displayName: Configure ImageBuilder Signing Options + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + # Write the signing env file for docker --env-file. + # Docker reads this file on the host before creating the container, + # so no volume mount is needed for the file itself. + $envFilePath = "$(Agent.TempDirectory)/imagebuilder-signing.env" + $envFileContent = @( + # MicroBuild plugin variables for DDSignFiles.dll + "MBSIGN_APPFOLDER=/microbuild" + "VSENGESRPSSL" + "USEESRPCLI" + "MBSIGN_CONNECTEDSERVICE" + + # Container-local temp/workspace paths (host paths aren't accessible inside the container) + "MBSIGNTEMPDIR=/tmp/MicroBuildSign" + "PIPELINE_WORKSPACE=$(Build.ArtifactStagingDirectory)" + "AGENT_TEMPDIRECTORY=/tmp" + + # Azure DevOps pipeline variables for ESRP bearer token auth (ESRPUtils.GetAccountInfo) + "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI" + "BUILD_BUILDID" + "SYSTEM_TEAMPROJECT" + "BUILD_SOURCEBRANCH" + + # Azure DevOps pipeline variables for ESRP CLI federated token (ESRPCliDll.GetFederatedTokenData) + "SYSTEM_JOBID" + "SYSTEM_PLANID" + "SYSTEM_TEAMPROJECTID" + "SYSTEM_HOSTTYPE" + "SYSTEM_COLLECTIONURI" + + # Azure DevOps pipeline variables for DDSignFilesConfiguration + "BUILD_DEFINITIONNAME" + "BUILD_BUILDNUMBER" + ) + + $envFileContent | Set-Content -Path $envFilePath -Encoding utf8NoBOM + + # Compose docker run options for signing: + # - Volume mount for MicroBuild plugin directory (DDSignFiles.dll and esrpcli.dll) + # - Env file with signing environment variables + $signingDockerRunOptions = "-v $env:MBSIGN_APPFOLDER`:/microbuild --env-file `"$envFilePath`"" + Write-Host "signingDockerRunOptions: $signingDockerRunOptions" + Write-Host "##vso[task.setvariable variable=${{ parameters.dockerRunOptionsVariableName }}]$signingDockerRunOptions" diff --git a/src/Infrastructure/Content/templates/steps/init-testrunner.yml b/src/Infrastructure/Content/templates/steps/init-testrunner.yml new file mode 100644 index 000000000..ba5e3d2bc --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/init-testrunner.yml @@ -0,0 +1,21 @@ +# Test Runner Setup (Linux only) +# Pulls the test runner image and builds a "withrepo" variant that includes +# the repository source code (test projects) in the container. +# This is Linux-only because Windows image tests run natively. + +parameters: +- name: condition + type: string + default: "true" + +steps: +- powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.testrunner) + displayName: Pull Test Runner + condition: and(succeeded(), ${{ parameters.condition }}) +- script: > + docker build + -t $(imageNames.testRunner.withrepo) + --build-arg IMAGE=$(imageNames.testrunner) + -f $(engDockerToolsPath)/Dockerfile.WithRepo . + displayName: Build Test Runner Image + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/src/Infrastructure/Content/templates/steps/parse-test-arg-arrays.yml b/src/Infrastructure/Content/templates/steps/parse-test-arg-arrays.yml new file mode 100644 index 000000000..5ab18214d --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/parse-test-arg-arrays.yml @@ -0,0 +1,15 @@ +steps: +- powershell: | + # Formats the OS versions in a compact human-readable form (e.g. "os1/os2") + $osVersionsDisplayName = '$(osVersions)' -Replace '--os-version ', '' -Replace ' ', '/' + + # Defines a PowerShell snippet in string-form that can be used to initialize an array of the OS versions + $osVersionsArrayInitStr = "@('" + $($osVersionsDisplayName -Replace "/", "', '") + "')" + + echo "##vso[task.setvariable variable=osVersionsDisplayName]$osVersionsDisplayName" + echo "##vso[task.setvariable variable=osVersionsArrayInitStr]$osVersionsArrayInitStr" + + # Defines a PowerShell snippet in string-form that can be used to initialize an array of the image builder paths + $pathInitStr = "@('" + $('$(imageBuilderPaths)' -Replace '--path', '' -Replace " ", "', '") + "')" + echo "##vso[task.setvariable variable=imageBuilderPathsArrayInitStr]$pathInitStr" + displayName: Parse Test Arg Arrays diff --git a/src/Infrastructure/Content/templates/steps/publish-artifact.yml b/src/Infrastructure/Content/templates/steps/publish-artifact.yml new file mode 100644 index 000000000..72ce47005 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/publish-artifact.yml @@ -0,0 +1,28 @@ +parameters: +- name: path + type: string +- name: artifactName + type: string +- name: displayName + type: string +- name: internalProjectName + type: string +- name: publicProjectName + type: string +- name: condition + type: string + default: 'true' + +steps: +- ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + path: ${{ parameters.path }} + artifact: ${{ parameters.artifactName }} + displayName: ${{ parameters.displayName }} + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + - publish: ${{ parameters.path }} + artifact: ${{ parameters.artifactName }} + displayName: ${{ parameters.displayName }} + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/src/Infrastructure/Content/templates/steps/publish-readmes.yml b/src/Infrastructure/Content/templates/steps/publish-readmes.yml new file mode 100644 index 000000000..c0daaf52f --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/publish-readmes.yml @@ -0,0 +1,29 @@ +parameters: + dryRunArg: "" + condition: true + +steps: +- script: > + $(runImageBuilderCmd) publishMcrDocs + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.PublishRegistry.server }}' + '$(mcrDocsRepoInfo.userName)' + '$(mcrDocsRepoInfo.email)' + $(mcrDocsRepoInfo.authArgs) + '$(publicGitRepoUri)' + ${{ parameters.dryRunArg }} + $(manifestVariables) + $(imageBuilder.queueArgs) + --git-owner 'Microsoft' + --git-repo 'mcrdocs' + --git-branch 'main' + --git-path 'teams' + $(additionalPublishMcrDocsArgs) + name: PublishReadmes + displayName: Publish Readmes + condition: ${{ parameters.condition }} +- template: /eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml@self + parameters: + commitDigest: $(PublishReadmes.readmeCommitDigest) + condition: and(${{ parameters.condition }}, ne(variables['PublishReadmes.readmeCommitDigest'], '')) + dryRunArg: ${{ parameters.dryRunArg }} diff --git a/src/Infrastructure/Content/templates/steps/reference-service-connections.yml b/src/Infrastructure/Content/templates/steps/reference-service-connections.yml new file mode 100644 index 000000000..ab6ec9805 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/reference-service-connections.yml @@ -0,0 +1,72 @@ +# Emit AzureCLI@2 steps that reference service connections in the current +# stage so that Azure DevOps will issue OIDC tokens for them. Azure DevOps +# requires each stage to explicitly reference (via an azureSubscription task +# input) any service connection it needs OIDC tokens for. +# +# This template must be included in every job that authenticates to Azure +# via AzurePipelinesCredential (i.e., any job that uses run-imagebuilder.yml +# with internalProjectName, or run-pwsh-with-auth.yml). +# +# Service connections can be specified in two ways: +# - Via publishConfig + usesRegistries: looks up service connections from +# publishConfig.RegistryAuthentication entries matching the given servers. +# Use this for registry-scoped connections (e.g., BuildRegistry, PublishRegistry). +# - Via serviceConnections: a direct list of { name: string } objects. +# Use this for non-registry connections (e.g., kusto, marStatus, cleanServiceConnection). +# +# Both can be used together in a single call. +parameters: +# Publishing configuration object. Only needed when using the usesRegistries parameter. +- name: publishConfig + type: object + default: {} +# List of registry server URLs to look up in publishConfig.RegistryAuthentication. +# Each matching entry's service connection will be referenced. +- name: usesRegistries + type: object + default: [] +# Direct list of service connections to reference. Shape: [{ name: string }] +- name: serviceConnections + type: object + default: [] +# The OS of the agent running this job. Determines whether to use PowerShell +# Core (pscore, Linux) or Windows PowerShell (ps, Windows) for the AzureCLI task. +- name: dockerClientOS + type: string + default: linux +# The internal Azure DevOps project name. Reference steps are only emitted +# for internal non-PR builds, since public projects don't have these service +# connections. +- name: internalProjectName + type: string + default: internal + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Guard on .name: null parameters passed through template layers can become + # empty objects that are truthy, so check the concrete property instead. + - ${{ each serviceConnection in parameters.serviceConnections }}: + - ${{ if serviceConnection.name }}: + - task: AzureCLI@2 + displayName: Reference ${{ serviceConnection.name }} + inputs: + azureSubscription: ${{ serviceConnection.name }} + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: ps + ${{ else }}: + scriptType: pscore + scriptLocation: inlineScript + inlineScript: Write-Host "Service connection referenced for OIDC" + - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}: + # Also guard on .name here for the same reason as the serviceConnections loop above. + - ${{ if and(containsValue(parameters.usesRegistries, auth.server), auth.serviceConnection.name) }}: + - task: AzureCLI@2 + displayName: Reference ${{ auth.serviceConnection.name }} + inputs: + azureSubscription: ${{ auth.serviceConnection.name }} + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: ps + ${{ else }}: + scriptType: pscore + scriptLocation: inlineScript + inlineScript: Write-Host "Service connection referenced for OIDC" diff --git a/src/Infrastructure/Content/templates/steps/retain-build.yml b/src/Infrastructure/Content/templates/steps/retain-build.yml new file mode 100644 index 000000000..bcf11d57e --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/retain-build.yml @@ -0,0 +1,9 @@ +steps: +- powershell: > + $(engDockerToolsPath)/Retain-Build.ps1 + -BuildId $(Build.BuildId) + -AzdoOrgUri '$(System.CollectionUri)' + -AzdoProject '$(System.TeamProject)' + -Token '$(System.AccessToken)' + displayName: Enable permanent build retention + condition: and(succeeded(), eq(variables.retainBuild, 'true')) diff --git a/src/Infrastructure/Content/templates/steps/run-imagebuilder.yml b/src/Infrastructure/Content/templates/steps/run-imagebuilder.yml new file mode 100644 index 000000000..8f2241d9a --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/run-imagebuilder.yml @@ -0,0 +1,81 @@ +parameters: +- name: name + type: string + default: "" +- name: displayName + type: string + default: "Run ImageBuilder" +- name: serviceConnections + type: object + default: [] + # - name: The name of the service connection argument that will be passed to the ImageBuilder command. + # For example, if the argument is --acr-service-connection, the name would be "acr". + # id: The service connection's ID (GUID). + # clientId: The client ID of the Managed Idendity backing the service connection (GUID). + # tenantId: The ID of the tenant that the Managed Identity is in (GUID). +- name: internalProjectName + type: string + default: null +- name: args + type: string + default: null +- name: condition + type: string + default: succeeded() +- name: continueOnError + type: boolean + default: false +- name: dockerClientOS + type: string + default: "linux" + +# Additional docker run args inserted before the image name (Linux only). +# These are flags passed to `docker run` for the containerized ImageBuilder +# and must not be set for Windows jobs where ImageBuilder runs as a native +# executable. Used for signing (volume mounts, env files) or other +# job-specific docker configuration. +- name: linuxOnlyExtraDockerRunArgs + type: string + default: "" + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) + inputs: + targetType: 'inline' + script: | + $serviceConnections = '${{ convertToJson(parameters.serviceConnections) }}' + + Write-Host "Service connections:" + Write-Host "${serviceConnections}" + + $serviceConnectionsJson = $serviceConnections | ConvertFrom-Json + $serviceConnectionsArgs = @() + foreach ($connection in $serviceConnectionsJson) { + $serviceConnectionsArgs += "--$($connection.name)-service-connection" + $serviceConnectionsArgs += "$($connection.tenantId):$($connection.clientId):$($connection.id)" + } + + $(runAuthedImageBuilderBaseCmd) ${{ parameters.linuxOnlyExtraDockerRunArgs }} $(imageBuilderImageName) ${{ parameters.args }} @serviceConnectionsArgs + +- ${{ else }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + inputs: + targetType: 'inline' + script: >- + $(runImageBuilderBaseCmd) ${{ parameters.linuxOnlyExtraDockerRunArgs }} $(imageBuilderImageName) ${{ parameters.args }} diff --git a/src/Infrastructure/Content/templates/steps/run-pwsh-with-auth.yml b/src/Infrastructure/Content/templates/steps/run-pwsh-with-auth.yml new file mode 100644 index 000000000..84de1e1f5 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/run-pwsh-with-auth.yml @@ -0,0 +1,39 @@ +parameters: +- name: name + type: string + default: "" +- name: displayName + type: string + default: "Run PowerShell" +- name: serviceConnection + type: string + default: "" +- name: command + type: string + default: null +- name: continueOnError + type: boolean + default: false +- name: dockerClientOS + type: string + default: "linux" +- name: condition + type: string + default: true + +steps: +- task: AzureCLI@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} (Authenticated) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + addSpnToEnvironment: true + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: 'ps' + ${{ else }}: + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: ${{ parameters.command }}; diff --git a/src/Infrastructure/Content/templates/steps/set-dry-run.yml b/src/Infrastructure/Content/templates/steps/set-dry-run.yml new file mode 100644 index 000000000..4879f7459 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/set-dry-run.yml @@ -0,0 +1,30 @@ +parameters: + name: publishConfig + type: object + +steps: +- powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + # Don't use dry-run mode for unofficial builds, since they publish to a + # non-production environment + $dryRunArg="" + } + elseif ("$(System.TeamProject)" -eq "$(publicProjectName)") + { + # Public builds need to use dry-run mode since they don't publish anywhere. + $dryRunArg="--dry-run" + } + elseif (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.PublishRegistry.repoPrefix }}")) + { + # If we're running an internal build on an official pipeline but not + # publishing to an official repo prefix, then use dry run mode. + $dryRunArg="--dry-run" + } + else + { + $dryRunArg="" + } + + echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg" + displayName: Set dry-run arg for non-prod diff --git a/src/Infrastructure/Content/templates/steps/set-image-info-path-var.yml b/src/Infrastructure/Content/templates/steps/set-image-info-path-var.yml new file mode 100644 index 000000000..71d317fa2 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/set-image-info-path-var.yml @@ -0,0 +1,19 @@ +parameters: + publicSourceBranch: null + +steps: +- powershell: | + $basePath = "$(gitHubVersionsRepoInfo.path)" + + $publicSourceBranch = "${{ parameters.publicSourceBranch }}" + + if ($publicSourceBranch -eq "") { + throw "publicSourceBranch variable is not set" + } + + $buildRepoName = "$(Build.Repository.Name)".Replace("/", "-") + $imageInfoName = "image-info.$buildRepoName-$publicSourceBranch$(imageInfoVariant).json" + + echo "##vso[task.setvariable variable=imageInfoVersionsPath]$basePath/$imageInfoName" + echo "##vso[task.setvariable variable=gitHubImageInfoVersionsPath]$(gitHubVersionsRepoInfo.path)/$imageInfoName" + displayName: Set Image Info Path Vars diff --git a/src/Infrastructure/Content/templates/steps/test-images-linux-client.yml b/src/Infrastructure/Content/templates/steps/test-images-linux-client.yml new file mode 100644 index 000000000..1806dbb7f --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/test-images-linux-client.yml @@ -0,0 +1,119 @@ +parameters: + preBuildValidation: false + internalProjectName: null + publishConfig: null + condition: true + customInitSteps: [] + sourceBuildPipelineRunId: "" + # Skip init-common when these steps are included inline in a job that already called init-common + skipCommonInit: false + +steps: +- ${{ if eq(parameters.skipCommonInit, false) }}: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + setupImageBuilder: false + # Clean only up when we're running an internal build, not a PR, and not doing pre-build validation. + # i.e. when we're building something important. + cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false')) }} + condition: ${{ parameters.condition }} +- template: /eng/docker-tools/templates/steps/init-testrunner.yml@self + parameters: + condition: ${{ parameters.condition }} +- ${{ parameters.customInitSteps }} +- script: | + echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" + + if [ "${{ parameters.preBuildValidation }}" == "true" ]; then + echo "##vso[task.setvariable variable=effectiveTestScriptPath]$(preBuildTestScriptPath)" + else + echo "##vso[task.setvariable variable=effectiveTestScriptPath]$(testScriptPath)" + fi + + additionalTestArgs="$ADDITIONALTESTARGS" + if [ "${{ parameters.preBuildValidation }}" == "true" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories pre-build" + else + if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then + additionalTestArgs="$additionalTestArgs -PullImages -Registry ${{ parameters.publishConfig.BuildRegistry.server }} -RepoPrefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" + if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" + fi + fi + fi + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" + displayName: Set Test Variables + condition: and(succeeded(), ${{ parameters.condition }}) +- script: > + docker run -t -d + -v /var/run/docker.sock:/var/run/docker.sock + -v $(Build.ArtifactStagingDirectory):$(artifactsPath) + -e DOCKER_BUILDKIT=1 + -e RUNNING_TESTS_IN_CONTAINER=true + --name $(testRunner.container) + $(imageNames.testRunner.withrepo) + displayName: Start Test Runner Container + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/run-pwsh-with-auth.yml@self + parameters: + displayName: Docker login + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} + condition: and(succeeded(), ${{ parameters.condition }}) + command: >- + $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; + docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh + -File $(engDockerToolsRelativePath)/Invoke-WithRetry.ps1 + "az login $azLoginArgs; az acr login -n ${{ parameters.publishConfig.BuildRegistry.server }}" + - ${{ if eq(parameters.preBuildValidation, 'false') }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + condition: ${{ parameters.condition }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} +- template: /eng/docker-tools/templates/steps/parse-test-arg-arrays.yml@self +- powershell: > + $(test.init); + docker exec + $(testRunner.options) + $(testRunner.container) + pwsh + -Command "$(effectiveTestScriptPath) + -Paths $(imageBuilderPathsArrayInitStr) + -OSVersions $(osVersionsArrayInitStr) + -Architecture '$(architecture)' + $(additionalTestArgs)" + displayName: Test Images + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker exec $(testRunner.container) docker logout ${{ parameters.publishConfig.BuildRegistry.server }} + displayName: Docker logout + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- powershell: > + docker cp + $(testRunner.container):/repo/$(testResultsDirectory) + $(Common.TestResultsDirectory)/. + displayName: Copy Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Common.TestResultsDirectory) + mergeTestResults: true + publishRunAttachments: true + ${{ if eq(parameters.preBuildValidation, 'false') }}: + testRunTitle: $(productVersion) $(osVersionsDisplayName) $(architecture) + ${{ if eq(parameters.preBuildValidation, 'true') }}: + testRunTitle: Pre-Build Validation +- script: docker rm -f $(testRunner.container) + displayName: Cleanup TestRunner Container + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/src/Infrastructure/Content/templates/steps/test-images-windows-client.yml b/src/Infrastructure/Content/templates/steps/test-images-windows-client.yml new file mode 100644 index 000000000..ccca2bbe2 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/test-images-windows-client.yml @@ -0,0 +1,69 @@ +parameters: + internalProjectName: null + publishConfig: null + condition: "true" + customInitSteps: [] + sourceBuildPipelineRunId: "" + # Skip init-common when these steps are included inline in a job that already called init-common + skipCommonInit: false + +steps: +- ${{ if and(eq(parameters.skipCommonInit, false), eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: windows + cleanupDocker: true + setupImageBuilder: false + condition: ${{ parameters.condition }} + - template: /eng/docker-tools/templates/steps/run-pwsh-with-auth.yml@self + parameters: + displayName: Docker login + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} + dockerClientOS: windows + condition: and(succeeded(), ${{ parameters.condition }}) + command: >- + az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; + $accessToken = $(az acr login -n ${{ parameters.publishConfig.BuildRegistry.server }} --expose-token --query accessToken --output tsv); + docker login ${{ parameters.publishConfig.BuildRegistry.server }} -u 00000000-0000-0000-0000-000000000000 -p $accessToken +- ${{ parameters.customInitSteps }} +- powershell: | + if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { + $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${{ parameters.publishConfig.BuildRegistry.server }} -RepoPrefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" + } + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" + displayName: Set Test Variables + condition: and(succeeded(), ${{ parameters.condition }}) +- powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item + displayName: Cleanup Old Test Results + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + condition: ${{ parameters.condition }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} +- template: /eng/docker-tools/templates/steps/parse-test-arg-arrays.yml@self +- powershell: > + $(test.init); + $(testScriptPath) + -Paths $(imageBuilderPathsArrayInitStr) + -OSVersions $(osVersionsArrayInitStr) + $(additionalTestArgs) + displayName: Test Images + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker logout ${{ parameters.publishConfig.BuildRegistry.server }} + displayName: Docker logout + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '$(testResultsDirectory)/**/*.trx' + mergeTestResults: true + publishRunAttachments: true + testRunTitle: $(productVersion) $(osVersionsDisplayName) amd64 diff --git a/src/Infrastructure/Content/templates/steps/validate-branch.yml b/src/Infrastructure/Content/templates/steps/validate-branch.yml new file mode 100644 index 000000000..945aefa68 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/validate-branch.yml @@ -0,0 +1,52 @@ +parameters: + publishConfig: null + internalProjectName: null + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + echo "Build is from an unofficial pipeline, continuing." + exit 0 + } + + $isOfficialRepoPrefix = "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.PublishRegistry.repoPrefix }}") + if (-not $isOfficialRepoPrefix) + { + echo "This build will not publish to an official repo prefix, continuing." + echo "Publish repo prefix: ${{ parameters.publishConfig.PublishRegistry.repoPrefix }}" + echo "Official repo prefixes: $(officialRepoPrefixes)" + exit 0 + } + + $isOfficialBranch = "$(officialBranches)".Split(',').Contains("$(sourceBranch)") + if ($isOfficialBranch) + { + echo "$(sourceBranch) is an official branch, continuing." + echo "Official branches: $(officialBranches)" + exit 0 + } + + $hasOfficialBranchPrefix = $false + foreach ($prefix in "$(officialBranchPrefixes)".Split(',')) { + if ("$(sourceBranch)".StartsWith($prefix)) { + $hasOfficialBranchPrefix = $true + break + } + } + + if ($hasOfficialBranchPrefix) + { + echo "$(sourceBranch) has an official branch prefix, continuing." + echo "Official branch prefixes: $(officialBranchPrefixes)" + exit 0 + } + + echo "##vso[task.logissue type=error]Official builds must be done from an official branch ($(officialBranches)) and repo prefix ($(officialRepoPrefixes))." + echo "Build definition: $(Build.DefinitionName)" + echo "1ESPT build type: $(OneESPT.BuildType)" + echo "Current branch: $(sourceBranch)" + echo "Publish repo prefix: ${{ parameters.publishConfig.PublishRegistry.repoPrefix }}" + exit 1 + displayName: Validate Branch diff --git a/src/Infrastructure/Content/templates/steps/wait-for-mcr-doc-ingestion.yml b/src/Infrastructure/Content/templates/steps/wait-for-mcr-doc-ingestion.yml new file mode 100644 index 000000000..5f64438bb --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/wait-for-mcr-doc-ingestion.yml @@ -0,0 +1,21 @@ +parameters: + commitDigest: null + condition: true + dryRunArg: "" + +steps: +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for MCR Doc Ingestion + condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: 'internal' + args: >- + waitForMcrDocIngestion + '${{ parameters.commitDigest }}' + --timeout '$(mcrDocIngestionTimeout)' + ${{ parameters.dryRunArg }} diff --git a/src/Infrastructure/Content/templates/steps/wait-for-mcr-image-ingestion.yml b/src/Infrastructure/Content/templates/steps/wait-for-mcr-image-ingestion.yml new file mode 100644 index 000000000..bffaf0378 --- /dev/null +++ b/src/Infrastructure/Content/templates/steps/wait-for-mcr-image-ingestion.yml @@ -0,0 +1,37 @@ +parameters: +- name: publishConfig + type: object + +- name: imageInfoPath + type: string + +- name: minQueueTime + type: string + +- name: dryRunArg + type: string + +- name: condition + type: string + default: "true" + +steps: +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Image Ingestion + condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: 'internal' + args: >- + waitForMcrImageIngestion + '${{ parameters.imageInfoPath }}' + --manifest '$(manifest)' + --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' + --min-queue-time '${{ parameters.minQueueTime }}' + --timeout '$(mcrImageIngestionTimeout)' + $(manifestVariables) + ${{ parameters.dryRunArg }} diff --git a/src/Infrastructure/Content/templates/task-prefix-decorator.yml b/src/Infrastructure/Content/templates/task-prefix-decorator.yml new file mode 100644 index 000000000..598bfbe24 --- /dev/null +++ b/src/Infrastructure/Content/templates/task-prefix-decorator.yml @@ -0,0 +1,63 @@ +# This Azure Pipelines template adds a prefix to the display name of each +# task passed through the `stages` parameter. When used in conjunction with +# an "extends" template which injects a lot of tasks into the pipeline, the +# added prefix helps to identify which tasks were passed through this template +# and which tasks were injected by the `baseTemplate`. +# +# This template assumes that `baseTemplate` uses the `stages` parameter. If it +# doesn't, this template likely won't work as expected. + +parameters: +# The pipeline will behave as if it were originally extended from this template, +# except with updated task display names. +- name: baseTemplate + type: string + default: "" + +# These parameters are passed directly to `baseTemplate` +- name: templateParameters + type: object + default: null + +# These stages will be modified and passed to the `baseTemplate` as the +# `stages` parameter. +- name: stages + type: stageList + default: [] + +# This prefix will be added to the display name of each task. +- name: taskPrefix + type: string + default: "🟪" + + +extends: + template: ${{ parameters.baseTemplate }} + parameters: + ${{ insert }}: ${{ parameters.templateParameters }} + stages: + - ${{ each stage in parameters.stages }}: + - stage: ${{ stage.stage }} + ${{ each property in stage }}: + ${{ if notIn(property.key, 'stage', 'jobs') }}: + ${{ property.key }} : ${{ property.value }} + jobs: + - ${{ each job in stage.jobs }}: + - job: ${{ job.job }} + ${{ each property in job }}: + ${{ if notIn(property.key, 'job', 'steps') }}: + ${{ property.key }} : ${{ property.value }} + steps: + - ${{ each step in job.steps }}: + # Special case for Azure Pipelines checkout task: + # https://learn.microsoft.com/azure/devops/extend/develop/pipeline-decorator-context?view=azure-devops#task-names-and-guids + # The checkout task does not have a name - it is special and built directly into the agent. + # Avoid modifying the checkout task, or else it will show up in the UI as a task with no name. + - ${{ if contains(step.task, '6d15af64-176c-496d-b583-fd2ae21d4df4') }}: + - ${{ step }} + - ${{ else }}: + - task: ${{ step.task }} + ${{ each property in step }}: + ${{ if notIn(property.key, 'task', 'displayName') }}: + ${{ property.key }} : ${{ property.value }} + displayName: ${{ parameters.taskPrefix }} ${{ step.displayName }} diff --git a/src/Infrastructure/Content/templates/variables/common-paths.yml b/src/Infrastructure/Content/templates/variables/common-paths.yml new file mode 100644 index 000000000..d676441b5 --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/common-paths.yml @@ -0,0 +1,6 @@ +variables: + engDockerToolsRelativePath: eng/docker-tools + engDockerToolsPath: $(Build.Repository.LocalPath)/$(engDockerToolsRelativePath) + engPath: $(Build.Repository.LocalPath)/eng + testScriptPath: "" + preBuildTestScriptPath: $(testScriptPath) diff --git a/src/Infrastructure/Content/templates/variables/common.yml b/src/Infrastructure/Content/templates/variables/common.yml new file mode 100644 index 000000000..680dc1293 --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/common.yml @@ -0,0 +1,89 @@ +variables: +- template: /eng/docker-tools/templates/variables/docker-images.yml@self +- template: /eng/docker-tools/templates/variables/common-paths.yml@self + +- name: publishReadme + value: true +- name: publishImageInfo + value: true +- name: ingestKustoImageInfo + value: true + # CG is disabled by default because projects are built within Dockerfiles and CG step do not scan artifacts + # that are built within Dockerfiles. A separate CG pipeline exists for this reason. +- name: skipComponentGovernanceDetection + value: false +- name: build.imageBuilderDockerRunExtraOptions + value: "" +- name: imageBuilderDockerRunExtraOptions + value: "" +- name: generateEolAnnotationDataExtraOptions + value: "" +- name: productVersionComponents + value: 2 +- name: imageInfoVariant + value: "" +- name: publishNotificationsEnabled + value: false +- name: manifestVariables + value: "" +- name: mcrImageIngestionTimeout + value: "00:20:00" +- name: mcrDocIngestionTimeout + value: "00:05:00" +- name: officialBranches + # comma-delimited list of branch names + value: main +- name: internalMirrorRepoPrefix + value: 'mirror/' +- name: publicMirrorRepoPrefix + value: '' +- name: cgBuildGrepArgs + value: "''" +- name: test.init + value: "" +- name: testRunner.options + value: "" +- name: customCopyBaseImagesArgs + value: "" +- name: additionalGenerateBuildMatrixOptions + value: "" +- name: trimCachedImagesForMatrix + value: false + +- name: defaultLinuxAmd64PoolImage + value: ubuntu-latest +- name: defaultLinuxArm32PoolImage + value: null +- name: defaultLinuxArm64PoolImage + value: null +- name: defaultWindows2016PoolImage + value: vs2017-win2016 +- name: defaultWindows1809PoolImage + value: windows-2019 +- name: defaultWindows2022PoolImage + value: windows-2022 +- name: defaultWindows2025PoolImage + value: windows-2025 + +- name: default1ESInternalPoolName + value: NetCore1ESPool-Internal +- name: default1ESInternalPoolImage + value: build.azurelinux.3.amd64 + +- template: /eng/docker-tools/templates/variables/sdl-pool.yml@self + +# Define these as placeholder values to allow string validation to succeed since we don't have the +# variable group with the actual values in public builds. For internal builds, the variable group +# will cause these values to be overridden with the real values. +- name: acr.subscription + value: 00000000-0000-0000-0000-000000000000 +- name: acr-staging.subscription + value: 00000000-0000-0000-0000-000000000000 + +# See https://devdiv.visualstudio.com/Engineering/_git/Sign?version=GBmain&path=/src/CertificateMappings.xml +- name: microBuildSigningKeyCode.containers + value: 4512 +- name: microBuildSigningKeyCode.attestations + value: 4571 +- name: microBuildSigningKeyCode.testing + value: 2151 diff --git a/src/Infrastructure/Content/templates/variables/dnceng-build-pools.yml b/src/Infrastructure/Content/templates/variables/dnceng-build-pools.yml new file mode 100644 index 000000000..29eb2c07f --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dnceng-build-pools.yml @@ -0,0 +1,58 @@ +# Build pool and image variables for dnceng pipelines. +# Reference this template in pipelines that run in the dnceng or dnceng-public Azure DevOps orgs. + +variables: +- name: linuxAmd64InternalPoolImage + value: build.azurelinux.3.amd64 +- name: linuxAmd64PublicPoolImage + value: build.azurelinux.3.amd64.open +- name: linuxAmd64PublicPoolName + value: NetCore-Public +- name: linuxAmd64InternalPoolName + value: NetCore1ESPool-Internal + +- name: linuxArm64PublicPoolImage + value: build.azurelinux.3.arm64.open +- name: linuxArm64InternalPoolImage + value: build.azurelinux.3.arm64 +- name: linuxArm64PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm64InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: linuxArm32PublicPoolImage + value: build.azurelinux.3.arm64.open +- name: linuxArm32InternalPoolImage + value: build.azurelinux.3.arm64 +- name: linuxArm32PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm32InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: windowsServer2016PublicPoolImage + value: Server2016-NESDockerBuilds +- name: windowsServer2016InternalPoolImage + value: Server2016-NESDockerBuilds-1ESPT +- name: windowsServer2016PoolName + value: Docker-2016-${{ variables['System.TeamProject'] }} + +- name: windowsServer2019PublicPoolImage + value: Server2019-1809-NESDockerBuilds +- name: windowsServer2019InternalPoolImage + value: Server2019-1809-NESDockerBuilds-1ESPT +- name: windowsServer2019PoolName + value: Docker-1809-${{ variables['System.TeamProject'] }} + +- name: windowsServer2022PublicPoolImage + value: Server2022-NESDockerBuilds +- name: windowsServer2022InternalPoolImage + value: Server2022-NESDockerBuilds-1ESPT +- name: windowsServer2022PoolName + value: Docker-2022-${{ variables['System.TeamProject'] }} + +- name: windowsServer2025PublicPoolImage + value: Server2025-NESDockerBuilds +- name: windowsServer2025InternalPoolImage + value: Server2025-NESDockerBuilds-1ESPT +- name: windowsServer2025PoolName + value: Docker-2025-${{ variables['System.TeamProject'] }} diff --git a/src/Infrastructure/Content/templates/variables/dnceng-project-names.yml b/src/Infrastructure/Content/templates/variables/dnceng-project-names.yml new file mode 100644 index 000000000..46871dd88 --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dnceng-project-names.yml @@ -0,0 +1,8 @@ +# Azure DevOps project name variables for dnceng pipelines. +# Reference this template in pipelines that run in the dnceng or dnceng-public Azure DevOps orgs. + +variables: +- name: publicProjectName + value: public +- name: internalProjectName + value: internal diff --git a/src/Infrastructure/Content/templates/variables/dnceng-signing.yml b/src/Infrastructure/Content/templates/variables/dnceng-signing.yml new file mode 100644 index 000000000..1f6c2d9ee --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dnceng-signing.yml @@ -0,0 +1,10 @@ +# MicroBuild signing variables for dnceng pipelines. +# Reference this template in pipelines that run in the dnceng or dnceng-public Azure DevOps orgs. + +variables: +- name: TeamName + value: DotNetCore +- name: MicroBuildFeedSource + value: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json +- name: MicroBuildPluginVersion + value: latest diff --git a/src/Infrastructure/Content/templates/variables/docker-images.yml b/src/Infrastructure/Content/templates/variables/docker-images.yml new file mode 100644 index 000000000..48ca5f36f --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/docker-images.yml @@ -0,0 +1,7 @@ +variables: + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:{{IMAGEBUILDER_TAG}} + imageNames.imageBuilder: $(imageNames.imageBuilderName) + imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner + imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.syft: anchore/syft:v1.31.0-debug diff --git a/src/Infrastructure/Content/templates/variables/dotnet/build-test-publish.yml b/src/Infrastructure/Content/templates/variables/dotnet/build-test-publish.yml new file mode 100644 index 000000000..96a0cf500 --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dotnet/build-test-publish.yml @@ -0,0 +1,48 @@ +# Common variables for building/testing/publishing in the .NET team's pipelines + +variables: +- template: /eng/docker-tools/templates/variables/dotnet/common.yml@self + +- name: commonVersionsImageInfoPath + value: build-info/docker +- name: publicGitRepoUri + value: https://github.com/dotnet/dotnet-docker +- name: testScriptPath + value: ./tests/run-tests.ps1 +- name: testResultsDirectory + value: tests/Microsoft.DotNet.Docker.Tests/TestResults/ + +- name: officialRepoPrefixes + value: public/,internal/private/,unlisted/ + readonly: true + +- name: mcrDocsRepoInfo.userName + value: $(gitHubApp.marDocsUpdater.userName) +- name: mcrDocsRepoInfo.email + value: $(gitHubApp.marDocsUpdater.email) + +- name: publishNotificationsEnabled + value: true +- name: gitHubNotificationsRepoInfo.org + value: dotnet +- name: gitHubNotificationsRepoInfo.repo + value: dotnet-docker-internal +# $(gitHubNotificationsRepoInfo.authArgs) is needed by the "Post Publish +# Notification" step in eng/docker-tools/templates/jobs/publish.yml#L271, even during +# a dry-run. This value is a placeholder that gets replaced when referencing +# the secrets.yml variable template. +- name: gitHubNotificationsRepoInfo.authArgs + value: --gh-token 'placeholder' + +- name: gitHubVersionsRepoInfo.org + value: dotnet +- name: gitHubVersionsRepoInfo.repo + value: versions +- name: gitHubVersionsRepoInfo.branch + value: main +- name: gitHubVersionsRepoInfo.path + value: ${{ variables.commonVersionsImageInfoPath }} +- name: gitHubVersionsRepoInfo.userName + value: $(dotnetDockerBot.userName) +- name: gitHubVersionsRepoInfo.email + value: $(dotnetDockerBot.email) diff --git a/src/Infrastructure/Content/templates/variables/dotnet/common.yml b/src/Infrastructure/Content/templates/variables/dotnet/common.yml new file mode 100644 index 000000000..03302cf3f --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dotnet/common.yml @@ -0,0 +1,14 @@ +variables: +- template: /eng/docker-tools/templates/variables/common.yml@self +- template: /eng/docker-tools/templates/variables/dnceng-build-pools.yml@self +- template: /eng/docker-tools/templates/variables/dnceng-signing.yml@self +- template: /eng/docker-tools/templates/variables/dnceng-project-names.yml@self + +# $(dockerHubRegistryCreds) is needed by the copy-base-images step in +# eng/docker-tools/templates/stages/build-and-test.yml#L73-L78, even during a dry-run. +# This is a placeholder that gets replaced when referencing the secrets.yml +# variable template. +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=placeholder;placeholder' + +- group: DotNet-Docker-Common-2 diff --git a/src/Infrastructure/Content/templates/variables/dotnet/secrets-unofficial.yml b/src/Infrastructure/Content/templates/variables/dotnet/secrets-unofficial.yml new file mode 100644 index 000000000..1744ad288 --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dotnet/secrets-unofficial.yml @@ -0,0 +1,5 @@ +variables: +- group: DotNet-Docker-Secrets-Low + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnet-dockerhub-bot-username);$(dotnet-dockerhub-bot-pat-low)' diff --git a/src/Infrastructure/Content/templates/variables/dotnet/secrets.yml b/src/Infrastructure/Content/templates/variables/dotnet/secrets.yml new file mode 100644 index 000000000..0224c441e --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/dotnet/secrets.yml @@ -0,0 +1,17 @@ +variables: +- group: DotNet-Docker-Secrets + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)' + +- name: gitHubNotificationsRepoInfo.authArgs + value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' + +- name: gitHubVersionsRepoInfo.authArgs + value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' + +- name: mcrDocsRepoInfo.authArgs + value: >- + --gh-private-key '$(GitHubApp-NET-Docker-MAR-Docs-Updater-PrivateKey)' + --gh-app-client-id '$(gitHubApp.marDocsUpdater.clientId)' + --gh-app-installation-id '$(gitHubApp.marDocsUpdater.microsoft.installationId)' diff --git a/src/Infrastructure/Content/templates/variables/sdl-pool.yml b/src/Infrastructure/Content/templates/variables/sdl-pool.yml new file mode 100644 index 000000000..18ae9b9fd --- /dev/null +++ b/src/Infrastructure/Content/templates/variables/sdl-pool.yml @@ -0,0 +1,9 @@ +# This template provides default variables for the source analysis pool and +# image used by 1ES pipeline templates for the "SDL Sources Analysis" stage. +variables: +- name: defaultSourceAnalysisPoolName + value: NetCore1ESPool-Internal + readonly: true +- name: defaultSourceAnalysisPoolImage + value: 1es-windows-2022 + readonly: true diff --git a/src/Infrastructure/InfrastructureContent.cs b/src/Infrastructure/InfrastructureContent.cs new file mode 100644 index 000000000..ff5b6fcc7 --- /dev/null +++ b/src/Infrastructure/InfrastructureContent.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +namespace Microsoft.DotNet.DockerTools.Infrastructure; + +/// +/// Provides access to the eng/docker-tools infrastructure files (pipeline templates, +/// PowerShell scripts, and docs) that are embedded into this assembly at build time. +/// +/// +/// A matching version of ImageBuilder ships these files so it can write them back out to disk, +/// keeping pipeline content coupled to the ImageBuilder version that consumes it. +/// +public static class InfrastructureContent +{ + /// + /// Prefix applied to the LogicalName of every embedded content resource. + /// + private const string ResourcePrefix = "Content/"; + + private static readonly Assembly s_assembly = typeof(InfrastructureContent).Assembly; + + /// + /// Maps each content file's path (relative to the embedded content root, using '/' separators) + /// to its underlying manifest resource name. + /// + private static readonly IReadOnlyDictionary s_resourceNamesByPath = BuildIndex(); + + /// + /// Gets the paths of all embedded content files, relative to the content root and using + /// '/' as the directory separator (for example, templates/jobs/build-images.yml). + /// + public static IReadOnlyList GetRelativePaths() => [.. s_resourceNamesByPath.Keys]; + + /// + /// Opens a stream over the raw bytes of an embedded content file. The caller owns the returned + /// stream and is responsible for disposing it. Returning a stream avoids buffering whole files + /// in memory, since the embedded content can be arbitrarily large. + /// + /// + /// The file's path relative to the content root, using '/' separators, exactly as returned by + /// . + /// + public static Stream OpenRead(string relativePath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(relativePath); + + if (!s_resourceNamesByPath.TryGetValue(relativePath, out string? resourceName)) + { + throw new KeyNotFoundException($"No embedded infrastructure content found for path '{relativePath}'."); + } + + return s_assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded resource '{resourceName}' could not be opened."); + } + + private static Dictionary BuildIndex() + { + Dictionary resourceNamesByPath = new(StringComparer.Ordinal); + + // GetManifestResourceNames/GetManifestResourceStream are part of the AOT/trim-safe reflection subset + // (embedded resources are preserved by the trimmer), so this is intentionally AOT compatible. Don't + // "fix" it by switching to a source generator without a concrete reason - it would only add an analyzer + // project and a Roslyn dependency for no AOT benefit. + foreach (string resourceName in s_assembly.GetManifestResourceNames()) + { + // Resource names use the build OS directory separator, so normalize before matching. + string normalizedName = resourceName.Replace('\\', '/'); + if (!normalizedName.StartsWith(ResourcePrefix, StringComparison.Ordinal)) + { + continue; + } + + string relativePath = normalizedName[ResourcePrefix.Length..]; + resourceNamesByPath[relativePath] = resourceName; + } + + return resourceNamesByPath; + } +} diff --git a/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj b/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj new file mode 100644 index 000000000..2cf0fb2b0 --- /dev/null +++ b/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj @@ -0,0 +1,33 @@ + + + + net9.0 + Microsoft.DotNet.DockerTools.Infrastructure + true + + + false + + + true + + + enable + + + + + + Content/%(RecursiveDir)%(Filename)%(Extension) + + + + + diff --git a/src/Infrastructure/README.md b/src/Infrastructure/README.md new file mode 100644 index 000000000..ce39b7e0c --- /dev/null +++ b/src/Infrastructure/README.md @@ -0,0 +1,34 @@ +# Microsoft.DotNet.DockerTools.Infrastructure + +This project carries a copy of the repository's `eng/docker-tools/` infrastructure (Azure +Pipelines templates, PowerShell scripts, and docs) and embeds it into the assembly as +resources. ImageBuilder references this project so it can ship these files inside its build +and write them back out to disk via the `update` command. + +## Why + +This repo both **produces** and **consumes** the `eng/docker-tools/` infrastructure: +`eng/docker-tools/` is the source of truth it authors, and ImageBuilder — which is built here — +must ship those same files so downstream repos receive them. That dual role is why the content +exists twice in this repo. Embedding the copy in ImageBuilder also collapses what used to be a +two-phase change process (ship a new ImageBuilder, then a follow-up PR to change the pipelines) +into a single commit, since source-code changes and pipeline-template changes now travel together. + +See [issue #2130](https://github.com/dotnet/docker-tools/issues/2130) for background. + +The embedded copy lives in `Content/`, under `src/`, because the ImageBuilder Docker build +context is `src/`, so `eng/docker-tools/` is not reachable from the container build. + +## Relationship to `eng/docker-tools/` + +`Content/` is the copy that ships inside ImageBuilder, so infrastructure changes are made here. +This repo's own `eng/docker-tools/` is not hand-synced to match: it is regenerated by the `update` +command when a newer ImageBuilder flows in through automatic dependency updates. The two therefore +differ much of the time — `eng/docker-tools/` reflects the ImageBuilder version this repo currently +consumes, while `Content/` reflects what the next ImageBuilder will ship. + +`Content/templates/variables/docker-images.yml` is a further, per-file difference: it stores the +ImageBuilder image tag as a `{{IMAGEBUILDER_TAG}}` Cottle expression rather than a concrete tag, +because a build cannot know its own future tag. The `update` command renders that one file +(substituting this build's tag, or `latest` for local/dotnet-tool builds with no baked-in tag), so +the rendered `eng/docker-tools/docker-images.yml` holds a concrete tag on that line. From ace15dd892726805107386eda2bfc8b6cf07c1bc Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 22 Jun 2026 11:08:16 -0700 Subject: [PATCH 4/7] Add ImageBuilder 'update' command to emit bundled docker-tools content Add the 'update' command, which writes ImageBuilder's embedded eng/docker-tools infrastructure to disk as a full mirror, rendering this build's ImageBuilder tag into docker-images.yml via Cottle. The tag is baked into assembly metadata through the IMAGEBUILDER_TAG build arg/env var that the Dockerfiles set and that manifest.json supplies as the pipeline UniqueId; it falls back to "latest" for local builds. Reference the Infrastructure project, register the command, and document the bundling in AGENTS.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 7 + src/Dockerfile.linux | 3 + src/Dockerfile.windows | 3 + src/ImageBuilder.Tests/UpdateCommandTests.cs | 211 ++++++++++++++++++ .../AssemblyImageBuilderTagProvider.cs | 23 ++ src/ImageBuilder/Commands/UpdateCommand.cs | 150 +++++++++++++ src/ImageBuilder/Commands/UpdateOptions.cs | 32 +++ src/ImageBuilder/IImageBuilderTagProvider.cs | 17 ++ src/ImageBuilder/ImageBuilder.cs | 2 + .../Microsoft.DotNet.ImageBuilder.csproj | 11 + src/manifest.json | 9 + 11 files changed, 468 insertions(+) create mode 100644 src/ImageBuilder.Tests/UpdateCommandTests.cs create mode 100644 src/ImageBuilder/AssemblyImageBuilderTagProvider.cs create mode 100644 src/ImageBuilder/Commands/UpdateCommand.cs create mode 100644 src/ImageBuilder/Commands/UpdateOptions.cs create mode 100644 src/ImageBuilder/IImageBuilderTagProvider.cs diff --git a/AGENTS.md b/AGENTS.md index 23c74230f..46ac0daac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,13 @@ Breaking changes to `eng/docker-tools/` must be documented in `eng/docker-tools/ For comprehensive documentation on the docker-tools infrastructure, pipeline architecture, image building workflows, and troubleshooting, see [eng/docker-tools/DEV-GUIDE.md](eng/docker-tools/DEV-GUIDE.md). +### Templates Bundled In ImageBuilder + +ImageBuilder ships a copy of the entire `eng/docker-tools/` directory so that source-code changes and pipeline-template changes can be made together in the same commit. See [issue #2130](https://github.com/dotnet/docker-tools/issues/2130) for background. + +- `src/Infrastructure/` (`Microsoft.DotNet.DockerTools.Infrastructure`) embeds a copy of `eng/docker-tools/` under `src/Infrastructure/Content/` as assembly resources, which ImageBuilder's `update` command writes back to disk. The copy must live under `src/` because the Docker build context is `src/`, so `eng/docker-tools/` is not reachable from the container build. +- **The two copies are not kept identical.** Make infrastructure changes in `src/Infrastructure/Content/` — that copy ships in ImageBuilder. This repo's own `eng/docker-tools/` is regenerated by the `update` command when a newer ImageBuilder arrives via automatic dependency updates, so the two legitimately differ in between: `eng/docker-tools/` tracks the ImageBuilder version this repo currently consumes, while `Content/` tracks what the next one will ship. See [src/Infrastructure/README.md](src/Infrastructure/README.md). + ### Service Connections and Authentication `publishConfig` is the source of truth for registry authentication. Registry service connections live in `publishConfig.RegistryAuthentication`. Non-registry connections (e.g., kusto, marStatus, cleanServiceConnection) are separate fields or passed via `additionalServiceConnections`. diff --git a/src/Dockerfile.linux b/src/Dockerfile.linux index ee8ce4112..05f23dba6 100644 --- a/src/Dockerfile.linux +++ b/src/Dockerfile.linux @@ -5,6 +5,8 @@ # build Microsoft.DotNet.ImageBuilder FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-azurelinux3.0 AS build-env ARG TARGETARCH +ARG IMAGEBUILDER_TAG= +ENV IMAGEBUILDER_TAG=${IMAGEBUILDER_TAG} # download root CA certificates for signature verification WORKDIR / @@ -20,6 +22,7 @@ WORKDIR /image-builder COPY NuGet.config ./ COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/ COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/ +COPY Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj ./Infrastructure/ RUN dotnet restore -r linux-$TARGETARCH ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj # copy everything else and publish diff --git a/src/Dockerfile.windows b/src/Dockerfile.windows index ac395518e..e50e0e2c4 100644 --- a/src/Dockerfile.windows +++ b/src/Dockerfile.windows @@ -5,6 +5,8 @@ ARG WINDOWS_SDK # build Microsoft.DotNet.ImageBuilder FROM mcr.microsoft.com/dotnet/sdk:9.0-$WINDOWS_SDK AS build-env +ARG IMAGEBUILDER_TAG= +ENV IMAGEBUILDER_TAG=${IMAGEBUILDER_TAG} WORKDIR /image-builder # Work around Windows path length limitations by moving NuGet package cache to @@ -15,6 +17,7 @@ ENV NUGET_PACKAGES=/nuget COPY NuGet.config ./ COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/ COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/ +COPY Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj ./Infrastructure/ RUN dotnet restore -r win-x64 ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj # copy everything else and publish diff --git a/src/ImageBuilder.Tests/UpdateCommandTests.cs b/src/ImageBuilder.Tests/UpdateCommandTests.cs new file mode 100644 index 000000000..709b5690d --- /dev/null +++ b/src/ImageBuilder.Tests/UpdateCommandTests.cs @@ -0,0 +1,211 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.DotNet.DockerTools.Infrastructure; +using Microsoft.DotNet.ImageBuilder.Commands; +using Microsoft.DotNet.ImageBuilder.Tests.Helpers; +using Microsoft.Extensions.Logging; +using Moq; +using Shouldly; + +namespace Microsoft.DotNet.ImageBuilder.Tests; + +[TestClass] +public class UpdateCommandTests +{ + // Use the platform's directory separator for the fake root so that paths derived via + // Path.Combine and Path.GetDirectoryName stay consistent (Path.GetDirectoryName normalizes + // a leading '/' to '\' on Windows, which would otherwise not match the in-memory entries). + private static readonly string RepoRoot = $"{Path.DirectorySeparatorChar}repo"; + + private static readonly string OutputPath = Path.Combine(RepoRoot, "eng", "docker-tools"); + + [TestMethod] + public async Task UpdateCommand_WritesAllEmbeddedFiles() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(OutputPath); + UpdateCommand command = CreateCommand(fileSystem); + + await command.ExecuteAsync(); + + IReadOnlyList expectedPaths = InfrastructureContent.GetRelativePaths(); + expectedPaths.ShouldNotBeEmpty(); + List renderedDestinations = []; + + foreach (string relativePath in expectedPaths) + { + string expectedDestination = Path.Combine(OutputPath, relativePath.Replace('/', Path.DirectorySeparatorChar)); + fileSystem.FileExists(expectedDestination).ShouldBeTrue(); + + using Stream expectedStream = InfrastructureContent.OpenRead(relativePath); + using MemoryStream expectedBytes = new(); + expectedStream.CopyTo(expectedBytes); + if (fileSystem.GetFileBytes(expectedDestination).SequenceEqual(expectedBytes.ToArray())) + { + continue; + } + + renderedDestinations.Add(expectedDestination); + } + + renderedDestinations.Count.ShouldBe(1); + } + + [TestMethod] + public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataSet_RendersDockerImagesTemplate() + { + const string tag = "1234567"; + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(OutputPath); + UpdateCommand command = CreateCommand(fileSystem, tag); + + await command.ExecuteAsync(); + + string content = GetRenderedImageBuilderVariables(fileSystem, $"image-builder:{tag}"); + content.ShouldContain($"image-builder:{tag}"); + content.ShouldNotContain("{{"); + } + + [TestMethod] + public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataMissing_FallsBackToLatestWithWarning() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(OutputPath); + Mock> logger = new(); + UpdateCommand command = CreateCommand(fileSystem, tag: null, logger: logger); + + await command.ExecuteAsync(); + + string content = GetRenderedImageBuilderVariables(fileSystem, "image-builder:latest"); + content.ShouldContain("image-builder:latest"); + content.ShouldNotContain("{{"); + + logger.Verify( + log => log.Log( + LogLevel.Warning, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.Once); + } + + [TestMethod] + public async Task UpdateCommand_DeletesStaleFiles() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(OutputPath); + string staleFile = Path.Combine(OutputPath, "templates", "removed-template.yml"); + fileSystem.AddFile(staleFile, "stale"); + UpdateCommand command = CreateCommand(fileSystem); + + await command.ExecuteAsync(); + + fileSystem.FileExists(staleFile).ShouldBeFalse(); + fileSystem.FilesDeleted.ShouldContain(staleFile); + } + + [TestMethod] + public async Task UpdateCommand_DeletesStaleDirectories() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(OutputPath); + string staleDirectory = Path.Combine(OutputPath, "obsolete"); + string staleFile = Path.Combine(staleDirectory, "old.yml"); + fileSystem.AddFile(staleFile, "stale"); + UpdateCommand command = CreateCommand(fileSystem); + + await command.ExecuteAsync(); + + fileSystem.DirectoryExists(staleDirectory).ShouldBeFalse(); + fileSystem.DirectoriesDeleted.ShouldContain(staleDirectory); + } + + [TestMethod] + public async Task UpdateCommand_NotGitRoot_Throws() + { + InMemoryFileSystem fileSystem = new() { CurrentDirectory = RepoRoot }; + fileSystem.AddDirectory(OutputPath); + UpdateCommand command = CreateCommand(fileSystem); + + InvalidOperationException exception = + await Should.ThrowAsync(() => command.ExecuteAsync()); + exception.Message.ShouldContain("root of a git repository"); + } + + [TestMethod] + public async Task UpdateCommand_OutputMissingWithoutInit_Throws() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + UpdateCommand command = CreateCommand(fileSystem); + + InvalidOperationException exception = + await Should.ThrowAsync(() => command.ExecuteAsync()); + exception.Message.ShouldContain("--init"); + fileSystem.FilesWritten.ShouldBeEmpty(); + } + + [TestMethod] + public async Task UpdateCommand_OutputMissingWithInit_CreatesAndWrites() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + UpdateCommand command = CreateCommand(fileSystem); + command.Options.Init = true; + + await command.ExecuteAsync(); + + fileSystem.DirectoriesCreated.ShouldContain(OutputPath); + fileSystem.FilesWritten.ShouldNotBeEmpty(); + } + + [TestMethod] + public async Task UpdateCommand_DryRun_MakesNoChanges() + { + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(OutputPath); + string staleFile = Path.Combine(OutputPath, "templates", "removed-template.yml"); + fileSystem.AddFile(staleFile, "stale"); + UpdateCommand command = CreateCommand(fileSystem); + command.Options.IsDryRun = true; + + await command.ExecuteAsync(); + + fileSystem.FilesWritten.ShouldBeEmpty(); + fileSystem.FilesDeleted.ShouldBeEmpty(); + fileSystem.DirectoriesDeleted.ShouldBeEmpty(); + fileSystem.FileExists(staleFile).ShouldBeTrue(); + } + + private static InMemoryFileSystem CreateRepoFileSystem() + { + InMemoryFileSystem fileSystem = new() { CurrentDirectory = RepoRoot }; + fileSystem.AddDirectory(Path.Combine(RepoRoot, ".git")); + return fileSystem; + } + + private static UpdateCommand CreateCommand( + IFileSystem fileSystem, + string? tag = null, + Mock>? logger = null) + { + Mock tagProvider = new(); + tagProvider.Setup(provider => provider.GetTag()).Returns(tag); + return new UpdateCommand( + fileSystem, + tagProvider.Object, + (logger ?? new Mock>()).Object); + } + + private static string GetRenderedImageBuilderVariables(InMemoryFileSystem fileSystem, string expectedImageBuilderReference) => + fileSystem.FilesWritten + .Select(path => Encoding.UTF8.GetString(fileSystem.GetFileBytes(path))) + .Single(content => content.Contains(expectedImageBuilderReference)); +} diff --git a/src/ImageBuilder/AssemblyImageBuilderTagProvider.cs b/src/ImageBuilder/AssemblyImageBuilderTagProvider.cs new file mode 100644 index 000000000..01c048ca6 --- /dev/null +++ b/src/ImageBuilder/AssemblyImageBuilderTagProvider.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Linq; +using System.Reflection; + +namespace Microsoft.DotNet.ImageBuilder; + +/// +/// Resolves the ImageBuilder tag from the baked into this +/// assembly at build time (via the ImageBuilderTag MSBuild property). +/// +internal sealed class AssemblyImageBuilderTagProvider : IImageBuilderTagProvider +{ + private const string ImageBuilderTagMetadataKey = "ImageBuilderTag"; + + /// + public string? GetTag() => + typeof(AssemblyImageBuilderTagProvider).Assembly + .GetCustomAttributes() + .FirstOrDefault(attribute => attribute.Key == ImageBuilderTagMetadataKey)?.Value; +} diff --git a/src/ImageBuilder/Commands/UpdateCommand.cs b/src/ImageBuilder/Commands/UpdateCommand.cs new file mode 100644 index 000000000..5e34d7f48 --- /dev/null +++ b/src/ImageBuilder/Commands/UpdateCommand.cs @@ -0,0 +1,150 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Cottle; +using Microsoft.DotNet.DockerTools.Infrastructure; +using Microsoft.DotNet.ImageBuilder.Templating; + +namespace Microsoft.DotNet.ImageBuilder.Commands; + +/// +/// Writes the eng/docker-tools infrastructure files (pipeline templates, scripts, and docs) +/// that are embedded in this ImageBuilder build to disk. This keeps the pipeline content in a +/// consuming repo coupled to the ImageBuilder version that uses it. +/// +/// +/// The command must be run from the root of a git repository and always targets +/// eng/docker-tools relative to that root. It performs a full mirror: files under the +/// target directory that ImageBuilder no longer ships are removed so the output exactly matches +/// what is embedded. +/// +public class UpdateCommand : Command +{ + private static readonly string s_outputRelativePath = Path.Combine("eng", "docker-tools"); + // InfrastructureContent.GetRelativePaths always returns '/'-separated paths, so this is kept in + // the same form to compare directly without re-normalizing per file. + private const string DockerImagesRelativePath = "templates/variables/docker-images.yml"; + private const string ImageBuilderTagTemplateVariableName = "IMAGEBUILDER_TAG"; + + private static readonly DocumentConfiguration s_templateConfiguration = CottleDocumentConfiguration.Create(); + + private readonly IFileSystem _fileSystem; + private readonly IImageBuilderTagProvider _tagProvider; + private readonly ILogger _logger; + + public UpdateCommand(IFileSystem fileSystem, IImageBuilderTagProvider tagProvider, ILogger logger) + { + _fileSystem = fileSystem; + _tagProvider = tagProvider; + _logger = logger; + } + + protected override string Description => "Writes ImageBuilder's bundled docker-tools infrastructure files to disk"; + + public override Task ExecuteAsync() + { + string currentDirectory = _fileSystem.GetCurrentDirectory(); + + // Ensure we are in a git directory. + string gitPath = Path.Combine(currentDirectory, ".git"); + if (!_fileSystem.DirectoryExists(gitPath) && !_fileSystem.FileExists(gitPath)) + { + throw new InvalidOperationException( + $"{nameof(UpdateCommand)} must be run from the root of a git repository."); + } + + // Ensure we are running inside a repo that actually uses docker-tools. + // Require --init to skip this check / onboard a new repo. + string outputPath = Path.Combine(currentDirectory, s_outputRelativePath); + if (!Options.Init && !_fileSystem.DirectoryExists(outputPath)) + { + throw new InvalidOperationException( + $"The output directory '{outputPath}' does not exist. " + + $"Pass --init to create it (use this only when onboarding a repo to docker-tools)."); + } + + // Resolve ImageBuilder's image tag. + // If ImageBuilder doesn't know about its own tag, then use `latest` as a fallback. + string imageBuilderTag; + if (_tagProvider.GetTag() is { } resolvedImageBuilderTag && !string.IsNullOrWhiteSpace(resolvedImageBuilderTag)) + { + imageBuilderTag = resolvedImageBuilderTag; + } + else + { + _logger.LogWarning( + "This build of ImageBuilder was not built with the \"IMAGEBUILDER_TAG\" MSBuild property set. " + + "ImageBuilder tag will fall back to \"latest\"."); + imageBuilderTag = "latest"; + } + + // Clear the existing infrastructure content and re-write it all. + // This prevents stale files/directories from being left behind. + if (!Options.IsDryRun) + { + try + { + _fileSystem.DeleteDirectory(outputPath, recursive: true); + _logger.LogInformation("Deleted existing directory '{OutputPath}'", outputPath); + } + catch (DirectoryNotFoundException) + { + // Already absent — the mirror writes everything fresh below, so there's nothing to remove. + } + } + + foreach (string relativePath in InfrastructureContent.GetRelativePaths()) + { + string destinationPath = Path.Combine(outputPath, relativePath.Replace('/', Path.DirectorySeparatorChar)); + + if (Options.IsDryRun) + { + _logger.LogInformation("[Dry run] Would write '{DestinationPath}'", destinationPath); + continue; + } + + string? destinationDirectory = Path.GetDirectoryName(destinationPath); + if (!string.IsNullOrEmpty(destinationDirectory)) + { + _fileSystem.CreateDirectory(destinationDirectory); + } + + _logger.LogInformation("Writing '{DestinationPath}'", destinationPath); + + // docker-images.yml is templated because ImageBuilder needs to self-reference its own image tag. + if (relativePath == DockerImagesRelativePath) + { + string renderedContent = RenderDockerImagesTemplate(relativePath, imageBuilderTag); + _fileSystem.WriteAllText(destinationPath, renderedContent); + } + else + { + using Stream source = InfrastructureContent.OpenRead(relativePath); + using Stream destination = _fileSystem.CreateFile(destinationPath); + source.CopyTo(destination); + } + } + + return Task.CompletedTask; + } + + private static string RenderDockerImagesTemplate(string relativePath, string imageBuilderTag) + { + using Stream source = InfrastructureContent.OpenRead(relativePath); + using StreamReader reader = new(source); + string template = reader.ReadToEnd(); + + IDocument document = Document.CreateDefault(template, s_templateConfiguration).DocumentOrThrow; + Dictionary symbols = new() + { + [ImageBuilderTagTemplateVariableName] = imageBuilderTag + }; + + return document.Render(Context.CreateBuiltin(symbols)); + } +} diff --git a/src/ImageBuilder/Commands/UpdateOptions.cs b/src/ImageBuilder/Commands/UpdateOptions.cs new file mode 100644 index 000000000..f9789e32d --- /dev/null +++ b/src/ImageBuilder/Commands/UpdateOptions.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.Parsing; + +namespace Microsoft.DotNet.ImageBuilder.Commands; + +public class UpdateOptions : Options +{ + /// + /// When true, creates the eng/docker-tools directory if it does not already exist. + /// Otherwise, the command fails when the directory is missing. + /// + public bool Init { get; set; } + + private static readonly Option InitOption = new("--init") + { + Description = "Create the eng/docker-tools directory if it does not already exist", + }; + + public override IEnumerable diff --git a/src/manifest.json b/src/manifest.json index a37289be9..662dab934 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -12,6 +12,9 @@ }, "platforms": [ { + "buildArgs": { + "IMAGEBUILDER_TAG": "$(UniqueId)" + }, "dockerfile": "Dockerfile.linux", "os": "linux", "osVersion": "azurelinux", @@ -22,6 +25,9 @@ }, { "architecture": "arm64", + "buildArgs": { + "IMAGEBUILDER_TAG": "$(UniqueId)" + }, "dockerfile": "Dockerfile.linux", "os": "linux", "osVersion": "azurelinux", @@ -33,6 +39,7 @@ }, { "buildArgs": { + "IMAGEBUILDER_TAG": "$(UniqueId)", "WINDOWS_BASE": "servercore:ltsc2016-amd64", "WINDOWS_SDK": "nanoserver-1809" }, @@ -46,6 +53,7 @@ }, { "buildArgs": { + "IMAGEBUILDER_TAG": "$(UniqueId)", "WINDOWS_BASE": "nanoserver:1809-amd64", "WINDOWS_SDK": "nanoserver-1809" }, @@ -59,6 +67,7 @@ }, { "buildArgs": { + "IMAGEBUILDER_TAG": "$(UniqueId)", "WINDOWS_BASE": "nanoserver:ltsc2022-amd64", "WINDOWS_SDK": "nanoserver-ltsc2022" }, From 8a428c9a9cb1107193b8333732585ed31af47173 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 22 Jun 2026 14:23:56 -0700 Subject: [PATCH 5/7] Add PathHelper.SafeCombine --- src/ImageBuilder.Tests/PathHelperTests.cs | 68 ++++++++++++++++++++ src/ImageBuilder.Tests/UpdateCommandTests.cs | 14 ++-- src/ImageBuilder/Commands/UpdateCommand.cs | 8 +-- src/ImageBuilder/PathHelper.cs | 43 +++++++++++++ 4 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 src/ImageBuilder.Tests/PathHelperTests.cs diff --git a/src/ImageBuilder.Tests/PathHelperTests.cs b/src/ImageBuilder.Tests/PathHelperTests.cs new file mode 100644 index 000000000..5133fc534 --- /dev/null +++ b/src/ImageBuilder.Tests/PathHelperTests.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Shouldly; + +namespace Microsoft.DotNet.ImageBuilder.Tests; + +[TestClass] +public class PathHelperTests +{ + [TestMethod] + public void SafeCombine_RelativeSegments_CombinesUnderBasePath() + { + string basePath = $"{Path.DirectorySeparatorChar}repo"; + + string result = PathHelper.SafeCombine(basePath, "eng", "docker-tools"); + + result.ShouldBe(Path.Combine(basePath, "eng", "docker-tools")); + } + + [TestMethod] + public void SafeCombine_ForwardSlashSegment_ConvertsToPlatformSeparator() + { + string basePath = $"{Path.DirectorySeparatorChar}repo"; + + string result = PathHelper.SafeCombine(basePath, "templates/variables/docker-images.yml"); + + result.ShouldBe(Path.Combine(basePath, "templates", "variables", "docker-images.yml")); + } + + [TestMethod] + public void SafeCombine_RootedSegment_Throws() + { + string basePath = $"{Path.DirectorySeparatorChar}repo"; + string rootedSegment = $"{Path.DirectorySeparatorChar}etc{Path.DirectorySeparatorChar}passwd"; + + ArgumentException exception = + Should.Throw(() => PathHelper.SafeCombine(basePath, rootedSegment)); + exception.Message.ShouldContain("rooted"); + } + + [TestMethod] + [DataRow("..")] + [DataRow("../escape")] + [DataRow("nested/../../escape")] + public void SafeCombine_TraversalSegment_Throws(string traversalSegment) + { + string basePath = $"{Path.DirectorySeparatorChar}repo"; + + ArgumentException exception = + Should.Throw(() => PathHelper.SafeCombine(basePath, traversalSegment)); + exception.Message.ShouldContain("traversal"); + } + + [TestMethod] + [DataRow("")] + [DataRow(" ")] + [DataRow(null)] + public void SafeCombine_NullOrWhitespaceSegment_Throws(string? segment) + { + string basePath = $"{Path.DirectorySeparatorChar}repo"; + + Should.Throw(() => PathHelper.SafeCombine(basePath, segment!)); + } +} diff --git a/src/ImageBuilder.Tests/UpdateCommandTests.cs b/src/ImageBuilder.Tests/UpdateCommandTests.cs index 709b5690d..2e383ea4b 100644 --- a/src/ImageBuilder.Tests/UpdateCommandTests.cs +++ b/src/ImageBuilder.Tests/UpdateCommandTests.cs @@ -25,7 +25,7 @@ public class UpdateCommandTests // a leading '/' to '\' on Windows, which would otherwise not match the in-memory entries). private static readonly string RepoRoot = $"{Path.DirectorySeparatorChar}repo"; - private static readonly string OutputPath = Path.Combine(RepoRoot, "eng", "docker-tools"); + private static readonly string OutputPath = PathHelper.SafeCombine(RepoRoot, "eng", "docker-tools"); [TestMethod] public async Task UpdateCommand_WritesAllEmbeddedFiles() @@ -42,7 +42,7 @@ public async Task UpdateCommand_WritesAllEmbeddedFiles() foreach (string relativePath in expectedPaths) { - string expectedDestination = Path.Combine(OutputPath, relativePath.Replace('/', Path.DirectorySeparatorChar)); + string expectedDestination = PathHelper.SafeCombine(OutputPath, relativePath); fileSystem.FileExists(expectedDestination).ShouldBeTrue(); using Stream expectedStream = InfrastructureContent.OpenRead(relativePath); @@ -103,7 +103,7 @@ public async Task UpdateCommand_DeletesStaleFiles() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); fileSystem.AddDirectory(OutputPath); - string staleFile = Path.Combine(OutputPath, "templates", "removed-template.yml"); + string staleFile = PathHelper.SafeCombine(OutputPath, "templates", "removed-template.yml"); fileSystem.AddFile(staleFile, "stale"); UpdateCommand command = CreateCommand(fileSystem); @@ -118,8 +118,8 @@ public async Task UpdateCommand_DeletesStaleDirectories() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); fileSystem.AddDirectory(OutputPath); - string staleDirectory = Path.Combine(OutputPath, "obsolete"); - string staleFile = Path.Combine(staleDirectory, "old.yml"); + string staleDirectory = PathHelper.SafeCombine(OutputPath, "obsolete"); + string staleFile = PathHelper.SafeCombine(staleDirectory, "old.yml"); fileSystem.AddFile(staleFile, "stale"); UpdateCommand command = CreateCommand(fileSystem); @@ -171,7 +171,7 @@ public async Task UpdateCommand_DryRun_MakesNoChanges() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); fileSystem.AddDirectory(OutputPath); - string staleFile = Path.Combine(OutputPath, "templates", "removed-template.yml"); + string staleFile = PathHelper.SafeCombine(OutputPath, "templates", "removed-template.yml"); fileSystem.AddFile(staleFile, "stale"); UpdateCommand command = CreateCommand(fileSystem); command.Options.IsDryRun = true; @@ -187,7 +187,7 @@ public async Task UpdateCommand_DryRun_MakesNoChanges() private static InMemoryFileSystem CreateRepoFileSystem() { InMemoryFileSystem fileSystem = new() { CurrentDirectory = RepoRoot }; - fileSystem.AddDirectory(Path.Combine(RepoRoot, ".git")); + fileSystem.AddDirectory(PathHelper.SafeCombine(RepoRoot, ".git")); return fileSystem; } diff --git a/src/ImageBuilder/Commands/UpdateCommand.cs b/src/ImageBuilder/Commands/UpdateCommand.cs index 5e34d7f48..2f1a8fb60 100644 --- a/src/ImageBuilder/Commands/UpdateCommand.cs +++ b/src/ImageBuilder/Commands/UpdateCommand.cs @@ -25,7 +25,7 @@ namespace Microsoft.DotNet.ImageBuilder.Commands; /// public class UpdateCommand : Command { - private static readonly string s_outputRelativePath = Path.Combine("eng", "docker-tools"); + private static readonly string s_outputRelativePath = PathHelper.SafeCombine("eng", "docker-tools"); // InfrastructureContent.GetRelativePaths always returns '/'-separated paths, so this is kept in // the same form to compare directly without re-normalizing per file. private const string DockerImagesRelativePath = "templates/variables/docker-images.yml"; @@ -51,7 +51,7 @@ public override Task ExecuteAsync() string currentDirectory = _fileSystem.GetCurrentDirectory(); // Ensure we are in a git directory. - string gitPath = Path.Combine(currentDirectory, ".git"); + string gitPath = PathHelper.SafeCombine(currentDirectory, ".git"); if (!_fileSystem.DirectoryExists(gitPath) && !_fileSystem.FileExists(gitPath)) { throw new InvalidOperationException( @@ -60,7 +60,7 @@ public override Task ExecuteAsync() // Ensure we are running inside a repo that actually uses docker-tools. // Require --init to skip this check / onboard a new repo. - string outputPath = Path.Combine(currentDirectory, s_outputRelativePath); + string outputPath = PathHelper.SafeCombine(currentDirectory, s_outputRelativePath); if (!Options.Init && !_fileSystem.DirectoryExists(outputPath)) { throw new InvalidOperationException( @@ -100,7 +100,7 @@ public override Task ExecuteAsync() foreach (string relativePath in InfrastructureContent.GetRelativePaths()) { - string destinationPath = Path.Combine(outputPath, relativePath.Replace('/', Path.DirectorySeparatorChar)); + string destinationPath = PathHelper.SafeCombine(outputPath, relativePath); if (Options.IsDryRun) { diff --git a/src/ImageBuilder/PathHelper.cs b/src/ImageBuilder/PathHelper.cs index 78f00aeeb..797178116 100644 --- a/src/ImageBuilder/PathHelper.cs +++ b/src/ImageBuilder/PathHelper.cs @@ -5,6 +5,7 @@ using System; using System.IO; +using System.Linq; namespace Microsoft.DotNet.ImageBuilder { @@ -12,6 +13,48 @@ public static class PathHelper { public static string NormalizePath(string path) => path.Replace(@"\", "/"); + /// + /// Combines two strings into a path. Same as but protects against path traversal. + /// + /// The base path that the result will remain under. + /// Relative segments to append, in order. + /// Thrown when path traversal is blocked. + public static string SafeCombine(string basePath, params string[] paths) + { + ArgumentException.ThrowIfNullOrWhiteSpace(basePath); + ArgumentNullException.ThrowIfNull(paths); + + string combined = basePath; + foreach (string relativePath in paths) + { + ArgumentException.ThrowIfNullOrWhiteSpace(relativePath); + + string platformRelativePath = relativePath.Replace('/', Path.DirectorySeparatorChar); + + if (Path.IsPathRooted(platformRelativePath)) + { + throw new ArgumentException( + $"Path segment '{relativePath}' must be relative, but it is rooted.", + nameof(paths)); + } + + bool hasTraversal = platformRelativePath + .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + .Any(segment => segment == ".."); + + if (hasTraversal) + { + throw new ArgumentException( + $"Path segment '{relativePath}' must not contain a '..' traversal component.", + nameof(paths)); + } + + combined = Path.Combine(combined, platformRelativePath); + } + + return combined; + } + /// /// Trims the string from . /// From 7d222db0c2f3df6a505be26540e69954ff45fe22 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 22 Jun 2026 14:24:46 -0700 Subject: [PATCH 6/7] Fix naming violations --- src/ImageBuilder.Tests/UpdateCommandTests.cs | 35 ++++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/ImageBuilder.Tests/UpdateCommandTests.cs b/src/ImageBuilder.Tests/UpdateCommandTests.cs index 2e383ea4b..8719a8360 100644 --- a/src/ImageBuilder.Tests/UpdateCommandTests.cs +++ b/src/ImageBuilder.Tests/UpdateCommandTests.cs @@ -23,15 +23,14 @@ public class UpdateCommandTests // Use the platform's directory separator for the fake root so that paths derived via // Path.Combine and Path.GetDirectoryName stay consistent (Path.GetDirectoryName normalizes // a leading '/' to '\' on Windows, which would otherwise not match the in-memory entries). - private static readonly string RepoRoot = $"{Path.DirectorySeparatorChar}repo"; - - private static readonly string OutputPath = PathHelper.SafeCombine(RepoRoot, "eng", "docker-tools"); + private static readonly string s_repoRoot = $"{Path.DirectorySeparatorChar}repo"; + private static readonly string s_outputPath = PathHelper.SafeCombine(s_repoRoot, "eng", "docker-tools"); [TestMethod] public async Task UpdateCommand_WritesAllEmbeddedFiles() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); - fileSystem.AddDirectory(OutputPath); + fileSystem.AddDirectory(s_outputPath); UpdateCommand command = CreateCommand(fileSystem); await command.ExecuteAsync(); @@ -42,7 +41,7 @@ public async Task UpdateCommand_WritesAllEmbeddedFiles() foreach (string relativePath in expectedPaths) { - string expectedDestination = PathHelper.SafeCombine(OutputPath, relativePath); + string expectedDestination = PathHelper.SafeCombine(s_outputPath, relativePath); fileSystem.FileExists(expectedDestination).ShouldBeTrue(); using Stream expectedStream = InfrastructureContent.OpenRead(relativePath); @@ -64,7 +63,7 @@ public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataSet_RendersDocker { const string tag = "1234567"; InMemoryFileSystem fileSystem = CreateRepoFileSystem(); - fileSystem.AddDirectory(OutputPath); + fileSystem.AddDirectory(s_outputPath); UpdateCommand command = CreateCommand(fileSystem, tag); await command.ExecuteAsync(); @@ -78,7 +77,7 @@ public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataSet_RendersDocker public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataMissing_FallsBackToLatestWithWarning() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); - fileSystem.AddDirectory(OutputPath); + fileSystem.AddDirectory(s_outputPath); Mock> logger = new(); UpdateCommand command = CreateCommand(fileSystem, tag: null, logger: logger); @@ -102,8 +101,8 @@ public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataMissing_FallsBack public async Task UpdateCommand_DeletesStaleFiles() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); - fileSystem.AddDirectory(OutputPath); - string staleFile = PathHelper.SafeCombine(OutputPath, "templates", "removed-template.yml"); + fileSystem.AddDirectory(s_outputPath); + string staleFile = PathHelper.SafeCombine(s_outputPath, "templates", "removed-template.yml"); fileSystem.AddFile(staleFile, "stale"); UpdateCommand command = CreateCommand(fileSystem); @@ -117,8 +116,8 @@ public async Task UpdateCommand_DeletesStaleFiles() public async Task UpdateCommand_DeletesStaleDirectories() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); - fileSystem.AddDirectory(OutputPath); - string staleDirectory = PathHelper.SafeCombine(OutputPath, "obsolete"); + fileSystem.AddDirectory(s_outputPath); + string staleDirectory = PathHelper.SafeCombine(s_outputPath, "obsolete"); string staleFile = PathHelper.SafeCombine(staleDirectory, "old.yml"); fileSystem.AddFile(staleFile, "stale"); UpdateCommand command = CreateCommand(fileSystem); @@ -132,8 +131,8 @@ public async Task UpdateCommand_DeletesStaleDirectories() [TestMethod] public async Task UpdateCommand_NotGitRoot_Throws() { - InMemoryFileSystem fileSystem = new() { CurrentDirectory = RepoRoot }; - fileSystem.AddDirectory(OutputPath); + InMemoryFileSystem fileSystem = new() { CurrentDirectory = s_repoRoot }; + fileSystem.AddDirectory(s_outputPath); UpdateCommand command = CreateCommand(fileSystem); InvalidOperationException exception = @@ -162,7 +161,7 @@ public async Task UpdateCommand_OutputMissingWithInit_CreatesAndWrites() await command.ExecuteAsync(); - fileSystem.DirectoriesCreated.ShouldContain(OutputPath); + fileSystem.DirectoriesCreated.ShouldContain(s_outputPath); fileSystem.FilesWritten.ShouldNotBeEmpty(); } @@ -170,8 +169,8 @@ public async Task UpdateCommand_OutputMissingWithInit_CreatesAndWrites() public async Task UpdateCommand_DryRun_MakesNoChanges() { InMemoryFileSystem fileSystem = CreateRepoFileSystem(); - fileSystem.AddDirectory(OutputPath); - string staleFile = PathHelper.SafeCombine(OutputPath, "templates", "removed-template.yml"); + fileSystem.AddDirectory(s_outputPath); + string staleFile = PathHelper.SafeCombine(s_outputPath, "templates", "removed-template.yml"); fileSystem.AddFile(staleFile, "stale"); UpdateCommand command = CreateCommand(fileSystem); command.Options.IsDryRun = true; @@ -186,8 +185,8 @@ public async Task UpdateCommand_DryRun_MakesNoChanges() private static InMemoryFileSystem CreateRepoFileSystem() { - InMemoryFileSystem fileSystem = new() { CurrentDirectory = RepoRoot }; - fileSystem.AddDirectory(PathHelper.SafeCombine(RepoRoot, ".git")); + InMemoryFileSystem fileSystem = new() { CurrentDirectory = s_repoRoot }; + fileSystem.AddDirectory(PathHelper.SafeCombine(s_repoRoot, ".git")); return fileSystem; } From e4c2df8b057b99acd058844ede2cc152a04c81df Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Tue, 23 Jun 2026 11:25:47 -0700 Subject: [PATCH 7/7] Pass ImageBuilder reference as an argument to the update command The update command now accepts the ImageBuilder image reference as an optional argument instead of resolving a tag baked into the build via assembly metadata. When omitted, it falls back to the published 'latest' reference. Removes the embedded-tag mechanism (IImageBuilderTagProvider, the ImageBuilderTag assembly metadata, the IMAGEBUILDER_TAG Docker build arg/env, and the manifest buildArgs) and renames the docker-images.yml template placeholder to {{IMAGEBUILDER_REF}}, which now holds the full image reference. Adds an example Update-ImageBuilder.ps1 (bundled in ImageBuilder and mirrored to eng/docker-tools) that resolves the multi-platform image index digest of the latest image via 'docker buildx imagetools inspect' and passes it to the update command. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/docker-tools/Update-ImageBuilder.ps1 | 89 +++++++++++++++++++ src/Dockerfile.linux | 2 - src/Dockerfile.windows | 2 - src/ImageBuilder.Tests/UpdateCommandTests.cs | 37 ++++---- .../AssemblyImageBuilderTagProvider.cs | 23 ----- src/ImageBuilder/Commands/UpdateCommand.cs | 34 +++---- src/ImageBuilder/Commands/UpdateOptions.cs | 20 +++++ src/ImageBuilder/IImageBuilderTagProvider.cs | 17 ---- src/ImageBuilder/ImageBuilder.cs | 1 - .../Microsoft.DotNet.ImageBuilder.csproj | 10 --- .../Content/Update-ImageBuilder.ps1 | 89 +++++++++++++++++++ .../templates/variables/docker-images.yml | 2 +- src/Infrastructure/README.md | 8 +- src/manifest.json | 9 -- 14 files changed, 240 insertions(+), 103 deletions(-) create mode 100644 eng/docker-tools/Update-ImageBuilder.ps1 delete mode 100644 src/ImageBuilder/AssemblyImageBuilderTagProvider.cs delete mode 100644 src/ImageBuilder/IImageBuilderTagProvider.cs create mode 100644 src/Infrastructure/Content/Update-ImageBuilder.ps1 diff --git a/eng/docker-tools/Update-ImageBuilder.ps1 b/eng/docker-tools/Update-ImageBuilder.ps1 new file mode 100644 index 000000000..88f0361c7 --- /dev/null +++ b/eng/docker-tools/Update-ImageBuilder.ps1 @@ -0,0 +1,89 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Example script that updates the bundled docker-tools infrastructure in the current repo to a specific +ImageBuilder image. + +.DESCRIPTION +ImageBuilder ships a copy of the eng/docker-tools infrastructure and writes it back to disk via its +'update' command. The reference that 'update' records into +eng/docker-tools/templates/variables/docker-images.yml is supplied as an argument rather than being +baked into the build, so the caller decides exactly which image the repo should pin to. + +This example resolves the multi-platform (manifest list / image index) digest of an ImageBuilder image +(the published 'latest' tag by default) and passes that digest reference to the 'update' command, which +runs inside the same image with the repository mounted so it can rewrite eng/docker-tools on disk. + +.PARAMETER ImageBuilderImage +The ImageBuilder image to resolve and run. Defaults to the published 'latest' tag. + +.PARAMETER RepoRoot +The root of the git repository to update. Defaults to the current directory. + +.NOTES +To exercise an unpublished ImageBuilder (for example, the 'update' command before it is released), +build the image, push it to a registry it can be pulled from, and pass its reference via +-ImageBuilderImage. The digest is read from the registry, so the image must be pushed first. +#> +[CmdletBinding()] +param( + [string] + $ImageBuilderImage = "mcr.microsoft.com/dotnet-buildtools/image-builder:latest", + + [string] + $RepoRoot = (Get-Location).Path +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Exec { + param ([string] $Cmd) + + Write-Output "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + throw "Failed: '$Cmd'" + } +} + +# Strip any existing tag or digest so the resolved digest can be appended to the bare repository name. +# A tag is a ':' within the final path segment; a registry host's ':port' precedes the last '/', so it +# must not be mistaken for a tag. +function Get-RepositoryName { + param ([string] $Reference) + + $withoutDigest = $Reference.Split('@')[0] + $lastSlash = $withoutDigest.LastIndexOf('/') + $lastColon = $withoutDigest.LastIndexOf(':') + if ($lastColon -gt $lastSlash) { + return $withoutDigest.Substring(0, $lastColon) + } + + return $withoutDigest +} + +# Resolve the multi-platform digest. 'docker buildx imagetools inspect' reads the top-level manifest +# straight from the registry, so for a multi-arch image this is the manifest list (image index) digest +# rather than a single platform's digest. Pinning the index keeps the reference valid on every +# platform the pipeline runs on. +$digest = (docker buildx imagetools inspect $ImageBuilderImage --format '{{.Manifest.Digest}}') +if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($digest)) { + throw "Unable to resolve a multi-platform digest for '$ImageBuilderImage'." +} + +$repository = Get-RepositoryName $ImageBuilderImage +$imageBuilderRef = "$repository@$($digest.Trim())" + +Write-Output "Resolved ImageBuilder reference: $imageBuilderRef" + +# Run 'update' from the resolved digest, mounting the repository so it can write eng/docker-tools to +# disk. The command must run from the repository root, which is why $RepoRoot is the mounted working +# directory. Running by the same digest that gets recorded keeps the writer and the pinned reference +# identical. +Exec ("docker run --rm " ` + + "-v `"${RepoRoot}:/repo`" " ` + + "-w /repo " ` + + "$imageBuilderRef " ` + + "update $imageBuilderRef") diff --git a/src/Dockerfile.linux b/src/Dockerfile.linux index 05f23dba6..59fb4c978 100644 --- a/src/Dockerfile.linux +++ b/src/Dockerfile.linux @@ -5,8 +5,6 @@ # build Microsoft.DotNet.ImageBuilder FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-azurelinux3.0 AS build-env ARG TARGETARCH -ARG IMAGEBUILDER_TAG= -ENV IMAGEBUILDER_TAG=${IMAGEBUILDER_TAG} # download root CA certificates for signature verification WORKDIR / diff --git a/src/Dockerfile.windows b/src/Dockerfile.windows index e50e0e2c4..ff19a2ec8 100644 --- a/src/Dockerfile.windows +++ b/src/Dockerfile.windows @@ -5,8 +5,6 @@ ARG WINDOWS_SDK # build Microsoft.DotNet.ImageBuilder FROM mcr.microsoft.com/dotnet/sdk:9.0-$WINDOWS_SDK AS build-env -ARG IMAGEBUILDER_TAG= -ENV IMAGEBUILDER_TAG=${IMAGEBUILDER_TAG} WORKDIR /image-builder # Work around Windows path length limitations by moving NuGet package cache to diff --git a/src/ImageBuilder.Tests/UpdateCommandTests.cs b/src/ImageBuilder.Tests/UpdateCommandTests.cs index 8719a8360..0b23de4ff 100644 --- a/src/ImageBuilder.Tests/UpdateCommandTests.cs +++ b/src/ImageBuilder.Tests/UpdateCommandTests.cs @@ -59,32 +59,33 @@ public async Task UpdateCommand_WritesAllEmbeddedFiles() } [TestMethod] - public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataSet_RendersDockerImagesTemplate() + public async Task UpdateCommand_ImageBuilderRefProvided_RendersDockerImagesTemplate() { - const string tag = "1234567"; + const string imageBuilderRef = "mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:abc123"; InMemoryFileSystem fileSystem = CreateRepoFileSystem(); fileSystem.AddDirectory(s_outputPath); - UpdateCommand command = CreateCommand(fileSystem, tag); + UpdateCommand command = CreateCommand(fileSystem, imageBuilderRef); await command.ExecuteAsync(); - string content = GetRenderedImageBuilderVariables(fileSystem, $"image-builder:{tag}"); - content.ShouldContain($"image-builder:{tag}"); + string content = GetRenderedDockerImagesContent(fileSystem); + content.ShouldContain(imageBuilderRef); content.ShouldNotContain("{{"); } [TestMethod] - public async Task UpdateCommand_ImageBuilderTagAssemblyMetadataMissing_FallsBackToLatestWithWarning() + public async Task UpdateCommand_ImageBuilderRefMissing_FallsBackToLatestWithWarning() { + const string latestRef = "mcr.microsoft.com/dotnet-buildtools/image-builder:latest"; InMemoryFileSystem fileSystem = CreateRepoFileSystem(); fileSystem.AddDirectory(s_outputPath); Mock> logger = new(); - UpdateCommand command = CreateCommand(fileSystem, tag: null, logger: logger); + UpdateCommand command = CreateCommand(fileSystem, imageBuilderRef: null, logger: logger); await command.ExecuteAsync(); - string content = GetRenderedImageBuilderVariables(fileSystem, "image-builder:latest"); - content.ShouldContain("image-builder:latest"); + string content = GetRenderedDockerImagesContent(fileSystem); + content.ShouldContain(latestRef); content.ShouldNotContain("{{"); logger.Verify( @@ -192,19 +193,19 @@ private static InMemoryFileSystem CreateRepoFileSystem() private static UpdateCommand CreateCommand( IFileSystem fileSystem, - string? tag = null, + string? imageBuilderRef = null, Mock>? logger = null) { - Mock tagProvider = new(); - tagProvider.Setup(provider => provider.GetTag()).Returns(tag); - return new UpdateCommand( + UpdateCommand command = new( fileSystem, - tagProvider.Object, (logger ?? new Mock>()).Object); + command.Options.ImageBuilderRef = imageBuilderRef; + return command; } - private static string GetRenderedImageBuilderVariables(InMemoryFileSystem fileSystem, string expectedImageBuilderReference) => - fileSystem.FilesWritten - .Select(path => Encoding.UTF8.GetString(fileSystem.GetFileBytes(path))) - .Single(content => content.Contains(expectedImageBuilderReference)); + private static string GetRenderedDockerImagesContent(InMemoryFileSystem fileSystem) + { + string dockerImagesPath = PathHelper.SafeCombine(s_outputPath, "templates/variables/docker-images.yml"); + return Encoding.UTF8.GetString(fileSystem.GetFileBytes(dockerImagesPath)); + } } diff --git a/src/ImageBuilder/AssemblyImageBuilderTagProvider.cs b/src/ImageBuilder/AssemblyImageBuilderTagProvider.cs deleted file mode 100644 index 01c048ca6..000000000 --- a/src/ImageBuilder/AssemblyImageBuilderTagProvider.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Linq; -using System.Reflection; - -namespace Microsoft.DotNet.ImageBuilder; - -/// -/// Resolves the ImageBuilder tag from the baked into this -/// assembly at build time (via the ImageBuilderTag MSBuild property). -/// -internal sealed class AssemblyImageBuilderTagProvider : IImageBuilderTagProvider -{ - private const string ImageBuilderTagMetadataKey = "ImageBuilderTag"; - - /// - public string? GetTag() => - typeof(AssemblyImageBuilderTagProvider).Assembly - .GetCustomAttributes() - .FirstOrDefault(attribute => attribute.Key == ImageBuilderTagMetadataKey)?.Value; -} diff --git a/src/ImageBuilder/Commands/UpdateCommand.cs b/src/ImageBuilder/Commands/UpdateCommand.cs index 2f1a8fb60..3a1420181 100644 --- a/src/ImageBuilder/Commands/UpdateCommand.cs +++ b/src/ImageBuilder/Commands/UpdateCommand.cs @@ -29,18 +29,19 @@ public class UpdateCommand : Command // InfrastructureContent.GetRelativePaths always returns '/'-separated paths, so this is kept in // the same form to compare directly without re-normalizing per file. private const string DockerImagesRelativePath = "templates/variables/docker-images.yml"; - private const string ImageBuilderTagTemplateVariableName = "IMAGEBUILDER_TAG"; + private const string ImageBuilderRefTemplateVariableName = "IMAGEBUILDER_REF"; + + // Used when no reference is supplied. Mirrors the published image's 'latest' shared tag. + private const string DefaultImageBuilderRef = "mcr.microsoft.com/dotnet-buildtools/image-builder:latest"; private static readonly DocumentConfiguration s_templateConfiguration = CottleDocumentConfiguration.Create(); private readonly IFileSystem _fileSystem; - private readonly IImageBuilderTagProvider _tagProvider; private readonly ILogger _logger; - public UpdateCommand(IFileSystem fileSystem, IImageBuilderTagProvider tagProvider, ILogger logger) + public UpdateCommand(IFileSystem fileSystem, ILogger logger) { _fileSystem = fileSystem; - _tagProvider = tagProvider; _logger = logger; } @@ -68,19 +69,20 @@ public override Task ExecuteAsync() $"Pass --init to create it (use this only when onboarding a repo to docker-tools)."); } - // Resolve ImageBuilder's image tag. - // If ImageBuilder doesn't know about its own tag, then use `latest` as a fallback. - string imageBuilderTag; - if (_tagProvider.GetTag() is { } resolvedImageBuilderTag && !string.IsNullOrWhiteSpace(resolvedImageBuilderTag)) + // Resolve ImageBuilder's image reference from the command argument. + // If the caller didn't provide one, fall back to the 'latest' reference. + string imageBuilderRef; + if (!string.IsNullOrWhiteSpace(Options.ImageBuilderRef)) { - imageBuilderTag = resolvedImageBuilderTag; + imageBuilderRef = Options.ImageBuilderRef; } else { _logger.LogWarning( - "This build of ImageBuilder was not built with the \"IMAGEBUILDER_TAG\" MSBuild property set. " + - "ImageBuilder tag will fall back to \"latest\"."); - imageBuilderTag = "latest"; + "No ImageBuilder reference was provided. " + + "The ImageBuilder reference will fall back to \"{DefaultImageBuilderRef}\".", + DefaultImageBuilderRef); + imageBuilderRef = DefaultImageBuilderRef; } // Clear the existing infrastructure content and re-write it all. @@ -116,10 +118,10 @@ public override Task ExecuteAsync() _logger.LogInformation("Writing '{DestinationPath}'", destinationPath); - // docker-images.yml is templated because ImageBuilder needs to self-reference its own image tag. + // docker-images.yml is templated because ImageBuilder needs to record its own image reference. if (relativePath == DockerImagesRelativePath) { - string renderedContent = RenderDockerImagesTemplate(relativePath, imageBuilderTag); + string renderedContent = RenderDockerImagesTemplate(relativePath, imageBuilderRef); _fileSystem.WriteAllText(destinationPath, renderedContent); } else @@ -133,7 +135,7 @@ public override Task ExecuteAsync() return Task.CompletedTask; } - private static string RenderDockerImagesTemplate(string relativePath, string imageBuilderTag) + private static string RenderDockerImagesTemplate(string relativePath, string imageBuilderRef) { using Stream source = InfrastructureContent.OpenRead(relativePath); using StreamReader reader = new(source); @@ -142,7 +144,7 @@ private static string RenderDockerImagesTemplate(string relativePath, string ima IDocument document = Document.CreateDefault(template, s_templateConfiguration).DocumentOrThrow; Dictionary symbols = new() { - [ImageBuilderTagTemplateVariableName] = imageBuilderTag + [ImageBuilderRefTemplateVariableName] = imageBuilderRef }; return document.Render(Context.CreateBuiltin(symbols)); diff --git a/src/ImageBuilder/Commands/UpdateOptions.cs b/src/ImageBuilder/Commands/UpdateOptions.cs index f9789e32d..c65210875 100644 --- a/src/ImageBuilder/Commands/UpdateOptions.cs +++ b/src/ImageBuilder/Commands/UpdateOptions.cs @@ -16,17 +16,37 @@ public class UpdateOptions : Options /// public bool Init { get; set; } + /// + /// The fully-qualified ImageBuilder image reference (for example, + /// mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:... or + /// mcr.microsoft.com/dotnet-buildtools/image-builder:<tag>) to write into + /// docker-images.yml. When omitted, the command falls back to the latest reference. + /// + public string? ImageBuilderRef { get; set; } + private static readonly Option InitOption = new("--init") { Description = "Create the eng/docker-tools directory if it does not already exist", }; + private static readonly Argument ImageBuilderRefArgument = new(nameof(ImageBuilderRef)) + { + Description = + "Fully-qualified ImageBuilder image reference (digest or tag) to record in docker-images.yml. " + + "Defaults to the 'latest' reference when not specified.", + Arity = ArgumentArity.ZeroOrOne, + }; + public override IEnumerable