From d4718f26b9bada4fff842cb554b8fa68a9971fbb Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 3 Jun 2026 17:12:46 +0000 Subject: [PATCH 01/13] feat: add tarball overlays for source archive modification Add three new overlay types (tarball-file-remove, tarball-search-replace, tarball-patch) that modify files inside source tarballs during source preparation. Operations are performed in pure Go on the host. Includes: - internal/utils/tarball: reusable deterministic tar extract/repack library - Overlay type registration, validation, and fingerprinting - Source prep integration with sources file hash rehashing - User documentation and TOML examples --- docs/user/reference/config/overlays.md | 72 +++- .../azldev/cmds/component/preparesources.go | 28 +- .../app/azldev/core/sources/sourceprep.go | 154 ++++++-- .../azldev/core/sources/sourceprep_test.go | 2 +- .../azldev/core/sources/tarballoverlays.go | 358 ++++++++++++++++++ .../sources/tarballoverlays_internal_test.go | 137 +++++++ internal/projectconfig/overlay.go | 64 +++- internal/projectconfig/overlay_test.go | 156 ++++++++ 8 files changed, 921 insertions(+), 50 deletions(-) create mode 100644 internal/app/azldev/core/sources/tarballoverlays.go create mode 100644 internal/app/azldev/core/sources/tarballoverlays_internal_test.go diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index 9a3abc9f..af4f485d 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -47,21 +47,34 @@ successfully makes a replacement to at least one matching file. | `file-remove` | Removes a file | `file` | Glob pattern for files to remove | | `file-rename` | Renames a file within the same directory | `file`, `replacement` | Name of file to rename | +### Tarball Overlays + +These overlays modify files **inside** source tarballs. The tarball is extracted into a temporary directory, modifications are applied, and the tarball is repacked with the same compression format. Extraction and repacking are handled natively; patch application requires the `patch` command on the host. + +> **Note:** Tarball overlays are applied before spec and file overlays, so subsequent overlays see the modified tarball. The `tarball-patch` overlay type requires the `patch` command to be installed on the host. + +| Type | Description | Required Fields | +|------|-------------|-----------------| +| `tarball-file-remove` | Removes file(s) matching a glob pattern from inside a tarball | `tarball`, `file` | +| `tarball-search-replace` | Regex-based search and replace on file(s) inside a tarball | `tarball`, `file`, `regex` | +| `tarball-patch` | Applies a unified diff patch to the extracted tarball contents | `tarball`, `source` | + ## Field Reference | Field | TOML Key | Description | Used By | |-------|----------|-------------|---------| | Type | `type` | **Required.** The overlay type to apply | All overlays | | Description | `description` | Human-readable explanation documenting the need for the change; helps identify overlays in error messages | All (optional) | +| Tarball | `tarball` | The source tarball filename to modify (must be a basename, not a path) | `tarball-file-remove`, `tarball-search-replace`, `tarball-patch` | | Tag | `tag` | The spec tag name (e.g., `BuildRequires`, `Requires`, `Version`) | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` | -| Value | `value` | The tag value to set, or value to match for removal | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` (optional for matching) | +| Value | `value` | The tag value to set, or value to match for removal. For `tarball-patch`, sets the patch strip level (default: `1`, equivalent to `patch -p1`). | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` (optional for matching), `tarball-patch` (optional) | | Section | `section` | The spec section to target (e.g., `%build`, `%install`, `%files`, `%description`) | `spec-prepend-lines`, `spec-append-lines`, `spec-search-replace` (optional), `spec-remove-section` | | Package | `package` | The sub-package name for multi-package specs; omit to target the main package | All spec overlays (optional, except `spec-remove-subpackage` which **requires** it) | -| Regex | `regex` | Regular expression pattern to match | `spec-search-replace`, `file-search-replace` | -| Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename` | +| Regex | `regex` | Regular expression pattern to match | `spec-search-replace`, `file-search-replace`, `tarball-search-replace` | +| Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename`, `tarball-search-replace` | | Lines | `lines` | Array of text lines to insert | `spec-prepend-lines`, `spec-append-lines`, `file-prepend-lines` | -| File | `file` | The name of the non-spec file to modify or add | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | -| Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add` | +| File | `file` | The name of the non-spec file to modify or add | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove`, `tarball-file-remove`, `tarball-search-replace` | +| Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add`, `tarball-patch` | | Metadata | `metadata` | Documentation table describing intent and provenance — see [Overlay Metadata](#overlay-metadata). Not allowed inside an overlay file loaded via `overlay-files` (the file-level `[metadata]` block applies to every overlay in the file). | All (optional) | > **Note:** For `file-rename`, the `replacement` field is a **filename only** (not a path). The file is renamed within its current directory. @@ -417,6 +430,55 @@ description = "Remove CVE patches that are now upstream" > `PatchN` tags. Macro-based tag numbering (e.g., `Patch%{n}`) is not expanded and may > conflict with auto-assigned numbers. +### Removing a File from a Tarball + +The `tarball-file-remove` overlay deletes files matching a glob pattern from inside a source +tarball. The tarball is extracted, matching files are removed, and the tarball is repacked. + +```toml +[[components.mypackage.overlays]] +type = "tarball-file-remove" +tarball = "mypackage-1.0.tar.gz" +file = "vendor/**" +description = "Remove bundled vendor directory" +``` + +### Search and Replace Inside a Tarball + +```toml +[[components.mypackage.overlays]] +type = "tarball-search-replace" +tarball = "mypackage-1.0.tar.xz" +file = "configure.ac" +regex = "AC_CHECK_LIB\\(old_lib" +replacement = "AC_CHECK_LIB(new_lib" +description = "Update library reference in configure script" +``` + +### Applying a Patch to Tarball Contents + +The `tarball-patch` overlay applies a unified diff patch to the extracted tarball contents. +By default, it uses `patch -p1`. Use the `value` field to change the strip level. + +```toml +[[components.mypackage.overlays]] +type = "tarball-patch" +tarball = "mypackage-1.0.tar.gz" +source = "patches/fix-build.patch" +description = "Fix build issue in upstream source" +``` + +With a custom strip level: + +```toml +[[components.mypackage.overlays]] +type = "tarball-patch" +tarball = "mypackage-1.0.tar.gz" +source = "patches/fix-build.patch" +value = "0" +description = "Apply patch with -p0 strip level" +``` + ### Removing a Section The `spec-remove-section` overlay removes an entire section from the spec, including its diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 2f0ffcaa..3dc7d687 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -138,13 +138,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er ) } - if options.AllowNoHashes { - preparerOpts = append(preparerOpts, sources.WithAllowNoHashes()) - } - - if options.SkipSources { - preparerOpts = append(preparerOpts, sources.WithSkipLookaside()) - } + preparerOpts = appendPrepareSourcesOptions(env, preparerOpts, options, distro) preparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env, preparerOpts...) if err != nil { @@ -194,3 +188,23 @@ func CheckOutputDir(env *azldev.Env, options *PrepareSourcesOptions) error { "use --force to delete and recreate it", options.OutputDir) } + +// appendPrepareSourcesOptions appends conditional preparer options that control +// hashing and lookaside behavior. Extracted from +// [PrepareComponentSources] to keep cyclomatic complexity within limits. +func appendPrepareSourcesOptions( + _ *azldev.Env, + opts []sources.PreparerOption, + options *PrepareSourcesOptions, + _ sourceproviders.ResolvedDistro, +) []sources.PreparerOption { + if options.AllowNoHashes { + opts = append(opts, sources.WithAllowNoHashes()) + } + + if options.SkipSources { + opts = append(opts, sources.WithSkipLookaside()) + } + + return opts +} diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 617dcbd8..62649e22 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -246,8 +246,7 @@ func (p *sourcePreparerImpl) PrepareSources( } if applyOverlays { - err := p.applyOverlaysToSources(ctx, component, outputDir) - if err != nil { + if err := p.applyOverlaysToSources(ctx, component, outputDir); err != nil { return err } @@ -276,9 +275,6 @@ func (p *sourcePreparerImpl) PrepareSources( func (p *sourcePreparerImpl) applyOverlaysToSources( ctx context.Context, component components.Component, outputDir string, ) error { - // Emit computed macros to a macros file in the output directory. - // If the build configuration produces no macros, no file is written and - // macrosFileName will be empty. var macrosFileName string macrosFilePath, err := p.writeMacrosFile(component, outputDir) @@ -291,32 +287,27 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( macrosFileName = filepath.Base(macrosFilePath) } - // Apply all overlays to prepared sources. if err := p.applyOverlays(ctx, component, outputDir, macrosFileName); err != nil { - return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) + return fmt.Errorf("failed to apply overlays for component %#q:\n%w", + component.GetName(), err) } return nil } // applyOverlays applies all overlays (user-defined and system-generated) to the -// component sources. Overlay application is decoupled from git history generation: -// overlays modify the working tree; synthetic history is recorded separately by -// [trySyntheticHistory]. +// component sources. func (p *sourcePreparerImpl) applyOverlays( - _ context.Context, component components.Component, sourcesDirPath, macrosFileName string, + ctx context.Context, component components.Component, sourcesDirPath, macrosFileName string, ) error { event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) defer event.End() - // Resolve the spec path once for all overlay operations in this call. absSpecPath, err := p.resolveSpecPath(component, sourcesDirPath) if err != nil { return err } - // Collect all overlays in application order. This ensures every change is - // captured in the synthetic history, including build configuration changes. allOverlays, err := p.collectOverlays(component, macrosFileName) if err != nil { return fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err) @@ -326,7 +317,13 @@ func (p *sourcePreparerImpl) applyOverlays( return nil } - // Apply all overlays to the working tree. + // Tarball overlays are applied first (they modify archived source files + // in-place), followed by spec and loose-file overlays. Each function + // self-filters to the overlay types it handles. + if err := p.applyTarballOverlayGroup(ctx, component, sourcesDirPath, allOverlays); err != nil { + return err + } + if err := p.applyOverlayList(allOverlays, sourcesDirPath, absSpecPath); err != nil { return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } @@ -334,6 +331,40 @@ func (p *sourcePreparerImpl) applyOverlays( return nil } +// applyTarballOverlayGroup applies tarball overlays. Skipped when source +// downloads were not performed. +func (p *sourcePreparerImpl) applyTarballOverlayGroup( + ctx context.Context, component components.Component, + sourcesDirPath string, tarballOverlays []projectconfig.ComponentOverlay, +) error { + if len(tarballOverlays) == 0 { + return nil + } + + if p.skipLookaside { + slog.Warn("Skipping tarball overlays because source downloads were skipped (--skip-sources)", + "component", component.GetName(), + "count", len(tarballOverlays)) + + return nil + } + + cmdFactory, ok := p.dryRunnable.(opctx.CmdFactory) + if !ok { + return errors.New( + "tarball overlays require a CmdFactory; the provided DryRunnable does not implement it") + } + + if err := applyTarballOverlays( + ctx, cmdFactory, p.fs, p.eventListener, sourcesDirPath, tarballOverlays, + ); err != nil { + return fmt.Errorf("failed to apply tarball overlays for component %#q:\n%w", + component.GetName(), err) + } + + return nil +} + // collectOverlays gathers all overlays for a component into a single ordered slice: // macros-load first, then user overlays, followed by check-skip and file-header overlays. func (p *sourcePreparerImpl) collectOverlays( @@ -634,9 +665,17 @@ func (p *sourcePreparerImpl) DiffSources( // enforced by [projectconfig.ConfigFile.Validate]). Setting `ReplaceUpstream` = true without // a matching upstream entry is also an error: the user expressed intent to replace something // that isn't there, which almost certainly indicates a stale config or filename typo. -func (p *sourcePreparerImpl) updateSourcesFile(component components.Component, outputDir string) error { - sourceFiles := component.GetConfig().SourceFiles - if len(sourceFiles) == 0 { +func (p *sourcePreparerImpl) updateSourcesFile( + component components.Component, outputDir string, +) error { + config := component.GetConfig() + sourceFiles := config.SourceFiles + + // Derive tarball names from the component's overlays — no need to thread + // them through the overlay application chain. + modifiedTarballs := tarballNamesFromOverlays(config.Overlays) + + if len(sourceFiles) == 0 && len(modifiedTarballs) == 0 { return nil } @@ -647,7 +686,19 @@ func (p *sourcePreparerImpl) updateSourcesFile(component components.Component, o return err } - mergedLines, err := p.buildSourceEntries(sourceFiles, existingContent, component.GetName(), outputDir) + // Parse once, then rehash modified tarballs and merge source-files entries + // on the parsed representation — single parse, single write. + existingLines, err := fedorasource.ReadSourcesFile(existingContent) + if err != nil { + return fmt.Errorf("failed to parse 'sources' file %#q:\n%w", sourcesFilePath, err) + } + + // Rehash tarballs that were modified by tarball overlays in-place. + if err := p.rehashModifiedEntries(existingLines, outputDir, modifiedTarballs); err != nil { + return err + } + + mergedLines, err := p.buildSourceEntries(sourceFiles, existingLines, component.GetName(), outputDir) if err != nil { return err } @@ -666,6 +717,47 @@ func (p *sourcePreparerImpl) updateSourcesFile(component components.Component, o return nil } +// rehashModifiedEntries updates the Raw and Entry fields of parsed 'sources' lines +// for tarballs that were modified by tarball overlays. The hash is recomputed using +// the same hash type as the original entry. +func (p *sourcePreparerImpl) rehashModifiedEntries( + lines []fedorasource.SourcesFileLine, outputDir string, modifiedTarballs []string, +) error { + if len(modifiedTarballs) == 0 { + return nil + } + + modified := make(map[string]bool, len(modifiedTarballs)) + for _, name := range modifiedTarballs { + modified[name] = true + } + + for idx, line := range lines { + if line.Entry == nil || !modified[line.Entry.Filename] { + continue + } + + tarballPath := filepath.Join(outputDir, line.Entry.Filename) + + newHash, err := fileutils.ComputeFileHash(p.fs, line.Entry.HashType, tarballPath) + if err != nil { + return fmt.Errorf("rehashing modified tarball %#q:\n%w", line.Entry.Filename, err) + } + + slog.Debug("Rehashed modified tarball in 'sources' file", + "tarball", line.Entry.Filename, + "hashType", line.Entry.HashType, + "oldHash", line.Entry.Hash, + "newHash", newHash, + ) + + lines[idx].Raw = fedorasource.FormatSourcesEntry(line.Entry.Filename, line.Entry.HashType, newHash) + lines[idx].Entry.Hash = newHash + } + + return nil +} + // readSourcesFileIfExists reads the 'sources' file content if it exists, returning empty string if not. func (p *sourcePreparerImpl) readSourcesFileIfExists(sourcesFilePath string) (string, error) { exists, err := fileutils.Exists(p.fs, sourcesFilePath) @@ -685,32 +777,24 @@ func (p *sourcePreparerImpl) readSourcesFileIfExists(sourcesFilePath string) (st return string(data), nil } -// buildSourceEntries validates [projectconfig.SourceFileReference] entries and returns -// the merged set of lines ready to be written to the 'sources' file. Before returning, -// it logs an INFO-level event indicating that the 'sources' file will be updated, -// including the counts of newly added and replaced entries. +// buildSourceEntries merges user-declared [projectconfig.SourceFileReference] entries +// into the parsed 'sources' lines. Returns the final set of raw lines ready to be +// written to the 'sources' file. // // Output ordering and preservation: -// - Each line of [existingContent] is emitted verbatim, except for entry lines whose +// - Each existing line is emitted verbatim, except for entry lines whose // filename matches a replacement, which are swapped for the new formatted entry. -// Comments and blank lines from the original file are kept in their original positions. -// - Brand-new entries (no upstream filename collision) are appended after the upstream -// content in the order they appear in [sourceFiles]. +// Comments and blank lines are kept in their original positions. +// - Brand-new entries (no upstream filename collision) are appended after the +// existing content in the order they appear in [sourceFiles]. // // Collision rules and hash resolution are documented on [sourcePreparerImpl.processSourceRef]. func (p *sourcePreparerImpl) buildSourceEntries( sourceFiles []projectconfig.SourceFileReference, - existingContent string, + existingLines []fedorasource.SourcesFileLine, componentName string, outputDir string, ) (mergedLines []string, err error) { - existingLines, err := fedorasource.ReadSourcesFile(existingContent) - if err != nil { - return nil, fmt.Errorf( - "failed to parse existing 'sources' file at %#q:\n%w", - filepath.Join(outputDir, fedorasource.SourcesFileName), err) - } - // Index upstream entries by filename for O(1) collision lookup. The parser // (fedorasource.ReadSourcesFile) errors on duplicate filenames, so the // entries are guaranteed unique by the time we get here. diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 45a9d286..d21aa52d 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -854,7 +854,7 @@ func TestPrepareSources_UpdatesSourcesFile(t *testing.T) { existingSourcesContent: "SHA512 (dup.tar.gz) = aaaa1111\nSHA512 (dup.tar.gz) = bbbb2222\n", expectError: true, errorContains: []string{ - "failed to parse existing 'sources' file", + "failed to parse 'sources' file", "duplicate filename", "dup.tar.gz", }, diff --git a/internal/app/azldev/core/sources/tarballoverlays.go b/internal/app/azldev/core/sources/tarballoverlays.go new file mode 100644 index 00000000..5f9d4fe8 --- /dev/null +++ b/internal/app/azldev/core/sources/tarballoverlays.go @@ -0,0 +1,358 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/tarball" +) + +// applyTarballOverlays groups tarball overlays by target archive and processes +// them in order. Multiple overlays targeting the same tarball are batched into +// a single extract/modify/repack cycle. All operations (extract, modify, repack) +// are performed in pure Go on the host, except for patch application which +// shells out to the host's `patch` command. +func applyTarballOverlays( + ctx context.Context, + cmdFactory opctx.CmdFactory, + fs opctx.FS, + eventListener opctx.EventListener, + sourcesDirPath string, + overlays []projectconfig.ComponentOverlay, +) error { + groups := groupOverlaysByTarball(overlays) + + if len(groups) == 0 { + return nil + } + + event := eventListener.StartEvent("Applying tarball overlays", + "tarballs", len(groups), + "operations", len(overlays), + ) + defer event.End() + + for _, group := range groups { + if err := processTarball(ctx, cmdFactory, fs, sourcesDirPath, group); err != nil { + return fmt.Errorf("tarball overlay failed for %#q:\n%w", group.tarball, err) + } + } + + return nil +} + +// tarballGroup holds overlays targeting the same tarball, preserving order. +type tarballGroup struct { + tarball string + overlays []projectconfig.ComponentOverlay +} + +// groupOverlaysByTarball groups tarball overlays by their +// [projectconfig.ComponentOverlay.Tarball] field, preserving insertion order +// within each group and across groups. Non-tarball overlays are silently skipped. +func groupOverlaysByTarball(overlays []projectconfig.ComponentOverlay) []tarballGroup { + orderMap := make(map[string]int) + + var groups []tarballGroup + + for _, overlay := range overlays { + if !overlay.ModifiesTarball() { + continue + } + + idx, exists := orderMap[overlay.Tarball] + if !exists { + idx = len(groups) + orderMap[overlay.Tarball] = idx + + groups = append(groups, tarballGroup{tarball: overlay.Tarball}) + } + + groups[idx].overlays = append(groups[idx].overlays, overlay) + } + + return groups +} + +// processTarball extracts a tarball to a temp directory, applies all overlays, +// and deterministically repacks it in-place with the original compression. +func processTarball( + ctx context.Context, + cmdFactory opctx.CmdFactory, + fs opctx.FS, + sourcesDirPath string, + group tarballGroup, +) error { + archivePath := filepath.Join(sourcesDirPath, group.tarball) + + // Create a temporary directory for extraction. + workDir, err := os.MkdirTemp("", "tarball-overlay-") + if err != nil { + return fmt.Errorf("creating temp directory:\n%w", err) + } + + defer func() { + if removeErr := os.RemoveAll(workDir); removeErr != nil { + slog.Warn("Failed to clean up tarball work directory", "error", removeErr) + } + }() + + // Detect compression and extract. + compression, err := tarball.DetectCompression(group.tarball) + if err != nil { + return fmt.Errorf("detecting compression for %#q:\n%w", group.tarball, err) + } + + if err := tarball.Extract(fs, archivePath, workDir, compression); err != nil { + return fmt.Errorf("extracting tarball:\n%w", err) + } + + // Determine the root of the extracted content. Most source tarballs have + // a single top-level directory (e.g., "pkg-1.0/"). + extractRoot, err := tarball.ResolveExtractRoot(workDir) + if err != nil { + return fmt.Errorf("resolving extract root:\n%w", err) + } + + // Apply each overlay operation in order. + for _, overlay := range group.overlays { + if err := applyTarballOperation(ctx, cmdFactory, fs, extractRoot, overlay); err != nil { + return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) + } + } + + // Deterministically repack the tarball in-place. + if err := tarball.RepackDeterministic(fs, archivePath, workDir, compression); err != nil { + return fmt.Errorf("repacking tarball:\n%w", err) + } + + slog.Info("Tarball overlay applied", "tarball", group.tarball) + + return nil +} + +// applyTarballOperation dispatches a single overlay to the appropriate handler. +func applyTarballOperation( + ctx context.Context, + cmdFactory opctx.CmdFactory, + fs opctx.FS, + extractRoot string, + overlay projectconfig.ComponentOverlay, +) error { + //nolint:exhaustive // Only tarball overlay types are valid here; the default catches the rest. + switch overlay.Type { + case projectconfig.ComponentOverlayTarballFileRemove: + return tarballFileRemove(extractRoot, overlay.Filename) + + case projectconfig.ComponentOverlayTarballSearchReplace: + return tarballSearchReplace(extractRoot, overlay.Filename, overlay.Regex, overlay.Replacement) + + case projectconfig.ComponentOverlayTarballPatch: + stripLevel := 1 + + if overlay.Value != "" { + parsed, err := strconv.Atoi(overlay.Value) + if err != nil { + return fmt.Errorf("invalid strip level %#q:\n%w", overlay.Value, err) + } + + stripLevel = parsed + } + + return tarballApplyPatch(ctx, cmdFactory, fs, extractRoot, overlay.Source, stripLevel) + + default: + return fmt.Errorf("unsupported tarball overlay type %#q", overlay.Type) + } +} + +// tarballFileRemove removes files matching a glob pattern from the extracted tree. +func tarballFileRemove(extractRoot, pattern string) error { + matches, err := globFilesInDir(extractRoot, pattern) + if err != nil { + return err + } + + if len(matches) == 0 { + return fmt.Errorf("no files match pattern %#q:\n%w", pattern, ErrOverlayDidNotApply) + } + + for _, path := range matches { + if err := os.Remove(path); err != nil { + return fmt.Errorf("failed to remove %#q:\n%w", path, err) + } + } + + return nil +} + +// tarballSearchReplace applies regex search-and-replace to files matching a glob +// pattern inside the extracted tree. +func tarballSearchReplace(extractRoot, pattern, regex, replacement string) error { + matches, err := globFilesInDir(extractRoot, pattern) + if err != nil { + return err + } + + if len(matches) == 0 { + return fmt.Errorf("no files match pattern %#q:\n%w", pattern, ErrOverlayDidNotApply) + } + + compiled, err := regexp.Compile(regex) + if err != nil { + return fmt.Errorf("invalid regex %#q:\n%w", regex, err) + } + + anyReplaced := false + + for _, path := range matches { + content, readErr := os.ReadFile(path) + if readErr != nil { + return fmt.Errorf("reading %#q:\n%w", path, readErr) + } + + newContent := compiled.ReplaceAll(content, []byte(replacement)) + if string(newContent) != string(content) { + anyReplaced = true + + if writeErr := os.WriteFile(path, newContent, fileperms.PublicFile); writeErr != nil { + return fmt.Errorf("writing %#q:\n%w", path, writeErr) + } + } + } + + if !anyReplaced { + return fmt.Errorf("regex %#q did not match any content in files matching %#q:\n%w", + regex, pattern, ErrOverlayDidNotApply) + } + + return nil +} + +// tarballApplyPatch applies a unified diff patch to the extracted tree by +// shelling out to the host's `patch` command. +func tarballApplyPatch( + ctx context.Context, + cmdFactory opctx.CmdFactory, + fs opctx.FS, + extractRoot, patchSource string, + stripLevel int, +) error { + if !cmdFactory.CommandInSearchPath("patch") { + return errors.New("'patch' command not found in PATH; " + + "install the 'patch' package to use tarball-patch overlays") + } + + // Read the patch file content via the abstract FS (supports both real and test FSs). + patchData, err := fileutils.ReadFile(fs, patchSource) + if err != nil { + return fmt.Errorf("reading patch file %#q:\n%w", patchSource, err) + } + + // Write to a temp file on the real filesystem so the patch command can read it. + tmpPatch, err := os.CreateTemp("", "tarball-patch-*.patch") + if err != nil { + return fmt.Errorf("creating temp patch file:\n%w", err) + } + + defer os.Remove(tmpPatch.Name()) + + if _, err := tmpPatch.Write(patchData); err != nil { + tmpPatch.Close() + + return fmt.Errorf("writing temp patch file:\n%w", err) + } + + tmpPatch.Close() + + var stderr strings.Builder + + rawCmd := exec.CommandContext(ctx, "patch", + fmt.Sprintf("-p%d", stripLevel), + "-i", tmpPatch.Name(), + ) + rawCmd.Dir = extractRoot + rawCmd.Stderr = &stderr + + cmd, err := cmdFactory.Command(rawCmd) + if err != nil { + return fmt.Errorf("creating patch command:\n%w", err) + } + + if runErr := cmd.Run(ctx); runErr != nil { + return fmt.Errorf("patch failed:\n%s\n%w", stderr.String(), runErr) + } + + return nil +} + +// globFilesInDir finds files under root matching a glob pattern. +// Supports doublestar patterns (e.g., "**/*.md"). +func globFilesInDir(root, pattern string) ([]string, error) { + var matches []string + + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if entry.IsDir() { + return nil + } + + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return fmt.Errorf("computing relative path for %#q:\n%w", path, relErr) + } + + matched, matchErr := doublestar.Match(pattern, rel) + if matchErr != nil { + return fmt.Errorf("invalid glob pattern %#q:\n%w", pattern, matchErr) + } + + if matched { + matches = append(matches, path) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking directory for glob %#q:\n%w", pattern, err) + } + + return matches, nil +} + +// tarballNamesFromOverlays returns the unique tarball filenames targeted by +// tarball overlays in the given overlay list. Used by [updateSourcesFile] to +// determine which 'sources' entries need rehashing after overlay application. +func tarballNamesFromOverlays(overlays []projectconfig.ComponentOverlay) []string { + seen := make(map[string]bool) + + var names []string + + for _, overlay := range overlays { + if overlay.ModifiesTarball() && !seen[overlay.Tarball] { + seen[overlay.Tarball] = true + names = append(names, overlay.Tarball) + } + } + + return names +} diff --git a/internal/app/azldev/core/sources/tarballoverlays_internal_test.go b/internal/app/azldev/core/sources/tarballoverlays_internal_test.go new file mode 100644 index 00000000..ac5a666e --- /dev/null +++ b/internal/app/azldev/core/sources/tarballoverlays_internal_test.go @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "os" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGroupOverlaysByTarball(t *testing.T) { + t.Run("groups overlays by tarball name preserving order", func(t *testing.T) { + overlays := []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlayTarballFileRemove, + Tarball: "pkg-1.0.tar.gz", + Filename: "unwanted.conf", + }, + { + Type: projectconfig.ComponentOverlayTarballSearchReplace, + Tarball: "pkg-1.0.tar.gz", + Filename: "config.h", + Regex: "old", + Replacement: "new", + }, + { + Type: projectconfig.ComponentOverlayTarballFileRemove, + Tarball: "other-2.0.tar.xz", + Filename: "docs/*.md", + }, + } + + groups := groupOverlaysByTarball(overlays) + + require.Len(t, groups, 2) + + assert.Equal(t, "pkg-1.0.tar.gz", groups[0].tarball) + require.Len(t, groups[0].overlays, 2) + assert.Equal(t, projectconfig.ComponentOverlayTarballFileRemove, groups[0].overlays[0].Type) + assert.Equal(t, projectconfig.ComponentOverlayTarballSearchReplace, groups[0].overlays[1].Type) + + assert.Equal(t, "other-2.0.tar.xz", groups[1].tarball) + require.Len(t, groups[1].overlays, 1) + }) + + t.Run("skips non-tarball overlays", func(t *testing.T) { + overlays := []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1.0"}, + {Type: projectconfig.ComponentOverlayTarballFileRemove, Tarball: "pkg.tar.gz", Filename: "f"}, + {Type: projectconfig.ComponentOverlayAddFile, Filename: "new.txt", Source: "src"}, + } + + groups := groupOverlaysByTarball(overlays) + + require.Len(t, groups, 1) + assert.Equal(t, "pkg.tar.gz", groups[0].tarball) + require.Len(t, groups[0].overlays, 1) + }) +} + +func TestGlobFilesInDir(t *testing.T) { + workDir := t.TempDir() + + require.NoError(t, os.MkdirAll(workDir+"/sub", 0o755)) + require.NoError(t, os.WriteFile(workDir+"/file.txt", nil, fileperms.PrivateFile)) + require.NoError(t, os.WriteFile(workDir+"/sub/deep.txt", nil, fileperms.PrivateFile)) + require.NoError(t, os.WriteFile(workDir+"/sub/other.md", nil, fileperms.PrivateFile)) + + t.Run("simple glob", func(t *testing.T) { + matches, err := globFilesInDir(workDir, "*.txt") + require.NoError(t, err) + require.Len(t, matches, 1) + }) + + t.Run("doublestar glob", func(t *testing.T) { + matches, err := globFilesInDir(workDir, "**/*.txt") + require.NoError(t, err) + require.Len(t, matches, 2) + }) + + t.Run("no matches", func(t *testing.T) { + matches, err := globFilesInDir(workDir, "*.rs") + require.NoError(t, err) + assert.Empty(t, matches) + }) +} + +func TestTarballFileRemove(t *testing.T) { + workDir := t.TempDir() + + require.NoError(t, os.WriteFile(workDir+"/keep.txt", []byte("keep"), fileperms.PrivateFile)) + require.NoError(t, os.WriteFile(workDir+"/remove.conf", []byte("remove"), fileperms.PrivateFile)) + + err := tarballFileRemove(workDir, "*.conf") + require.NoError(t, err) + + assert.FileExists(t, workDir+"/keep.txt") + assert.NoFileExists(t, workDir+"/remove.conf") +} + +func TestTarballFileRemoveNoMatch(t *testing.T) { + workDir := t.TempDir() + + require.NoError(t, os.WriteFile(workDir+"/file.txt", nil, fileperms.PrivateFile)) + + err := tarballFileRemove(workDir, "*.conf") + require.Error(t, err) + assert.ErrorIs(t, err, ErrOverlayDidNotApply) +} + +func TestTarballSearchReplace(t *testing.T) { + workDir := t.TempDir() + + require.NoError(t, os.WriteFile(workDir+"/config.h", []byte("#define OLD_VALUE 1\n"), fileperms.PrivateFile)) + + err := tarballSearchReplace(workDir, "config.h", "OLD_VALUE", "NEW_VALUE") + require.NoError(t, err) + + content, readErr := os.ReadFile(workDir + "/config.h") + require.NoError(t, readErr) + assert.Equal(t, "#define NEW_VALUE 1\n", string(content)) +} + +func TestTarballSearchReplaceNoMatch(t *testing.T) { + workDir := t.TempDir() + + require.NoError(t, os.WriteFile(workDir+"/config.h", []byte("#define SOMETHING 1\n"), fileperms.PrivateFile)) + + err := tarballSearchReplace(workDir, "config.h", "NONEXISTENT", "new") + require.Error(t, err) + assert.ErrorIs(t, err, ErrOverlayDidNotApply) +} diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index d6a13c76..3a02251e 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -17,10 +17,13 @@ import ( // ComponentOverlay represents an overlay that may be applied to a component's spec and/or its sources. type ComponentOverlay struct { // The type of overlay to apply. - Type ComponentOverlayType `toml:"type" json:"type" validate:"required" jsonschema:"enum=spec-add-tag,enum=spec-insert-tag,enum=spec-set-tag,enum=spec-update-tag,enum=spec-remove-tag,enum=spec-prepend-lines,enum=spec-append-lines,enum=spec-search-replace,enum=spec-remove-section,enum=spec-remove-subpackage,enum=patch-add,enum=patch-remove,enum=file-prepend-lines,enum=file-search-replace,enum=file-add,enum=file-remove,enum=file-rename,title=Overlay type,description=The type of overlay to apply"` + Type ComponentOverlayType `toml:"type" json:"type" validate:"required" jsonschema:"enum=spec-add-tag,enum=spec-insert-tag,enum=spec-set-tag,enum=spec-update-tag,enum=spec-remove-tag,enum=spec-prepend-lines,enum=spec-append-lines,enum=spec-search-replace,enum=spec-remove-section,enum=spec-remove-subpackage,enum=patch-add,enum=patch-remove,enum=file-prepend-lines,enum=file-search-replace,enum=file-add,enum=file-remove,enum=file-rename,enum=tarball-file-remove,enum=tarball-search-replace,enum=tarball-patch,title=Overlay type,description=The type of overlay to apply"` // Human readable description of overlay; primarily present to document the need for the change. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Human readable description of overlay" fingerprint:"-"` + // For overlays that target files inside a source tarball, identifies the tarball to modify. + // Must be a filename (not a path) matching a source archive in the component's sources directory. + Tarball string `toml:"tarball,omitempty" json:"tarball,omitempty" jsonschema:"title=Tarball,description=The source tarball to modify (e.g. pkg-1.0.tar.gz)"` // For overlays that apply to non-spec files, indicates the filename. For overlays that can // apply to multiple files, supports glob patterns (including globstar). Filename string `toml:"file,omitempty" json:"file,omitempty" jsonschema:"title=Filename,description=The name of the non-spec file to which this overlay applies, or a glob pattern matching multiple files"` @@ -126,6 +129,14 @@ func (c *ComponentOverlay) ModifiesSpec() bool { c.Type == ComponentOverlayRemovePatch } +// ModifiesTarball returns true if the overlay modifies files inside a source tarball. +// These overlays require a mock chroot for extraction and repacking. +func (c *ComponentOverlay) ModifiesTarball() bool { + return c.Type == ComponentOverlayTarballFileRemove || + c.Type == ComponentOverlayTarballSearchReplace || + c.Type == ComponentOverlayTarballPatch +} + // ModifiesNonSpecFiles returns true if the overlay modifies non-spec files. This includes // hybrid overlays that modify both spec and source files (e.g., patch overlays), since // those also require non-spec modifications. @@ -189,12 +200,21 @@ const ( ComponentOverlayRemoveFile ComponentOverlayType = "file-remove" // ComponentOverlayRenameFile is an overlay that renames a non-spec file. ComponentOverlayRenameFile ComponentOverlayType = "file-rename" + // ComponentOverlayTarballFileRemove is an overlay that removes file(s) from inside a source tarball. + // The tarball is extracted, matching files are deleted, and the tarball is repacked. + ComponentOverlayTarballFileRemove ComponentOverlayType = "tarball-file-remove" + // ComponentOverlayTarballSearchReplace is an overlay that performs regex search-and-replace + // on file(s) inside a source tarball. + ComponentOverlayTarballSearchReplace ComponentOverlayType = "tarball-search-replace" + // ComponentOverlayTarballPatch is an overlay that applies a unified diff patch to the + // extracted contents of a source tarball. + ComponentOverlayTarballPatch ComponentOverlayType = "tarball-patch" ) // Validate checks that required fields are set based on the overlay type. This catches // configuration errors at load time rather than at apply time. // -//nolint:cyclop,gocognit,gocyclo,funlen // complexity is inherent to the number of overlay types. +//nolint:cyclop,gocognit,gocyclo,funlen,maintidx // complexity is inherent to the number of overlay types. func (c *ComponentOverlay) Validate() error { desc := c.Description if desc == "" { @@ -336,6 +356,46 @@ func (c *ComponentOverlay) Validate() error { if err := validateGlobPattern(c.Filename, desc); err != nil { return err } + case ComponentOverlayTarballFileRemove: + if err := requireFileBasename("tarball", c.Tarball); err != nil { + return err + } + + if err := requireRelativePath("file", c.Filename); err != nil { + return err + } + + if err := validateGlobPattern(c.Filename, desc); err != nil { + return err + } + case ComponentOverlayTarballSearchReplace: + if err := requireFileBasename("tarball", c.Tarball); err != nil { + return err + } + + if err := requireRelativePath("file", c.Filename); err != nil { + return err + } + + if err := validateGlobPattern(c.Filename, desc); err != nil { + return err + } + + if c.Regex == "" { + return missingField("regex") + } + + if err := validateRegex(c.Regex, desc); err != nil { + return err + } + case ComponentOverlayTarballPatch: + if err := requireFileBasename("tarball", c.Tarball); err != nil { + return err + } + + if c.Source == "" { + return missingField("source") + } default: return fmt.Errorf("unknown overlay type %#q: %#q", c.Type, desc) } diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index 84ccf200..7b75f0e2 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -412,6 +412,147 @@ func TestComponentOverlay_Validate(t *testing.T) { errorExpected: true, errorContains: "section", }, + // tarball-file-remove tests + { + name: "tarball-file-remove valid", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballFileRemove, + Tarball: "pkg-1.0.tar.gz", + Filename: "unwanted.conf", + }, + errorExpected: false, + }, + { + name: "tarball-file-remove valid with glob", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballFileRemove, + Tarball: "pkg-1.0.tar.gz", + Filename: "docs/**/*.md", + }, + errorExpected: false, + }, + { + name: "tarball-file-remove missing tarball", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballFileRemove, + Filename: "unwanted.conf", + }, + errorExpected: true, + errorContains: "tarball", + }, + { + name: "tarball-file-remove missing file", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballFileRemove, + Tarball: "pkg-1.0.tar.gz", + }, + errorExpected: true, + errorContains: "file", + }, + { + name: "tarball-file-remove rejects tarball path", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballFileRemove, + Tarball: "subdir/pkg-1.0.tar.gz", + Filename: "unwanted.conf", + }, + errorExpected: true, + errorContains: "tarball", + }, + // tarball-search-replace tests + { + name: "tarball-search-replace valid", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballSearchReplace, + Tarball: "pkg-1.0.tar.gz", + Filename: "config.h", + Regex: "old_value", + Replacement: "new_value", + }, + errorExpected: false, + }, + { + name: "tarball-search-replace missing tarball", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballSearchReplace, + Filename: "config.h", + Regex: "old_value", + Replacement: "new_value", + }, + errorExpected: true, + errorContains: "tarball", + }, + { + name: "tarball-search-replace missing file", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballSearchReplace, + Tarball: "pkg-1.0.tar.gz", + Regex: "old_value", + Replacement: "new_value", + }, + errorExpected: true, + errorContains: "file", + }, + { + name: "tarball-search-replace missing regex", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballSearchReplace, + Tarball: "pkg-1.0.tar.gz", + Filename: "config.h", + }, + errorExpected: true, + errorContains: "regex", + }, + { + name: "tarball-search-replace invalid regex", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballSearchReplace, + Tarball: "pkg-1.0.tar.gz", + Filename: "config.h", + Regex: "[invalid", + Replacement: "new_value", + }, + errorExpected: true, + errorContains: "regex", + }, + // tarball-patch tests + { + name: "tarball-patch valid", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballPatch, + Tarball: "pkg-1.0.tar.gz", + Source: "patches/fix.patch", + }, + errorExpected: false, + }, + { + name: "tarball-patch valid with strip level", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballPatch, + Tarball: "pkg-1.0.tar.gz", + Source: "patches/fix.patch", + Value: "2", + }, + errorExpected: false, + }, + { + name: "tarball-patch missing tarball", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballPatch, + Source: "patches/fix.patch", + }, + errorExpected: true, + errorContains: "tarball", + }, + { + name: "tarball-patch missing source", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayTarballPatch, + Tarball: "pkg-1.0.tar.gz", + }, + errorExpected: true, + errorContains: "source", + }, } for _, testCase := range testCases { @@ -455,6 +596,12 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { projectconfig.ComponentOverlayAddFile, } + tarballOverlayTypes := []projectconfig.ComponentOverlayType{ + projectconfig.ComponentOverlayTarballFileRemove, + projectconfig.ComponentOverlayTarballSearchReplace, + projectconfig.ComponentOverlayTarballPatch, + } + for _, overlayType := range specOverlayTypes { t.Run(string(overlayType)+"_is_spec_overlay", func(t *testing.T) { overlay := projectconfig.ComponentOverlay{Type: overlayType} @@ -468,4 +615,13 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { assert.False(t, overlay.ModifiesSpec(), "expected %s to not be a spec overlay", overlayType) }) } + + for _, overlayType := range tarballOverlayTypes { + t.Run(string(overlayType)+"_is_tarball_overlay", func(t *testing.T) { + overlay := projectconfig.ComponentOverlay{Type: overlayType} + assert.True(t, overlay.ModifiesTarball(), "expected %s to be a tarball overlay", overlayType) + assert.False(t, overlay.ModifiesSpec(), "expected %s to not be a spec overlay", overlayType) + assert.False(t, overlay.ModifiesNonSpecFiles(), "expected %s to not be a non-spec overlay", overlayType) + }) + } } From 7f8614e5e7c8867ca6b037672b9c4e88a828b2cc Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 4 Jun 2026 21:42:05 +0000 Subject: [PATCH 02/13] refactor: collapse tarball overlays into archive-scoped file overlays --- docs/user/reference/config/overlays.md | 46 +++- .../app/azldev/core/sources/sourceprep.go | 9 +- .../azldev/core/sources/tarballoverlays.go | 246 ++++++++---------- .../sources/tarballoverlays_internal_test.go | 222 ++++++++++++---- internal/projectconfig/overlay.go | 103 ++++---- internal/projectconfig/overlay_test.go | 121 ++++++--- internal/utils/archive/archive.go | 27 ++ schemas/azldev.schema.json | 13 +- 8 files changed, 496 insertions(+), 291 deletions(-) diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index af4f485d..0880c95d 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -47,16 +47,29 @@ successfully makes a replacement to at least one matching file. | `file-remove` | Removes a file | `file` | Glob pattern for files to remove | | `file-rename` | Renames a file within the same directory | `file`, `replacement` | Name of file to rename | -### Tarball Overlays +> **Tip:** `file-remove` and `file-search-replace` can also operate inside a source tarball by +> setting the `tarball` field — see [Archive (Tarball) Overlays](#archive-tarball-overlays). -These overlays modify files **inside** source tarballs. The tarball is extracted into a temporary directory, modifications are applied, and the tarball is repacked with the same compression format. Extraction and repacking are handled natively; patch application requires the `patch` command on the host. +### Archive (Tarball) Overlays -> **Note:** Tarball overlays are applied before spec and file overlays, so subsequent overlays see the modified tarball. The `tarball-patch` overlay type requires the `patch` command to be installed on the host. +File overlays can operate on files **inside** a source tarball instead of loose files in the +sources tree. Set the `tarball` field on a `file-remove` or `file-search-replace` overlay to +scope it to that archive; the dedicated `tarball-patch` type applies a unified diff to extracted +archive contents. The tarball is extracted into a temporary directory, the modifications are +applied with the same machinery as loose-file overlays, and the tarball is repacked with its +original compression format. Extraction and repacking are handled natively; `tarball-patch` +additionally requires the `patch` command on the host. + +> **Note:** Archive overlays are batched per tarball — all overlays targeting the same archive +> share a single extract/modify/repack cycle — and the `sources` file is rehashed afterward to +> reflect the repacked archive. They are processed independently of spec and loose-file overlays. + +> **Extraction root:** Paths in archive overlays (`file`, and the paths a patch targets) are interpreted relative to the archive's extraction root. By default the root is inferred: if the archive unpacks to a single top-level directory (the conventional `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. Set `tarball-root` to override this — the equivalent of rpmbuild's `%setup -n` — when an archive's top-level directory does not follow that convention. | Type | Description | Required Fields | |------|-------------|-----------------| -| `tarball-file-remove` | Removes file(s) matching a glob pattern from inside a tarball | `tarball`, `file` | -| `tarball-search-replace` | Regex-based search and replace on file(s) inside a tarball | `tarball`, `file`, `regex` | +| `file-remove` + `tarball` | Removes file(s) matching a glob pattern from inside a tarball | `tarball`, `file` | +| `file-search-replace` + `tarball` | Regex-based search and replace on file(s) inside a tarball | `tarball`, `file`, `regex` | | `tarball-patch` | Applies a unified diff patch to the extracted tarball contents | `tarball`, `source` | ## Field Reference @@ -65,15 +78,16 @@ These overlays modify files **inside** source tarballs. The tarball is extracted |-------|----------|-------------|---------| | Type | `type` | **Required.** The overlay type to apply | All overlays | | Description | `description` | Human-readable explanation documenting the need for the change; helps identify overlays in error messages | All (optional) | -| Tarball | `tarball` | The source tarball filename to modify (must be a basename, not a path) | `tarball-file-remove`, `tarball-search-replace`, `tarball-patch` | +| Tarball | `tarball` | The source tarball filename to scope an overlay to (must be a basename, not a path). When set, the overlay operates on files inside that archive. | `tarball-patch` (required); `file-remove`, `file-search-replace` (optional) | +| Tarball root | `tarball-root` | Top-level directory inside the tarball to treat as the extraction root (mirrors `%setup -n`); inferred when unset. Must be a local relative path (no `..` or absolute paths). When multiple overlays target the same tarball, any that set this must agree. | `tarball-patch`, and archive-scoped `file-remove` / `file-search-replace` (optional) | | Tag | `tag` | The spec tag name (e.g., `BuildRequires`, `Requires`, `Version`) | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` | | Value | `value` | The tag value to set, or value to match for removal. For `tarball-patch`, sets the patch strip level (default: `1`, equivalent to `patch -p1`). | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` (optional for matching), `tarball-patch` (optional) | | Section | `section` | The spec section to target (e.g., `%build`, `%install`, `%files`, `%description`) | `spec-prepend-lines`, `spec-append-lines`, `spec-search-replace` (optional), `spec-remove-section` | | Package | `package` | The sub-package name for multi-package specs; omit to target the main package | All spec overlays (optional, except `spec-remove-subpackage` which **requires** it) | -| Regex | `regex` | Regular expression pattern to match | `spec-search-replace`, `file-search-replace`, `tarball-search-replace` | -| Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename`, `tarball-search-replace` | +| Regex | `regex` | Regular expression pattern to match | `spec-search-replace`, `file-search-replace` | +| Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename` | | Lines | `lines` | Array of text lines to insert | `spec-prepend-lines`, `spec-append-lines`, `file-prepend-lines` | -| File | `file` | The name of the non-spec file to modify or add | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove`, `tarball-file-remove`, `tarball-search-replace` | +| File | `file` | The name of the non-spec file to modify or add, or a glob pattern. For archive-scoped overlays, it is matched against the tarball's extracted contents. | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | | Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add`, `tarball-patch` | | Metadata | `metadata` | Documentation table describing intent and provenance — see [Overlay Metadata](#overlay-metadata). Not allowed inside an overlay file loaded via `overlay-files` (the file-level `[metadata]` block applies to every overlay in the file). | All (optional) | @@ -432,22 +446,28 @@ description = "Remove CVE patches that are now upstream" ### Removing a File from a Tarball -The `tarball-file-remove` overlay deletes files matching a glob pattern from inside a source -tarball. The tarball is extracted, matching files are removed, and the tarball is repacked. +Set the `tarball` field on a `file-remove` overlay to delete files matching a glob pattern from +inside a source tarball. The tarball is extracted, matching files are removed, and the tarball is +repacked. ```toml [[components.mypackage.overlays]] -type = "tarball-file-remove" +type = "file-remove" tarball = "mypackage-1.0.tar.gz" file = "vendor/**" description = "Remove bundled vendor directory" ``` +> **Tip:** Without the `tarball` field, the same `file-remove` overlay removes a loose file from +> the sources tree instead. The `tarball` field is the only thing that scopes it to an archive. + ### Search and Replace Inside a Tarball +Set the `tarball` field on a `file-search-replace` overlay to rewrite content inside an archive: + ```toml [[components.mypackage.overlays]] -type = "tarball-search-replace" +type = "file-search-replace" tarball = "mypackage-1.0.tar.xz" file = "configure.ac" regex = "AC_CHECK_LIB\\(old_lib" diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 62649e22..1bbf7cf6 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -356,7 +356,7 @@ func (p *sourcePreparerImpl) applyTarballOverlayGroup( } if err := applyTarballOverlays( - ctx, cmdFactory, p.fs, p.eventListener, sourcesDirPath, tarballOverlays, + ctx, cmdFactory, p.dryRunnable, p.fs, p.eventListener, sourcesDirPath, tarballOverlays, ); err != nil { return fmt.Errorf("failed to apply tarball overlays for component %#q:\n%w", component.GetName(), err) @@ -1169,10 +1169,17 @@ func (p *sourcePreparerImpl) resolveSpecPath( } // applyOverlayList applies a list of overlays to the component sources sequentially. +// Archive-scoped overlays (see [projectconfig.ComponentOverlay.ModifiesTarball]) are +// skipped here; they are handled separately by [applyTarballOverlays], which batches +// extraction and repacking per archive. func (p *sourcePreparerImpl) applyOverlayList( overlays []projectconfig.ComponentOverlay, sourcesDirPath, absSpecPath string, ) error { for _, overlay := range overlays { + if overlay.ModifiesTarball() { + continue + } + if err := ApplyOverlayToSources( p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath, ); err != nil { diff --git a/internal/app/azldev/core/sources/tarballoverlays.go b/internal/app/azldev/core/sources/tarballoverlays.go index 5f9d4fe8..bbf6d4d0 100644 --- a/internal/app/azldev/core/sources/tarballoverlays.go +++ b/internal/app/azldev/core/sources/tarballoverlays.go @@ -11,32 +11,35 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "strconv" "strings" - "github.com/bmatcuk/doublestar/v4" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/tarball" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" ) // applyTarballOverlays groups tarball overlays by target archive and processes // them in order. Multiple overlays targeting the same tarball are batched into -// a single extract/modify/repack cycle. All operations (extract, modify, repack) -// are performed in pure Go on the host, except for patch application which -// shells out to the host's `patch` command. +// a single extract/modify/repack cycle. File modifications inside the archive +// reuse the same machinery as loose-file overlays ([applyNonSpecOverlay]); patch +// application shells out to the host's `patch` command. func applyTarballOverlays( ctx context.Context, cmdFactory opctx.CmdFactory, + dryRunnable opctx.DryRunnable, fs opctx.FS, eventListener opctx.EventListener, sourcesDirPath string, overlays []projectconfig.ComponentOverlay, ) error { - groups := groupOverlaysByTarball(overlays) + groups, err := groupOverlaysByTarball(overlays) + if err != nil { + return err + } if len(groups) == 0 { return nil @@ -49,7 +52,7 @@ func applyTarballOverlays( defer event.End() for _, group := range groups { - if err := processTarball(ctx, cmdFactory, fs, sourcesDirPath, group); err != nil { + if err := processTarball(ctx, cmdFactory, dryRunnable, fs, sourcesDirPath, group); err != nil { return fmt.Errorf("tarball overlay failed for %#q:\n%w", group.tarball, err) } } @@ -60,13 +63,19 @@ func applyTarballOverlays( // tarballGroup holds overlays targeting the same tarball, preserving order. type tarballGroup struct { tarball string + root string overlays []projectconfig.ComponentOverlay } // groupOverlaysByTarball groups tarball overlays by their // [projectconfig.ComponentOverlay.Tarball] field, preserving insertion order // within each group and across groups. Non-tarball overlays are silently skipped. -func groupOverlaysByTarball(overlays []projectconfig.ComponentOverlay) []tarballGroup { +// +// The optional [projectconfig.ComponentOverlay.TarballRoot] override (mirroring +// rpmbuild's `%setup -n`) is reconciled per tarball: all overlays targeting the +// same archive that set it must agree, otherwise the configuration is ambiguous +// and an error is returned. +func groupOverlaysByTarball(overlays []projectconfig.ComponentOverlay) ([]tarballGroup, error) { orderMap := make(map[string]int) var groups []tarballGroup @@ -84,10 +93,21 @@ func groupOverlaysByTarball(overlays []projectconfig.ComponentOverlay) []tarball groups = append(groups, tarballGroup{tarball: overlay.Tarball}) } + if overlay.TarballRoot != "" { + if groups[idx].root != "" && groups[idx].root != overlay.TarballRoot { + return nil, fmt.Errorf( + "conflicting %#q overrides for tarball %#q: %#q vs %#q", + "tarball-root", overlay.Tarball, groups[idx].root, overlay.TarballRoot, + ) + } + + groups[idx].root = overlay.TarballRoot + } + groups[idx].overlays = append(groups[idx].overlays, overlay) } - return groups + return groups, nil } // processTarball extracts a tarball to a temp directory, applies all overlays, @@ -95,50 +115,66 @@ func groupOverlaysByTarball(overlays []projectconfig.ComponentOverlay) []tarball func processTarball( ctx context.Context, cmdFactory opctx.CmdFactory, + dryRunnable opctx.DryRunnable, fs opctx.FS, sourcesDirPath string, group tarballGroup, ) error { archivePath := filepath.Join(sourcesDirPath, group.tarball) - // Create a temporary directory for extraction. - workDir, err := os.MkdirTemp("", "tarball-overlay-") + // Create a temporary directory for extraction. The injected FS is real-filesystem + // backed in production, so the returned path is a genuine on-disk path usable by + // the [archive] package and the host `patch` command. + workDir, err := fileutils.MkdirTempInTempDir(fs, "tarball-overlay-") if err != nil { return fmt.Errorf("creating temp directory:\n%w", err) } defer func() { - if removeErr := os.RemoveAll(workDir); removeErr != nil { + if removeErr := fs.RemoveAll(workDir); removeErr != nil { slog.Warn("Failed to clean up tarball work directory", "error", removeErr) } }() - // Detect compression and extract. - compression, err := tarball.DetectCompression(group.tarball) - if err != nil { - return fmt.Errorf("detecting compression for %#q:\n%w", group.tarball, err) - } - - if err := tarball.Extract(fs, archivePath, workDir, compression); err != nil { + // Extract the archive; compression is inferred from the filename extension. + if err := archive.ExtractAuto(archivePath, workDir); err != nil { return fmt.Errorf("extracting tarball:\n%w", err) } // Determine the root of the extracted content. Most source tarballs have - // a single top-level directory (e.g., "pkg-1.0/"). - extractRoot, err := tarball.ResolveExtractRoot(workDir) + // a single top-level directory (e.g., "pkg-1.0/"); group.root overrides this + // inference when set (mirrors rpmbuild's `%setup -n`). + extractRoot, err := resolveExtractRoot(workDir, group.root) if err != nil { return fmt.Errorf("resolving extract root:\n%w", err) } + // Confine an FS to the extract root so file overlays reuse the same machinery + // as loose-file overlays. The extracted tree is always on the real filesystem + // (written by the [archive] package), so root it on an OS-backed FS regardless + // of the injected fs implementation. + extractFS, err := rootfs.New(extractRoot) + if err != nil { + return fmt.Errorf("confining FS to extract root:\n%w", err) + } + + defer func() { + if closeErr := extractFS.Close(); closeErr != nil { + slog.Warn("Failed to close extract-root FS", "error", closeErr) + } + }() + // Apply each overlay operation in order. for _, overlay := range group.overlays { - if err := applyTarballOperation(ctx, cmdFactory, fs, extractRoot, overlay); err != nil { + if err := applyTarballOperation( + ctx, cmdFactory, dryRunnable, fs, extractFS, extractRoot, overlay, + ); err != nil { return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) } } - // Deterministically repack the tarball in-place. - if err := tarball.RepackDeterministic(fs, archivePath, workDir, compression); err != nil { + // Deterministically repack the tarball in-place, reusing the original compression. + if err := archive.CreateDeterministicArchiveAuto(archivePath, workDir); err != nil { return fmt.Errorf("repacking tarball:\n%w", err) } @@ -147,23 +183,19 @@ func processTarball( return nil } -// applyTarballOperation dispatches a single overlay to the appropriate handler. +// applyTarballOperation dispatches a single overlay against the extracted tree. +// The dedicated tarball-patch type shells out to the host `patch` command; all +// other (archive-scoped) types reuse [applyNonSpecOverlay], operating on the +// extract-root FS exactly as they would on the loose sources tree. func applyTarballOperation( ctx context.Context, cmdFactory opctx.CmdFactory, - fs opctx.FS, + dryRunnable opctx.DryRunnable, + sourceFS, extractFS opctx.FS, extractRoot string, overlay projectconfig.ComponentOverlay, ) error { - //nolint:exhaustive // Only tarball overlay types are valid here; the default catches the rest. - switch overlay.Type { - case projectconfig.ComponentOverlayTarballFileRemove: - return tarballFileRemove(extractRoot, overlay.Filename) - - case projectconfig.ComponentOverlayTarballSearchReplace: - return tarballSearchReplace(extractRoot, overlay.Filename, overlay.Regex, overlay.Replacement) - - case projectconfig.ComponentOverlayTarballPatch: + if overlay.Type == projectconfig.ComponentOverlayTarballPatch { stripLevel := 1 if overlay.Value != "" { @@ -175,74 +207,10 @@ func applyTarballOperation( stripLevel = parsed } - return tarballApplyPatch(ctx, cmdFactory, fs, extractRoot, overlay.Source, stripLevel) - - default: - return fmt.Errorf("unsupported tarball overlay type %#q", overlay.Type) - } -} - -// tarballFileRemove removes files matching a glob pattern from the extracted tree. -func tarballFileRemove(extractRoot, pattern string) error { - matches, err := globFilesInDir(extractRoot, pattern) - if err != nil { - return err - } - - if len(matches) == 0 { - return fmt.Errorf("no files match pattern %#q:\n%w", pattern, ErrOverlayDidNotApply) - } - - for _, path := range matches { - if err := os.Remove(path); err != nil { - return fmt.Errorf("failed to remove %#q:\n%w", path, err) - } - } - - return nil -} - -// tarballSearchReplace applies regex search-and-replace to files matching a glob -// pattern inside the extracted tree. -func tarballSearchReplace(extractRoot, pattern, regex, replacement string) error { - matches, err := globFilesInDir(extractRoot, pattern) - if err != nil { - return err - } - - if len(matches) == 0 { - return fmt.Errorf("no files match pattern %#q:\n%w", pattern, ErrOverlayDidNotApply) - } - - compiled, err := regexp.Compile(regex) - if err != nil { - return fmt.Errorf("invalid regex %#q:\n%w", regex, err) - } - - anyReplaced := false - - for _, path := range matches { - content, readErr := os.ReadFile(path) - if readErr != nil { - return fmt.Errorf("reading %#q:\n%w", path, readErr) - } - - newContent := compiled.ReplaceAll(content, []byte(replacement)) - if string(newContent) != string(content) { - anyReplaced = true - - if writeErr := os.WriteFile(path, newContent, fileperms.PublicFile); writeErr != nil { - return fmt.Errorf("writing %#q:\n%w", path, writeErr) - } - } - } - - if !anyReplaced { - return fmt.Errorf("regex %#q did not match any content in files matching %#q:\n%w", - regex, pattern, ErrOverlayDidNotApply) + return tarballApplyPatch(ctx, cmdFactory, sourceFS, extractRoot, overlay.Source, stripLevel) } - return nil + return applyNonSpecOverlay(dryRunnable, sourceFS, extractFS, overlay) } // tarballApplyPatch applies a unified diff patch to the extracted tree by @@ -265,27 +233,29 @@ func tarballApplyPatch( return fmt.Errorf("reading patch file %#q:\n%w", patchSource, err) } - // Write to a temp file on the real filesystem so the patch command can read it. - tmpPatch, err := os.CreateTemp("", "tarball-patch-*.patch") + // Stage the patch in a temp dir so the host `patch` command can read it. The + // injected FS is real-filesystem backed in production, so the path is on disk. + tmpDir, err := fileutils.MkdirTempInTempDir(fs, "tarball-patch-") if err != nil { - return fmt.Errorf("creating temp patch file:\n%w", err) + return fmt.Errorf("creating temp patch directory:\n%w", err) } - defer os.Remove(tmpPatch.Name()) - - if _, err := tmpPatch.Write(patchData); err != nil { - tmpPatch.Close() + defer func() { + if removeErr := fs.RemoveAll(tmpDir); removeErr != nil { + slog.Warn("Failed to clean up tarball patch directory", "error", removeErr) + } + }() + tmpPatchPath := filepath.Join(tmpDir, "overlay.patch") + if err := fileutils.WriteFile(fs, tmpPatchPath, patchData, fileperms.PublicFile); err != nil { return fmt.Errorf("writing temp patch file:\n%w", err) } - tmpPatch.Close() - var stderr strings.Builder rawCmd := exec.CommandContext(ctx, "patch", fmt.Sprintf("-p%d", stripLevel), - "-i", tmpPatch.Name(), + "-i", tmpPatchPath, ) rawCmd.Dir = extractRoot rawCmd.Stderr = &stderr @@ -302,41 +272,45 @@ func tarballApplyPatch( return nil } -// globFilesInDir finds files under root matching a glob pattern. -// Supports doublestar patterns (e.g., "**/*.md"). -func globFilesInDir(root, pattern string) ([]string, error) { - var matches []string - - err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr +// resolveExtractRoot returns the effective root of an extracted tarball. +// When rootOverride is set (the `%setup -n` equivalent), the named subdirectory +// of workDir is used; it must be a local path that exists as a directory. When +// rootOverride is empty, the root is inferred: if workDir contains exactly one +// entry and that entry is a directory (the common case for source tarballs like +// "pkg-1.0/"), that subdirectory is returned; otherwise workDir itself is +// returned. +func resolveExtractRoot(workDir, rootOverride string) (string, error) { + if rootOverride != "" { + // Defense in depth: validation already rejects non-local overrides, but + // re-check before joining so a malformed value can never escape workDir. + if !filepath.IsLocal(rootOverride) { + return "", fmt.Errorf("tarball root %#q is not a local path", rootOverride) } - if entry.IsDir() { - return nil - } + target := filepath.Join(workDir, rootOverride) - rel, relErr := filepath.Rel(root, path) - if relErr != nil { - return fmt.Errorf("computing relative path for %#q:\n%w", path, relErr) + info, err := os.Stat(target) + if err != nil { + return "", fmt.Errorf("tarball root %#q not found after extraction:\n%w", rootOverride, err) } - matched, matchErr := doublestar.Match(pattern, rel) - if matchErr != nil { - return fmt.Errorf("invalid glob pattern %#q:\n%w", pattern, matchErr) + if !info.IsDir() { + return "", fmt.Errorf("tarball root %#q is not a directory", rootOverride) } - if matched { - matches = append(matches, path) - } + return target, nil + } - return nil - }) + entries, err := os.ReadDir(workDir) if err != nil { - return nil, fmt.Errorf("walking directory for glob %#q:\n%w", pattern, err) + return "", fmt.Errorf("reading extracted directory:\n%w", err) + } + + if len(entries) == 1 && entries[0].IsDir() { + return filepath.Join(workDir, entries[0].Name()), nil } - return matches, nil + return workDir, nil } // tarballNamesFromOverlays returns the unique tarball filenames targeted by diff --git a/internal/app/azldev/core/sources/tarballoverlays_internal_test.go b/internal/app/azldev/core/sources/tarballoverlays_internal_test.go index ac5a666e..45acbcf8 100644 --- a/internal/app/azldev/core/sources/tarballoverlays_internal_test.go +++ b/internal/app/azldev/core/sources/tarballoverlays_internal_test.go @@ -4,11 +4,14 @@ package sources import ( + "context" "os" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,121 +20,228 @@ func TestGroupOverlaysByTarball(t *testing.T) { t.Run("groups overlays by tarball name preserving order", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ { - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", Filename: "unwanted.conf", }, { - Type: projectconfig.ComponentOverlayTarballSearchReplace, + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, Tarball: "pkg-1.0.tar.gz", Filename: "config.h", Regex: "old", Replacement: "new", }, { - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "other-2.0.tar.xz", Filename: "docs/*.md", }, } - groups := groupOverlaysByTarball(overlays) + groups, err := groupOverlaysByTarball(overlays) + require.NoError(t, err) require.Len(t, groups, 2) assert.Equal(t, "pkg-1.0.tar.gz", groups[0].tarball) require.Len(t, groups[0].overlays, 2) - assert.Equal(t, projectconfig.ComponentOverlayTarballFileRemove, groups[0].overlays[0].Type) - assert.Equal(t, projectconfig.ComponentOverlayTarballSearchReplace, groups[0].overlays[1].Type) + assert.Equal(t, projectconfig.ComponentOverlayRemoveFile, groups[0].overlays[0].Type) + assert.Equal(t, projectconfig.ComponentOverlaySearchAndReplaceInFile, groups[0].overlays[1].Type) assert.Equal(t, "other-2.0.tar.xz", groups[1].tarball) require.Len(t, groups[1].overlays, 1) }) - t.Run("skips non-tarball overlays", func(t *testing.T) { + t.Run("skips overlays that are not archive-scoped", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1.0"}, - {Type: projectconfig.ComponentOverlayTarballFileRemove, Tarball: "pkg.tar.gz", Filename: "f"}, + {Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg.tar.gz", Filename: "f"}, + // Plain (non-archive) file overlay: no Tarball set, so it must be skipped. + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "loose.txt"}, {Type: projectconfig.ComponentOverlayAddFile, Filename: "new.txt", Source: "src"}, } - groups := groupOverlaysByTarball(overlays) + groups, err := groupOverlaysByTarball(overlays) + require.NoError(t, err) require.Len(t, groups, 1) assert.Equal(t, "pkg.tar.gz", groups[0].tarball) require.Len(t, groups[0].overlays, 1) }) -} -func TestGlobFilesInDir(t *testing.T) { - workDir := t.TempDir() + t.Run("reconciles matching tarball-root overrides", func(t *testing.T) { + overlays := []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg-1.0.tar.gz", + TarballRoot: "custom-root", + Filename: "a.conf", + }, + { + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg-1.0.tar.gz", + Filename: "b.conf", + }, + } - require.NoError(t, os.MkdirAll(workDir+"/sub", 0o755)) - require.NoError(t, os.WriteFile(workDir+"/file.txt", nil, fileperms.PrivateFile)) - require.NoError(t, os.WriteFile(workDir+"/sub/deep.txt", nil, fileperms.PrivateFile)) - require.NoError(t, os.WriteFile(workDir+"/sub/other.md", nil, fileperms.PrivateFile)) + groups, err := groupOverlaysByTarball(overlays) + require.NoError(t, err) + + require.Len(t, groups, 1) + assert.Equal(t, "custom-root", groups[0].root) + }) - t.Run("simple glob", func(t *testing.T) { - matches, err := globFilesInDir(workDir, "*.txt") + t.Run("errors on conflicting tarball-root overrides", func(t *testing.T) { + overlays := []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg-1.0.tar.gz", + TarballRoot: "root-a", + Filename: "a.conf", + }, + { + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg-1.0.tar.gz", + TarballRoot: "root-b", + Filename: "b.conf", + }, + } + + _, err := groupOverlaysByTarball(overlays) + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicting") + }) +} + +func TestResolveExtractRoot(t *testing.T) { + t.Run("infers single top-level directory", func(t *testing.T) { + workDir := t.TempDir() + require.NoError(t, os.MkdirAll(workDir+"/pkg-1.0", 0o755)) + + root, err := resolveExtractRoot(workDir, "") require.NoError(t, err) - require.Len(t, matches, 1) + assert.Equal(t, workDir+"/pkg-1.0", root) }) - t.Run("doublestar glob", func(t *testing.T) { - matches, err := globFilesInDir(workDir, "**/*.txt") + t.Run("falls back to workDir for multiple top-level entries", func(t *testing.T) { + workDir := t.TempDir() + require.NoError(t, os.MkdirAll(workDir+"/dirA", 0o755)) + require.NoError(t, os.WriteFile(workDir+"/loose.txt", nil, fileperms.PrivateFile)) + + root, err := resolveExtractRoot(workDir, "") require.NoError(t, err) - require.Len(t, matches, 2) + assert.Equal(t, workDir, root) }) - t.Run("no matches", func(t *testing.T) { - matches, err := globFilesInDir(workDir, "*.rs") + t.Run("override selects named subdirectory", func(t *testing.T) { + workDir := t.TempDir() + // Two top-level dirs so the heuristic would not pick one. + require.NoError(t, os.MkdirAll(workDir+"/dirA", 0o755)) + require.NoError(t, os.MkdirAll(workDir+"/dirB", 0o755)) + + root, err := resolveExtractRoot(workDir, "dirB") require.NoError(t, err) - assert.Empty(t, matches) + assert.Equal(t, workDir+"/dirB", root) }) -} -func TestTarballFileRemove(t *testing.T) { - workDir := t.TempDir() + t.Run("override missing directory errors", func(t *testing.T) { + workDir := t.TempDir() - require.NoError(t, os.WriteFile(workDir+"/keep.txt", []byte("keep"), fileperms.PrivateFile)) - require.NoError(t, os.WriteFile(workDir+"/remove.conf", []byte("remove"), fileperms.PrivateFile)) + _, err := resolveExtractRoot(workDir, "does-not-exist") + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") + }) - err := tarballFileRemove(workDir, "*.conf") - require.NoError(t, err) + t.Run("override pointing at a file errors", func(t *testing.T) { + workDir := t.TempDir() + require.NoError(t, os.WriteFile(workDir+"/afile", nil, fileperms.PrivateFile)) - assert.FileExists(t, workDir+"/keep.txt") - assert.NoFileExists(t, workDir+"/remove.conf") + _, err := resolveExtractRoot(workDir, "afile") + require.Error(t, err) + assert.Contains(t, err.Error(), "not a directory") + }) + + t.Run("non-local override is rejected", func(t *testing.T) { + workDir := t.TempDir() + + _, err := resolveExtractRoot(workDir, "../escape") + require.Error(t, err) + assert.Contains(t, err.Error(), "not a local path") + }) } -func TestTarballFileRemoveNoMatch(t *testing.T) { - workDir := t.TempDir() +// TestApplyTarballOperation_FileOverlays verifies that archive-scoped file +// overlays are routed through the shared [applyNonSpecOverlay] machinery against +// the extract-root FS (i.e., the same code path as loose-file overlays). +func TestApplyTarballOperation_FileOverlays(t *testing.T) { + ctx := testctx.NewCtx() - require.NoError(t, os.WriteFile(workDir+"/file.txt", nil, fileperms.PrivateFile)) + t.Run("file-remove deletes matching files in the extracted tree", func(t *testing.T) { + extractRoot := t.TempDir() + require.NoError(t, os.WriteFile(extractRoot+"/keep.txt", []byte("keep"), fileperms.PrivateFile)) + require.NoError(t, os.WriteFile(extractRoot+"/remove.conf", []byte("x"), fileperms.PrivateFile)) - err := tarballFileRemove(workDir, "*.conf") - require.Error(t, err) - assert.ErrorIs(t, err, ErrOverlayDidNotApply) -} + extractFS, err := rootfs.New(extractRoot) + require.NoError(t, err) -func TestTarballSearchReplace(t *testing.T) { - workDir := t.TempDir() + defer extractFS.Close() - require.NoError(t, os.WriteFile(workDir+"/config.h", []byte("#define OLD_VALUE 1\n"), fileperms.PrivateFile)) + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg.tar.gz", + Filename: "*.conf", + } - err := tarballSearchReplace(workDir, "config.h", "OLD_VALUE", "NEW_VALUE") - require.NoError(t, err) + err = applyTarballOperation(context.Background(), nil, ctx, ctx.FS(), extractFS, extractRoot, overlay) + require.NoError(t, err) - content, readErr := os.ReadFile(workDir + "/config.h") - require.NoError(t, readErr) - assert.Equal(t, "#define NEW_VALUE 1\n", string(content)) -} + assert.FileExists(t, extractRoot+"/keep.txt") + assert.NoFileExists(t, extractRoot+"/remove.conf") + }) + + t.Run("file-search-replace rewrites matching content in the extracted tree", func(t *testing.T) { + extractRoot := t.TempDir() + require.NoError(t, os.WriteFile( + extractRoot+"/config.h", []byte("#define OLD_VALUE 1\n"), fileperms.PrivateFile)) -func TestTarballSearchReplaceNoMatch(t *testing.T) { - workDir := t.TempDir() + extractFS, err := rootfs.New(extractRoot) + require.NoError(t, err) + + defer extractFS.Close() + + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Tarball: "pkg.tar.gz", + Filename: "config.h", + Regex: "OLD_VALUE", + Replacement: "NEW_VALUE", + } + + err = applyTarballOperation(context.Background(), nil, ctx, ctx.FS(), extractFS, extractRoot, overlay) + require.NoError(t, err) + + content, readErr := os.ReadFile(extractRoot + "/config.h") + require.NoError(t, readErr) + assert.Equal(t, "#define NEW_VALUE 1\n", string(content)) + }) + + t.Run("file-remove with no match errors like a loose-file overlay", func(t *testing.T) { + extractRoot := t.TempDir() + require.NoError(t, os.WriteFile(extractRoot+"/file.txt", nil, fileperms.PrivateFile)) + + extractFS, err := rootfs.New(extractRoot) + require.NoError(t, err) - require.NoError(t, os.WriteFile(workDir+"/config.h", []byte("#define SOMETHING 1\n"), fileperms.PrivateFile)) + defer extractFS.Close() - err := tarballSearchReplace(workDir, "config.h", "NONEXISTENT", "new") - require.Error(t, err) - assert.ErrorIs(t, err, ErrOverlayDidNotApply) + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg.tar.gz", + Filename: "*.conf", + } + + err = applyTarballOperation(context.Background(), nil, ctx, ctx.FS(), extractFS, extractRoot, overlay) + require.Error(t, err) + assert.Contains(t, err.Error(), "did not match any files") + }) } diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index 3a02251e..b804a2fa 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -17,13 +17,21 @@ import ( // ComponentOverlay represents an overlay that may be applied to a component's spec and/or its sources. type ComponentOverlay struct { // The type of overlay to apply. - Type ComponentOverlayType `toml:"type" json:"type" validate:"required" jsonschema:"enum=spec-add-tag,enum=spec-insert-tag,enum=spec-set-tag,enum=spec-update-tag,enum=spec-remove-tag,enum=spec-prepend-lines,enum=spec-append-lines,enum=spec-search-replace,enum=spec-remove-section,enum=spec-remove-subpackage,enum=patch-add,enum=patch-remove,enum=file-prepend-lines,enum=file-search-replace,enum=file-add,enum=file-remove,enum=file-rename,enum=tarball-file-remove,enum=tarball-search-replace,enum=tarball-patch,title=Overlay type,description=The type of overlay to apply"` + Type ComponentOverlayType `toml:"type" json:"type" validate:"required" jsonschema:"enum=spec-add-tag,enum=spec-insert-tag,enum=spec-set-tag,enum=spec-update-tag,enum=spec-remove-tag,enum=spec-prepend-lines,enum=spec-append-lines,enum=spec-search-replace,enum=spec-remove-section,enum=spec-remove-subpackage,enum=patch-add,enum=patch-remove,enum=file-prepend-lines,enum=file-search-replace,enum=file-add,enum=file-remove,enum=file-rename,enum=tarball-patch,title=Overlay type,description=The type of overlay to apply"` // Human readable description of overlay; primarily present to document the need for the change. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Human readable description of overlay" fingerprint:"-"` // For overlays that target files inside a source tarball, identifies the tarball to modify. // Must be a filename (not a path) matching a source archive in the component's sources directory. + // Required for tarball-patch; optional for file-remove and file-search-replace, which operate + // inside the named archive instead of the loose sources tree when it is set. Tarball string `toml:"tarball,omitempty" json:"tarball,omitempty" jsonschema:"title=Tarball,description=The source tarball to modify (e.g. pkg-1.0.tar.gz)"` + // For overlays that target files inside a source tarball, optionally overrides the top-level + // directory to treat as the extraction root, mirroring rpmbuild's `%setup -n`. When unset, the + // root is inferred: if the archive unpacks to a single top-level directory (the conventional + // `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. + // Set this when an archive's top-level directory does not follow that convention. + TarballRoot string `toml:"tarball-root,omitempty" json:"tarballRoot,omitempty" jsonschema:"title=Tarball root,description=Top-level directory inside the tarball to treat as the extraction root (mirrors %setup -n); inferred when unset"` // For overlays that apply to non-spec files, indicates the filename. For overlays that can // apply to multiple files, supports glob patterns (including globstar). Filename string `toml:"file,omitempty" json:"file,omitempty" jsonschema:"title=Filename,description=The name of the non-spec file to which this overlay applies, or a glob pattern matching multiple files"` @@ -130,17 +138,27 @@ func (c *ComponentOverlay) ModifiesSpec() bool { } // ModifiesTarball returns true if the overlay modifies files inside a source tarball. -// These overlays require a mock chroot for extraction and repacking. +// These overlays require extraction and repacking of the archive. The dedicated +// tarball-patch type is always archive-scoped; the file-remove and file-search-replace +// types become archive-scoped when their [ComponentOverlay.Tarball] field is set. func (c *ComponentOverlay) ModifiesTarball() bool { - return c.Type == ComponentOverlayTarballFileRemove || - c.Type == ComponentOverlayTarballSearchReplace || - c.Type == ComponentOverlayTarballPatch + if c.Type == ComponentOverlayTarballPatch { + return true + } + + return c.Tarball != "" && + (c.Type == ComponentOverlayRemoveFile || c.Type == ComponentOverlaySearchAndReplaceInFile) } // ModifiesNonSpecFiles returns true if the overlay modifies non-spec files. This includes // hybrid overlays that modify both spec and source files (e.g., patch overlays), since -// those also require non-spec modifications. +// those also require non-spec modifications. Archive-scoped overlays (see [ModifiesTarball]) +// are excluded: they operate on files inside a tarball, not loose files in the sources tree. func (c *ComponentOverlay) ModifiesNonSpecFiles() bool { + if c.ModifiesTarball() { + return false + } + return c.Type == ComponentOverlayPrependLinesToFile || c.Type == ComponentOverlaySearchAndReplaceInFile || c.Type == ComponentOverlayAddFile || @@ -193,19 +211,17 @@ const ( // ComponentOverlayPrependLinesToFile is an overlay that prepends lines to a non-spec file. ComponentOverlayPrependLinesToFile ComponentOverlayType = "file-prepend-lines" // ComponentOverlaySearchAndReplaceInFile is an overlay that replaces text in a non-spec file. + // When its [ComponentOverlay.Tarball] field is set, it operates on file(s) inside that source + // tarball instead of loose files in the sources tree. ComponentOverlaySearchAndReplaceInFile ComponentOverlayType = "file-search-replace" // ComponentOverlayAddFile is an overlay that adds a non-spec file. ComponentOverlayAddFile ComponentOverlayType = "file-add" - // ComponentOverlayRemoveFile is an overlay that removes a non-spec file. + // ComponentOverlayRemoveFile is an overlay that removes a non-spec file. When its + // [ComponentOverlay.Tarball] field is set, it removes file(s) from inside that source + // tarball instead of loose files in the sources tree. ComponentOverlayRemoveFile ComponentOverlayType = "file-remove" // ComponentOverlayRenameFile is an overlay that renames a non-spec file. ComponentOverlayRenameFile ComponentOverlayType = "file-rename" - // ComponentOverlayTarballFileRemove is an overlay that removes file(s) from inside a source tarball. - // The tarball is extracted, matching files are deleted, and the tarball is repacked. - ComponentOverlayTarballFileRemove ComponentOverlayType = "tarball-file-remove" - // ComponentOverlayTarballSearchReplace is an overlay that performs regex search-and-replace - // on file(s) inside a source tarball. - ComponentOverlayTarballSearchReplace ComponentOverlayType = "tarball-search-replace" // ComponentOverlayTarballPatch is an overlay that applies a unified diff patch to the // extracted contents of a source tarball. ComponentOverlayTarballPatch ComponentOverlayType = "tarball-patch" @@ -259,6 +275,35 @@ func (c *ComponentOverlay) Validate() error { return nil } + // The tarball field scopes an overlay to operate inside a source archive. It is only + // accepted on the archive-capable types and must be a bare filename (not a path). + if c.Tarball != "" { + //nolint:exhaustive // Only archive-capable types accept the tarball field; the default rejects the rest. + switch c.Type { + case ComponentOverlayRemoveFile, ComponentOverlaySearchAndReplaceInFile, ComponentOverlayTarballPatch: + if err := requireFileBasename("tarball", c.Tarball); err != nil { + return err + } + default: + return unexpectedField("tarball") + } + } + + // The tarball-root override is only meaningful for archive-scoped overlays, and must be a + // local relative path so it cannot escape the extraction directory. + if c.TarballRoot != "" { + if !c.ModifiesTarball() { + return unexpectedField("tarball-root") + } + + if !filepath.IsLocal(c.TarballRoot) { + return fmt.Errorf( + "overlay type %#q requires %#q to be a local relative path (no %#q or absolute paths); found %#q", + c.Type, "tarball-root", "..", c.TarballRoot, + ) + } + } + switch c.Type { case ComponentOverlayAddSpecTag, ComponentOverlayInsertSpecTag, ComponentOverlaySetSpecTag, ComponentOverlayUpdateSpecTag: @@ -356,38 +401,6 @@ func (c *ComponentOverlay) Validate() error { if err := validateGlobPattern(c.Filename, desc); err != nil { return err } - case ComponentOverlayTarballFileRemove: - if err := requireFileBasename("tarball", c.Tarball); err != nil { - return err - } - - if err := requireRelativePath("file", c.Filename); err != nil { - return err - } - - if err := validateGlobPattern(c.Filename, desc); err != nil { - return err - } - case ComponentOverlayTarballSearchReplace: - if err := requireFileBasename("tarball", c.Tarball); err != nil { - return err - } - - if err := requireRelativePath("file", c.Filename); err != nil { - return err - } - - if err := validateGlobPattern(c.Filename, desc); err != nil { - return err - } - - if c.Regex == "" { - return missingField("regex") - } - - if err := validateRegex(c.Regex, desc); err != nil { - return err - } case ComponentOverlayTarballPatch: if err := requireFileBasename("tarball", c.Tarball); err != nil { return err diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index 7b75f0e2..1a760a7a 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -412,80 +412,121 @@ func TestComponentOverlay_Validate(t *testing.T) { errorExpected: true, errorContains: "section", }, - // tarball-file-remove tests + // archive-scoped file-remove tests (tarball modifier) { - name: "tarball-file-remove valid", + name: "file-remove with tarball valid (archive-scoped)", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", Filename: "unwanted.conf", }, errorExpected: false, }, { - name: "tarball-file-remove valid with glob", + name: "file-remove with tarball glob valid", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", Filename: "docs/**/*.md", }, errorExpected: false, }, { - name: "tarball-file-remove missing tarball", + name: "file-remove without tarball is a plain loose-file remove", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Filename: "unwanted.conf", }, - errorExpected: true, - errorContains: "tarball", + errorExpected: false, }, { - name: "tarball-file-remove missing file", + name: "file-remove with tarball missing file", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", }, errorExpected: true, errorContains: "file", }, { - name: "tarball-file-remove rejects tarball path", + name: "file-remove rejects tarball path", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballFileRemove, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "subdir/pkg-1.0.tar.gz", Filename: "unwanted.conf", }, errorExpected: true, errorContains: "tarball", }, - // tarball-search-replace tests { - name: "tarball-search-replace valid", + name: "tarball rejected on overlay type that cannot be archive-scoped", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayAddFile, + Tarball: "pkg-1.0.tar.gz", + Filename: "new.conf", + Source: "files/new.conf", + }, + errorExpected: true, + errorContains: "tarball", + }, + { + name: "file-remove with tarball-root override valid", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballSearchReplace, + Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", - Filename: "config.h", - Regex: "old_value", - Replacement: "new_value", + TarballRoot: "src-root", + Filename: "unwanted.conf", }, errorExpected: false, }, { - name: "tarball-search-replace missing tarball", + name: "file-remove rejects non-local tarball-root", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, + Tarball: "pkg-1.0.tar.gz", + TarballRoot: "../escape", + Filename: "unwanted.conf", + }, + errorExpected: true, + errorContains: "tarball-root", + }, + { + name: "tarball-root rejected when tarball unset", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballSearchReplace, + Type: projectconfig.ComponentOverlayRemoveFile, + TarballRoot: "src-root", + Filename: "unwanted.conf", + }, + errorExpected: true, + errorContains: "tarball-root", + }, + { + name: "tarball-root rejected on non-archive overlay", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlaySetSpecTag, + Tag: "Version", + Value: "1.0.0", + TarballRoot: "src-root", + }, + errorExpected: true, + errorContains: "tarball-root", + }, + // archive-scoped file-search-replace tests (tarball modifier) + { + name: "file-search-replace with tarball valid", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Tarball: "pkg-1.0.tar.gz", Filename: "config.h", Regex: "old_value", Replacement: "new_value", }, - errorExpected: true, - errorContains: "tarball", + errorExpected: false, }, { - name: "tarball-search-replace missing file", + name: "file-search-replace with tarball missing file", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballSearchReplace, + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, Tarball: "pkg-1.0.tar.gz", Regex: "old_value", Replacement: "new_value", @@ -494,9 +535,9 @@ func TestComponentOverlay_Validate(t *testing.T) { errorContains: "file", }, { - name: "tarball-search-replace missing regex", + name: "file-search-replace with tarball missing regex", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballSearchReplace, + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, Tarball: "pkg-1.0.tar.gz", Filename: "config.h", }, @@ -504,9 +545,9 @@ func TestComponentOverlay_Validate(t *testing.T) { errorContains: "regex", }, { - name: "tarball-search-replace invalid regex", + name: "file-search-replace with tarball invalid regex", overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballSearchReplace, + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, Tarball: "pkg-1.0.tar.gz", Filename: "config.h", Regex: "[invalid", @@ -596,10 +637,12 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { projectconfig.ComponentOverlayAddFile, } - tarballOverlayTypes := []projectconfig.ComponentOverlayType{ - projectconfig.ComponentOverlayTarballFileRemove, - projectconfig.ComponentOverlayTarballSearchReplace, - projectconfig.ComponentOverlayTarballPatch, + // Archive-scoped overlays: tarball-patch is always archive-scoped, while file-remove + // and file-search-replace become archive-scoped only when their Tarball field is set. + tarballOverlays := []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayTarballPatch, Tarball: "pkg-1.0.tar.gz", Source: "p.patch"}, + {Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", Filename: "f"}, + {Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, Tarball: "pkg-1.0.tar.gz", Filename: "f"}, } for _, overlayType := range specOverlayTypes { @@ -616,12 +659,12 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { }) } - for _, overlayType := range tarballOverlayTypes { - t.Run(string(overlayType)+"_is_tarball_overlay", func(t *testing.T) { - overlay := projectconfig.ComponentOverlay{Type: overlayType} - assert.True(t, overlay.ModifiesTarball(), "expected %s to be a tarball overlay", overlayType) - assert.False(t, overlay.ModifiesSpec(), "expected %s to not be a spec overlay", overlayType) - assert.False(t, overlay.ModifiesNonSpecFiles(), "expected %s to not be a non-spec overlay", overlayType) + for _, overlay := range tarballOverlays { + t.Run(string(overlay.Type)+"_is_archive_scoped", func(t *testing.T) { + assert.True(t, overlay.ModifiesTarball(), "expected %s to be an archive-scoped overlay", overlay.Type) + assert.False(t, overlay.ModifiesSpec(), "expected %s to not be a spec overlay", overlay.Type) + assert.False(t, overlay.ModifiesNonSpecFiles(), + "expected %s to not be a loose non-spec overlay", overlay.Type) }) } } diff --git a/internal/utils/archive/archive.go b/internal/utils/archive/archive.go index 65be9a18..a50dff61 100644 --- a/internal/utils/archive/archive.go +++ b/internal/utils/archive/archive.go @@ -79,6 +79,20 @@ func DetectCompression(filename string) (Compression, error) { } } +// ExtractAuto is a convenience wrapper that infers the compression from +// archivePath's extension via [DetectCompression] and then calls [Extract]. +// Most callers should prefer this over the explicit-compression [Extract], +// which exists for cases where the compression cannot be derived from the +// filename. +func ExtractAuto(archivePath, destDir string) error { + comp, err := DetectCompression(archivePath) + if err != nil { + return fmt.Errorf("detecting compression for %#q:\n%w", archivePath, err) + } + + return Extract(archivePath, destDir, comp) +} + // Extract reads a tar archive, decompresses it, and extracts all entries into // destDir. Supported entry types are regular files, directories, and symlinks; // other entry types are skipped. Entry paths are confined to destDir via @@ -174,6 +188,19 @@ func (r readerCloser) Close() error { return nil } +// CreateDeterministicArchiveAuto is a convenience wrapper that infers the +// compression from archivePath's extension via [DetectCompression] and then +// calls [CreateDeterministicArchive]. Most callers should prefer this over the +// explicit-compression [CreateDeterministicArchive]. +func CreateDeterministicArchiveAuto(archivePath, sourceDir string) error { + comp, err := DetectCompression(archivePath) + if err != nil { + return fmt.Errorf("detecting compression for %#q:\n%w", archivePath, err) + } + + return CreateDeterministicArchive(archivePath, sourceDir, comp) +} + // CreateDeterministicArchive creates a new tar archive from the contents of sourceDir // and writes it to archivePath on the OS filesystem, replacing any existing file. // diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7b824f4e..2ba3065a 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -245,7 +245,8 @@ "file-search-replace", "file-add", "file-remove", - "file-rename" + "file-rename", + "tarball-patch" ], "title": "Overlay type", "description": "The type of overlay to apply" @@ -255,6 +256,16 @@ "title": "Description", "description": "Human readable description of overlay" }, + "tarball": { + "type": "string", + "title": "Tarball", + "description": "The source tarball to modify (e.g. pkg-1.0.tar.gz)" + }, + "tarball-root": { + "type": "string", + "title": "Tarball root", + "description": "Top-level directory inside the tarball to treat as the extraction root (mirrors %setup -n); inferred when unset" + }, "file": { "type": "string", "title": "Filename", From a7a1c005fa63e1c2d0d8c880b27a924340204690 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Mon, 8 Jun 2026 23:30:57 +0000 Subject: [PATCH 03/13] refactor: rename to archive, limit archive overlays to file-remove/search-replace --- docs/user/reference/config/overlays.md | 79 ++--- .../azldev/core/sources/archiveoverlays.go | 231 ++++++++++++ ...st.go => archiveoverlays_internal_test.go} | 103 ++---- .../app/azldev/core/sources/sourceprep.go | 78 ++-- .../azldev/core/sources/tarballoverlays.go | 332 ------------------ internal/projectconfig/overlay.go | 85 ++--- internal/projectconfig/overlay_test.go | 145 ++------ ...ainer_config_generate-schema_stdout_1.snap | 10 + ...shots_config_generate-schema_stdout_1.snap | 10 + schemas/azldev.schema.json | 15 +- 10 files changed, 426 insertions(+), 662 deletions(-) create mode 100644 internal/app/azldev/core/sources/archiveoverlays.go rename internal/app/azldev/core/sources/{tarballoverlays_internal_test.go => archiveoverlays_internal_test.go} (62%) delete mode 100644 internal/app/azldev/core/sources/tarballoverlays.go diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index 0880c95d..671331ce 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -47,30 +47,27 @@ successfully makes a replacement to at least one matching file. | `file-remove` | Removes a file | `file` | Glob pattern for files to remove | | `file-rename` | Renames a file within the same directory | `file`, `replacement` | Name of file to rename | -> **Tip:** `file-remove` and `file-search-replace` can also operate inside a source tarball by -> setting the `tarball` field — see [Archive (Tarball) Overlays](#archive-tarball-overlays). +> **Tip:** `file-remove` and `file-search-replace` can also operate inside a source archive by +> setting the `archive` field — see [Archive Overlays](#archive-overlays). -### Archive (Tarball) Overlays +### Archive Overlays -File overlays can operate on files **inside** a source tarball instead of loose files in the -sources tree. Set the `tarball` field on a `file-remove` or `file-search-replace` overlay to -scope it to that archive; the dedicated `tarball-patch` type applies a unified diff to extracted -archive contents. The tarball is extracted into a temporary directory, the modifications are -applied with the same machinery as loose-file overlays, and the tarball is repacked with its -original compression format. Extraction and repacking are handled natively; `tarball-patch` -additionally requires the `patch` command on the host. +A `file-remove` or `file-search-replace` overlay can modify files **inside** a source archive +instead of loose files in the sources tree. Set the `archive` field to scope it to that archive. +The archive is extracted into a temporary directory, the matching files are modified with the +same machinery as loose-file overlays, and the archive is repacked with its original compression +format. Extraction and repacking are handled natively. -> **Note:** Archive overlays are batched per tarball — all overlays targeting the same archive +> **Note:** Archive overlays are batched per archive — all overlays targeting the same archive > share a single extract/modify/repack cycle — and the `sources` file is rehashed afterward to > reflect the repacked archive. They are processed independently of spec and loose-file overlays. -> **Extraction root:** Paths in archive overlays (`file`, and the paths a patch targets) are interpreted relative to the archive's extraction root. By default the root is inferred: if the archive unpacks to a single top-level directory (the conventional `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. Set `tarball-root` to override this — the equivalent of rpmbuild's `%setup -n` — when an archive's top-level directory does not follow that convention. +> **Extraction root:** The `file` glob in an archive overlay is interpreted relative to the archive's extraction root. By default the root is inferred: if the archive unpacks to a single top-level directory (the conventional `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. Set `archive-root` to override this — the equivalent of rpmbuild's `%setup -n` — when an archive's top-level directory does not follow that convention. | Type | Description | Required Fields | |------|-------------|-----------------| -| `file-remove` + `tarball` | Removes file(s) matching a glob pattern from inside a tarball | `tarball`, `file` | -| `file-search-replace` + `tarball` | Regex-based search and replace on file(s) inside a tarball | `tarball`, `file`, `regex` | -| `tarball-patch` | Applies a unified diff patch to the extracted tarball contents | `tarball`, `source` | +| `file-remove` + `archive` | Removes file(s) matching a glob pattern from inside an archive | `archive`, `file` | +| `file-search-replace` + `archive` | Regex-based search and replace on file(s) inside an archive | `archive`, `file`, `regex` | ## Field Reference @@ -78,17 +75,17 @@ additionally requires the `patch` command on the host. |-------|----------|-------------|---------| | Type | `type` | **Required.** The overlay type to apply | All overlays | | Description | `description` | Human-readable explanation documenting the need for the change; helps identify overlays in error messages | All (optional) | -| Tarball | `tarball` | The source tarball filename to scope an overlay to (must be a basename, not a path). When set, the overlay operates on files inside that archive. | `tarball-patch` (required); `file-remove`, `file-search-replace` (optional) | -| Tarball root | `tarball-root` | Top-level directory inside the tarball to treat as the extraction root (mirrors `%setup -n`); inferred when unset. Must be a local relative path (no `..` or absolute paths). When multiple overlays target the same tarball, any that set this must agree. | `tarball-patch`, and archive-scoped `file-remove` / `file-search-replace` (optional) | +| Archive | `archive` | The source archive filename to scope an overlay to (must be a basename, not a path). When set, the overlay operates on files inside that archive. | `file-remove`, `file-search-replace` (optional) | +| Archive root | `archive-root` | Top-level directory inside the archive to treat as the extraction root (mirrors `%setup -n`); inferred when unset. Must be a local relative path (no `..` or absolute paths). When multiple overlays target the same archive, any that set this must agree. | archive-scoped `file-remove` / `file-search-replace` (optional) | | Tag | `tag` | The spec tag name (e.g., `BuildRequires`, `Requires`, `Version`) | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` | -| Value | `value` | The tag value to set, or value to match for removal. For `tarball-patch`, sets the patch strip level (default: `1`, equivalent to `patch -p1`). | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` (optional for matching), `tarball-patch` (optional) | +| Value | `value` | The tag value to set, or value to match for removal. | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` (optional for matching) | | Section | `section` | The spec section to target (e.g., `%build`, `%install`, `%files`, `%description`) | `spec-prepend-lines`, `spec-append-lines`, `spec-search-replace` (optional), `spec-remove-section` | | Package | `package` | The sub-package name for multi-package specs; omit to target the main package | All spec overlays (optional, except `spec-remove-subpackage` which **requires** it) | | Regex | `regex` | Regular expression pattern to match | `spec-search-replace`, `file-search-replace` | | Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename` | | Lines | `lines` | Array of text lines to insert | `spec-prepend-lines`, `spec-append-lines`, `file-prepend-lines` | -| File | `file` | The name of the non-spec file to modify or add, or a glob pattern. For archive-scoped overlays, it is matched against the tarball's extracted contents. | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | -| Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add`, `tarball-patch` | +| File | `file` | The name of the non-spec file to modify or add, or a glob pattern. For an archive-scoped overlay, it is matched against the archive's extracted contents. | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | +| Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add` | | Metadata | `metadata` | Documentation table describing intent and provenance — see [Overlay Metadata](#overlay-metadata). Not allowed inside an overlay file loaded via `overlay-files` (the file-level `[metadata]` block applies to every overlay in the file). | All (optional) | > **Note:** For `file-rename`, the `replacement` field is a **filename only** (not a path). The file is renamed within its current directory. @@ -444,61 +441,37 @@ description = "Remove CVE patches that are now upstream" > `PatchN` tags. Macro-based tag numbering (e.g., `Patch%{n}`) is not expanded and may > conflict with auto-assigned numbers. -### Removing a File from a Tarball +### Removing a File from an Archive -Set the `tarball` field on a `file-remove` overlay to delete files matching a glob pattern from -inside a source tarball. The tarball is extracted, matching files are removed, and the tarball is +Set the `archive` field on a `file-remove` overlay to delete files matching a glob pattern from +inside a source archive. The archive is extracted, matching files are removed, and the archive is repacked. ```toml [[components.mypackage.overlays]] type = "file-remove" -tarball = "mypackage-1.0.tar.gz" +archive = "mypackage-1.0.tar.gz" file = "vendor/**" description = "Remove bundled vendor directory" ``` -> **Tip:** Without the `tarball` field, the same `file-remove` overlay removes a loose file from -> the sources tree instead. The `tarball` field is the only thing that scopes it to an archive. +> **Tip:** Without the `archive` field, the same `file-remove` overlay removes a loose file from +> the sources tree instead. The `archive` field is the only thing that scopes it to an archive. -### Search and Replace Inside a Tarball +### Search and Replace Inside an Archive -Set the `tarball` field on a `file-search-replace` overlay to rewrite content inside an archive: +Set the `archive` field on a `file-search-replace` overlay to rewrite content inside an archive: ```toml [[components.mypackage.overlays]] type = "file-search-replace" -tarball = "mypackage-1.0.tar.xz" +archive = "mypackage-1.0.tar.xz" file = "configure.ac" regex = "AC_CHECK_LIB\\(old_lib" replacement = "AC_CHECK_LIB(new_lib" description = "Update library reference in configure script" ``` -### Applying a Patch to Tarball Contents - -The `tarball-patch` overlay applies a unified diff patch to the extracted tarball contents. -By default, it uses `patch -p1`. Use the `value` field to change the strip level. - -```toml -[[components.mypackage.overlays]] -type = "tarball-patch" -tarball = "mypackage-1.0.tar.gz" -source = "patches/fix-build.patch" -description = "Fix build issue in upstream source" -``` - -With a custom strip level: - -```toml -[[components.mypackage.overlays]] -type = "tarball-patch" -tarball = "mypackage-1.0.tar.gz" -source = "patches/fix-build.patch" -value = "0" -description = "Apply patch with -p0 strip level" -``` - ### Removing a Section The `spec-remove-section` overlay removes an entire section from the spec, including its diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go new file mode 100644 index 00000000..c1907e00 --- /dev/null +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" +) + +// applyArchiveOverlays groups archive overlays by target archive and processes +// them in order. Multiple overlays targeting the same archive are batched into +// a single extract/modify/repack cycle. File removals inside the archive reuse +// the same machinery as loose-file overlays ([applyNonSpecOverlay]). +func applyArchiveOverlays( + dryRunnable opctx.DryRunnable, + fs opctx.FS, + eventListener opctx.EventListener, + sourcesDirPath string, + overlays []projectconfig.ComponentOverlay, +) error { + groups, err := groupOverlaysByArchive(overlays) + if err != nil { + return err + } + + if len(groups) == 0 { + return nil + } + + event := eventListener.StartEvent("Applying archive overlays", + "archives", len(groups), + "operations", len(overlays), + ) + defer event.End() + + for _, group := range groups { + if err := processArchive(dryRunnable, fs, sourcesDirPath, group); err != nil { + return fmt.Errorf("archive overlay failed for %#q:\n%w", group.archive, err) + } + } + + return nil +} + +// archiveGroup holds overlays targeting the same archive, preserving order. +type archiveGroup struct { + archive string + root string + overlays []projectconfig.ComponentOverlay +} + +// groupOverlaysByArchive groups archive overlays by their +// [projectconfig.ComponentOverlay.Archive] field, preserving insertion order +// within each group and across groups. Non-archive overlays are silently skipped. +// +// The optional [projectconfig.ComponentOverlay.ArchiveRoot] override (mirroring +// rpmbuild's `%setup -n`) is reconciled per archive: all overlays targeting the +// same archive that set it must agree, otherwise the configuration is ambiguous +// and an error is returned. +func groupOverlaysByArchive(overlays []projectconfig.ComponentOverlay) ([]archiveGroup, error) { + orderMap := make(map[string]int) + + var groups []archiveGroup + + for _, overlay := range overlays { + if !overlay.ModifiesArchive() { + continue + } + + idx, exists := orderMap[overlay.Archive] + if !exists { + idx = len(groups) + orderMap[overlay.Archive] = idx + + groups = append(groups, archiveGroup{archive: overlay.Archive}) + } + + if overlay.ArchiveRoot != "" { + if groups[idx].root != "" && groups[idx].root != overlay.ArchiveRoot { + return nil, fmt.Errorf( + "conflicting %#q overrides for archive %#q: %#q vs %#q", + "archive-root", overlay.Archive, groups[idx].root, overlay.ArchiveRoot, + ) + } + + groups[idx].root = overlay.ArchiveRoot + } + + groups[idx].overlays = append(groups[idx].overlays, overlay) + } + + return groups, nil +} + +// processArchive extracts an archive to a temp directory, applies all overlays, +// and deterministically repacks it in-place with the original compression. +func processArchive( + dryRunnable opctx.DryRunnable, + fs opctx.FS, + sourcesDirPath string, + group archiveGroup, +) error { + archivePath := filepath.Join(sourcesDirPath, group.archive) + + // Create a temporary directory for extraction. The injected FS is real-filesystem + // backed in production, so the returned path is a genuine on-disk path usable by + // the [archive] package. + workDir, err := fileutils.MkdirTempInTempDir(fs, "archive-overlay-") + if err != nil { + return fmt.Errorf("creating temp directory:\n%w", err) + } + + defer func() { + if removeErr := fs.RemoveAll(workDir); removeErr != nil { + slog.Warn("Failed to clean up archive work directory", "error", removeErr) + } + }() + + // Extract the archive; compression is inferred from the filename extension. + if err := archive.ExtractAuto(archivePath, workDir); err != nil { + return fmt.Errorf("extracting archive:\n%w", err) + } + + // Determine the root of the extracted content. Most source archives have + // a single top-level directory (e.g., "pkg-1.0/"); group.root overrides this + // inference when set (mirrors rpmbuild's `%setup -n`). + extractRoot, err := resolveExtractRoot(workDir, group.root) + if err != nil { + return fmt.Errorf("resolving extract root:\n%w", err) + } + + // Confine an FS to the extract root so file overlays reuse the same machinery + // as loose-file overlays. The extracted tree is always on the real filesystem + // (written by the [archive] package), so root it on an OS-backed FS regardless + // of the injected fs implementation. + extractFS, err := rootfs.New(extractRoot) + if err != nil { + return fmt.Errorf("confining FS to extract root:\n%w", err) + } + + defer func() { + if closeErr := extractFS.Close(); closeErr != nil { + slog.Warn("Failed to close extract-root FS", "error", closeErr) + } + }() + + // Apply each overlay operation in order. Archive overlays are file removals + // scoped to the extracted tree, routed through the shared loose-file machinery. + for _, overlay := range group.overlays { + if err := applyNonSpecOverlay(dryRunnable, fs, extractFS, overlay); err != nil { + return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) + } + } + + // Deterministically repack the archive in-place, reusing the original compression. + if err := archive.CreateDeterministicArchiveAuto(archivePath, workDir); err != nil { + return fmt.Errorf("repacking archive:\n%w", err) + } + + slog.Info("Archive overlay applied", "archive", group.archive) + + return nil +} + +// resolveExtractRoot returns the effective root of an extracted archive. +// When rootOverride is set (the `%setup -n` equivalent), the named subdirectory +// of workDir is used; it must be a local path that exists as a directory. When +// rootOverride is empty, the root is inferred: if workDir contains exactly one +// entry and that entry is a directory (the common case for source archives like +// "pkg-1.0/"), that subdirectory is returned; otherwise workDir itself is +// returned. +func resolveExtractRoot(workDir, rootOverride string) (string, error) { + if rootOverride != "" { + // Defense in depth: validation already rejects non-local overrides, but + // re-check before joining so a malformed value can never escape workDir. + if !filepath.IsLocal(rootOverride) { + return "", fmt.Errorf("archive root %#q is not a local path", rootOverride) + } + + target := filepath.Join(workDir, rootOverride) + + info, err := os.Stat(target) + if err != nil { + return "", fmt.Errorf("archive root %#q not found after extraction:\n%w", rootOverride, err) + } + + if !info.IsDir() { + return "", fmt.Errorf("archive root %#q is not a directory", rootOverride) + } + + return target, nil + } + + entries, err := os.ReadDir(workDir) + if err != nil { + return "", fmt.Errorf("reading extracted directory:\n%w", err) + } + + if len(entries) == 1 && entries[0].IsDir() { + return filepath.Join(workDir, entries[0].Name()), nil + } + + return workDir, nil +} + +// archiveNamesFromOverlays returns the unique archive filenames targeted by +// archive overlays in the given overlay list. Used by [updateSourcesFile] to +// determine which 'sources' entries need rehashing after overlay application. +func archiveNamesFromOverlays(overlays []projectconfig.ComponentOverlay) []string { + seen := make(map[string]bool) + + var names []string + + for _, overlay := range overlays { + if overlay.ModifiesArchive() && !seen[overlay.Archive] { + seen[overlay.Archive] = true + names = append(names, overlay.Archive) + } + } + + return names +} diff --git a/internal/app/azldev/core/sources/tarballoverlays_internal_test.go b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go similarity index 62% rename from internal/app/azldev/core/sources/tarballoverlays_internal_test.go rename to internal/app/azldev/core/sources/archiveoverlays_internal_test.go index 45acbcf8..1a188f8f 100644 --- a/internal/app/azldev/core/sources/tarballoverlays_internal_test.go +++ b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go @@ -4,7 +4,6 @@ package sources import ( - "context" "os" "testing" @@ -16,98 +15,96 @@ import ( "github.com/stretchr/testify/require" ) -func TestGroupOverlaysByTarball(t *testing.T) { - t.Run("groups overlays by tarball name preserving order", func(t *testing.T) { +func TestGroupOverlaysByArchive(t *testing.T) { + t.Run("groups overlays by archive name preserving order", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ { Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", Filename: "unwanted.conf", }, { - Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Tarball: "pkg-1.0.tar.gz", - Filename: "config.h", - Regex: "old", - Replacement: "new", + Type: projectconfig.ComponentOverlayRemoveFile, + Archive: "pkg-1.0.tar.gz", + Filename: "config.h", }, { Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "other-2.0.tar.xz", + Archive: "other-2.0.tar.xz", Filename: "docs/*.md", }, } - groups, err := groupOverlaysByTarball(overlays) + groups, err := groupOverlaysByArchive(overlays) require.NoError(t, err) require.Len(t, groups, 2) - assert.Equal(t, "pkg-1.0.tar.gz", groups[0].tarball) + assert.Equal(t, "pkg-1.0.tar.gz", groups[0].archive) require.Len(t, groups[0].overlays, 2) - assert.Equal(t, projectconfig.ComponentOverlayRemoveFile, groups[0].overlays[0].Type) - assert.Equal(t, projectconfig.ComponentOverlaySearchAndReplaceInFile, groups[0].overlays[1].Type) + assert.Equal(t, "unwanted.conf", groups[0].overlays[0].Filename) + assert.Equal(t, "config.h", groups[0].overlays[1].Filename) - assert.Equal(t, "other-2.0.tar.xz", groups[1].tarball) + assert.Equal(t, "other-2.0.tar.xz", groups[1].archive) require.Len(t, groups[1].overlays, 1) }) t.Run("skips overlays that are not archive-scoped", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1.0"}, - {Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg.tar.gz", Filename: "f"}, - // Plain (non-archive) file overlay: no Tarball set, so it must be skipped. + {Type: projectconfig.ComponentOverlayRemoveFile, Archive: "pkg.tar.gz", Filename: "f"}, + // Plain (non-archive) file overlay: no Archive set, so it must be skipped. {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "loose.txt"}, {Type: projectconfig.ComponentOverlayAddFile, Filename: "new.txt", Source: "src"}, } - groups, err := groupOverlaysByTarball(overlays) + groups, err := groupOverlaysByArchive(overlays) require.NoError(t, err) require.Len(t, groups, 1) - assert.Equal(t, "pkg.tar.gz", groups[0].tarball) + assert.Equal(t, "pkg.tar.gz", groups[0].archive) require.Len(t, groups[0].overlays, 1) }) - t.Run("reconciles matching tarball-root overrides", func(t *testing.T) { + t.Run("reconciles matching archive-root overrides", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ { Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", - TarballRoot: "custom-root", + Archive: "pkg-1.0.tar.gz", + ArchiveRoot: "custom-root", Filename: "a.conf", }, { Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", Filename: "b.conf", }, } - groups, err := groupOverlaysByTarball(overlays) + groups, err := groupOverlaysByArchive(overlays) require.NoError(t, err) require.Len(t, groups, 1) assert.Equal(t, "custom-root", groups[0].root) }) - t.Run("errors on conflicting tarball-root overrides", func(t *testing.T) { + t.Run("errors on conflicting archive-root overrides", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ { Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", - TarballRoot: "root-a", + Archive: "pkg-1.0.tar.gz", + ArchiveRoot: "root-a", Filename: "a.conf", }, { Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", - TarballRoot: "root-b", + Archive: "pkg-1.0.tar.gz", + ArchiveRoot: "root-b", Filename: "b.conf", }, } - _, err := groupOverlaysByTarball(overlays) + _, err := groupOverlaysByArchive(overlays) require.Error(t, err) assert.Contains(t, err.Error(), "conflicting") }) @@ -170,13 +167,13 @@ func TestResolveExtractRoot(t *testing.T) { }) } -// TestApplyTarballOperation_FileOverlays verifies that archive-scoped file -// overlays are routed through the shared [applyNonSpecOverlay] machinery against -// the extract-root FS (i.e., the same code path as loose-file overlays). -func TestApplyTarballOperation_FileOverlays(t *testing.T) { +// TestArchiveFileRemove verifies that archive-scoped file-remove overlays are +// routed through the shared [applyNonSpecOverlay] machinery against the +// extract-root FS (i.e., the same code path that [processArchive] uses). +func TestArchiveFileRemove(t *testing.T) { ctx := testctx.NewCtx() - t.Run("file-remove deletes matching files in the extracted tree", func(t *testing.T) { + t.Run("deletes matching files in the extracted tree", func(t *testing.T) { extractRoot := t.TempDir() require.NoError(t, os.WriteFile(extractRoot+"/keep.txt", []byte("keep"), fileperms.PrivateFile)) require.NoError(t, os.WriteFile(extractRoot+"/remove.conf", []byte("x"), fileperms.PrivateFile)) @@ -188,44 +185,18 @@ func TestApplyTarballOperation_FileOverlays(t *testing.T) { overlay := projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg.tar.gz", + Archive: "pkg.tar.gz", Filename: "*.conf", } - err = applyTarballOperation(context.Background(), nil, ctx, ctx.FS(), extractFS, extractRoot, overlay) + err = applyNonSpecOverlay(ctx, ctx.FS(), extractFS, overlay) require.NoError(t, err) assert.FileExists(t, extractRoot+"/keep.txt") assert.NoFileExists(t, extractRoot+"/remove.conf") }) - t.Run("file-search-replace rewrites matching content in the extracted tree", func(t *testing.T) { - extractRoot := t.TempDir() - require.NoError(t, os.WriteFile( - extractRoot+"/config.h", []byte("#define OLD_VALUE 1\n"), fileperms.PrivateFile)) - - extractFS, err := rootfs.New(extractRoot) - require.NoError(t, err) - - defer extractFS.Close() - - overlay := projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Tarball: "pkg.tar.gz", - Filename: "config.h", - Regex: "OLD_VALUE", - Replacement: "NEW_VALUE", - } - - err = applyTarballOperation(context.Background(), nil, ctx, ctx.FS(), extractFS, extractRoot, overlay) - require.NoError(t, err) - - content, readErr := os.ReadFile(extractRoot + "/config.h") - require.NoError(t, readErr) - assert.Equal(t, "#define NEW_VALUE 1\n", string(content)) - }) - - t.Run("file-remove with no match errors like a loose-file overlay", func(t *testing.T) { + t.Run("with no match errors like a loose-file overlay", func(t *testing.T) { extractRoot := t.TempDir() require.NoError(t, os.WriteFile(extractRoot+"/file.txt", nil, fileperms.PrivateFile)) @@ -236,11 +207,11 @@ func TestApplyTarballOperation_FileOverlays(t *testing.T) { overlay := projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg.tar.gz", + Archive: "pkg.tar.gz", Filename: "*.conf", } - err = applyTarballOperation(context.Background(), nil, ctx, ctx.FS(), extractFS, extractRoot, overlay) + err = applyNonSpecOverlay(ctx, ctx.FS(), extractFS, overlay) require.Error(t, err) assert.Contains(t, err.Error(), "did not match any files") }) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 1bbf7cf6..a759904b 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -246,7 +246,7 @@ func (p *sourcePreparerImpl) PrepareSources( } if applyOverlays { - if err := p.applyOverlaysToSources(ctx, component, outputDir); err != nil { + if err := p.applyOverlaysToSources(component, outputDir); err != nil { return err } @@ -273,7 +273,7 @@ func (p *sourcePreparerImpl) PrepareSources( // applyOverlaysToSources writes the macros file and then applies all overlays. func (p *sourcePreparerImpl) applyOverlaysToSources( - ctx context.Context, component components.Component, outputDir string, + component components.Component, outputDir string, ) error { var macrosFileName string @@ -287,7 +287,7 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( macrosFileName = filepath.Base(macrosFilePath) } - if err := p.applyOverlays(ctx, component, outputDir, macrosFileName); err != nil { + if err := p.applyOverlays(component, outputDir, macrosFileName); err != nil { return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } @@ -298,7 +298,7 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( // applyOverlays applies all overlays (user-defined and system-generated) to the // component sources. func (p *sourcePreparerImpl) applyOverlays( - ctx context.Context, component components.Component, sourcesDirPath, macrosFileName string, + component components.Component, sourcesDirPath, macrosFileName string, ) error { event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) defer event.End() @@ -317,10 +317,10 @@ func (p *sourcePreparerImpl) applyOverlays( return nil } - // Tarball overlays are applied first (they modify archived source files + // Archive overlays are applied first (they modify archived source files // in-place), followed by spec and loose-file overlays. Each function // self-filters to the overlay types it handles. - if err := p.applyTarballOverlayGroup(ctx, component, sourcesDirPath, allOverlays); err != nil { + if err := p.applyArchiveOverlayGroup(component, sourcesDirPath, allOverlays); err != nil { return err } @@ -331,34 +331,28 @@ func (p *sourcePreparerImpl) applyOverlays( return nil } -// applyTarballOverlayGroup applies tarball overlays. Skipped when source +// applyArchiveOverlayGroup applies archive overlays. Skipped when source // downloads were not performed. -func (p *sourcePreparerImpl) applyTarballOverlayGroup( - ctx context.Context, component components.Component, - sourcesDirPath string, tarballOverlays []projectconfig.ComponentOverlay, +func (p *sourcePreparerImpl) applyArchiveOverlayGroup( + component components.Component, + sourcesDirPath string, archiveOverlays []projectconfig.ComponentOverlay, ) error { - if len(tarballOverlays) == 0 { + if len(archiveOverlays) == 0 { return nil } if p.skipLookaside { - slog.Warn("Skipping tarball overlays because source downloads were skipped (--skip-sources)", + slog.Warn("Skipping archive overlays because source downloads were skipped (--skip-sources)", "component", component.GetName(), - "count", len(tarballOverlays)) + "count", len(archiveOverlays)) return nil } - cmdFactory, ok := p.dryRunnable.(opctx.CmdFactory) - if !ok { - return errors.New( - "tarball overlays require a CmdFactory; the provided DryRunnable does not implement it") - } - - if err := applyTarballOverlays( - ctx, cmdFactory, p.dryRunnable, p.fs, p.eventListener, sourcesDirPath, tarballOverlays, + if err := applyArchiveOverlays( + p.dryRunnable, p.fs, p.eventListener, sourcesDirPath, archiveOverlays, ); err != nil { - return fmt.Errorf("failed to apply tarball overlays for component %#q:\n%w", + return fmt.Errorf("failed to apply archive overlays for component %#q:\n%w", component.GetName(), err) } @@ -636,7 +630,7 @@ func (p *sourcePreparerImpl) DiffSources( } // Apply overlays in-place to the copied directory only. - if err := p.applyOverlaysToSources(ctx, component, overlaidDir); err != nil { + if err := p.applyOverlaysToSources(component, overlaidDir); err != nil { return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } @@ -671,11 +665,11 @@ func (p *sourcePreparerImpl) updateSourcesFile( config := component.GetConfig() sourceFiles := config.SourceFiles - // Derive tarball names from the component's overlays — no need to thread + // Derive archive names from the component's overlays — no need to thread // them through the overlay application chain. - modifiedTarballs := tarballNamesFromOverlays(config.Overlays) + modifiedArchives := archiveNamesFromOverlays(config.Overlays) - if len(sourceFiles) == 0 && len(modifiedTarballs) == 0 { + if len(sourceFiles) == 0 && len(modifiedArchives) == 0 { return nil } @@ -686,15 +680,15 @@ func (p *sourcePreparerImpl) updateSourcesFile( return err } - // Parse once, then rehash modified tarballs and merge source-files entries + // Parse once, then rehash modified archives and merge source-files entries // on the parsed representation — single parse, single write. existingLines, err := fedorasource.ReadSourcesFile(existingContent) if err != nil { return fmt.Errorf("failed to parse 'sources' file %#q:\n%w", sourcesFilePath, err) } - // Rehash tarballs that were modified by tarball overlays in-place. - if err := p.rehashModifiedEntries(existingLines, outputDir, modifiedTarballs); err != nil { + // Rehash archives that were modified by archive overlays in-place. + if err := p.rehashModifiedEntries(existingLines, outputDir, modifiedArchives); err != nil { return err } @@ -718,17 +712,17 @@ func (p *sourcePreparerImpl) updateSourcesFile( } // rehashModifiedEntries updates the Raw and Entry fields of parsed 'sources' lines -// for tarballs that were modified by tarball overlays. The hash is recomputed using +// for archives that were modified by archive overlays. The hash is recomputed using // the same hash type as the original entry. func (p *sourcePreparerImpl) rehashModifiedEntries( - lines []fedorasource.SourcesFileLine, outputDir string, modifiedTarballs []string, + lines []fedorasource.SourcesFileLine, outputDir string, modifiedArchives []string, ) error { - if len(modifiedTarballs) == 0 { + if len(modifiedArchives) == 0 { return nil } - modified := make(map[string]bool, len(modifiedTarballs)) - for _, name := range modifiedTarballs { + modified := make(map[string]bool, len(modifiedArchives)) + for _, name := range modifiedArchives { modified[name] = true } @@ -737,15 +731,15 @@ func (p *sourcePreparerImpl) rehashModifiedEntries( continue } - tarballPath := filepath.Join(outputDir, line.Entry.Filename) + archivePath := filepath.Join(outputDir, line.Entry.Filename) - newHash, err := fileutils.ComputeFileHash(p.fs, line.Entry.HashType, tarballPath) + newHash, err := fileutils.ComputeFileHash(p.fs, line.Entry.HashType, archivePath) if err != nil { - return fmt.Errorf("rehashing modified tarball %#q:\n%w", line.Entry.Filename, err) + return fmt.Errorf("rehashing modified archive %#q:\n%w", line.Entry.Filename, err) } - slog.Debug("Rehashed modified tarball in 'sources' file", - "tarball", line.Entry.Filename, + slog.Debug("Rehashed modified archive in 'sources' file", + "archive", line.Entry.Filename, "hashType", line.Entry.HashType, "oldHash", line.Entry.Hash, "newHash", newHash, @@ -1169,14 +1163,14 @@ func (p *sourcePreparerImpl) resolveSpecPath( } // applyOverlayList applies a list of overlays to the component sources sequentially. -// Archive-scoped overlays (see [projectconfig.ComponentOverlay.ModifiesTarball]) are -// skipped here; they are handled separately by [applyTarballOverlays], which batches +// Archive-scoped overlays (see [projectconfig.ComponentOverlay.ModifiesArchive]) are +// skipped here; they are handled separately by [applyArchiveOverlays], which batches // extraction and repacking per archive. func (p *sourcePreparerImpl) applyOverlayList( overlays []projectconfig.ComponentOverlay, sourcesDirPath, absSpecPath string, ) error { for _, overlay := range overlays { - if overlay.ModifiesTarball() { + if overlay.ModifiesArchive() { continue } diff --git a/internal/app/azldev/core/sources/tarballoverlays.go b/internal/app/azldev/core/sources/tarballoverlays.go deleted file mode 100644 index bbf6d4d0..00000000 --- a/internal/app/azldev/core/sources/tarballoverlays.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package sources - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" - "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" -) - -// applyTarballOverlays groups tarball overlays by target archive and processes -// them in order. Multiple overlays targeting the same tarball are batched into -// a single extract/modify/repack cycle. File modifications inside the archive -// reuse the same machinery as loose-file overlays ([applyNonSpecOverlay]); patch -// application shells out to the host's `patch` command. -func applyTarballOverlays( - ctx context.Context, - cmdFactory opctx.CmdFactory, - dryRunnable opctx.DryRunnable, - fs opctx.FS, - eventListener opctx.EventListener, - sourcesDirPath string, - overlays []projectconfig.ComponentOverlay, -) error { - groups, err := groupOverlaysByTarball(overlays) - if err != nil { - return err - } - - if len(groups) == 0 { - return nil - } - - event := eventListener.StartEvent("Applying tarball overlays", - "tarballs", len(groups), - "operations", len(overlays), - ) - defer event.End() - - for _, group := range groups { - if err := processTarball(ctx, cmdFactory, dryRunnable, fs, sourcesDirPath, group); err != nil { - return fmt.Errorf("tarball overlay failed for %#q:\n%w", group.tarball, err) - } - } - - return nil -} - -// tarballGroup holds overlays targeting the same tarball, preserving order. -type tarballGroup struct { - tarball string - root string - overlays []projectconfig.ComponentOverlay -} - -// groupOverlaysByTarball groups tarball overlays by their -// [projectconfig.ComponentOverlay.Tarball] field, preserving insertion order -// within each group and across groups. Non-tarball overlays are silently skipped. -// -// The optional [projectconfig.ComponentOverlay.TarballRoot] override (mirroring -// rpmbuild's `%setup -n`) is reconciled per tarball: all overlays targeting the -// same archive that set it must agree, otherwise the configuration is ambiguous -// and an error is returned. -func groupOverlaysByTarball(overlays []projectconfig.ComponentOverlay) ([]tarballGroup, error) { - orderMap := make(map[string]int) - - var groups []tarballGroup - - for _, overlay := range overlays { - if !overlay.ModifiesTarball() { - continue - } - - idx, exists := orderMap[overlay.Tarball] - if !exists { - idx = len(groups) - orderMap[overlay.Tarball] = idx - - groups = append(groups, tarballGroup{tarball: overlay.Tarball}) - } - - if overlay.TarballRoot != "" { - if groups[idx].root != "" && groups[idx].root != overlay.TarballRoot { - return nil, fmt.Errorf( - "conflicting %#q overrides for tarball %#q: %#q vs %#q", - "tarball-root", overlay.Tarball, groups[idx].root, overlay.TarballRoot, - ) - } - - groups[idx].root = overlay.TarballRoot - } - - groups[idx].overlays = append(groups[idx].overlays, overlay) - } - - return groups, nil -} - -// processTarball extracts a tarball to a temp directory, applies all overlays, -// and deterministically repacks it in-place with the original compression. -func processTarball( - ctx context.Context, - cmdFactory opctx.CmdFactory, - dryRunnable opctx.DryRunnable, - fs opctx.FS, - sourcesDirPath string, - group tarballGroup, -) error { - archivePath := filepath.Join(sourcesDirPath, group.tarball) - - // Create a temporary directory for extraction. The injected FS is real-filesystem - // backed in production, so the returned path is a genuine on-disk path usable by - // the [archive] package and the host `patch` command. - workDir, err := fileutils.MkdirTempInTempDir(fs, "tarball-overlay-") - if err != nil { - return fmt.Errorf("creating temp directory:\n%w", err) - } - - defer func() { - if removeErr := fs.RemoveAll(workDir); removeErr != nil { - slog.Warn("Failed to clean up tarball work directory", "error", removeErr) - } - }() - - // Extract the archive; compression is inferred from the filename extension. - if err := archive.ExtractAuto(archivePath, workDir); err != nil { - return fmt.Errorf("extracting tarball:\n%w", err) - } - - // Determine the root of the extracted content. Most source tarballs have - // a single top-level directory (e.g., "pkg-1.0/"); group.root overrides this - // inference when set (mirrors rpmbuild's `%setup -n`). - extractRoot, err := resolveExtractRoot(workDir, group.root) - if err != nil { - return fmt.Errorf("resolving extract root:\n%w", err) - } - - // Confine an FS to the extract root so file overlays reuse the same machinery - // as loose-file overlays. The extracted tree is always on the real filesystem - // (written by the [archive] package), so root it on an OS-backed FS regardless - // of the injected fs implementation. - extractFS, err := rootfs.New(extractRoot) - if err != nil { - return fmt.Errorf("confining FS to extract root:\n%w", err) - } - - defer func() { - if closeErr := extractFS.Close(); closeErr != nil { - slog.Warn("Failed to close extract-root FS", "error", closeErr) - } - }() - - // Apply each overlay operation in order. - for _, overlay := range group.overlays { - if err := applyTarballOperation( - ctx, cmdFactory, dryRunnable, fs, extractFS, extractRoot, overlay, - ); err != nil { - return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) - } - } - - // Deterministically repack the tarball in-place, reusing the original compression. - if err := archive.CreateDeterministicArchiveAuto(archivePath, workDir); err != nil { - return fmt.Errorf("repacking tarball:\n%w", err) - } - - slog.Info("Tarball overlay applied", "tarball", group.tarball) - - return nil -} - -// applyTarballOperation dispatches a single overlay against the extracted tree. -// The dedicated tarball-patch type shells out to the host `patch` command; all -// other (archive-scoped) types reuse [applyNonSpecOverlay], operating on the -// extract-root FS exactly as they would on the loose sources tree. -func applyTarballOperation( - ctx context.Context, - cmdFactory opctx.CmdFactory, - dryRunnable opctx.DryRunnable, - sourceFS, extractFS opctx.FS, - extractRoot string, - overlay projectconfig.ComponentOverlay, -) error { - if overlay.Type == projectconfig.ComponentOverlayTarballPatch { - stripLevel := 1 - - if overlay.Value != "" { - parsed, err := strconv.Atoi(overlay.Value) - if err != nil { - return fmt.Errorf("invalid strip level %#q:\n%w", overlay.Value, err) - } - - stripLevel = parsed - } - - return tarballApplyPatch(ctx, cmdFactory, sourceFS, extractRoot, overlay.Source, stripLevel) - } - - return applyNonSpecOverlay(dryRunnable, sourceFS, extractFS, overlay) -} - -// tarballApplyPatch applies a unified diff patch to the extracted tree by -// shelling out to the host's `patch` command. -func tarballApplyPatch( - ctx context.Context, - cmdFactory opctx.CmdFactory, - fs opctx.FS, - extractRoot, patchSource string, - stripLevel int, -) error { - if !cmdFactory.CommandInSearchPath("patch") { - return errors.New("'patch' command not found in PATH; " + - "install the 'patch' package to use tarball-patch overlays") - } - - // Read the patch file content via the abstract FS (supports both real and test FSs). - patchData, err := fileutils.ReadFile(fs, patchSource) - if err != nil { - return fmt.Errorf("reading patch file %#q:\n%w", patchSource, err) - } - - // Stage the patch in a temp dir so the host `patch` command can read it. The - // injected FS is real-filesystem backed in production, so the path is on disk. - tmpDir, err := fileutils.MkdirTempInTempDir(fs, "tarball-patch-") - if err != nil { - return fmt.Errorf("creating temp patch directory:\n%w", err) - } - - defer func() { - if removeErr := fs.RemoveAll(tmpDir); removeErr != nil { - slog.Warn("Failed to clean up tarball patch directory", "error", removeErr) - } - }() - - tmpPatchPath := filepath.Join(tmpDir, "overlay.patch") - if err := fileutils.WriteFile(fs, tmpPatchPath, patchData, fileperms.PublicFile); err != nil { - return fmt.Errorf("writing temp patch file:\n%w", err) - } - - var stderr strings.Builder - - rawCmd := exec.CommandContext(ctx, "patch", - fmt.Sprintf("-p%d", stripLevel), - "-i", tmpPatchPath, - ) - rawCmd.Dir = extractRoot - rawCmd.Stderr = &stderr - - cmd, err := cmdFactory.Command(rawCmd) - if err != nil { - return fmt.Errorf("creating patch command:\n%w", err) - } - - if runErr := cmd.Run(ctx); runErr != nil { - return fmt.Errorf("patch failed:\n%s\n%w", stderr.String(), runErr) - } - - return nil -} - -// resolveExtractRoot returns the effective root of an extracted tarball. -// When rootOverride is set (the `%setup -n` equivalent), the named subdirectory -// of workDir is used; it must be a local path that exists as a directory. When -// rootOverride is empty, the root is inferred: if workDir contains exactly one -// entry and that entry is a directory (the common case for source tarballs like -// "pkg-1.0/"), that subdirectory is returned; otherwise workDir itself is -// returned. -func resolveExtractRoot(workDir, rootOverride string) (string, error) { - if rootOverride != "" { - // Defense in depth: validation already rejects non-local overrides, but - // re-check before joining so a malformed value can never escape workDir. - if !filepath.IsLocal(rootOverride) { - return "", fmt.Errorf("tarball root %#q is not a local path", rootOverride) - } - - target := filepath.Join(workDir, rootOverride) - - info, err := os.Stat(target) - if err != nil { - return "", fmt.Errorf("tarball root %#q not found after extraction:\n%w", rootOverride, err) - } - - if !info.IsDir() { - return "", fmt.Errorf("tarball root %#q is not a directory", rootOverride) - } - - return target, nil - } - - entries, err := os.ReadDir(workDir) - if err != nil { - return "", fmt.Errorf("reading extracted directory:\n%w", err) - } - - if len(entries) == 1 && entries[0].IsDir() { - return filepath.Join(workDir, entries[0].Name()), nil - } - - return workDir, nil -} - -// tarballNamesFromOverlays returns the unique tarball filenames targeted by -// tarball overlays in the given overlay list. Used by [updateSourcesFile] to -// determine which 'sources' entries need rehashing after overlay application. -func tarballNamesFromOverlays(overlays []projectconfig.ComponentOverlay) []string { - seen := make(map[string]bool) - - var names []string - - for _, overlay := range overlays { - if overlay.ModifiesTarball() && !seen[overlay.Tarball] { - seen[overlay.Tarball] = true - names = append(names, overlay.Tarball) - } - } - - return names -} diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index b804a2fa..adb11fbe 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -17,21 +17,21 @@ import ( // ComponentOverlay represents an overlay that may be applied to a component's spec and/or its sources. type ComponentOverlay struct { // The type of overlay to apply. - Type ComponentOverlayType `toml:"type" json:"type" validate:"required" jsonschema:"enum=spec-add-tag,enum=spec-insert-tag,enum=spec-set-tag,enum=spec-update-tag,enum=spec-remove-tag,enum=spec-prepend-lines,enum=spec-append-lines,enum=spec-search-replace,enum=spec-remove-section,enum=spec-remove-subpackage,enum=patch-add,enum=patch-remove,enum=file-prepend-lines,enum=file-search-replace,enum=file-add,enum=file-remove,enum=file-rename,enum=tarball-patch,title=Overlay type,description=The type of overlay to apply"` + Type ComponentOverlayType `toml:"type" json:"type" validate:"required" jsonschema:"enum=spec-add-tag,enum=spec-insert-tag,enum=spec-set-tag,enum=spec-update-tag,enum=spec-remove-tag,enum=spec-prepend-lines,enum=spec-append-lines,enum=spec-search-replace,enum=spec-remove-section,enum=spec-remove-subpackage,enum=patch-add,enum=patch-remove,enum=file-prepend-lines,enum=file-search-replace,enum=file-add,enum=file-remove,enum=file-rename,title=Overlay type,description=The type of overlay to apply"` // Human readable description of overlay; primarily present to document the need for the change. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Human readable description of overlay" fingerprint:"-"` - // For overlays that target files inside a source tarball, identifies the tarball to modify. + // For overlays that target files inside a source archive, identifies the archive to modify. // Must be a filename (not a path) matching a source archive in the component's sources directory. - // Required for tarball-patch; optional for file-remove and file-search-replace, which operate - // inside the named archive instead of the loose sources tree when it is set. - Tarball string `toml:"tarball,omitempty" json:"tarball,omitempty" jsonschema:"title=Tarball,description=The source tarball to modify (e.g. pkg-1.0.tar.gz)"` - // For overlays that target files inside a source tarball, optionally overrides the top-level + // Only file-remove supports archive scoping; when set, it removes file(s) from inside the named + // archive instead of the loose sources tree. + Archive string `toml:"archive,omitempty" json:"archive,omitempty" jsonschema:"title=Archive,description=The source archive to modify (e.g. pkg-1.0.tar.gz)"` + // For overlays that target files inside a source archive, optionally overrides the top-level // directory to treat as the extraction root, mirroring rpmbuild's `%setup -n`. When unset, the // root is inferred: if the archive unpacks to a single top-level directory (the conventional // `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. // Set this when an archive's top-level directory does not follow that convention. - TarballRoot string `toml:"tarball-root,omitempty" json:"tarballRoot,omitempty" jsonschema:"title=Tarball root,description=Top-level directory inside the tarball to treat as the extraction root (mirrors %setup -n); inferred when unset"` + ArchiveRoot string `toml:"archive-root,omitempty" json:"archiveRoot,omitempty" jsonschema:"title=Archive root,description=Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset"` // For overlays that apply to non-spec files, indicates the filename. For overlays that can // apply to multiple files, supports glob patterns (including globstar). Filename string `toml:"file,omitempty" json:"file,omitempty" jsonschema:"title=Filename,description=The name of the non-spec file to which this overlay applies, or a glob pattern matching multiple files"` @@ -137,25 +137,21 @@ func (c *ComponentOverlay) ModifiesSpec() bool { c.Type == ComponentOverlayRemovePatch } -// ModifiesTarball returns true if the overlay modifies files inside a source tarball. -// These overlays require extraction and repacking of the archive. The dedicated -// tarball-patch type is always archive-scoped; the file-remove and file-search-replace -// types become archive-scoped when their [ComponentOverlay.Tarball] field is set. -func (c *ComponentOverlay) ModifiesTarball() bool { - if c.Type == ComponentOverlayTarballPatch { - return true - } - - return c.Tarball != "" && +// ModifiesArchive returns true if the overlay modifies files inside a source archive. +// These overlays require extraction and repacking of the archive. Only file-remove and +// file-search-replace support archive scoping, and only when their +// [ComponentOverlay.Archive] field is set. +func (c *ComponentOverlay) ModifiesArchive() bool { + return c.Archive != "" && (c.Type == ComponentOverlayRemoveFile || c.Type == ComponentOverlaySearchAndReplaceInFile) } // ModifiesNonSpecFiles returns true if the overlay modifies non-spec files. This includes // hybrid overlays that modify both spec and source files (e.g., patch overlays), since -// those also require non-spec modifications. Archive-scoped overlays (see [ModifiesTarball]) -// are excluded: they operate on files inside a tarball, not loose files in the sources tree. +// those also require non-spec modifications. Archive-scoped overlays (see [ModifiesArchive]) +// are excluded: they operate on files inside an archive, not loose files in the sources tree. func (c *ComponentOverlay) ModifiesNonSpecFiles() bool { - if c.ModifiesTarball() { + if c.ModifiesArchive() { return false } @@ -211,20 +207,15 @@ const ( // ComponentOverlayPrependLinesToFile is an overlay that prepends lines to a non-spec file. ComponentOverlayPrependLinesToFile ComponentOverlayType = "file-prepend-lines" // ComponentOverlaySearchAndReplaceInFile is an overlay that replaces text in a non-spec file. - // When its [ComponentOverlay.Tarball] field is set, it operates on file(s) inside that source - // tarball instead of loose files in the sources tree. ComponentOverlaySearchAndReplaceInFile ComponentOverlayType = "file-search-replace" // ComponentOverlayAddFile is an overlay that adds a non-spec file. ComponentOverlayAddFile ComponentOverlayType = "file-add" // ComponentOverlayRemoveFile is an overlay that removes a non-spec file. When its - // [ComponentOverlay.Tarball] field is set, it removes file(s) from inside that source - // tarball instead of loose files in the sources tree. + // [ComponentOverlay.Archive] field is set, it removes file(s) from inside that source + // archive instead of loose files in the sources tree. ComponentOverlayRemoveFile ComponentOverlayType = "file-remove" // ComponentOverlayRenameFile is an overlay that renames a non-spec file. ComponentOverlayRenameFile ComponentOverlayType = "file-rename" - // ComponentOverlayTarballPatch is an overlay that applies a unified diff patch to the - // extracted contents of a source tarball. - ComponentOverlayTarballPatch ComponentOverlayType = "tarball-patch" ) // Validate checks that required fields are set based on the overlay type. This catches @@ -275,31 +266,29 @@ func (c *ComponentOverlay) Validate() error { return nil } - // The tarball field scopes an overlay to operate inside a source archive. It is only - // accepted on the archive-capable types and must be a bare filename (not a path). - if c.Tarball != "" { - //nolint:exhaustive // Only archive-capable types accept the tarball field; the default rejects the rest. - switch c.Type { - case ComponentOverlayRemoveFile, ComponentOverlaySearchAndReplaceInFile, ComponentOverlayTarballPatch: - if err := requireFileBasename("tarball", c.Tarball); err != nil { - return err - } - default: - return unexpectedField("tarball") + // The archive field scopes an overlay to operate inside a source archive. It is only + // accepted on file-remove and file-search-replace, and must be a bare filename (not a path). + if c.Archive != "" { + if c.Type != ComponentOverlayRemoveFile && c.Type != ComponentOverlaySearchAndReplaceInFile { + return unexpectedField("archive") + } + + if err := requireFileBasename("archive", c.Archive); err != nil { + return err } } - // The tarball-root override is only meaningful for archive-scoped overlays, and must be a + // The archive-root override is only meaningful for archive-scoped overlays, and must be a // local relative path so it cannot escape the extraction directory. - if c.TarballRoot != "" { - if !c.ModifiesTarball() { - return unexpectedField("tarball-root") + if c.ArchiveRoot != "" { + if !c.ModifiesArchive() { + return unexpectedField("archive-root") } - if !filepath.IsLocal(c.TarballRoot) { + if !filepath.IsLocal(c.ArchiveRoot) { return fmt.Errorf( "overlay type %#q requires %#q to be a local relative path (no %#q or absolute paths); found %#q", - c.Type, "tarball-root", "..", c.TarballRoot, + c.Type, "archive-root", "..", c.ArchiveRoot, ) } } @@ -401,14 +390,6 @@ func (c *ComponentOverlay) Validate() error { if err := validateGlobPattern(c.Filename, desc); err != nil { return err } - case ComponentOverlayTarballPatch: - if err := requireFileBasename("tarball", c.Tarball); err != nil { - return err - } - - if c.Source == "" { - return missingField("source") - } default: return fmt.Errorf("unknown overlay type %#q: %#q", c.Type, desc) } diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index 1a760a7a..ece5a574 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -412,27 +412,27 @@ func TestComponentOverlay_Validate(t *testing.T) { errorExpected: true, errorContains: "section", }, - // archive-scoped file-remove tests (tarball modifier) + // archive-scoped file-remove tests (archive modifier) { - name: "file-remove with tarball valid (archive-scoped)", + name: "file-remove with archive valid (archive-scoped)", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", Filename: "unwanted.conf", }, errorExpected: false, }, { - name: "file-remove with tarball glob valid", + name: "file-remove with archive glob valid", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", Filename: "docs/**/*.md", }, errorExpected: false, }, { - name: "file-remove without tarball is a plain loose-file remove", + name: "file-remove without archive is a plain loose-file remove", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, Filename: "unwanted.conf", @@ -440,160 +440,89 @@ func TestComponentOverlay_Validate(t *testing.T) { errorExpected: false, }, { - name: "file-remove with tarball missing file", + name: "file-remove with archive missing file", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", }, errorExpected: true, errorContains: "file", }, { - name: "file-remove rejects tarball path", + name: "file-remove rejects archive path", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "subdir/pkg-1.0.tar.gz", + Archive: "subdir/pkg-1.0.tar.gz", Filename: "unwanted.conf", }, errorExpected: true, - errorContains: "tarball", + errorContains: "archive", }, { - name: "tarball rejected on overlay type that cannot be archive-scoped", + name: "archive rejected on overlay type that cannot be archive-scoped", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayAddFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", Filename: "new.conf", Source: "files/new.conf", }, errorExpected: true, - errorContains: "tarball", + errorContains: "archive", }, { - name: "file-remove with tarball-root override valid", + name: "file-remove with archive-root override valid", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", - TarballRoot: "src-root", + Archive: "pkg-1.0.tar.gz", + ArchiveRoot: "src-root", Filename: "unwanted.conf", }, errorExpected: false, }, { - name: "file-remove rejects non-local tarball-root", + name: "file-remove rejects non-local archive-root", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Tarball: "pkg-1.0.tar.gz", - TarballRoot: "../escape", + Archive: "pkg-1.0.tar.gz", + ArchiveRoot: "../escape", Filename: "unwanted.conf", }, errorExpected: true, - errorContains: "tarball-root", + errorContains: "archive-root", }, { - name: "tarball-root rejected when tarball unset", + name: "archive-root rejected when archive unset", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - TarballRoot: "src-root", + ArchiveRoot: "src-root", Filename: "unwanted.conf", }, errorExpected: true, - errorContains: "tarball-root", + errorContains: "archive-root", }, { - name: "tarball-root rejected on non-archive overlay", + name: "archive-root rejected on non-archive overlay", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1.0.0", - TarballRoot: "src-root", + ArchiveRoot: "src-root", }, errorExpected: true, - errorContains: "tarball-root", + errorContains: "archive-root", }, - // archive-scoped file-search-replace tests (tarball modifier) + // file-search-replace supports archive scoping { - name: "file-search-replace with tarball valid", + name: "file-search-replace with archive valid (archive-scoped)", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Tarball: "pkg-1.0.tar.gz", + Archive: "pkg-1.0.tar.gz", Filename: "config.h", Regex: "old_value", Replacement: "new_value", }, errorExpected: false, }, - { - name: "file-search-replace with tarball missing file", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Tarball: "pkg-1.0.tar.gz", - Regex: "old_value", - Replacement: "new_value", - }, - errorExpected: true, - errorContains: "file", - }, - { - name: "file-search-replace with tarball missing regex", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Tarball: "pkg-1.0.tar.gz", - Filename: "config.h", - }, - errorExpected: true, - errorContains: "regex", - }, - { - name: "file-search-replace with tarball invalid regex", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Tarball: "pkg-1.0.tar.gz", - Filename: "config.h", - Regex: "[invalid", - Replacement: "new_value", - }, - errorExpected: true, - errorContains: "regex", - }, - // tarball-patch tests - { - name: "tarball-patch valid", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballPatch, - Tarball: "pkg-1.0.tar.gz", - Source: "patches/fix.patch", - }, - errorExpected: false, - }, - { - name: "tarball-patch valid with strip level", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballPatch, - Tarball: "pkg-1.0.tar.gz", - Source: "patches/fix.patch", - Value: "2", - }, - errorExpected: false, - }, - { - name: "tarball-patch missing tarball", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballPatch, - Source: "patches/fix.patch", - }, - errorExpected: true, - errorContains: "tarball", - }, - { - name: "tarball-patch missing source", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayTarballPatch, - Tarball: "pkg-1.0.tar.gz", - }, - errorExpected: true, - errorContains: "source", - }, } for _, testCase := range testCases { @@ -637,12 +566,10 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { projectconfig.ComponentOverlayAddFile, } - // Archive-scoped overlays: tarball-patch is always archive-scoped, while file-remove - // and file-search-replace become archive-scoped only when their Tarball field is set. - tarballOverlays := []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayTarballPatch, Tarball: "pkg-1.0.tar.gz", Source: "p.patch"}, - {Type: projectconfig.ComponentOverlayRemoveFile, Tarball: "pkg-1.0.tar.gz", Filename: "f"}, - {Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, Tarball: "pkg-1.0.tar.gz", Filename: "f"}, + // Archive-scoped overlays: only file-remove becomes archive-scoped, and only + // when its Archive field is set. + archiveOverlays := []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Archive: "pkg-1.0.tar.gz", Filename: "f"}, } for _, overlayType := range specOverlayTypes { @@ -659,9 +586,9 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { }) } - for _, overlay := range tarballOverlays { + for _, overlay := range archiveOverlays { t.Run(string(overlay.Type)+"_is_archive_scoped", func(t *testing.T) { - assert.True(t, overlay.ModifiesTarball(), "expected %s to be an archive-scoped overlay", overlay.Type) + assert.True(t, overlay.ModifiesArchive(), "expected %s to be an archive-scoped overlay", overlay.Type) assert.False(t, overlay.ModifiesSpec(), "expected %s to not be a spec overlay", overlay.Type) assert.False(t, overlay.ModifiesNonSpecFiles(), "expected %s to not be a loose non-spec overlay", overlay.Type) diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 7b824f4e..7727e40c 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -255,6 +255,16 @@ "title": "Description", "description": "Human readable description of overlay" }, + "archive": { + "type": "string", + "title": "Archive", + "description": "The source archive to modify (e.g. pkg-1.0.tar.gz)" + }, + "archive-root": { + "type": "string", + "title": "Archive root", + "description": "Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset" + }, "file": { "type": "string", "title": "Filename", diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 7b824f4e..7727e40c 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -255,6 +255,16 @@ "title": "Description", "description": "Human readable description of overlay" }, + "archive": { + "type": "string", + "title": "Archive", + "description": "The source archive to modify (e.g. pkg-1.0.tar.gz)" + }, + "archive-root": { + "type": "string", + "title": "Archive root", + "description": "Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset" + }, "file": { "type": "string", "title": "Filename", diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 2ba3065a..7727e40c 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -245,8 +245,7 @@ "file-search-replace", "file-add", "file-remove", - "file-rename", - "tarball-patch" + "file-rename" ], "title": "Overlay type", "description": "The type of overlay to apply" @@ -256,15 +255,15 @@ "title": "Description", "description": "Human readable description of overlay" }, - "tarball": { + "archive": { "type": "string", - "title": "Tarball", - "description": "The source tarball to modify (e.g. pkg-1.0.tar.gz)" + "title": "Archive", + "description": "The source archive to modify (e.g. pkg-1.0.tar.gz)" }, - "tarball-root": { + "archive-root": { "type": "string", - "title": "Tarball root", - "description": "Top-level directory inside the tarball to treat as the extraction root (mirrors %setup -n); inferred when unset" + "title": "Archive root", + "description": "Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset" }, "file": { "type": "string", From 72e00b26ce9f707696245623353ca2942d8e9d7e Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 10 Jun 2026 20:40:51 +0000 Subject: [PATCH 04/13] Added additional tests --- docs/user/reference/config/overlays.md | 2 +- .../azldev/core/sources/archiveoverlays.go | 33 ++++-- .../app/azldev/core/sources/sourceprep.go | 14 ++- .../azldev/core/sources/sourceprep_test.go | 111 ++++++++++++++++++ internal/fingerprint/fingerprint_test.go | 39 ++++++ internal/projectconfig/fingerprint_test.go | 25 +++- internal/projectconfig/overlay.go | 14 +-- internal/projectconfig/overlay_test.go | 8 +- 8 files changed, 212 insertions(+), 34 deletions(-) diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index 671331ce..d01e3a4d 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -452,7 +452,7 @@ repacked. type = "file-remove" archive = "mypackage-1.0.tar.gz" file = "vendor/**" -description = "Remove bundled vendor directory" +description = "Remove all bundled vendor files" ``` > **Tip:** Without the `archive` field, the same `file-remove` overlay removes a loose file from diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go index c1907e00..f54711ae 100644 --- a/internal/app/azldev/core/sources/archiveoverlays.go +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -12,7 +12,6 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" ) @@ -22,7 +21,6 @@ import ( // the same machinery as loose-file overlays ([applyNonSpecOverlay]). func applyArchiveOverlays( dryRunnable opctx.DryRunnable, - fs opctx.FS, eventListener opctx.EventListener, sourcesDirPath string, overlays []projectconfig.ComponentOverlay, @@ -36,14 +34,19 @@ func applyArchiveOverlays( return nil } + operationCount := 0 + for _, group := range groups { + operationCount += len(group.overlays) + } + event := eventListener.StartEvent("Applying archive overlays", "archives", len(groups), - "operations", len(overlays), + "operations", operationCount, ) defer event.End() for _, group := range groups { - if err := processArchive(dryRunnable, fs, sourcesDirPath, group); err != nil { + if err := processArchive(dryRunnable, sourcesDirPath, group); err != nil { return fmt.Errorf("archive overlay failed for %#q:\n%w", group.archive, err) } } @@ -105,22 +108,24 @@ func groupOverlaysByArchive(overlays []projectconfig.ComponentOverlay) ([]archiv // and deterministically repacks it in-place with the original compression. func processArchive( dryRunnable opctx.DryRunnable, - fs opctx.FS, sourcesDirPath string, group archiveGroup, ) error { archivePath := filepath.Join(sourcesDirPath, group.archive) - // Create a temporary directory for extraction. The injected FS is real-filesystem - // backed in production, so the returned path is a genuine on-disk path usable by - // the [archive] package. - workDir, err := fileutils.MkdirTempInTempDir(fs, "archive-overlay-") + // Create a temporary directory for extraction directly on the real filesystem. + // The [archive] package operates exclusively through OS primitives ([os.Root], + // os.*), so the work directory must be a genuine on-disk path regardless of the + // injected FS implementation. Using os.MkdirTemp here (instead of the injected + // FS) makes that requirement explicit and keeps the path valid even when fs is + // an in-memory or otherwise non-OS-backed FS (e.g., in tests or alternate runners). + workDir, err := os.MkdirTemp("", "archive-overlay-") if err != nil { return fmt.Errorf("creating temp directory:\n%w", err) } defer func() { - if removeErr := fs.RemoveAll(workDir); removeErr != nil { + if removeErr := os.RemoveAll(workDir); removeErr != nil { slog.Warn("Failed to clean up archive work directory", "error", removeErr) } }() @@ -153,10 +158,12 @@ func processArchive( } }() - // Apply each overlay operation in order. Archive overlays are file removals - // scoped to the extracted tree, routed through the shared loose-file machinery. + // Apply each overlay operation in order. Archive overlays are restricted to + // file-remove / file-search-replace (see [projectconfig.ComponentOverlay.ModifiesArchive]), + // which operate solely on the destination tree, so the extract-root FS is passed as + // both the source and destination FS — there is no component-source FS to read from. for _, overlay := range group.overlays { - if err := applyNonSpecOverlay(dryRunnable, fs, extractFS, overlay); err != nil { + if err := applyNonSpecOverlay(dryRunnable, extractFS, extractFS, overlay); err != nil { return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) } } diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index a759904b..526158aa 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -331,12 +331,18 @@ func (p *sourcePreparerImpl) applyOverlays( return nil } -// applyArchiveOverlayGroup applies archive overlays. Skipped when source -// downloads were not performed. +// applyArchiveOverlayGroup applies the archive-scoped overlays contained in the +// given overlay list. The list may hold overlays of any type; only those for +// which [projectconfig.ComponentOverlay.ModifiesArchive] reports true are +// processed here. Skipped when source downloads were not performed. func (p *sourcePreparerImpl) applyArchiveOverlayGroup( component components.Component, - sourcesDirPath string, archiveOverlays []projectconfig.ComponentOverlay, + sourcesDirPath string, overlays []projectconfig.ComponentOverlay, ) error { + archiveOverlays := lo.Filter(overlays, func(overlay projectconfig.ComponentOverlay, _ int) bool { + return overlay.ModifiesArchive() + }) + if len(archiveOverlays) == 0 { return nil } @@ -350,7 +356,7 @@ func (p *sourcePreparerImpl) applyArchiveOverlayGroup( } if err := applyArchiveOverlays( - p.dryRunnable, p.fs, p.eventListener, sourcesDirPath, archiveOverlays, + p.dryRunnable, p.eventListener, sourcesDirPath, archiveOverlays, ); err != nil { return fmt.Errorf("failed to apply archive overlays for component %#q:\n%w", component.GetName(), err) diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index d21aa52d..704bea69 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -5,6 +5,7 @@ package sources_test import ( "errors" + "os" "path/filepath" "strings" "testing" @@ -16,6 +17,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" @@ -99,6 +101,115 @@ func TestPrepareSources_Success(t *testing.T) { assert.NotContains(t, string(specContents), "Source9999") } +// TestPrepareSources_ArchiveOverlayRehashesSourcesEntry is an end-to-end check +// of the key correctness behavior introduced with archive overlays: when an +// archive-scoped overlay mutates an archive's contents, the matching 'sources' +// entry must be re-hashed in place so the recorded digest reflects the repacked +// archive (while keeping the original hash *type*). +// +// This runs against the host filesystem with a real temp dir because archive +// overlays extract/repack through the [archive] package, which uses OS +// primitives ([os.Root], os.*) and therefore requires genuine on-disk paths — +// an in-memory FS would not be visible to extraction/repacking. This mirrors +// the existing archive internal tests, which likewise use t.TempDir(). +func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { + const ( + componentName = "test-component" + archiveName = "pkg-1.0.tar.gz" + ) + + // Host FS + real temp dir: archive extraction/repacking happens on disk. + ctx := testctx.NewCtx(testctx.WithHostFS()) + outputDir := t.TempDir() + + archivePath := filepath.Join(outputDir, archiveName) + specPath := filepath.Join(outputDir, componentName+".spec") + sourcesPath := filepath.Join(outputDir, fedorasource.SourcesFileName) + + // Build a deterministic archive whose single top-level directory follows the + // conventional "%{name}-%{version}/" layout, containing a file we will remove + // and one we will keep. + stagingDir := t.TempDir() + pkgRoot := filepath.Join(stagingDir, "pkg-1.0") + require.NoError(t, os.MkdirAll(pkgRoot, fileperms.PublicDir)) + require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "keep.txt"), []byte("keep me"), fileperms.PrivateFile)) + require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "remove-me.txt"), []byte("delete me"), fileperms.PrivateFile)) + require.NoError(t, archive.CreateDeterministicArchiveAuto(archivePath, stagingDir)) + + // Record the original hash of the archive and seed a 'sources' file with it. + // Use SHA256 (not the SHA512 default) so the test also proves the hash *type* + // is preserved rather than coincidentally matching a default. + originalHash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, archivePath) + require.NoError(t, err) + + originalEntry := fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA256, originalHash) + require.NoError(t, fileutils.WriteFile( + ctx.FS(), sourcesPath, []byte(originalEntry+"\n"), fileperms.PublicFile)) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) + + component.EXPECT().GetName().AnyTimes().Return(componentName) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlayRemoveFile, + Archive: archiveName, + Filename: "remove-me.txt", + }, + }, + }) + + // The archive and 'sources' file already exist on disk; the source manager + // only needs to provide the spec file (FetchFiles is a no-op download). + sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).Return(nil) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile( + ctx.FS(), filepath.Join(dir, componentName+".spec"), + []byte("# test spec"), fileperms.PublicFile) + }, + ) + + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) + require.NoError(t, err) + + err = preparer.PrepareSources(ctx, component, outputDir, true /*applyOverlays*/) + require.NoError(t, err) + + // The overlay must have actually mutated the archive on disk. + assert.FileExists(t, specPath) + + repackedHash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, archivePath) + require.NoError(t, err) + require.NotEqual(t, originalHash, repackedHash, + "precondition: removing a file from the archive should change its hash") + + // The 'sources' entry must have been rewritten to the repacked archive's hash, + // preserving the original SHA256 hash type. + sourcesContent, err := fileutils.ReadFile(ctx.FS(), sourcesPath) + require.NoError(t, err) + + parsedLines, err := fedorasource.ReadSourcesFile(string(sourcesContent)) + require.NoError(t, err) + + var entry *fedorasource.SourcesFileEntry + + for i := range parsedLines { + if parsedLines[i].Entry != nil && parsedLines[i].Entry.Filename == archiveName { + entry = parsedLines[i].Entry + + break + } + } + + require.NotNil(t, entry, "rewritten 'sources' file should still contain an entry for %q", archiveName) + assert.Equal(t, fileutils.HashTypeSHA256, entry.HashType, "original hash type must be preserved") + assert.Equal(t, repackedHash, entry.Hash, "'sources' entry must record the repacked archive's hash") + assert.NotEqual(t, originalHash, entry.Hash, "'sources' entry hash must have been updated") +} + func TestPrepareSources_SourceManagerError(t *testing.T) { ctrl := gomock.NewController(t) component := components_testutils.NewMockComponent(ctrl) diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 4a6cf1eb..3c748a46 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -251,6 +251,45 @@ func TestComputeIdentity_OverlaySourceFileChange(t *testing.T) { assert.NotEqual(t, fp1, fp2, "different overlay source content must produce different fingerprints") } +func TestComputeIdentity_OverlayArchiveScopingChangesFP(t *testing.T) { + // Archive scoping fields are hashed with `,omitempty`: an unset scope must + // not perturb the fingerprint (so the overwhelming majority of overlays stay + // idempotent against already-rendered specs), but a SET scope genuinely + // changes the build output (a file removed from inside an archive) and so + // must change the fingerprint. This test pins the second half of that + // contract — guarding against an over-correction that excludes the fields + // entirely (e.g. `fingerprint:"-"`), which would let an archive overlay slip + // through without triggering a rebuild. + ctx := newTestFS(t, map[string]string{ + "/specs/test.spec": "Name: testpkg\nVersion: 1.0", + }) + releaseVer := testReleaseVer + + base := baseComponent() + base.Overlays = []projectconfig.ComponentOverlay{ + {Type: "file-remove", Filename: "bundled.conf"}, + } + fpBase := computeFingerprint(t, ctx, base, releaseVer, 0) + + withArchive := baseComponent() + withArchive.Overlays = []projectconfig.ComponentOverlay{ + {Type: "file-remove", Filename: "bundled.conf", Archive: "pkg-1.0.tar.gz"}, + } + fpArchive := computeFingerprint(t, ctx, withArchive, releaseVer, 0) + + assert.NotEqual(t, fpBase, fpArchive, + "setting the archive scope must change the fingerprint") + + withRoot := baseComponent() + withRoot.Overlays = []projectconfig.ComponentOverlay{ + {Type: "file-remove", Filename: "bundled.conf", Archive: "pkg-1.0.tar.gz", ArchiveRoot: "custom-root"}, + } + fpRoot := computeFingerprint(t, ctx, withRoot, releaseVer, 0) + + assert.NotEqual(t, fpArchive, fpRoot, + "setting the archive-root override must change the fingerprint") +} + func TestComputeIdentity_PatchAddRenameChangesFP(t *testing.T) { // When patch-add omits 'file', the destination filename is derived from // filepath.Base(Source). Renaming the source file changes the rendered diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 9fe56870..7c8cc966 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -5,6 +5,7 @@ package projectconfig_test import ( "reflect" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" @@ -110,15 +111,29 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { tag := field.Tag.Get("fingerprint") - switch tag { + // hashstructure tags are `name,option,...`; the name part decides + // inclusion ("-" excludes, anything else includes) and the options + // tune how an included field is hashed. + name, options, _ := strings.Cut(tag, ",") + + switch name { case "": - // No tag — included by default (the safe default). + // Empty name — included by default (the safe default). The only + // option we permit is `omitempty`, which makes hashstructure skip + // the field when it holds its zero value (so an unset field never + // perturbs the hash) while still hashing it when set. Reject any + // other option as a likely typo. + if options != "" && options != "omitempty" { + assert.Failf(t, "invalid fingerprint tag", + "field %q has unrecognised fingerprint tag option %q — "+ + "only `omitempty` is supported on included fields", key, options) + } case "-": actualExclusions[key] = true default: - // hashstructure only recognises "" (include) and "-" (exclude). - // Any other value is silently treated as included, which is - // almost certainly a typo. + // hashstructure only recognises "" (include) and "-" (exclude) + // for the name part. Any other value is silently treated as + // included, which is almost certainly a typo. assert.Failf(t, "invalid fingerprint tag", "field %q has unrecognised fingerprint tag value %q — "+ "only `fingerprint:\"-\"` (exclude) is valid; "+ diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index adb11fbe..b6f5ee03 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -21,16 +21,12 @@ type ComponentOverlay struct { // Human readable description of overlay; primarily present to document the need for the change. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Human readable description of overlay" fingerprint:"-"` - // For overlays that target files inside a source archive, identifies the archive to modify. - // Must be a filename (not a path) matching a source archive in the component's sources directory. - // Only file-remove supports archive scoping; when set, it removes file(s) from inside the named - // archive instead of the loose sources tree. + // Scopes the overlay to files inside this source archive (a bare filename, not a path). + // Only file-remove and file-search-replace honor it; when set, the overlay operates inside + // the named archive instead of the loose sources tree. Archive string `toml:"archive,omitempty" json:"archive,omitempty" jsonschema:"title=Archive,description=The source archive to modify (e.g. pkg-1.0.tar.gz)"` - // For overlays that target files inside a source archive, optionally overrides the top-level - // directory to treat as the extraction root, mirroring rpmbuild's `%setup -n`. When unset, the - // root is inferred: if the archive unpacks to a single top-level directory (the conventional - // `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. - // Set this when an archive's top-level directory does not follow that convention. + // Overrides the archive's extraction root (rpmbuild's `%setup -n` equivalent). When unset, the + // root is inferred: a single top-level directory is used, otherwise the archive root. ArchiveRoot string `toml:"archive-root,omitempty" json:"archiveRoot,omitempty" jsonschema:"title=Archive root,description=Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset"` // For overlays that apply to non-spec files, indicates the filename. For overlays that can // apply to multiple files, supports glob patterns (including globstar). diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index ece5a574..63734bc1 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -566,10 +566,14 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { projectconfig.ComponentOverlayAddFile, } - // Archive-scoped overlays: only file-remove becomes archive-scoped, and only - // when its Archive field is set. + // Archive-scoped overlays: only file-remove/file-search-replace becomes archive-scoped, + // and only when its Archive field is set. archiveOverlays := []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlayRemoveFile, Archive: "pkg-1.0.tar.gz", Filename: "f"}, + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Archive: "pkg-1.0.tar.gz", Filename: "f", Regex: "old", Replacement: "new", + }, } for _, overlayType := range specOverlayTypes { From 4ecd8fec1d7b3603702c922189e356c62ae569f8 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 10 Jun 2026 21:42:56 +0000 Subject: [PATCH 05/13] Fixed test pipeline profile conflict --- .github/workflows/test.yml | 12 ++++++++++-- internal/app/azldev/core/sources/sourceprep.go | 12 +++++++++--- internal/fingerprint/fingerprint_test.go | 11 +++-------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f7acc036..a9a5dad6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,11 @@ jobs: set -euxo pipefail sudo apt-get update sudo apt-get purge firefox passt - sudo rm -f /etc/apparmor.d/microsoft-edge-stable + # Microsoft Edge (preinstalled on the runner image) ships two AppArmor + # profiles attached to the same binary (/etc/apparmor.d/msedge and + # /etc/apparmor.d/microsoft-edge-stable). That duplicate attachment makes + # aa-disable abort while parsing all profiles, so remove them first. + sudo rm -f /etc/apparmor.d/msedge /etc/apparmor.d/microsoft-edge-stable sudo systemctl reload apparmor.service sudo apt-get install apparmor-utils sudo aa-disable /usr/sbin/unix_chkpwd @@ -91,7 +95,11 @@ jobs: set -euxo pipefail sudo apt-get update sudo apt-get purge firefox passt - sudo rm -f /etc/apparmor.d/microsoft-edge-stable + # Microsoft Edge (preinstalled on the runner image) ships two AppArmor + # profiles attached to the same binary (/etc/apparmor.d/msedge and + # /etc/apparmor.d/microsoft-edge-stable). That duplicate attachment makes + # aa-disable abort while parsing all profiles, so remove them first. + sudo rm -f /etc/apparmor.d/msedge /etc/apparmor.d/microsoft-edge-stable sudo systemctl reload apparmor.service sudo apt-get install apparmor-utils sudo aa-disable /usr/sbin/unix_chkpwd diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 526158aa..46781805 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -671,9 +671,15 @@ func (p *sourcePreparerImpl) updateSourcesFile( config := component.GetConfig() sourceFiles := config.SourceFiles - // Derive archive names from the component's overlays — no need to thread - // them through the overlay application chain. - modifiedArchives := archiveNamesFromOverlays(config.Overlays) + // Derive the archives whose 'sources' hash needs refreshing because an archive + // overlay repacked them. Only meaningful when archive overlays actually ran: + // when skipLookaside is set, archive overlays are skipped (see + // [applyArchiveOverlayGroup]) and the archives may not even be present on disk, + // so rehashing would either fail or pointlessly rewrite an unchanged hash. + var modifiedArchives []string + if !p.skipLookaside { + modifiedArchives = archiveNamesFromOverlays(config.Overlays) + } if len(sourceFiles) == 0 && len(modifiedArchives) == 0 { return nil diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 3c748a46..2248cd0c 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -252,14 +252,9 @@ func TestComputeIdentity_OverlaySourceFileChange(t *testing.T) { } func TestComputeIdentity_OverlayArchiveScopingChangesFP(t *testing.T) { - // Archive scoping fields are hashed with `,omitempty`: an unset scope must - // not perturb the fingerprint (so the overwhelming majority of overlays stay - // idempotent against already-rendered specs), but a SET scope genuinely - // changes the build output (a file removed from inside an archive) and so - // must change the fingerprint. This test pins the second half of that - // contract — guarding against an over-correction that excludes the fields - // entirely (e.g. `fingerprint:"-"`), which would let an archive overlay slip - // through without triggering a rebuild. + // Archive-scoping fields are part of the hashed config, so setting them must + // change the fingerprint. Guards against excluding them (e.g. `fingerprint:"-"`), + // which would let an archive overlay skip a rebuild. ctx := newTestFS(t, map[string]string{ "/specs/test.spec": "Name: testpkg\nVersion: 1.0", }) From 379b09776a97bd4d26fee825fde7dba589c6b2ac Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 11 Jun 2026 22:03:09 +0000 Subject: [PATCH 06/13] refactor: autodetect archive in file overlay field --- .github/workflows/test.yml | 12 +- docs/user/reference/config/overlays.md | 44 +++--- .../azldev/core/sources/archiveoverlays.go | 128 ++++++---------- .../sources/archiveoverlays_internal_test.go | 143 ++++++------------ .../app/azldev/core/sources/sourceprep.go | 36 ++++- .../azldev/core/sources/sourceprep_test.go | 65 +++++++- internal/fingerprint/fingerprint_test.go | 20 +-- internal/projectconfig/overlay.go | 75 +++++---- internal/projectconfig/overlay_test.go | 99 ++---------- internal/utils/archive/archive.go | 10 ++ ...ainer_config_generate-schema_stdout_1.snap | 10 -- ...shots_config_generate-schema_stdout_1.snap | 10 -- schemas/azldev.schema.json | 10 -- 13 files changed, 283 insertions(+), 379 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9a5dad6..f7acc036 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,11 +55,7 @@ jobs: set -euxo pipefail sudo apt-get update sudo apt-get purge firefox passt - # Microsoft Edge (preinstalled on the runner image) ships two AppArmor - # profiles attached to the same binary (/etc/apparmor.d/msedge and - # /etc/apparmor.d/microsoft-edge-stable). That duplicate attachment makes - # aa-disable abort while parsing all profiles, so remove them first. - sudo rm -f /etc/apparmor.d/msedge /etc/apparmor.d/microsoft-edge-stable + sudo rm -f /etc/apparmor.d/microsoft-edge-stable sudo systemctl reload apparmor.service sudo apt-get install apparmor-utils sudo aa-disable /usr/sbin/unix_chkpwd @@ -95,11 +91,7 @@ jobs: set -euxo pipefail sudo apt-get update sudo apt-get purge firefox passt - # Microsoft Edge (preinstalled on the runner image) ships two AppArmor - # profiles attached to the same binary (/etc/apparmor.d/msedge and - # /etc/apparmor.d/microsoft-edge-stable). That duplicate attachment makes - # aa-disable abort while parsing all profiles, so remove them first. - sudo rm -f /etc/apparmor.d/msedge /etc/apparmor.d/microsoft-edge-stable + sudo rm -f /etc/apparmor.d/microsoft-edge-stable sudo systemctl reload apparmor.service sudo apt-get install apparmor-utils sudo aa-disable /usr/sbin/unix_chkpwd diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index d01e3a4d..6f44ceb0 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -48,26 +48,33 @@ successfully makes a replacement to at least one matching file. | `file-rename` | Renames a file within the same directory | `file`, `replacement` | Name of file to rename | > **Tip:** `file-remove` and `file-search-replace` can also operate inside a source archive by -> setting the `archive` field — see [Archive Overlays](#archive-overlays). +> prefixing the `file` path with the archive name — see [Archive Overlays](#archive-overlays). ### Archive Overlays A `file-remove` or `file-search-replace` overlay can modify files **inside** a source archive -instead of loose files in the sources tree. Set the `archive` field to scope it to that archive. -The archive is extracted into a temporary directory, the matching files are modified with the -same machinery as loose-file overlays, and the archive is repacked with its original compression -format. Extraction and repacking are handled natively. +instead of loose files in the sources tree. This is detected from the `file` path: when its first +segment is a source archive (e.g. `pkg-1.0.tar.gz`) followed by an inner path, the overlay is +scoped to that archive and the remainder is matched against its contents. The archive is extracted, +the matching files are modified with the same machinery as loose-file overlays, and the archive is +repacked with its original compression format. + +``` +file = "pkg-1.0.tar.gz/vendor/**" # inside the archive (glob = vendor/**) +file = "vendor/**" # loose files in the sources tree +file = "old.tar.gz" # removes the archive file itself (bare name, no inner path) +``` > **Note:** Archive overlays are batched per archive — all overlays targeting the same archive > share a single extract/modify/repack cycle — and the `sources` file is rehashed afterward to > reflect the repacked archive. They are processed independently of spec and loose-file overlays. -> **Extraction root:** The `file` glob in an archive overlay is interpreted relative to the archive's extraction root. By default the root is inferred: if the archive unpacks to a single top-level directory (the conventional `%{name}-%{version}` layout) that directory is used; otherwise the archive root is used. Set `archive-root` to override this — the equivalent of rpmbuild's `%setup -n` — when an archive's top-level directory does not follow that convention. +> **Extraction root:** The inner path is interpreted relative to the archive's extraction root: if the archive unpacks to a single top-level directory (the conventional `%{name}-%{version}` layout) that directory is the root; otherwise the archive root is used. | Type | Description | Required Fields | |------|-------------|-----------------| -| `file-remove` + `archive` | Removes file(s) matching a glob pattern from inside an archive | `archive`, `file` | -| `file-search-replace` + `archive` | Regex-based search and replace on file(s) inside an archive | `archive`, `file`, `regex` | +| `file-remove` (archive-scoped path) | Removes file(s) matching a glob pattern from inside an archive | `file` (e.g. `pkg-1.0.tar.gz/vendor/**`) | +| `file-search-replace` (archive-scoped path) | Regex-based search and replace on file(s) inside an archive | `file`, `regex` | ## Field Reference @@ -75,8 +82,6 @@ format. Extraction and repacking are handled natively. |-------|----------|-------------|---------| | Type | `type` | **Required.** The overlay type to apply | All overlays | | Description | `description` | Human-readable explanation documenting the need for the change; helps identify overlays in error messages | All (optional) | -| Archive | `archive` | The source archive filename to scope an overlay to (must be a basename, not a path). When set, the overlay operates on files inside that archive. | `file-remove`, `file-search-replace` (optional) | -| Archive root | `archive-root` | Top-level directory inside the archive to treat as the extraction root (mirrors `%setup -n`); inferred when unset. Must be a local relative path (no `..` or absolute paths). When multiple overlays target the same archive, any that set this must agree. | archive-scoped `file-remove` / `file-search-replace` (optional) | | Tag | `tag` | The spec tag name (e.g., `BuildRequires`, `Requires`, `Version`) | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` | | Value | `value` | The tag value to set, or value to match for removal. | `spec-add-tag`, `spec-insert-tag`, `spec-set-tag`, `spec-update-tag`, `spec-remove-tag` (optional for matching) | | Section | `section` | The spec section to target (e.g., `%build`, `%install`, `%files`, `%description`) | `spec-prepend-lines`, `spec-append-lines`, `spec-search-replace` (optional), `spec-remove-section` | @@ -84,7 +89,7 @@ format. Extraction and repacking are handled natively. | Regex | `regex` | Regular expression pattern to match | `spec-search-replace`, `file-search-replace` | | Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename` | | Lines | `lines` | Array of text lines to insert | `spec-prepend-lines`, `spec-append-lines`, `file-prepend-lines` | -| File | `file` | The name of the non-spec file to modify or add, or a glob pattern. For an archive-scoped overlay, it is matched against the archive's extracted contents. | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | +| File | `file` | The name of the non-spec file to modify or add, or a glob pattern. An archive-scoped path (e.g. `pkg-1.0.tar.gz/vendor/**`) targets files inside that source archive. | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | | Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add` | | Metadata | `metadata` | Documentation table describing intent and provenance — see [Overlay Metadata](#overlay-metadata). Not allowed inside an overlay file loaded via `overlay-files` (the file-level `[metadata]` block applies to every overlay in the file). | All (optional) | @@ -443,30 +448,27 @@ description = "Remove CVE patches that are now upstream" ### Removing a File from an Archive -Set the `archive` field on a `file-remove` overlay to delete files matching a glob pattern from -inside a source archive. The archive is extracted, matching files are removed, and the archive is -repacked. +Prefix the `file` path with the archive name to delete files matching a glob pattern from inside a +source archive. The archive is extracted, matching files are removed, and the archive is repacked. ```toml [[components.mypackage.overlays]] type = "file-remove" -archive = "mypackage-1.0.tar.gz" -file = "vendor/**" +file = "mypackage-1.0.tar.gz/vendor/**" description = "Remove all bundled vendor files" ``` -> **Tip:** Without the `archive` field, the same `file-remove` overlay removes a loose file from -> the sources tree instead. The `archive` field is the only thing that scopes it to an archive. +> **Tip:** Without the archive-name prefix, the same `file-remove` overlay removes a loose file +> from the sources tree instead. ### Search and Replace Inside an Archive -Set the `archive` field on a `file-search-replace` overlay to rewrite content inside an archive: +Prefix the `file` path with the archive name to rewrite content inside an archive: ```toml [[components.mypackage.overlays]] type = "file-search-replace" -archive = "mypackage-1.0.tar.xz" -file = "configure.ac" +file = "mypackage-1.0.tar.xz/configure.ac" regex = "AC_CHECK_LIB\\(old_lib" replacement = "AC_CHECK_LIB(new_lib" description = "Update library reference in configure script" diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go index f54711ae..052ed527 100644 --- a/internal/app/azldev/core/sources/archiveoverlays.go +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -25,10 +25,7 @@ func applyArchiveOverlays( sourcesDirPath string, overlays []projectconfig.ComponentOverlay, ) error { - groups, err := groupOverlaysByArchive(overlays) - if err != nil { - return err - } + groups := groupOverlaysByArchive(overlays) if len(groups) == 0 { return nil @@ -57,19 +54,17 @@ func applyArchiveOverlays( // archiveGroup holds overlays targeting the same archive, preserving order. type archiveGroup struct { archive string - root string overlays []projectconfig.ComponentOverlay } -// groupOverlaysByArchive groups archive overlays by their -// [projectconfig.ComponentOverlay.Archive] field, preserving insertion order -// within each group and across groups. Non-archive overlays are silently skipped. +// groupOverlaysByArchive groups archive overlays by the archive named in the first +// path segment of [projectconfig.ComponentOverlay.Filename] (see +// [projectconfig.ComponentOverlay.ArchiveTarget]), preserving insertion order within +// each group and across groups. Non-archive overlays are silently skipped. // -// The optional [projectconfig.ComponentOverlay.ArchiveRoot] override (mirroring -// rpmbuild's `%setup -n`) is reconciled per archive: all overlays targeting the -// same archive that set it must agree, otherwise the configuration is ambiguous -// and an error is returned. -func groupOverlaysByArchive(overlays []projectconfig.ComponentOverlay) ([]archiveGroup, error) { +// Each grouped overlay's Filename is rewritten to the in-archive glob (the archive +// prefix stripped) so that application globs relative to the extracted tree. +func groupOverlaysByArchive(overlays []projectconfig.ComponentOverlay) []archiveGroup { orderMap := make(map[string]int) var groups []archiveGroup @@ -79,29 +74,25 @@ func groupOverlaysByArchive(overlays []projectconfig.ComponentOverlay) ([]archiv continue } - idx, exists := orderMap[overlay.Archive] + // ok is guaranteed true here: ModifiesArchive() implies ArchiveTarget() ok. + archiveName, innerGlob, _ := overlay.ArchiveTarget() + + idx, exists := orderMap[archiveName] if !exists { idx = len(groups) - orderMap[overlay.Archive] = idx - - groups = append(groups, archiveGroup{archive: overlay.Archive}) - } - - if overlay.ArchiveRoot != "" { - if groups[idx].root != "" && groups[idx].root != overlay.ArchiveRoot { - return nil, fmt.Errorf( - "conflicting %#q overrides for archive %#q: %#q vs %#q", - "archive-root", overlay.Archive, groups[idx].root, overlay.ArchiveRoot, - ) - } + orderMap[archiveName] = idx - groups[idx].root = overlay.ArchiveRoot + groups = append(groups, archiveGroup{archive: archiveName}) } - groups[idx].overlays = append(groups[idx].overlays, overlay) + // Rewrite Filename to the in-archive glob so the in-archive application + // (which globs relative to the extract root) does not see the archive prefix. + scoped := overlay + scoped.Filename = innerGlob + groups[idx].overlays = append(groups[idx].overlays, scoped) } - return groups, nil + return groups } // processArchive extracts an archive to a temp directory, applies all overlays, @@ -113,12 +104,16 @@ func processArchive( ) error { archivePath := filepath.Join(sourcesDirPath, group.archive) - // Create a temporary directory for extraction directly on the real filesystem. + if dryRunnable.DryRun() { + slog.Info("Dry run; would apply archive overlays", + "archive", group.archive, "operations", len(group.overlays)) + + return nil + } + // The [archive] package operates exclusively through OS primitives ([os.Root], - // os.*), so the work directory must be a genuine on-disk path regardless of the - // injected FS implementation. Using os.MkdirTemp here (instead of the injected - // FS) makes that requirement explicit and keeps the path valid even when fs is - // an in-memory or otherwise non-OS-backed FS (e.g., in tests or alternate runners). + // os.*), so extraction must use a genuine on-disk path regardless of the injected + // FS implementation (which may be in-memory or otherwise non-OS-backed). workDir, err := os.MkdirTemp("", "archive-overlay-") if err != nil { return fmt.Errorf("creating temp directory:\n%w", err) @@ -135,18 +130,15 @@ func processArchive( return fmt.Errorf("extracting archive:\n%w", err) } - // Determine the root of the extracted content. Most source archives have - // a single top-level directory (e.g., "pkg-1.0/"); group.root overrides this - // inference when set (mirrors rpmbuild's `%setup -n`). - extractRoot, err := resolveExtractRoot(workDir, group.root) + // Determine the root of the extracted content. Most source archives unpack to + // a single top-level directory (e.g., "pkg-1.0/"), which is used as the root. + extractRoot, err := resolveExtractRoot(workDir) if err != nil { return fmt.Errorf("resolving extract root:\n%w", err) } - // Confine an FS to the extract root so file overlays reuse the same machinery - // as loose-file overlays. The extracted tree is always on the real filesystem - // (written by the [archive] package), so root it on an OS-backed FS regardless - // of the injected fs implementation. + // Confine an OS-backed FS to the extract root so file overlays reuse the same + // machinery as loose-file overlays. extractFS, err := rootfs.New(extractRoot) if err != nil { return fmt.Errorf("confining FS to extract root:\n%w", err) @@ -158,10 +150,9 @@ func processArchive( } }() - // Apply each overlay operation in order. Archive overlays are restricted to - // file-remove / file-search-replace (see [projectconfig.ComponentOverlay.ModifiesArchive]), - // which operate solely on the destination tree, so the extract-root FS is passed as - // both the source and destination FS — there is no component-source FS to read from. + // Apply each overlay in order. Archive overlays (file-remove / file-search-replace, + // see [projectconfig.ComponentOverlay.ModifiesArchive]) operate solely on the + // destination tree, so extractFS is passed as both source and destination FS. for _, overlay := range group.overlays { if err := applyNonSpecOverlay(dryRunnable, extractFS, extractFS, overlay); err != nil { return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) @@ -178,35 +169,11 @@ func processArchive( return nil } -// resolveExtractRoot returns the effective root of an extracted archive. -// When rootOverride is set (the `%setup -n` equivalent), the named subdirectory -// of workDir is used; it must be a local path that exists as a directory. When -// rootOverride is empty, the root is inferred: if workDir contains exactly one -// entry and that entry is a directory (the common case for source archives like -// "pkg-1.0/"), that subdirectory is returned; otherwise workDir itself is -// returned. -func resolveExtractRoot(workDir, rootOverride string) (string, error) { - if rootOverride != "" { - // Defense in depth: validation already rejects non-local overrides, but - // re-check before joining so a malformed value can never escape workDir. - if !filepath.IsLocal(rootOverride) { - return "", fmt.Errorf("archive root %#q is not a local path", rootOverride) - } - - target := filepath.Join(workDir, rootOverride) - - info, err := os.Stat(target) - if err != nil { - return "", fmt.Errorf("archive root %#q not found after extraction:\n%w", rootOverride, err) - } - - if !info.IsDir() { - return "", fmt.Errorf("archive root %#q is not a directory", rootOverride) - } - - return target, nil - } - +// resolveExtractRoot returns the effective root of an extracted archive. If workDir +// contains exactly one entry and that entry is a directory (the common case for +// source archives like "pkg-1.0/"), that subdirectory is returned; otherwise workDir +// itself is returned. +func resolveExtractRoot(workDir string) (string, error) { entries, err := os.ReadDir(workDir) if err != nil { return "", fmt.Errorf("reading extracted directory:\n%w", err) @@ -228,9 +195,14 @@ func archiveNamesFromOverlays(overlays []projectconfig.ComponentOverlay) []strin var names []string for _, overlay := range overlays { - if overlay.ModifiesArchive() && !seen[overlay.Archive] { - seen[overlay.Archive] = true - names = append(names, overlay.Archive) + if !overlay.ModifiesArchive() { + continue + } + + archiveName, _, _ := overlay.ArchiveTarget() + if !seen[archiveName] { + seen[archiveName] = true + names = append(names, archiveName) } } diff --git a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go index 1a188f8f..de3d566b 100644 --- a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go +++ b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go @@ -16,97 +16,54 @@ import ( ) func TestGroupOverlaysByArchive(t *testing.T) { - t.Run("groups overlays by archive name preserving order", func(t *testing.T) { + t.Run("groups overlays by archive name preserving order and strips the prefix", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ { Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - Filename: "unwanted.conf", + Filename: "pkg-1.0.tar.gz/unwanted.conf", }, { Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - Filename: "config.h", + Filename: "pkg-1.0.tar.gz/config.h", }, { Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "other-2.0.tar.xz", - Filename: "docs/*.md", + Filename: "other-2.0.tar.xz/docs/*.md", }, } - groups, err := groupOverlaysByArchive(overlays) - require.NoError(t, err) + groups := groupOverlaysByArchive(overlays) require.Len(t, groups, 2) assert.Equal(t, "pkg-1.0.tar.gz", groups[0].archive) require.Len(t, groups[0].overlays, 2) + // Filename is rewritten to the in-archive glob (archive prefix stripped). assert.Equal(t, "unwanted.conf", groups[0].overlays[0].Filename) assert.Equal(t, "config.h", groups[0].overlays[1].Filename) assert.Equal(t, "other-2.0.tar.xz", groups[1].archive) require.Len(t, groups[1].overlays, 1) + assert.Equal(t, "docs/*.md", groups[1].overlays[0].Filename) }) t.Run("skips overlays that are not archive-scoped", func(t *testing.T) { overlays := []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1.0"}, - {Type: projectconfig.ComponentOverlayRemoveFile, Archive: "pkg.tar.gz", Filename: "f"}, - // Plain (non-archive) file overlay: no Archive set, so it must be skipped. + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "pkg.tar.gz/f"}, + // Plain (non-archive) file overlay: no archive prefix, so it must be skipped. {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "loose.txt"}, + // Bare archive name with no inner path: a loose removal of the archive itself. + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "drop-me.tar.gz"}, {Type: projectconfig.ComponentOverlayAddFile, Filename: "new.txt", Source: "src"}, } - groups, err := groupOverlaysByArchive(overlays) - require.NoError(t, err) + groups := groupOverlaysByArchive(overlays) require.Len(t, groups, 1) assert.Equal(t, "pkg.tar.gz", groups[0].archive) require.Len(t, groups[0].overlays, 1) - }) - - t.Run("reconciles matching archive-root overrides", func(t *testing.T) { - overlays := []projectconfig.ComponentOverlay{ - { - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - ArchiveRoot: "custom-root", - Filename: "a.conf", - }, - { - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - Filename: "b.conf", - }, - } - - groups, err := groupOverlaysByArchive(overlays) - require.NoError(t, err) - - require.Len(t, groups, 1) - assert.Equal(t, "custom-root", groups[0].root) - }) - - t.Run("errors on conflicting archive-root overrides", func(t *testing.T) { - overlays := []projectconfig.ComponentOverlay{ - { - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - ArchiveRoot: "root-a", - Filename: "a.conf", - }, - { - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - ArchiveRoot: "root-b", - Filename: "b.conf", - }, - } - - _, err := groupOverlaysByArchive(overlays) - require.Error(t, err) - assert.Contains(t, err.Error(), "conflicting") + assert.Equal(t, "f", groups[0].overlays[0].Filename) }) } @@ -115,7 +72,7 @@ func TestResolveExtractRoot(t *testing.T) { workDir := t.TempDir() require.NoError(t, os.MkdirAll(workDir+"/pkg-1.0", 0o755)) - root, err := resolveExtractRoot(workDir, "") + root, err := resolveExtractRoot(workDir) require.NoError(t, err) assert.Equal(t, workDir+"/pkg-1.0", root) }) @@ -125,46 +82,10 @@ func TestResolveExtractRoot(t *testing.T) { require.NoError(t, os.MkdirAll(workDir+"/dirA", 0o755)) require.NoError(t, os.WriteFile(workDir+"/loose.txt", nil, fileperms.PrivateFile)) - root, err := resolveExtractRoot(workDir, "") + root, err := resolveExtractRoot(workDir) require.NoError(t, err) assert.Equal(t, workDir, root) }) - - t.Run("override selects named subdirectory", func(t *testing.T) { - workDir := t.TempDir() - // Two top-level dirs so the heuristic would not pick one. - require.NoError(t, os.MkdirAll(workDir+"/dirA", 0o755)) - require.NoError(t, os.MkdirAll(workDir+"/dirB", 0o755)) - - root, err := resolveExtractRoot(workDir, "dirB") - require.NoError(t, err) - assert.Equal(t, workDir+"/dirB", root) - }) - - t.Run("override missing directory errors", func(t *testing.T) { - workDir := t.TempDir() - - _, err := resolveExtractRoot(workDir, "does-not-exist") - require.Error(t, err) - assert.Contains(t, err.Error(), "not found") - }) - - t.Run("override pointing at a file errors", func(t *testing.T) { - workDir := t.TempDir() - require.NoError(t, os.WriteFile(workDir+"/afile", nil, fileperms.PrivateFile)) - - _, err := resolveExtractRoot(workDir, "afile") - require.Error(t, err) - assert.Contains(t, err.Error(), "not a directory") - }) - - t.Run("non-local override is rejected", func(t *testing.T) { - workDir := t.TempDir() - - _, err := resolveExtractRoot(workDir, "../escape") - require.Error(t, err) - assert.Contains(t, err.Error(), "not a local path") - }) } // TestArchiveFileRemove verifies that archive-scoped file-remove overlays are @@ -185,7 +106,6 @@ func TestArchiveFileRemove(t *testing.T) { overlay := projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg.tar.gz", Filename: "*.conf", } @@ -207,7 +127,6 @@ func TestArchiveFileRemove(t *testing.T) { overlay := projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg.tar.gz", Filename: "*.conf", } @@ -216,3 +135,35 @@ func TestArchiveFileRemove(t *testing.T) { assert.Contains(t, err.Error(), "did not match any files") }) } + +// TestProcessArchive_DryRunDoesNotModifyArchive verifies that, in dry-run mode, +// processArchive skips the extract/repack cycle entirely and leaves the original +// archive on disk byte-for-byte unchanged (repacking would otherwise rewrite it). +func TestProcessArchive_DryRunDoesNotModifyArchive(t *testing.T) { + ctx := testctx.NewCtx() + ctx.DryRunValue = true + + sourcesDir := t.TempDir() + + const archiveName = "pkg-1.0.tar.gz" + + archivePath := sourcesDir + "/" + archiveName + + // Content need not be a valid archive: dry-run returns before extraction, and + // the test only asserts the bytes are untouched. + original := []byte("original archive bytes") + require.NoError(t, os.WriteFile(archivePath, original, fileperms.PrivateFile)) + + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove.conf"}, + }, + } + + require.NoError(t, processArchive(ctx, sourcesDir, group)) + + after, err := os.ReadFile(archivePath) + require.NoError(t, err) + assert.Equal(t, original, after, "dry-run must not modify the archive on disk") +} diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 46781805..b7a61f2d 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -725,7 +725,8 @@ func (p *sourcePreparerImpl) updateSourcesFile( // rehashModifiedEntries updates the Raw and Entry fields of parsed 'sources' lines // for archives that were modified by archive overlays. The hash is recomputed using -// the same hash type as the original entry. +// the same hash type as the original entry. It returns an error if any modified +// archive has no matching 'sources' entry, since that would leave a stale digest. func (p *sourcePreparerImpl) rehashModifiedEntries( lines []fedorasource.SourcesFileLine, outputDir string, modifiedArchives []string, ) error { @@ -733,13 +734,20 @@ func (p *sourcePreparerImpl) rehashModifiedEntries( return nil } - modified := make(map[string]bool, len(modifiedArchives)) + // Track which archives we actually rehashed so we can detect any that were + // repacked by an overlay but have no matching 'sources' entry. Leaving such + // an archive unrehashed would record a stale digest, so it is treated as an error. + rehashed := make(map[string]bool, len(modifiedArchives)) for _, name := range modifiedArchives { - modified[name] = true + rehashed[name] = false } for idx, line := range lines { - if line.Entry == nil || !modified[line.Entry.Filename] { + if line.Entry == nil { + continue + } + + if _, ok := rehashed[line.Entry.Filename]; !ok { continue } @@ -759,6 +767,26 @@ func (p *sourcePreparerImpl) rehashModifiedEntries( lines[idx].Raw = fedorasource.FormatSourcesEntry(line.Entry.Filename, line.Entry.HashType, newHash) lines[idx].Entry.Hash = newHash + rehashed[line.Entry.Filename] = true + } + + // Any archive that an overlay repacked but that has no 'sources' entry would + // silently keep a stale digest. Surface this as an error identifying the + // missing filenames rather than producing an inconsistent 'sources' file. + var missing []string + + for name, done := range rehashed { + if !done { + missing = append(missing, name) + } + } + + if len(missing) > 0 { + slices.Sort(missing) + + return fmt.Errorf( + "archive overlay(s) modified %d archive(s) with no matching 'sources' entry to rehash: %s", + len(missing), strings.Join(missing, ", ")) } return nil diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 704bea69..44401519 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -155,8 +155,7 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { Overlays: []projectconfig.ComponentOverlay{ { Type: projectconfig.ComponentOverlayRemoveFile, - Archive: archiveName, - Filename: "remove-me.txt", + Filename: archiveName + "/remove-me.txt", }, }, }) @@ -210,6 +209,68 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { assert.NotEqual(t, originalHash, entry.Hash, "'sources' entry hash must have been updated") } +// TestPrepareSources_ArchiveOverlayMissingSourcesEntryErrors verifies that when an +// archive-scoped overlay repacks an archive that has no matching 'sources' entry, +// preparation fails instead of silently leaving a stale (or absent) digest. +func TestPrepareSources_ArchiveOverlayMissingSourcesEntryErrors(t *testing.T) { + const ( + componentName = "test-component" + archiveName = "pkg-1.0.tar.gz" + ) + + // Host FS + real temp dir: archive extraction/repacking happens on disk. + ctx := testctx.NewCtx(testctx.WithHostFS()) + outputDir := t.TempDir() + + archivePath := filepath.Join(outputDir, archiveName) + sourcesPath := filepath.Join(outputDir, fedorasource.SourcesFileName) + + // Build a deterministic archive with a file to remove, but deliberately seed a + // 'sources' file that has NO entry for this archive. + stagingDir := t.TempDir() + pkgRoot := filepath.Join(stagingDir, "pkg-1.0") + require.NoError(t, os.MkdirAll(pkgRoot, fileperms.PublicDir)) + require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "remove-me.txt"), []byte("delete me"), fileperms.PrivateFile)) + require.NoError(t, archive.CreateDeterministicArchiveAuto(archivePath, stagingDir)) + + // 'sources' file references some unrelated file, not the archive being modified. + require.NoError(t, fileutils.WriteFile( + ctx.FS(), sourcesPath, + []byte("SHA256 (unrelated.tar.gz) = 0000000000000000000000000000000000000000000000000000000000000000\n"), + fileperms.PublicFile)) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) + + component.EXPECT().GetName().AnyTimes().Return(componentName) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlayRemoveFile, + Filename: archiveName + "/remove-me.txt", + }, + }, + }) + + sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).Return(nil) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile( + ctx.FS(), filepath.Join(dir, componentName+".spec"), + []byte("# test spec"), fileperms.PublicFile) + }, + ) + + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) + require.NoError(t, err) + + err = preparer.PrepareSources(ctx, component, outputDir, true /*applyOverlays*/) + require.Error(t, err, "preparing sources should fail when a modified archive has no 'sources' entry") + assert.Contains(t, err.Error(), archiveName, + "error should identify the archive missing from the 'sources' file") +} + func TestPrepareSources_SourceManagerError(t *testing.T) { ctrl := gomock.NewController(t) component := components_testutils.NewMockComponent(ctrl) diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 2248cd0c..6f3883b9 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -252,9 +252,10 @@ func TestComputeIdentity_OverlaySourceFileChange(t *testing.T) { } func TestComputeIdentity_OverlayArchiveScopingChangesFP(t *testing.T) { - // Archive-scoping fields are part of the hashed config, so setting them must - // change the fingerprint. Guards against excluding them (e.g. `fingerprint:"-"`), - // which would let an archive overlay skip a rebuild. + // Archive scoping comes from the overlay's file path, which is part of the hashed + // config, so scoping an overlay to an archive must change the fingerprint. Guards + // against excluding the path (e.g. `fingerprint:"-"`), which would let an archive + // overlay skip a rebuild. ctx := newTestFS(t, map[string]string{ "/specs/test.spec": "Name: testpkg\nVersion: 1.0", }) @@ -268,21 +269,12 @@ func TestComputeIdentity_OverlayArchiveScopingChangesFP(t *testing.T) { withArchive := baseComponent() withArchive.Overlays = []projectconfig.ComponentOverlay{ - {Type: "file-remove", Filename: "bundled.conf", Archive: "pkg-1.0.tar.gz"}, + {Type: "file-remove", Filename: "pkg-1.0.tar.gz/bundled.conf"}, } fpArchive := computeFingerprint(t, ctx, withArchive, releaseVer, 0) assert.NotEqual(t, fpBase, fpArchive, - "setting the archive scope must change the fingerprint") - - withRoot := baseComponent() - withRoot.Overlays = []projectconfig.ComponentOverlay{ - {Type: "file-remove", Filename: "bundled.conf", Archive: "pkg-1.0.tar.gz", ArchiveRoot: "custom-root"}, - } - fpRoot := computeFingerprint(t, ctx, withRoot, releaseVer, 0) - - assert.NotEqual(t, fpArchive, fpRoot, - "setting the archive-root override must change the fingerprint") + "scoping the overlay to an archive (via the path prefix) must change the fingerprint") } func TestComputeIdentity_PatchAddRenameChangesFP(t *testing.T) { diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index b6f5ee03..e64cfb43 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -7,10 +7,12 @@ import ( "fmt" "path/filepath" "regexp" + "strings" "github.com/bmatcuk/doublestar/v4" "github.com/brunoga/deep" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" ) @@ -21,13 +23,6 @@ type ComponentOverlay struct { // Human readable description of overlay; primarily present to document the need for the change. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Human readable description of overlay" fingerprint:"-"` - // Scopes the overlay to files inside this source archive (a bare filename, not a path). - // Only file-remove and file-search-replace honor it; when set, the overlay operates inside - // the named archive instead of the loose sources tree. - Archive string `toml:"archive,omitempty" json:"archive,omitempty" jsonschema:"title=Archive,description=The source archive to modify (e.g. pkg-1.0.tar.gz)"` - // Overrides the archive's extraction root (rpmbuild's `%setup -n` equivalent). When unset, the - // root is inferred: a single top-level directory is used, otherwise the archive root. - ArchiveRoot string `toml:"archive-root,omitempty" json:"archiveRoot,omitempty" jsonschema:"title=Archive root,description=Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset"` // For overlays that apply to non-spec files, indicates the filename. For overlays that can // apply to multiple files, supports glob patterns (including globstar). Filename string `toml:"file,omitempty" json:"file,omitempty" jsonschema:"title=Filename,description=The name of the non-spec file to which this overlay applies, or a glob pattern matching multiple files"` @@ -133,13 +128,37 @@ func (c *ComponentOverlay) ModifiesSpec() bool { c.Type == ComponentOverlayRemovePatch } +// ArchiveTarget inspects the overlay's file path and, when it targets files inside a source +// archive, returns the archive filename and the glob to match within the extracted archive. +// +// An overlay is archive-scoped when the first path segment of [ComponentOverlay.Filename] is a +// recognized archive name (see [archive.IsArchiveName]) and a non-empty inner path follows it, +// e.g. "pkg-1.0.tar.gz/vendor/**" yields archive="pkg-1.0.tar.gz", inner="vendor/**". +// +// Returns ok=false for loose-file overlays: a path with no archive prefix (e.g. "vendor/**"), +// or a bare archive name with no inner path (e.g. "old.tar.gz", which removes the archive file +// itself from the loose sources tree). +func (c *ComponentOverlay) ArchiveTarget() (archiveName, innerPath string, ok bool) { + before, after, found := strings.Cut(c.Filename, "/") + if !found || after == "" || !archive.IsArchiveName(before) { + return "", "", false + } + + return before, after, true +} + // ModifiesArchive returns true if the overlay modifies files inside a source archive. // These overlays require extraction and repacking of the archive. Only file-remove and -// file-search-replace support archive scoping, and only when their -// [ComponentOverlay.Archive] field is set. +// file-search-replace support archive scoping, and only when their [ComponentOverlay.Filename] +// is an archive-scoped path (see [ComponentOverlay.ArchiveTarget]). func (c *ComponentOverlay) ModifiesArchive() bool { - return c.Archive != "" && - (c.Type == ComponentOverlayRemoveFile || c.Type == ComponentOverlaySearchAndReplaceInFile) + if c.Type != ComponentOverlayRemoveFile && c.Type != ComponentOverlaySearchAndReplaceInFile { + return false + } + + _, _, ok := c.ArchiveTarget() + + return ok } // ModifiesNonSpecFiles returns true if the overlay modifies non-spec files. This includes @@ -207,8 +226,9 @@ const ( // ComponentOverlayAddFile is an overlay that adds a non-spec file. ComponentOverlayAddFile ComponentOverlayType = "file-add" // ComponentOverlayRemoveFile is an overlay that removes a non-spec file. When its - // [ComponentOverlay.Archive] field is set, it removes file(s) from inside that source - // archive instead of loose files in the sources tree. + // [ComponentOverlay.Filename] is an archive-scoped path (e.g. "pkg-1.0.tar.gz/vendor/**"), + // it removes file(s) from inside that source archive instead of loose files in the + // sources tree (see [ComponentOverlay.ArchiveTarget]). ComponentOverlayRemoveFile ComponentOverlayType = "file-remove" // ComponentOverlayRenameFile is an overlay that renames a non-spec file. ComponentOverlayRenameFile ComponentOverlayType = "file-rename" @@ -217,7 +237,7 @@ const ( // Validate checks that required fields are set based on the overlay type. This catches // configuration errors at load time rather than at apply time. // -//nolint:cyclop,gocognit,gocyclo,funlen,maintidx // complexity is inherent to the number of overlay types. +//nolint:cyclop,gocognit,gocyclo,funlen // complexity is inherent to the number of overlay types. func (c *ComponentOverlay) Validate() error { desc := c.Description if desc == "" { @@ -262,33 +282,6 @@ func (c *ComponentOverlay) Validate() error { return nil } - // The archive field scopes an overlay to operate inside a source archive. It is only - // accepted on file-remove and file-search-replace, and must be a bare filename (not a path). - if c.Archive != "" { - if c.Type != ComponentOverlayRemoveFile && c.Type != ComponentOverlaySearchAndReplaceInFile { - return unexpectedField("archive") - } - - if err := requireFileBasename("archive", c.Archive); err != nil { - return err - } - } - - // The archive-root override is only meaningful for archive-scoped overlays, and must be a - // local relative path so it cannot escape the extraction directory. - if c.ArchiveRoot != "" { - if !c.ModifiesArchive() { - return unexpectedField("archive-root") - } - - if !filepath.IsLocal(c.ArchiveRoot) { - return fmt.Errorf( - "overlay type %#q requires %#q to be a local relative path (no %#q or absolute paths); found %#q", - c.Type, "archive-root", "..", c.ArchiveRoot, - ) - } - } - switch c.Type { case ComponentOverlayAddSpecTag, ComponentOverlayInsertSpecTag, ComponentOverlaySetSpecTag, ComponentOverlayUpdateSpecTag: diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index 63734bc1..7600c011 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -412,112 +412,45 @@ func TestComponentOverlay_Validate(t *testing.T) { errorExpected: true, errorContains: "section", }, - // archive-scoped file-remove tests (archive modifier) + // archive-scoped file-remove tests (archive derived from path prefix) { - name: "file-remove with archive valid (archive-scoped)", + name: "file-remove archive-scoped valid", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - Filename: "unwanted.conf", + Filename: "pkg-1.0.tar.gz/unwanted.conf", }, errorExpected: false, }, { - name: "file-remove with archive glob valid", + name: "file-remove archive-scoped glob valid", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - Filename: "docs/**/*.md", + Filename: "pkg-1.0.tar.gz/docs/**/*.md", }, errorExpected: false, }, { - name: "file-remove without archive is a plain loose-file remove", + name: "file-remove of a bare archive name is a plain loose-file remove", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Filename: "unwanted.conf", + Filename: "old.tar.gz", }, errorExpected: false, }, { - name: "file-remove with archive missing file", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - }, - errorExpected: true, - errorContains: "file", - }, - { - name: "file-remove rejects archive path", + name: "file-remove without archive prefix is a plain loose-file remove", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "subdir/pkg-1.0.tar.gz", Filename: "unwanted.conf", }, - errorExpected: true, - errorContains: "archive", - }, - { - name: "archive rejected on overlay type that cannot be archive-scoped", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayAddFile, - Archive: "pkg-1.0.tar.gz", - Filename: "new.conf", - Source: "files/new.conf", - }, - errorExpected: true, - errorContains: "archive", - }, - { - name: "file-remove with archive-root override valid", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - ArchiveRoot: "src-root", - Filename: "unwanted.conf", - }, errorExpected: false, }, + // file-search-replace supports archive scoping via the path prefix { - name: "file-remove rejects non-local archive-root", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayRemoveFile, - Archive: "pkg-1.0.tar.gz", - ArchiveRoot: "../escape", - Filename: "unwanted.conf", - }, - errorExpected: true, - errorContains: "archive-root", - }, - { - name: "archive-root rejected when archive unset", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayRemoveFile, - ArchiveRoot: "src-root", - Filename: "unwanted.conf", - }, - errorExpected: true, - errorContains: "archive-root", - }, - { - name: "archive-root rejected on non-archive overlay", - overlay: projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlaySetSpecTag, - Tag: "Version", - Value: "1.0.0", - ArchiveRoot: "src-root", - }, - errorExpected: true, - errorContains: "archive-root", - }, - // file-search-replace supports archive scoping - { - name: "file-search-replace with archive valid (archive-scoped)", + name: "file-search-replace archive-scoped valid", overlay: projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Archive: "pkg-1.0.tar.gz", - Filename: "config.h", + Filename: "pkg-1.0.tar.gz/config.h", Regex: "old_value", Replacement: "new_value", }, @@ -566,13 +499,13 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { projectconfig.ComponentOverlayAddFile, } - // Archive-scoped overlays: only file-remove/file-search-replace becomes archive-scoped, - // and only when its Archive field is set. + // Archive-scoped overlays: only file-remove/file-search-replace become archive-scoped, + // and only when their file path carries an archive prefix (e.g. "pkg-1.0.tar.gz/..."). archiveOverlays := []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayRemoveFile, Archive: "pkg-1.0.tar.gz", Filename: "f"}, + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "pkg-1.0.tar.gz/f"}, { - Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, - Archive: "pkg-1.0.tar.gz", Filename: "f", Regex: "old", Replacement: "new", + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Filename: "pkg-1.0.tar.gz/f", Regex: "old", Replacement: "new", }, } diff --git a/internal/utils/archive/archive.go b/internal/utils/archive/archive.go index a50dff61..f5c8a962 100644 --- a/internal/utils/archive/archive.go +++ b/internal/utils/archive/archive.go @@ -79,6 +79,16 @@ func DetectCompression(filename string) (Compression, error) { } } +// IsArchiveName reports whether filename has a recognized archive extension +// (i.e. [DetectCompression] succeeds for it). It is the cheap, error-free +// predicate form of [DetectCompression], useful for classifying a path as an +// archive without needing the specific compression type. +func IsArchiveName(filename string) bool { + _, err := DetectCompression(filename) + + return err == nil +} + // ExtractAuto is a convenience wrapper that infers the compression from // archivePath's extension via [DetectCompression] and then calls [Extract]. // Most callers should prefer this over the explicit-compression [Extract], diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 7727e40c..7b824f4e 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -255,16 +255,6 @@ "title": "Description", "description": "Human readable description of overlay" }, - "archive": { - "type": "string", - "title": "Archive", - "description": "The source archive to modify (e.g. pkg-1.0.tar.gz)" - }, - "archive-root": { - "type": "string", - "title": "Archive root", - "description": "Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset" - }, "file": { "type": "string", "title": "Filename", diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 7727e40c..7b824f4e 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -255,16 +255,6 @@ "title": "Description", "description": "Human readable description of overlay" }, - "archive": { - "type": "string", - "title": "Archive", - "description": "The source archive to modify (e.g. pkg-1.0.tar.gz)" - }, - "archive-root": { - "type": "string", - "title": "Archive root", - "description": "Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset" - }, "file": { "type": "string", "title": "Filename", diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7727e40c..7b824f4e 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -255,16 +255,6 @@ "title": "Description", "description": "Human readable description of overlay" }, - "archive": { - "type": "string", - "title": "Archive", - "description": "The source archive to modify (e.g. pkg-1.0.tar.gz)" - }, - "archive-root": { - "type": "string", - "title": "Archive root", - "description": "Top-level directory inside the archive to treat as the extraction root (mirrors %setup -n); inferred when unset" - }, "file": { "type": "string", "title": "Filename", From 5a3a2063511968063fcabc17dfeda7f14c9f17c6 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 11 Jun 2026 22:53:25 +0000 Subject: [PATCH 07/13] Addressed feedback again --- .../azldev/core/sources/archiveoverlays.go | 122 ++++++++++++------ .../sources/archiveoverlays_internal_test.go | 4 +- .../app/azldev/core/sources/sourceprep.go | 80 ++++++------ internal/projectconfig/overlay.go | 2 +- 4 files changed, 130 insertions(+), 78 deletions(-) diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go index 052ed527..3ad4a3c8 100644 --- a/internal/app/azldev/core/sources/archiveoverlays.go +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -19,16 +19,20 @@ import ( // them in order. Multiple overlays targeting the same archive are batched into // a single extract/modify/repack cycle. File removals inside the archive reuse // the same machinery as loose-file overlays ([applyNonSpecOverlay]). +// +// It returns the names of the archives that were actually repacked. In dry-run +// mode no archive is repacked, so the returned slice is empty even when archive +// overlays were present. func applyArchiveOverlays( dryRunnable opctx.DryRunnable, eventListener opctx.EventListener, sourcesDirPath string, overlays []projectconfig.ComponentOverlay, -) error { +) ([]string, error) { groups := groupOverlaysByArchive(overlays) if len(groups) == 0 { - return nil + return nil, nil } operationCount := 0 @@ -42,13 +46,20 @@ func applyArchiveOverlays( ) defer event.End() + var repacked []string + for _, group := range groups { - if err := processArchive(dryRunnable, sourcesDirPath, group); err != nil { - return fmt.Errorf("archive overlay failed for %#q:\n%w", group.archive, err) + didRepack, err := processArchive(dryRunnable, sourcesDirPath, group) + if err != nil { + return nil, fmt.Errorf("archive overlay failed for %#q:\n%w", group.archive, err) + } + + if didRepack { + repacked = append(repacked, group.archive) } } - return nil + return repacked, nil } // archiveGroup holds overlays targeting the same archive, preserving order. @@ -96,19 +107,22 @@ func groupOverlaysByArchive(overlays []projectconfig.ComponentOverlay) []archive } // processArchive extracts an archive to a temp directory, applies all overlays, -// and deterministically repacks it in-place with the original compression. +// and deterministically repacks it with the original compression, atomically +// replacing the original via a temp file + rename. It returns true when the +// archive was repacked. In dry-run mode it returns (false, nil) without touching +// the archive on disk. func processArchive( dryRunnable opctx.DryRunnable, sourcesDirPath string, group archiveGroup, -) error { +) (repacked bool, err error) { archivePath := filepath.Join(sourcesDirPath, group.archive) if dryRunnable.DryRun() { slog.Info("Dry run; would apply archive overlays", "archive", group.archive, "operations", len(group.overlays)) - return nil + return false, nil } // The [archive] package operates exclusively through OS primitives ([os.Root], @@ -116,7 +130,7 @@ func processArchive( // FS implementation (which may be in-memory or otherwise non-OS-backed). workDir, err := os.MkdirTemp("", "archive-overlay-") if err != nil { - return fmt.Errorf("creating temp directory:\n%w", err) + return false, fmt.Errorf("creating temp directory:\n%w", err) } defer func() { @@ -127,21 +141,21 @@ func processArchive( // Extract the archive; compression is inferred from the filename extension. if err := archive.ExtractAuto(archivePath, workDir); err != nil { - return fmt.Errorf("extracting archive:\n%w", err) + return false, fmt.Errorf("extracting archive:\n%w", err) } // Determine the root of the extracted content. Most source archives unpack to // a single top-level directory (e.g., "pkg-1.0/"), which is used as the root. extractRoot, err := resolveExtractRoot(workDir) if err != nil { - return fmt.Errorf("resolving extract root:\n%w", err) + return false, fmt.Errorf("resolving extract root:\n%w", err) } // Confine an OS-backed FS to the extract root so file overlays reuse the same // machinery as loose-file overlays. extractFS, err := rootfs.New(extractRoot) if err != nil { - return fmt.Errorf("confining FS to extract root:\n%w", err) + return false, fmt.Errorf("confining FS to extract root:\n%w", err) } defer func() { @@ -155,17 +169,70 @@ func processArchive( // destination tree, so extractFS is passed as both source and destination FS. for _, overlay := range group.overlays { if err := applyNonSpecOverlay(dryRunnable, extractFS, extractFS, overlay); err != nil { - return fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) + return false, fmt.Errorf("applying %#q operation:\n%w", overlay.Type, err) } } - // Deterministically repack the archive in-place, reusing the original compression. - if err := archive.CreateDeterministicArchiveAuto(archivePath, workDir); err != nil { - return fmt.Errorf("repacking archive:\n%w", err) + // Deterministically repack the archive, reusing the original compression, and + // atomically replace the original (see [repackArchiveAtomic]). + if err := repackArchiveAtomic(archivePath, group.archive, workDir); err != nil { + return false, err } slog.Info("Archive overlay applied", "archive", group.archive) + return true, nil +} + +// repackArchiveAtomic deterministically repacks workDir into an archive that +// replaces archivePath, reusing the compression inferred from archiveName. +// +// To avoid corrupting the fetched source archive on a mid-write failure (disk +// full, permission error, etc.), it repacks to a temp file in the same directory +// and atomically renames it over the original only on success. Repacking directly +// over archivePath would truncate it first, leaving the workspace unrecoverable +// without refetching if the repack then failed. +func repackArchiveAtomic(archivePath, archiveName, workDir string) (err error) { + comp, err := archive.DetectCompression(archiveName) + if err != nil { + return fmt.Errorf("detecting compression for %#q:\n%w", archiveName, err) + } + + tmpFile, err := os.CreateTemp(filepath.Dir(archivePath), "."+filepath.Base(archiveName)+".repack-*") + if err != nil { + return fmt.Errorf("creating temp archive:\n%w", err) + } + + tmpPath := tmpFile.Name() + + // Close the handle immediately; CreateDeterministicArchive reopens the path. + if closeErr := tmpFile.Close(); closeErr != nil { + _ = os.Remove(tmpPath) + + return fmt.Errorf("closing temp archive %#q:\n%w", tmpPath, closeErr) + } + + // Clean up the temp file unless it was successfully renamed over the original. + repackedOK := false + + defer func() { + if !repackedOK { + if removeErr := os.Remove(tmpPath); removeErr != nil && !os.IsNotExist(removeErr) { + slog.Warn("Failed to clean up temp archive", "path", tmpPath, "error", removeErr) + } + } + }() + + if err := archive.CreateDeterministicArchive(tmpPath, workDir, comp); err != nil { + return fmt.Errorf("repacking archive:\n%w", err) + } + + if err := os.Rename(tmpPath, archivePath); err != nil { + return fmt.Errorf("replacing archive %#q with repacked archive:\n%w", archivePath, err) + } + + repackedOK = true + return nil } @@ -185,26 +252,3 @@ func resolveExtractRoot(workDir string) (string, error) { return workDir, nil } - -// archiveNamesFromOverlays returns the unique archive filenames targeted by -// archive overlays in the given overlay list. Used by [updateSourcesFile] to -// determine which 'sources' entries need rehashing after overlay application. -func archiveNamesFromOverlays(overlays []projectconfig.ComponentOverlay) []string { - seen := make(map[string]bool) - - var names []string - - for _, overlay := range overlays { - if !overlay.ModifiesArchive() { - continue - } - - archiveName, _, _ := overlay.ArchiveTarget() - if !seen[archiveName] { - seen[archiveName] = true - names = append(names, archiveName) - } - } - - return names -} diff --git a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go index de3d566b..f6e51261 100644 --- a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go +++ b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go @@ -161,7 +161,9 @@ func TestProcessArchive_DryRunDoesNotModifyArchive(t *testing.T) { }, } - require.NoError(t, processArchive(ctx, sourcesDir, group)) + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + assert.False(t, repacked, "dry-run must report that no archive was repacked") after, err := os.ReadFile(archivePath) require.NoError(t, err) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index b7a61f2d..fb16fd4c 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -246,11 +246,12 @@ func (p *sourcePreparerImpl) PrepareSources( } if applyOverlays { - if err := p.applyOverlaysToSources(component, outputDir); err != nil { + repackedArchives, err := p.applyOverlaysToSources(component, outputDir) + if err != nil { return err } - if err := p.updateSourcesFile(component, outputDir); err != nil { + if err := p.updateSourcesFile(component, outputDir, repackedArchives); err != nil { return fmt.Errorf("failed to update 'sources' file for component %#q:\n%w", component.GetName(), err) } @@ -272,14 +273,17 @@ func (p *sourcePreparerImpl) PrepareSources( } // applyOverlaysToSources writes the macros file and then applies all overlays. +// It returns the names of any archives that were repacked by archive overlays +// (empty in dry-run mode or when no archive overlays ran), so the caller can +// rehash exactly those entries in the 'sources' file. func (p *sourcePreparerImpl) applyOverlaysToSources( component components.Component, outputDir string, -) error { +) ([]string, error) { var macrosFileName string macrosFilePath, err := p.writeMacrosFile(component, outputDir) if err != nil { - return fmt.Errorf("failed to write macros file for component %#q:\n%w", + return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", component.GetName(), err) } @@ -287,64 +291,69 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( macrosFileName = filepath.Base(macrosFilePath) } - if err := p.applyOverlays(component, outputDir, macrosFileName); err != nil { - return fmt.Errorf("failed to apply overlays for component %#q:\n%w", + repackedArchives, err := p.applyOverlays(component, outputDir, macrosFileName) + if err != nil { + return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } - return nil + return repackedArchives, nil } // applyOverlays applies all overlays (user-defined and system-generated) to the -// component sources. +// component sources. It returns the names of any archives that were repacked by +// archive overlays. func (p *sourcePreparerImpl) applyOverlays( component components.Component, sourcesDirPath, macrosFileName string, -) error { +) ([]string, error) { event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) defer event.End() absSpecPath, err := p.resolveSpecPath(component, sourcesDirPath) if err != nil { - return err + return nil, err } allOverlays, err := p.collectOverlays(component, macrosFileName) if err != nil { - return fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err) + return nil, fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err) } if len(allOverlays) == 0 { - return nil + return nil, nil } // Archive overlays are applied first (they modify archived source files // in-place), followed by spec and loose-file overlays. Each function // self-filters to the overlay types it handles. - if err := p.applyArchiveOverlayGroup(component, sourcesDirPath, allOverlays); err != nil { - return err + repackedArchives, err := p.applyArchiveOverlayGroup(component, sourcesDirPath, allOverlays) + if err != nil { + return nil, err } if err := p.applyOverlayList(allOverlays, sourcesDirPath, absSpecPath); err != nil { - return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) + return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } - return nil + return repackedArchives, nil } // applyArchiveOverlayGroup applies the archive-scoped overlays contained in the // given overlay list. The list may hold overlays of any type; only those for // which [projectconfig.ComponentOverlay.ModifiesArchive] reports true are -// processed here. Skipped when source downloads were not performed. +// processed here. Skipped when source downloads were not performed. It returns +// the names of the archives that were actually repacked (empty in dry-run mode +// or when source downloads were skipped). func (p *sourcePreparerImpl) applyArchiveOverlayGroup( component components.Component, sourcesDirPath string, overlays []projectconfig.ComponentOverlay, -) error { +) ([]string, error) { archiveOverlays := lo.Filter(overlays, func(overlay projectconfig.ComponentOverlay, _ int) bool { return overlay.ModifiesArchive() }) if len(archiveOverlays) == 0 { - return nil + return nil, nil } if p.skipLookaside { @@ -352,17 +361,18 @@ func (p *sourcePreparerImpl) applyArchiveOverlayGroup( "component", component.GetName(), "count", len(archiveOverlays)) - return nil + return nil, nil } - if err := applyArchiveOverlays( + repackedArchives, err := applyArchiveOverlays( p.dryRunnable, p.eventListener, sourcesDirPath, archiveOverlays, - ); err != nil { - return fmt.Errorf("failed to apply archive overlays for component %#q:\n%w", + ) + if err != nil { + return nil, fmt.Errorf("failed to apply archive overlays for component %#q:\n%w", component.GetName(), err) } - return nil + return repackedArchives, nil } // collectOverlays gathers all overlays for a component into a single ordered slice: @@ -635,8 +645,10 @@ func (p *sourcePreparerImpl) DiffSources( return nil, fmt.Errorf("failed to copy sources for component %#q:\n%w", component.GetName(), err) } - // Apply overlays in-place to the copied directory only. - if err := p.applyOverlaysToSources(component, overlaidDir); err != nil { + // Apply overlays in-place to the copied directory only. The repacked-archive + // list is unused here: DiffSources diffs the trees directly and does not + // rewrite a 'sources' file. + if _, err := p.applyOverlaysToSources(component, overlaidDir); err != nil { return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } @@ -666,21 +678,15 @@ func (p *sourcePreparerImpl) DiffSources( // a matching upstream entry is also an error: the user expressed intent to replace something // that isn't there, which almost certainly indicates a stale config or filename typo. func (p *sourcePreparerImpl) updateSourcesFile( - component components.Component, outputDir string, + component components.Component, outputDir string, modifiedArchives []string, ) error { config := component.GetConfig() sourceFiles := config.SourceFiles - // Derive the archives whose 'sources' hash needs refreshing because an archive - // overlay repacked them. Only meaningful when archive overlays actually ran: - // when skipLookaside is set, archive overlays are skipped (see - // [applyArchiveOverlayGroup]) and the archives may not even be present on disk, - // so rehashing would either fail or pointlessly rewrite an unchanged hash. - var modifiedArchives []string - if !p.skipLookaside { - modifiedArchives = archiveNamesFromOverlays(config.Overlays) - } - + // modifiedArchives lists the archives that archive overlays actually repacked + // during this run; their 'sources' digests must be refreshed. The list is empty + // when no archive overlays ran, in dry-run mode, or when source downloads were + // skipped, so rehashing is correctly avoided in those cases. if len(sourceFiles) == 0 && len(modifiedArchives) == 0 { return nil } diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index e64cfb43..5d43e5bd 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -139,7 +139,7 @@ func (c *ComponentOverlay) ModifiesSpec() bool { // or a bare archive name with no inner path (e.g. "old.tar.gz", which removes the archive file // itself from the loose sources tree). func (c *ComponentOverlay) ArchiveTarget() (archiveName, innerPath string, ok bool) { - before, after, found := strings.Cut(c.Filename, "/") + before, after, found := strings.Cut(filepath.ToSlash(c.Filename), "/") if !found || after == "" || !archive.IsArchiveName(before) { return "", "", false } From 42c49db399ee959a70f922f5b95025daaa506c81 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 11 Jun 2026 23:08:25 +0000 Subject: [PATCH 08/13] Addressed minor feedback fixes --- internal/app/azldev/core/sources/archiveoverlays.go | 2 +- internal/utils/archive/archive.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go index 3ad4a3c8..cf6ea521 100644 --- a/internal/app/azldev/core/sources/archiveoverlays.go +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -128,7 +128,7 @@ func processArchive( // The [archive] package operates exclusively through OS primitives ([os.Root], // os.*), so extraction must use a genuine on-disk path regardless of the injected // FS implementation (which may be in-memory or otherwise non-OS-backed). - workDir, err := os.MkdirTemp("", "archive-overlay-") + workDir, err := os.MkdirTemp(sourcesDirPath, ".archive-overlay-") if err != nil { return false, fmt.Errorf("creating temp directory:\n%w", err) } diff --git a/internal/utils/archive/archive.go b/internal/utils/archive/archive.go index f5c8a962..b5046e39 100644 --- a/internal/utils/archive/archive.go +++ b/internal/utils/archive/archive.go @@ -79,10 +79,9 @@ func DetectCompression(filename string) (Compression, error) { } } -// IsArchiveName reports whether filename has a recognized archive extension -// (i.e. [DetectCompression] succeeds for it). It is the cheap, error-free -// predicate form of [DetectCompression], useful for classifying a path as an -// archive without needing the specific compression type. +// IsArchiveName reports whether filename has a recognized archive extension. +// It is a convenience predicate over [DetectCompression] for classifying a path +// as an archive without needing the specific compression type. func IsArchiveName(filename string) bool { _, err := DetectCompression(filename) From ea8008b0365feb96240f2c70492a5fc99c863b06 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 17 Jun 2026 22:10:26 +0000 Subject: [PATCH 09/13] Made compression detection more accurate --- internal/utils/archive/archive.go | 38 +++++++++++++++++- internal/utils/archive/archive_test.go | 53 ++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/internal/utils/archive/archive.go b/internal/utils/archive/archive.go index b5046e39..32595d1e 100644 --- a/internal/utils/archive/archive.go +++ b/internal/utils/archive/archive.go @@ -20,6 +20,8 @@ package archive import ( "archive/tar" + "bufio" + "bytes" "compress/gzip" "errors" "fmt" @@ -50,6 +52,11 @@ const ( CompressionZstd ) +// compressionMagicLen is the number of leading bytes [sniffCompression] needs +// to recognize every supported compressed format (xz has the longest, 6-byte, +// signature). +const compressionMagicLen = 6 + // maxEntryBytes caps the decompressed size of any single regular-file entry // extracted by [Extract]. This prevents a decompression-bomb archive from // filling the destination filesystem. 10 GiB is well above any reasonable @@ -125,7 +132,13 @@ func Extract(archivePath, destDir string, comp Compression) (err error) { } defer defers.HandleDeferError(file.Close, &err) - decompressed, closer, err := newDecompressor(file, comp) + // Prefer the actual compression detected from the leading magic bytes over + // the caller-supplied (extension-derived) value: upstream archives are + // sometimes mislabeled, e.g. an uncompressed tar published as ".txz". + bufReader := bufio.NewReader(file) + magic, _ := bufReader.Peek(compressionMagicLen) + + decompressed, closer, err := newDecompressor(bufReader, sniffCompression(magic, comp)) if err != nil { return err } @@ -152,6 +165,29 @@ func Extract(archivePath, destDir string, comp Compression) (err error) { } } +// sniffCompression returns the compression format implied by the leading magic +// bytes, falling back to fallback when no known signature matches (an +// uncompressed tar carries no leading magic). All supported compressed formats +// have a fixed-position header, so this is authoritative over the filename +// extension. +func sniffCompression(magic []byte, fallback Compression) Compression { + switch { + case bytes.HasPrefix(magic, []byte{0xFD, '7', 'z', 'X', 'Z', 0x00}): + return CompressionXZ + case bytes.HasPrefix(magic, []byte{0x1F, 0x8B}): + return CompressionGzip + case bytes.HasPrefix(magic, []byte{0x28, 0xB5, 0x2F, 0xFD}): + return CompressionZstd + case len(magic) == 0: + // Unreadable/empty peek: defer to the extension-derived hint. + return fallback + default: + // No compression signature: an uncompressed tar (whatever the + // extension claims). A real tar's "ustar" magic sits at byte 257. + return CompressionNone + } +} + // newDecompressor wraps reader in the chosen decompressor. For // [CompressionNone] the returned closer is nil; otherwise it is the // decompressor itself. diff --git a/internal/utils/archive/archive_test.go b/internal/utils/archive/archive_test.go index fece1cd0..4fe9cc37 100644 --- a/internal/utils/archive/archive_test.go +++ b/internal/utils/archive/archive_test.go @@ -187,14 +187,61 @@ func TestRoundTrip_AllCompressions(t *testing.T) { } } +func TestExtract_MislabeledCompression(t *testing.T) { + // Some upstream archives are published with a compression extension that + // doesn't match their bytes (e.g. an uncompressed tar named ".txz"). Extract + // must sniff the real format from the magic bytes rather than trusting the + // caller-supplied (extension-derived) compression. + tests := []struct { + name string + // actual is the format the bytes are really written in. + actual archive.Compression + // claimed is the (wrong) compression Extract is told to use. + claimed archive.Compression + }{ + {"plain tar claimed as xz", archive.CompressionNone, archive.CompressionXZ}, + {"plain tar claimed as gzip", archive.CompressionNone, archive.CompressionGzip}, + {"gzip claimed as xz", archive.CompressionGzip, archive.CompressionXZ}, + {"xz claimed as none", archive.CompressionXZ, archive.CompressionNone}, + {"zstd claimed as gzip", archive.CompressionZstd, archive.CompressionGzip}, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + tmpDir := t.TempDir() + sourceDir := filepath.Join(tmpDir, "src") + extractDir := filepath.Join(tmpDir, "out") + + require.NoError(t, os.MkdirAll(sourceDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "a.txt"), []byte("alpha"), 0o600)) + + // Write the archive in its real format but with a mismatched name. + archivePath := filepath.Join(tmpDir, "pkg.txz") + require.NoError(t, archive.CreateDeterministicArchive(archivePath, sourceDir, testCase.actual)) + + // Extract is given the wrong compression; magic-byte sniffing must + // recover the real format and extract successfully. + require.NoError(t, archive.Extract(archivePath, extractDir, testCase.claimed)) + + got, err := os.ReadFile(filepath.Join(extractDir, "a.txt")) + require.NoError(t, err) + assert.Equal(t, "alpha", string(got)) + }) + } +} + func TestUnsupportedCompression(t *testing.T) { tmpDir := t.TempDir() - archivePath := filepath.Join(tmpDir, "archive.bin") - require.NoError(t, os.WriteFile(archivePath, []byte("dummy"), 0o600)) bogus := archive.Compression(99) - err := archive.Extract(archivePath, tmpDir, bogus) + // Extract sniffs real compression from magic bytes and only falls back to + // the caller-supplied value when there are no bytes to inspect. Use an empty + // file so the bogus value actually reaches the decompressor guard. + emptyPath := filepath.Join(tmpDir, "empty.bin") + require.NoError(t, os.WriteFile(emptyPath, nil, 0o600)) + + err := archive.Extract(emptyPath, tmpDir, bogus) require.Error(t, err) assert.Contains(t, err.Error(), "unsupported compression type") From 18238986b858b7e8fabbb90a90f653e76fa27a26 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 18 Jun 2026 16:45:19 +0000 Subject: [PATCH 10/13] Added additional tests --- docs/user/reference/config/overlays.md | 2 + .../azldev/cmds/component/preparesources.go | 7 +- .../azldev/core/sources/archiveoverlays.go | 4 +- .../sources/archiveoverlays_internal_test.go | 373 +++++++++++++-- .../core/sources/archiveoverlays_test.go | 441 ++++++++++++++++++ internal/app/azldev/core/sources/overlays.go | 4 +- .../azldev/core/sources/sourceprep_test.go | 4 +- internal/projectconfig/overlay.go | 12 +- internal/projectconfig/overlay_test.go | 4 +- internal/utils/archive/archive.go | 71 ++- internal/utils/archive/archive_test.go | 44 ++ 11 files changed, 880 insertions(+), 86 deletions(-) create mode 100644 internal/app/azldev/core/sources/archiveoverlays_test.go diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index 6f44ceb0..1358ca31 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -71,6 +71,8 @@ file = "old.tar.gz" # removes the archive file itself (bare name > **Extraction root:** The inner path is interpreted relative to the archive's extraction root: if the archive unpacks to a single top-level directory (the conventional `%{name}-%{version}` layout) that directory is the root; otherwise the archive root is used. +> **Supported entry types:** Only regular files, directories, and symlinks are supported inside an archive overlay's target. If the archive contains an entry that cannot be repacked safely (a hardlink, device node, FIFO, etc.), the overlay fails with an error rather than silently dropping the entry from the repacked archive. + | Type | Description | Required Fields | |------|-------------|-----------------| | `file-remove` (archive-scoped path) | Removes file(s) matching a glob pattern from inside an archive | `file` (e.g. `pkg-1.0.tar.gz/vendor/**`) | diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 3dc7d687..647d25bf 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -138,7 +138,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er ) } - preparerOpts = appendPrepareSourcesOptions(env, preparerOpts, options, distro) + preparerOpts = appendPrepareSourcesOptions(preparerOpts, options) preparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env, preparerOpts...) if err != nil { @@ -190,13 +190,10 @@ func CheckOutputDir(env *azldev.Env, options *PrepareSourcesOptions) error { } // appendPrepareSourcesOptions appends conditional preparer options that control -// hashing and lookaside behavior. Extracted from -// [PrepareComponentSources] to keep cyclomatic complexity within limits. +// hashing and lookaside behavior. func appendPrepareSourcesOptions( - _ *azldev.Env, opts []sources.PreparerOption, options *PrepareSourcesOptions, - _ sourceproviders.ResolvedDistro, ) []sources.PreparerOption { if options.AllowNoHashes { opts = append(opts, sources.WithAllowNoHashes()) diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go index cf6ea521..a7e7bf3b 100644 --- a/internal/app/azldev/core/sources/archiveoverlays.go +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -140,7 +140,9 @@ func processArchive( }() // Extract the archive; compression is inferred from the filename extension. - if err := archive.ExtractAuto(archivePath, workDir); err != nil { + // Fail on entry types we can't repack (hardlinks, devices, ...) so they + // aren't silently dropped from the repacked archive. + if err := archive.ExtractAuto(archivePath, workDir, archive.WithErrorOnUnsupportedEntry()); err != nil { return false, fmt.Errorf("extracting archive:\n%w", err) } diff --git a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go index f6e51261..c4e970c4 100644 --- a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go +++ b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go @@ -4,13 +4,17 @@ package sources import ( + "archive/tar" + "compress/gzip" "os" + "path/filepath" + "sort" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/rootfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -88,84 +92,361 @@ func TestResolveExtractRoot(t *testing.T) { }) } -// TestArchiveFileRemove verifies that archive-scoped file-remove overlays are -// routed through the shared [applyNonSpecOverlay] machinery against the -// extract-root FS (i.e., the same code path that [processArchive] uses). -func TestArchiveFileRemove(t *testing.T) { +// TestProcessArchive_DryRunDoesNotModifyArchive verifies that, in dry-run mode, +// processArchive skips the extract/repack cycle entirely and leaves the original +// archive on disk byte-for-byte unchanged (repacking would otherwise rewrite it). +func TestProcessArchive_DryRunDoesNotModifyArchive(t *testing.T) { ctx := testctx.NewCtx() + ctx.DryRunValue = true - t.Run("deletes matching files in the extracted tree", func(t *testing.T) { - extractRoot := t.TempDir() - require.NoError(t, os.WriteFile(extractRoot+"/keep.txt", []byte("keep"), fileperms.PrivateFile)) - require.NoError(t, os.WriteFile(extractRoot+"/remove.conf", []byte("x"), fileperms.PrivateFile)) + sourcesDir := t.TempDir() - extractFS, err := rootfs.New(extractRoot) - require.NoError(t, err) + const archiveName = "pkg-1.0.tar.gz" + + archivePath := sourcesDir + "/" + archiveName + + // Content need not be a valid archive: dry-run returns before extraction, and + // the test only asserts the bytes are untouched. + original := []byte("original archive bytes") + require.NoError(t, os.WriteFile(archivePath, original, fileperms.PrivateFile)) + + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove.conf"}, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + assert.False(t, repacked, "dry-run must report that no archive was repacked") - defer extractFS.Close() + after, err := os.ReadFile(archivePath) + require.NoError(t, err) + assert.Equal(t, original, after, "dry-run must not modify the archive on disk") +} + +// stageFiles writes the given slash-separated relative path -> content map under +// root, creating parent directories as needed. +func stageFiles(t *testing.T, root string, files map[string]string) { + t.Helper() + + for rel, content := range files { + full := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(full), fileperms.PublicDir)) + require.NoError(t, os.WriteFile(full, []byte(content), fileperms.PrivateFile)) + } +} - overlay := projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayRemoveFile, - Filename: "*.conf", +// listRegularFiles returns the sorted, slash-separated relative paths of all +// regular files under root (directories are ignored). +func listRegularFiles(t *testing.T, root string) []string { + t.Helper() + + var files []string + + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err } - err = applyNonSpecOverlay(ctx, ctx.FS(), extractFS, overlay) - require.NoError(t, err) + if d.IsDir() { + return nil + } + + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + + files = append(files, filepath.ToSlash(rel)) + + return nil + })) + + sort.Strings(files) + + return files +} + +// extractedRegularFiles extracts archivePath into a fresh temp dir and returns +// the sorted relative paths of the regular files it contains. +func extractedRegularFiles(t *testing.T, archivePath string) []string { + t.Helper() + + out := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, out)) + + return listRegularFiles(t, out) +} + +// rawTarEntry describes a single entry for [buildTarGz], which writes a tarball +// directly so that entry types [archive.CreateDeterministicArchive] never emits +// (e.g. hardlinks) can be constructed for tests. +type rawTarEntry struct { + name string + typeflag byte + content string + linkname string +} - assert.FileExists(t, extractRoot+"/keep.txt") - assert.NoFileExists(t, extractRoot+"/remove.conf") +// buildTarGz writes a gzip-compressed tar archive of the given entries to path. +func buildTarGz(t *testing.T, path string, entries []rawTarEntry) { + t.Helper() + + file, err := os.Create(path) + require.NoError(t, err) + + defer func() { require.NoError(t, file.Close()) }() + + gzWriter := gzip.NewWriter(file) + tarWriter := tar.NewWriter(gzWriter) + + for _, entry := range entries { + header := &tar.Header{Name: entry.name, Typeflag: entry.typeflag, Mode: 0o644} + + switch entry.typeflag { + case tar.TypeReg: + header.Size = int64(len(entry.content)) + case tar.TypeLink, tar.TypeSymlink: + header.Linkname = entry.linkname + } + + require.NoError(t, tarWriter.WriteHeader(header)) + + if entry.typeflag == tar.TypeReg { + _, writeErr := tarWriter.Write([]byte(entry.content)) + require.NoError(t, writeErr) + } + } + + require.NoError(t, tarWriter.Close()) + require.NoError(t, gzWriter.Close()) +} + +// TestProcessArchive_AppliesMultipleOverlaysInSingleCycle verifies that two +// overlays targeting the same archive are both applied within a single +// extract/repack cycle (the [processArchive] overlay loop). +func TestProcessArchive_AppliesMultipleOverlaysInSingleCycle(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) + + // Single top-level directory so the extract root is "pkg-1.0/". + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "pkg-1.0/remove-me.conf": "junk", + "pkg-1.0/keep.conf": "keep", + "pkg-1.0/version.txt": "version = old_value", }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) - t.Run("with no match errors like a loose-file overlay", func(t *testing.T) { - extractRoot := t.TempDir() - require.NoError(t, os.WriteFile(extractRoot+"/file.txt", nil, fileperms.PrivateFile)) + // A removal and a search-replace targeting the same archive. Both must be + // applied to the same extracted tree before the single repack. + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove-me.conf"}, + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Filename: "version.txt", + Regex: "old_value", + Replacement: "new_value", + }, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + assert.True(t, repacked) - extractFS, err := rootfs.New(extractRoot) + out := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, out)) + + // Removal applied; untouched sibling preserved. + assert.NoFileExists(t, filepath.Join(out, "pkg-1.0", "remove-me.conf")) + assert.FileExists(t, filepath.Join(out, "pkg-1.0", "keep.conf")) + + // Search-replace applied in the same cycle. + content, err := os.ReadFile(filepath.Join(out, "pkg-1.0", "version.txt")) + require.NoError(t, err) + assert.Equal(t, "version = new_value", string(content)) +} + +// TestProcessArchive_ResolveExtractRootFallback drives the extract-root +// resolution through the full [processArchive] cycle, confirming that overlay +// globs are matched relative to the resolved root in both the single-top-level- +// directory case and the multiple-top-level-entries fallback. +func TestProcessArchive_ResolveExtractRootFallback(t *testing.T) { + t.Run("single top-level directory: glob is relative to that directory", func(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) + + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "pkg-1.0/remove.conf": "x", + "pkg-1.0/keep.txt": "x", + }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + + group := archiveGroup{ + archive: archiveName, + // Path is relative to the extract root (pkg-1.0/), not the archive root. + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove.conf"}, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) require.NoError(t, err) + assert.True(t, repacked) + assert.Equal(t, []string{"pkg-1.0/keep.txt"}, extractedRegularFiles(t, archivePath)) + }) + + t.Run("multiple top-level entries: glob is relative to the archive root", func(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) - defer extractFS.Close() + // Two top-level entries (no single wrapping directory) => extract root is + // the archive root, so the glob is matched there. + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "remove.conf": "x", + "keep.txt": "x", + }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) - overlay := projectconfig.ComponentOverlay{ - Type: projectconfig.ComponentOverlayRemoveFile, - Filename: "*.conf", + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove.conf"}, + }, } - err = applyNonSpecOverlay(ctx, ctx.FS(), extractFS, overlay) - require.Error(t, err) - assert.Contains(t, err.Error(), "did not match any files") + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + assert.True(t, repacked) + assert.Equal(t, []string{"keep.txt"}, extractedRegularFiles(t, archivePath)) }) } -// TestProcessArchive_DryRunDoesNotModifyArchive verifies that, in dry-run mode, -// processArchive skips the extract/repack cycle entirely and leaves the original -// archive on disk byte-for-byte unchanged (repacking would otherwise rewrite it). -func TestProcessArchive_DryRunDoesNotModifyArchive(t *testing.T) { +// TestProcessArchive_UnsupportedEntryTypeErrors verifies the data-loss guard: +// an archive containing an entry that cannot be repacked (here a hardlink) must +// fail rather than silently drop the entry, and the original archive must be +// left untouched (extraction fails before the repack runs). +func TestProcessArchive_UnsupportedEntryTypeErrors(t *testing.T) { ctx := testctx.NewCtx() - ctx.DryRunValue = true - sourcesDir := t.TempDir() - const archiveName = "pkg-1.0.tar.gz" + const archiveName = "pkg.tar.gz" - archivePath := sourcesDir + "/" + archiveName + archivePath := filepath.Join(sourcesDir, archiveName) - // Content need not be a valid archive: dry-run returns before extraction, and - // the test only asserts the bytes are untouched. - original := []byte("original archive bytes") - require.NoError(t, os.WriteFile(archivePath, original, fileperms.PrivateFile)) + // CreateDeterministicArchive never emits hardlinks, so build the tarball raw. + buildTarGz(t, archivePath, []rawTarEntry{ + {name: "pkg-1.0/real.txt", typeflag: tar.TypeReg, content: "hello"}, + {name: "pkg-1.0/hard.txt", typeflag: tar.TypeLink, linkname: "pkg-1.0/real.txt"}, + }) + + before, err := os.ReadFile(archivePath) + require.NoError(t, err) group := archiveGroup{ archive: archiveName, overlays: []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove.conf"}, + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "real.txt"}, }, } repacked, err := processArchive(ctx, sourcesDir, group) - require.NoError(t, err) - assert.False(t, repacked, "dry-run must report that no archive was repacked") + require.Error(t, err, "an unsupported (hardlink) entry must fail rather than be silently dropped") + assert.False(t, repacked) + assert.Contains(t, err.Error(), "unsupported type") + // The original archive must be byte-for-byte intact (the repack never ran). after, err := os.ReadFile(archivePath) require.NoError(t, err) - assert.Equal(t, original, after, "dry-run must not modify the archive on disk") + assert.Equal(t, before, after, "a failed extraction must not modify the source archive") +} + +// TestProcessArchive_DirectoryHandling pins two documented behaviors of +// file-remove inside an archive: emptied directories survive (file-remove never +// deletes directories), and a bare directory pattern matches nothing and errors. +func TestProcessArchive_DirectoryHandling(t *testing.T) { + t.Run("emptied directory survives file removal", func(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) + + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "pkg-1.0/sub/a.txt": "x", + "pkg-1.0/sub/b.txt": "x", + "pkg-1.0/keep.txt": "x", + }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "sub/**"}, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + assert.True(t, repacked) + + out := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, out)) + + // Every file under sub/ is gone... + assert.Equal(t, []string{"pkg-1.0/keep.txt"}, listRegularFiles(t, out)) + + // ...but the now-empty directory survives (file-remove can't delete dirs). + info, err := os.Stat(filepath.Join(out, "pkg-1.0", "sub")) + require.NoError(t, err) + assert.True(t, info.IsDir(), "emptied directory should survive file removal") + }) + + t.Run("bare directory pattern matches nothing and errors", func(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) + + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "pkg-1.0/sub/a.txt": "x", + }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + + group := archiveGroup{ + archive: archiveName, + // A bare directory name: the files-only matcher matches nothing. + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "sub"}, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) + require.Error(t, err, "removing a directory is unsupported; the pattern should match no files") + assert.False(t, repacked) + }) } diff --git a/internal/app/azldev/core/sources/archiveoverlays_test.go b/internal/app/azldev/core/sources/archiveoverlays_test.go new file mode 100644 index 00000000..4f9db6a2 --- /dev/null +++ b/internal/app/azldev/core/sources/archiveoverlays_test.go @@ -0,0 +1,441 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources_test + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// globRemovalTree returns the source tree shared by the file-remove glob tests. +// Every entry is a regular file (the glob matcher used by file-remove is +// files-only). +func globRemovalTree() []string { + return []string{ + "keep.txt", + "some_folder/a.txt", + "some_folder/b.txt", + "some_folder/sub/c.txt", + "some_folder/sub/deep/d.txt", + "top1/nested/some_file.txt", + "top1/some_file.txt", + "top2/some_file.txt", + } +} + +// globRemovalCase describes the expected outcome of applying a file-remove +// overlay with a given glob pattern against [globRemovalTree]. +type globRemovalCase struct { + name string + // pattern is the in-archive / in-sources glob (no archive prefix). + pattern string + // wantErr is true when the pattern matches no files and the overlay errors. + wantErr bool + // remaining is the sorted set of regular files left after removal (equal to + // the whole tree when wantErr is true, since nothing is removed). + remaining []string +} + +// globRemovalCases enumerates the documented file-remove glob behaviors. The +// expectations were captured against the live doublestar matcher and encode two +// behaviors worth pinning: (1) the matcher is files-only, so a bare directory +// name matches nothing, and (2) `*` matches a single path segment while `**` +// matches any depth. Directories are never removed (only files), so emptied +// directories survive a removal; these assertions look at regular files only. +func globRemovalCases() []globRemovalCase { + return []globRemovalCase{ + { + name: "bare folder name matches nothing (files-only matcher)", + pattern: "some_folder", + wantErr: true, + remaining: globRemovalTree(), + }, + { + name: "single star removes immediate file children only", + pattern: "some_folder/*", + remaining: []string{ + "keep.txt", + "some_folder/sub/c.txt", + "some_folder/sub/deep/d.txt", + "top1/nested/some_file.txt", + "top1/some_file.txt", + "top2/some_file.txt", + }, + }, + { + name: "doublestar removes all files under the folder recursively", + pattern: "some_folder/**", + remaining: []string{ + "keep.txt", + "top1/nested/some_file.txt", + "top1/some_file.txt", + "top2/some_file.txt", + }, + }, + { + name: "single-star prefix matches only top-level folders", + pattern: "*/some_file.txt", + remaining: []string{ + "keep.txt", + "some_folder/a.txt", + "some_folder/b.txt", + "some_folder/sub/c.txt", + "some_folder/sub/deep/d.txt", + "top1/nested/some_file.txt", + }, + }, + { + name: "doublestar prefix matches files at any depth", + pattern: "**/some_file.txt", + remaining: []string{ + "keep.txt", + "some_folder/a.txt", + "some_folder/b.txt", + "some_folder/sub/c.txt", + "some_folder/sub/deep/d.txt", + }, + }, + } +} + +// writeTreeFiles materializes the given slash-separated relative file paths +// under root, each with placeholder content. +func writeTreeFiles(t *testing.T, root string, files []string) { + t.Helper() + + for _, f := range files { + full := filepath.Join(root, filepath.FromSlash(f)) + require.NoError(t, os.MkdirAll(filepath.Dir(full), fileperms.PublicDir)) + require.NoError(t, os.WriteFile(full, []byte("x"), fileperms.PrivateFile)) + } +} + +// collectRegularFiles returns the sorted, slash-separated relative paths of all +// regular files under root (directories are ignored). +func collectRegularFiles(t *testing.T, root string) []string { + t.Helper() + + var out []string + + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + + out = append(out, filepath.ToSlash(rel)) + + return nil + })) + + sort.Strings(out) + + return out +} + +// TestPrepareSources_RemoveFileGlob_Archive exercises archive-scoped file-remove +// globs through the real exported [sources.SourcePreparer.PrepareSources] path +// (which drives the extract/remove/repack cycle), asserting which files survive +// inside the repacked archive for each glob pattern. +func TestPrepareSources_RemoveFileGlob_Archive(t *testing.T) { + const ( + componentName = "test-component" + archiveName = "pkg.tar.gz" + ) + + for _, testCase := range globRemovalCases() { + t.Run(testCase.name, func(t *testing.T) { + // Host FS + real temp dir: archive extraction/repacking happens on disk. + ctx := testctx.NewCtx(testctx.WithHostFS()) + outputDir := t.TempDir() + archivePath := filepath.Join(outputDir, archiveName) + + // Stage a tree with multiple top-level entries so the extraction root + // is the archive root and the glob is matched relative to it. + staging := t.TempDir() + writeTreeFiles(t, staging, globRemovalTree()) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + + // Seed a 'sources' entry for the archive so the post-overlay rehash + // has an entry to update (a missing entry is itself an error). + originalHash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, archivePath) + require.NoError(t, err) + + sourcesPath := filepath.Join(outputDir, fedorasource.SourcesFileName) + entry := fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA256, originalHash) + require.NoError(t, fileutils.WriteFile( + ctx.FS(), sourcesPath, []byte(entry+"\n"), fileperms.PublicFile)) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) + + component.EXPECT().GetName().AnyTimes().Return(componentName) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlayRemoveFile, + Filename: archiveName + "/" + testCase.pattern, + }, + }, + }) + + sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).Return(nil) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile( + ctx.FS(), filepath.Join(dir, componentName+".spec"), + []byte("# test spec"), fileperms.PublicFile) + }, + ) + + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) + require.NoError(t, err) + + err = preparer.PrepareSources(ctx, component, outputDir, true /*applyOverlays*/) + + if testCase.wantErr { + // A no-match pattern errors and leaves the archive untouched. + require.Error(t, err) + + return + } + + require.NoError(t, err) + + out := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, out)) + assert.Equal(t, testCase.remaining, collectRegularFiles(t, out)) + }) + } +} + +// TestApplyOverlayToSources_RemoveFileGlob_LooseFiles exercises the same glob +// patterns against loose files in the sources tree (no archive prefix) through +// the real exported [sources.ApplyOverlayToSources] entry point, confirming +// archive-scoped and loose-file removal share identical glob semantics. +func TestApplyOverlayToSources_RemoveFileGlob_LooseFiles(t *testing.T) { + for _, testCase := range globRemovalCases() { + t.Run(testCase.name, func(t *testing.T) { + ctx := testctx.NewCtx(testctx.WithHostFS()) + sourcesDir := t.TempDir() + writeTreeFiles(t, sourcesDir, globRemovalTree()) + + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, + Filename: testCase.pattern, + } + + // file-remove does not touch the spec, so specPath is unused. + err := sources.ApplyOverlayToSources(ctx, ctx.FS(), overlay, sourcesDir, "") + + if testCase.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + assert.Equal(t, testCase.remaining, collectRegularFiles(t, sourcesDir)) + }) + } +} + +// findSourcesEntry returns the parsed 'sources' entry for filename, or nil. +func findSourcesEntry(t *testing.T, sourcesContent, filename string) *fedorasource.SourcesFileEntry { + t.Helper() + + parsed, err := fedorasource.ReadSourcesFile(sourcesContent) + require.NoError(t, err) + + for i := range parsed { + if parsed[i].Entry != nil && parsed[i].Entry.Filename == filename { + return parsed[i].Entry + } + } + + return nil +} + +// TestPrepareSources_SearchReplaceInArchiveRehashesEntry is an end-to-end check +// that a file-search-replace overlay scoped to an archive rewrites the file +// inside the archive, repacks it, and rehashes the matching 'sources' entry +// (preserving the original hash type). file-remove already has this coverage; +// this exercises the search-replace path through the exported PrepareSources. +func TestPrepareSources_SearchReplaceInArchiveRehashesEntry(t *testing.T) { + const ( + componentName = "test-component" + archiveName = "pkg.tar.gz" + ) + + // Host FS + real temp dir: archive extraction/repacking happens on disk. + ctx := testctx.NewCtx(testctx.WithHostFS()) + outputDir := t.TempDir() + archivePath := filepath.Join(outputDir, archiveName) + + // Single top-level directory => extract root is "pkg-1.0/", so the inner + // path "configure.ac" resolves relative to that root. + staging := t.TempDir() + pkgRoot := filepath.Join(staging, "pkg-1.0") + require.NoError(t, os.MkdirAll(pkgRoot, fileperms.PublicDir)) + require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "configure.ac"), + []byte("AC_CHECK_LIB(old_lib, main)\n"), fileperms.PrivateFile)) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + + // Seed a SHA256 'sources' entry (not the SHA512 default) so the test also + // proves the hash type is preserved. + originalHash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, archivePath) + require.NoError(t, err) + + sourcesPath := filepath.Join(outputDir, fedorasource.SourcesFileName) + entry := fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA256, originalHash) + require.NoError(t, fileutils.WriteFile(ctx.FS(), sourcesPath, []byte(entry+"\n"), fileperms.PublicFile)) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) + + component.EXPECT().GetName().AnyTimes().Return(componentName) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{ + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Filename: archiveName + "/configure.ac", + Regex: "old_lib", + Replacement: "new_lib", + }, + }, + }) + + sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).Return(nil) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(ctx.FS(), filepath.Join(dir, componentName+".spec"), + []byte("# test spec"), fileperms.PublicFile) + }, + ) + + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) + require.NoError(t, err) + + require.NoError(t, preparer.PrepareSources(ctx, component, outputDir, true /*applyOverlays*/)) + + // Rewrite: the file content inside the repacked archive reflects the replacement. + out := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, out)) + content, err := os.ReadFile(filepath.Join(out, "pkg-1.0", "configure.ac")) + require.NoError(t, err) + assert.Equal(t, "AC_CHECK_LIB(new_lib, main)\n", string(content)) + + // Repack: the archive's hash changed. + repackedHash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, archivePath) + require.NoError(t, err) + require.NotEqual(t, originalHash, repackedHash, + "precondition: rewriting a file in the archive should change its hash") + + // Rehash: the 'sources' entry was rewritten to the repacked hash, type preserved. + sourcesContent, err := fileutils.ReadFile(ctx.FS(), sourcesPath) + require.NoError(t, err) + + got := findSourcesEntry(t, string(sourcesContent), archiveName) + require.NotNil(t, got, "rewritten 'sources' file should still contain an entry for %q", archiveName) + assert.Equal(t, fileutils.HashTypeSHA256, got.HashType, "original hash type must be preserved") + assert.Equal(t, repackedHash, got.Hash, "'sources' entry must record the repacked archive's hash") +} + +// TestPrepareSources_SkipSourcesSkipsArchiveOverlays verifies the --skip-sources +// branch: when source downloads are skipped, archive overlays are skipped (with a +// warning) instead of applied, leaving both the archive and its 'sources' entry +// untouched. +func TestPrepareSources_SkipSourcesSkipsArchiveOverlays(t *testing.T) { + const ( + componentName = "test-component" + archiveName = "pkg.tar.gz" + ) + + ctx := testctx.NewCtx(testctx.WithHostFS()) + outputDir := t.TempDir() + archivePath := filepath.Join(outputDir, archiveName) + + staging := t.TempDir() + pkgRoot := filepath.Join(staging, "pkg-1.0") + require.NoError(t, os.MkdirAll(pkgRoot, fileperms.PublicDir)) + require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "remove-me.txt"), + []byte("delete me"), fileperms.PrivateFile)) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + + // Snapshot the archive and seed its 'sources' entry so we can assert both + // are left untouched. + originalArchive, err := os.ReadFile(archivePath) + require.NoError(t, err) + + originalHash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, archivePath) + require.NoError(t, err) + + sourcesPath := filepath.Join(outputDir, fedorasource.SourcesFileName) + entry := fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA256, originalHash) + require.NoError(t, fileutils.WriteFile(ctx.FS(), sourcesPath, []byte(entry+"\n"), fileperms.PublicFile)) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) + + component.EXPECT().GetName().AnyTimes().Return(componentName) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: archiveName + "/remove-me.txt"}, + }, + }) + + // With --skip-sources, FetchFiles must NOT be called (no EXPECT for it); + // FetchComponent is still called to provide the spec. + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(ctx.FS(), filepath.Join(dir, componentName+".spec"), + []byte("# test spec"), fileperms.PublicFile) + }, + ) + + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx, sources.WithSkipLookaside()) + require.NoError(t, err) + + require.NoError(t, preparer.PrepareSources(ctx, component, outputDir, true /*applyOverlays*/)) + + // The archive overlay is skipped: the archive is byte-for-byte unchanged... + afterArchive, err := os.ReadFile(archivePath) + require.NoError(t, err) + assert.Equal(t, originalArchive, afterArchive, + "archive must not be modified when --skip-sources skips archive overlays") + + // ...and its 'sources' entry keeps the original hash. + sourcesContent, err := fileutils.ReadFile(ctx.FS(), sourcesPath) + require.NoError(t, err) + + got := findSourcesEntry(t, string(sourcesContent), archiveName) + require.NotNil(t, got) + assert.Equal(t, originalHash, got.Hash, "'sources' entry hash must be unchanged when overlays are skipped") +} diff --git a/internal/app/azldev/core/sources/overlays.go b/internal/app/azldev/core/sources/overlays.go index cfc23069..8d45ad03 100644 --- a/internal/app/azldev/core/sources/overlays.go +++ b/internal/app/azldev/core/sources/overlays.go @@ -53,8 +53,8 @@ func ApplyOverlayToSources( } } - // Apply the non-spec file component, if any. - if !overlay.ModifiesNonSpecFiles() { + // Apply the loose-file component, if any. + if !overlay.ModifiesLooseFiles() { return nil } diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 44401519..35522a66 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -134,7 +134,7 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { require.NoError(t, os.MkdirAll(pkgRoot, fileperms.PublicDir)) require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "keep.txt"), []byte("keep me"), fileperms.PrivateFile)) require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "remove-me.txt"), []byte("delete me"), fileperms.PrivateFile)) - require.NoError(t, archive.CreateDeterministicArchiveAuto(archivePath, stagingDir)) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, stagingDir, archive.CompressionGzip)) // Record the original hash of the archive and seed a 'sources' file with it. // Use SHA256 (not the SHA512 default) so the test also proves the hash *type* @@ -231,7 +231,7 @@ func TestPrepareSources_ArchiveOverlayMissingSourcesEntryErrors(t *testing.T) { pkgRoot := filepath.Join(stagingDir, "pkg-1.0") require.NoError(t, os.MkdirAll(pkgRoot, fileperms.PublicDir)) require.NoError(t, os.WriteFile(filepath.Join(pkgRoot, "remove-me.txt"), []byte("delete me"), fileperms.PrivateFile)) - require.NoError(t, archive.CreateDeterministicArchiveAuto(archivePath, stagingDir)) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, stagingDir, archive.CompressionGzip)) // 'sources' file references some unrelated file, not the archive being modified. require.NoError(t, fileutils.WriteFile( diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index 5d43e5bd..f85a9d50 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -161,11 +161,13 @@ func (c *ComponentOverlay) ModifiesArchive() bool { return ok } -// ModifiesNonSpecFiles returns true if the overlay modifies non-spec files. This includes -// hybrid overlays that modify both spec and source files (e.g., patch overlays), since -// those also require non-spec modifications. Archive-scoped overlays (see [ModifiesArchive]) -// are excluded: they operate on files inside an archive, not loose files in the sources tree. -func (c *ComponentOverlay) ModifiesNonSpecFiles() bool { +// ModifiesLooseFiles returns true if the overlay modifies loose files in the +// sources tree (as opposed to the spec or files inside an archive). This includes +// hybrid overlays that modify both the spec and loose source files (e.g., patch +// overlays), since those also require loose-file modifications. Archive-scoped +// overlays (see [ModifiesArchive]) are excluded: they operate on files inside an +// archive, not loose files in the sources tree. +func (c *ComponentOverlay) ModifiesLooseFiles() bool { if c.ModifiesArchive() { return false } diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index 7600c011..0fb19f7d 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -527,8 +527,8 @@ func TestComponentOverlay_ModifiesSpec(t *testing.T) { t.Run(string(overlay.Type)+"_is_archive_scoped", func(t *testing.T) { assert.True(t, overlay.ModifiesArchive(), "expected %s to be an archive-scoped overlay", overlay.Type) assert.False(t, overlay.ModifiesSpec(), "expected %s to not be a spec overlay", overlay.Type) - assert.False(t, overlay.ModifiesNonSpecFiles(), - "expected %s to not be a loose non-spec overlay", overlay.Type) + assert.False(t, overlay.ModifiesLooseFiles(), + "expected %s to not be a loose-file overlay", overlay.Type) }) } } diff --git a/internal/utils/archive/archive.go b/internal/utils/archive/archive.go index 32595d1e..a4a30dcc 100644 --- a/internal/utils/archive/archive.go +++ b/internal/utils/archive/archive.go @@ -11,6 +11,11 @@ // are not policed by [os.Root]; this package additionally rejects any symlink // whose target is non-local via [filepath.IsLocal]. // +// Only regular files, directories, and symlinks are extracted. Other entry +// types (hardlinks, devices, FIFOs, etc.) are skipped by default, or cause a +// failure when [WithErrorOnUnsupportedEntry] is set — callers that repack the +// tree must set it so such entries aren't silently dropped. +// // Archive creation is designed for reproducible builds: file ordering is // lexicographic, timestamps are pinned to Unix epoch, and owner/group metadata // is zeroed out. This matches the @@ -100,22 +105,47 @@ func IsArchiveName(filename string) bool { // Most callers should prefer this over the explicit-compression [Extract], // which exists for cases where the compression cannot be derived from the // filename. -func ExtractAuto(archivePath, destDir string) error { +func ExtractAuto(archivePath, destDir string, opts ...ExtractOption) error { comp, err := DetectCompression(archivePath) if err != nil { return fmt.Errorf("detecting compression for %#q:\n%w", archivePath, err) } - return Extract(archivePath, destDir, comp) + return Extract(archivePath, destDir, comp, opts...) +} + +// ExtractOption configures the behavior of [Extract] and [ExtractAuto]. +type ExtractOption func(*extractConfig) + +// extractConfig holds the resolved options for an extraction. +type extractConfig struct { + errorOnUnsupportedEntry bool +} + +// WithErrorOnUnsupportedEntry makes [Extract] fail on tar entries it cannot +// materialize (anything but a regular file, directory, or symlink). Without it +// such entries are skipped; callers that repack the tree should set it so the +// entries aren't silently dropped from the rebuilt archive. +func WithErrorOnUnsupportedEntry() ExtractOption { + return func(c *extractConfig) { + c.errorOnUnsupportedEntry = true + } } // Extract reads a tar archive, decompresses it, and extracts all entries into -// destDir. Supported entry types are regular files, directories, and symlinks; -// other entry types are skipped. Entry paths are confined to destDir via -// [os.Root]: any path that would escape destDir is rejected by the runtime. -// Symlink targets are validated separately by this package — see the package -// doc for details. -func Extract(archivePath, destDir string, comp Compression) (err error) { +// destDir. Entry paths are confined to destDir via [os.Root]: any path that +// would escape destDir is rejected by the runtime. Symlink targets are +// validated separately by this package — see the package doc for details. +// +// Only regular files, directories, and symlinks are supported. Other entry +// types are skipped by default, or fail when [WithErrorOnUnsupportedEntry] is +// set (required by callers that repack the tree). +func Extract(archivePath, destDir string, comp Compression, opts ...ExtractOption) (err error) { + var cfg extractConfig + for _, opt := range opts { + opt(&cfg) + } + if err := os.MkdirAll(destDir, fileperms.PublicDir); err != nil { return fmt.Errorf("creating destination %#q:\n%w", destDir, err) } @@ -159,7 +189,7 @@ func Extract(archivePath, destDir string, comp Compression) (err error) { return fmt.Errorf("reading tar entry from %#q:\n%w", archivePath, readErr) } - if err := extractEntry(root, header, tarReader); err != nil { + if err := extractEntry(root, header, tarReader, cfg); err != nil { return fmt.Errorf("extracting %#q from %#q:\n%w", header.Name, archivePath, err) } } @@ -233,19 +263,6 @@ func (r readerCloser) Close() error { return nil } -// CreateDeterministicArchiveAuto is a convenience wrapper that infers the -// compression from archivePath's extension via [DetectCompression] and then -// calls [CreateDeterministicArchive]. Most callers should prefer this over the -// explicit-compression [CreateDeterministicArchive]. -func CreateDeterministicArchiveAuto(archivePath, sourceDir string) error { - comp, err := DetectCompression(archivePath) - if err != nil { - return fmt.Errorf("detecting compression for %#q:\n%w", archivePath, err) - } - - return CreateDeterministicArchive(archivePath, sourceDir, comp) -} - // CreateDeterministicArchive creates a new tar archive from the contents of sourceDir // and writes it to archivePath on the OS filesystem, replacing any existing file. // @@ -307,7 +324,7 @@ func deterministicEpoch() time.Time { return time.Unix(0, 0).UTC() } -func extractEntry(root *os.Root, header *tar.Header, tarReader io.Reader) error { +func extractEntry(root *os.Root, header *tar.Header, tarReader io.Reader, cfg extractConfig) error { name := header.Name if header.Typeflag == tar.TypeDir { @@ -344,6 +361,14 @@ func extractEntry(root *os.Root, header *tar.Header, tarReader io.Reader) error case tar.TypeReg: return extractRegularFile(root, header, tarReader) default: + // Unsupported entry type (hardlink, device, FIFO, ...): fail when the + // caller is strict (repacking would drop it), otherwise skip and log. + if cfg.errorOnUnsupportedEntry { + return fmt.Errorf( + "tar entry %#q has unsupported type (typeflag %d); only regular files, directories, and symlinks are supported", + name, header.Typeflag) + } + slog.Debug("Skipping unsupported tar entry type", "name", name, "typeflag", header.Typeflag) return nil diff --git a/internal/utils/archive/archive_test.go b/internal/utils/archive/archive_test.go index 4fe9cc37..8b32763e 100644 --- a/internal/utils/archive/archive_test.go +++ b/internal/utils/archive/archive_test.go @@ -121,6 +121,8 @@ func createTestTarGz(t *testing.T, path string, entries []testTarEntry) { header.Size = int64(len(entry.content)) case tar.TypeSymlink: header.Linkname = entry.linkname + case tar.TypeLink: + header.Linkname = entry.linkname } require.NoError(t, tarWriter.WriteHeader(header)) @@ -250,6 +252,48 @@ func TestUnsupportedCompression(t *testing.T) { assert.Contains(t, err.Error(), "unsupported compression type") } +// TestExtract_UnsupportedEntryType verifies the handling of tar entries whose +// type cannot be materialized (here, a hardlink). By default such entries are +// skipped; with [archive.WithErrorOnUnsupportedEntry] they cause a failure so +// callers that repack the tree don't silently drop them. +func TestExtract_UnsupportedEntryType(t *testing.T) { + makeArchive := func(t *testing.T) string { + t.Helper() + + archivePath := filepath.Join(t.TempDir(), "pkg.tar.gz") + createTestTarGz(t, archivePath, []testTarEntry{ + {name: "pkg/real.txt", typeflag: tar.TypeReg, content: "hello"}, + // A hardlink to the regular file above: a valid tar entry type that + // Extract cannot materialize. + {name: "pkg/link.txt", typeflag: tar.TypeLink, linkname: "pkg/real.txt"}, + }) + + return archivePath + } + + t.Run("skipped by default", func(t *testing.T) { + extractDir := filepath.Join(t.TempDir(), "out") + + require.NoError(t, archive.Extract(makeArchive(t), extractDir, archive.CompressionGzip)) + + // The regular file is extracted; the unsupported hardlink is skipped. + assert.FileExists(t, filepath.Join(extractDir, "pkg", "real.txt")) + assert.NoFileExists(t, filepath.Join(extractDir, "pkg", "link.txt")) + }) + + t.Run("errors with WithErrorOnUnsupportedEntry", func(t *testing.T) { + extractDir := filepath.Join(t.TempDir(), "out") + + err := archive.Extract( + makeArchive(t), extractDir, archive.CompressionGzip, + archive.WithErrorOnUnsupportedEntry(), + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported type") + assert.Contains(t, err.Error(), "link.txt") + }) +} + func TestCreateDeterministicArchive_PreservesSymlinks(t *testing.T) { tmpDir := t.TempDir() sourceDir := filepath.Join(tmpDir, "src") From c244d0c0fbd32d106a11bac0b97785e4b6c94092 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 18 Jun 2026 20:56:41 +0000 Subject: [PATCH 11/13] Except globalheaders & mutation gap tests --- .../azldev/core/sources/archiveoverlays.go | 12 +- .../sources/archiveoverlays_internal_test.go | 104 ++++++++++++++++ internal/utils/archive/archive.go | 26 ++++ internal/utils/archive/archive_test.go | 112 ++++++++++++++++++ 4 files changed, 253 insertions(+), 1 deletion(-) diff --git a/internal/app/azldev/core/sources/archiveoverlays.go b/internal/app/azldev/core/sources/archiveoverlays.go index a7e7bf3b..25890112 100644 --- a/internal/app/azldev/core/sources/archiveoverlays.go +++ b/internal/app/azldev/core/sources/archiveoverlays.go @@ -195,11 +195,17 @@ func processArchive( // over archivePath would truncate it first, leaving the workspace unrecoverable // without refetching if the repack then failed. func repackArchiveAtomic(archivePath, archiveName, workDir string) (err error) { - comp, err := archive.DetectCompression(archiveName) + // Fall back to the extension only if the archive is empty (nothing to sniff). + extComp, err := archive.DetectCompression(archiveName) if err != nil { return fmt.Errorf("detecting compression for %#q:\n%w", archiveName, err) } + comp, err := archive.SniffCompressionFromFile(archivePath, extComp) + if err != nil { + return fmt.Errorf("sniffing compression for %#q:\n%w", archiveName, err) + } + tmpFile, err := os.CreateTemp(filepath.Dir(archivePath), "."+filepath.Base(archiveName)+".repack-*") if err != nil { return fmt.Errorf("creating temp archive:\n%w", err) @@ -214,6 +220,10 @@ func repackArchiveAtomic(archivePath, archiveName, workDir string) (err error) { return fmt.Errorf("closing temp archive %#q:\n%w", tmpPath, closeErr) } + if removeErr := os.Remove(tmpPath); removeErr != nil && !os.IsNotExist(removeErr) { + return fmt.Errorf("removing temp archive placeholder %#q:\n%w", tmpPath, removeErr) + } + // Clean up the temp file unless it was successfully renamed over the original. repackedOK := false diff --git a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go index c4e970c4..609c9c09 100644 --- a/internal/app/azldev/core/sources/archiveoverlays_internal_test.go +++ b/internal/app/azldev/core/sources/archiveoverlays_internal_test.go @@ -450,3 +450,107 @@ func TestProcessArchive_DirectoryHandling(t *testing.T) { assert.False(t, repacked) }) } + +// TestProcessArchive_PreservesMislabeledFormatOnRepack verifies that when an +// archive's extension lies about its real format (here a plain, uncompressed tar +// named ".tar.gz"), the repacked archive keeps the real on-disk format (plain +// tar) rather than being "healed" to match the extension. The misleading name is +// preserved; only the contents change. +func TestProcessArchive_PreservesMislabeledFormatOnRepack(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + // Misleading name: ".tar.gz" extension, but written as an uncompressed tar. + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) + + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "pkg-1.0/remove-me.txt": "junk", + "pkg-1.0/keep.txt": "keep", + }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionNone)) + + // Precondition: the file really is an uncompressed tar despite its name. + preComp, err := archive.SniffCompressionFromFile(archivePath, archive.CompressionGzip) + require.NoError(t, err) + require.Equal(t, archive.CompressionNone, preComp) + + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove-me.txt"}, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + assert.True(t, repacked) + + // The overlay was applied (file removed)... + assert.Equal(t, []string{"pkg-1.0/keep.txt"}, extractedRegularFiles(t, archivePath)) + + // ...and the repacked archive preserves the real format: still an uncompressed + // tar, NOT re-compressed to gzip to match the ".tar.gz" extension. + postComp, err := archive.SniffCompressionFromFile(archivePath, archive.CompressionGzip) + require.NoError(t, err) + assert.Equal(t, archive.CompressionNone, postComp, + "repack must preserve the original (sniffed) format, keeping the misleading extension") +} + +// TestProcessArchive_RepackedArchiveHasNormalPermissions is a regression test: +// the repack writes to an os.CreateTemp placeholder (mode 0600) before renaming +// it over the original. Because CreateDeterministicArchive only truncates an +// existing file, reusing that placeholder would leave the repacked archive at +// 0600. processArchive removes the placeholder first so the archive is recreated +// with normal permissions; assert it does not regress to 0600. +func TestProcessArchive_RepackedArchiveHasNormalPermissions(t *testing.T) { + ctx := testctx.NewCtx() + sourcesDir := t.TempDir() + + const archiveName = "pkg.tar.gz" + + archivePath := filepath.Join(sourcesDir, archiveName) + + staging := t.TempDir() + stageFiles(t, staging, map[string]string{ + "pkg-1.0/remove-me.txt": "junk", + "pkg-1.0/keep.txt": "keep", + }) + require.NoError(t, archive.CreateDeterministicArchive(archivePath, staging, archive.CompressionGzip)) + // Seed the original archive at the restrictive 0600 so the test would fail if + // the repack inherited the original's (or the temp placeholder's) mode instead + // of recreating the file with normal permissions. + require.NoError(t, os.Chmod(archivePath, 0o600)) + + group := archiveGroup{ + archive: archiveName, + overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayRemoveFile, Filename: "remove-me.txt"}, + }, + } + + repacked, err := processArchive(ctx, sourcesDir, group) + require.NoError(t, err) + require.True(t, repacked) + + // Compare against a freshly os.Create'd file in the same dir: that is exactly + // the mode a normal (non-placeholder) create yields under this umask. This is + // umask-independent and proves the archive was recreated rather than inheriting + // the 0600 placeholder. + refPath := filepath.Join(sourcesDir, "reference") + refFile, err := os.Create(refPath) + require.NoError(t, err) + require.NoError(t, refFile.Close()) + + refInfo, err := os.Stat(refPath) + require.NoError(t, err) + + info, err := os.Stat(archivePath) + require.NoError(t, err) + assert.Equal(t, refInfo.Mode().Perm(), info.Mode().Perm(), + "repacked archive must have normal create permissions, not the 0600 temp-file mode") + assert.NotEqual(t, os.FileMode(0o600), info.Mode().Perm(), + "repacked archive must not inherit the 0600 temp-file permissions") +} diff --git a/internal/utils/archive/archive.go b/internal/utils/archive/archive.go index a4a30dcc..39497261 100644 --- a/internal/utils/archive/archive.go +++ b/internal/utils/archive/archive.go @@ -218,6 +218,28 @@ func sniffCompression(magic []byte, fallback Compression) Compression { } } +// SniffCompressionFromFile determines an existing archive's compression by +// inspecting its leading magic bytes — authoritative over the filename +// extension. +func SniffCompressionFromFile(archivePath string, fallback Compression) (comp Compression, err error) { + file, err := os.Open(archivePath) + if err != nil { + return fallback, fmt.Errorf("opening %#q for compression sniffing:\n%w", archivePath, err) + } + defer defers.HandleDeferError(file.Close, &err) + + magic := make([]byte, compressionMagicLen) + + bytesRead, readErr := io.ReadFull(file, magic) + // Short files are fine — a tiny tar may be under compressionMagicLen bytes. + // Only a genuine read error (not EOF/short read) is fatal. + if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { + return fallback, fmt.Errorf("reading %#q header:\n%w", archivePath, readErr) + } + + return sniffCompression(magic[:bytesRead], fallback), nil +} + // newDecompressor wraps reader in the chosen decompressor. For // [CompressionNone] the returned closer is nil; otherwise it is the // decompressor itself. @@ -327,6 +349,10 @@ func deterministicEpoch() time.Time { func extractEntry(root *os.Root, header *tar.Header, tarReader io.Reader, cfg extractConfig) error { name := header.Name + if header.Typeflag == tar.TypeXGlobalHeader || header.Typeflag == tar.TypeXHeader { + return nil + } + if header.Typeflag == tar.TypeDir { if err := root.MkdirAll(name, fileperms.PublicDir); err != nil { return fmt.Errorf("creating directory %#q:\n%w", name, err) diff --git a/internal/utils/archive/archive_test.go b/internal/utils/archive/archive_test.go index 8b32763e..082e0b03 100644 --- a/internal/utils/archive/archive_test.go +++ b/internal/utils/archive/archive_test.go @@ -52,6 +52,63 @@ func TestDetectCompression(t *testing.T) { } } +// TestSniffCompressionFromFile verifies that compression is detected from the +// file's real magic bytes, ignoring (and overriding) the filename extension. +func TestSniffCompressionFromFile(t *testing.T) { + tmpDir := t.TempDir() + + // Write a real archive of each format under a deliberately misleading ".txz" + // name, then assert the sniffer reports the real format, not what the name claims. + write := func(t *testing.T, name string, comp archive.Compression) string { + t.Helper() + + src := filepath.Join(t.TempDir(), "src") + require.NoError(t, os.MkdirAll(src, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("hi"), 0o600)) + + path := filepath.Join(tmpDir, name) + require.NoError(t, archive.CreateDeterministicArchive(path, src, comp)) + + return path + } + + tests := []struct { + name string + comp archive.Compression + }{ + {"plain-tar-named.txz", archive.CompressionNone}, + {"gzip-named.txz", archive.CompressionGzip}, + {"xz-named.tar", archive.CompressionXZ}, + {"zstd-named.tgz", archive.CompressionZstd}, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + path := write(t, testCase.name, testCase.comp) + + // Fallback is deliberately a wrong value to prove it is never used + // for a non-empty file. + got, err := archive.SniffCompressionFromFile(path, archive.CompressionGzip) + require.NoError(t, err) + assert.Equal(t, testCase.comp, got) + }) + } + + t.Run("empty file uses fallback", func(t *testing.T) { + path := filepath.Join(tmpDir, "empty.txz") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + got, err := archive.SniffCompressionFromFile(path, archive.CompressionXZ) + require.NoError(t, err) + assert.Equal(t, archive.CompressionXZ, got) + }) + + t.Run("missing file errors", func(t *testing.T) { + _, err := archive.SniffCompressionFromFile(filepath.Join(tmpDir, "nope.tar"), archive.CompressionNone) + require.Error(t, err) + }) +} + func TestExtractAndRepack(t *testing.T) { tmpDir := t.TempDir() @@ -294,6 +351,61 @@ func TestExtract_UnsupportedEntryType(t *testing.T) { }) } +// TestExtract_PaxGlobalHeaderIgnored is a regression test: git-generated +// tarballs (e.g. GitHub archives) begin with a "pax_global_header" entry +// (typeflag 'g'). It is tar metadata, not file content, so it must be skipped +// even under [archive.WithErrorOnUnsupportedEntry] — otherwise repack-strict +// callers cannot extract any git-generated source archive. +func TestExtract_PaxGlobalHeaderIgnored(t *testing.T) { + archivePath := filepath.Join(t.TempDir(), "pkg.tar.gz") + + file, err := os.Create(archivePath) + require.NoError(t, err) + + gzWriter := gzip.NewWriter(file) + tarWriter := tar.NewWriter(gzWriter) + + // Leading PAX global header carrying a commit hash, exactly as `git archive` + // emits. Go names it "pax_global_header" on read-back. + require.NoError(t, tarWriter.WriteHeader(&tar.Header{ + Typeflag: tar.TypeXGlobalHeader, + Name: "pax_global_header", + PAXRecords: map[string]string{ + "comment": "0123456789abcdef0123456789abcdef01234567", + }, + })) + + const fileContent = "hello" + + require.NoError(t, tarWriter.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: "pkg-1.0/real.txt", + Mode: 0o644, + Size: int64(len(fileContent)), + })) + _, err = tarWriter.Write([]byte(fileContent)) + require.NoError(t, err) + + require.NoError(t, tarWriter.Close()) + require.NoError(t, gzWriter.Close()) + require.NoError(t, file.Close()) + + extractDir := filepath.Join(t.TempDir(), "out") + + // Strict mode must succeed: the global header is skipped, the file extracted. + require.NoError(t, archive.Extract( + archivePath, extractDir, archive.CompressionGzip, + archive.WithErrorOnUnsupportedEntry(), + )) + + content, err := os.ReadFile(filepath.Join(extractDir, "pkg-1.0", "real.txt")) + require.NoError(t, err) + assert.Equal(t, fileContent, string(content)) + + // The pax header must not have produced a file on disk. + assert.NoFileExists(t, filepath.Join(extractDir, "pax_global_header")) +} + func TestCreateDeterministicArchive_PreservesSymlinks(t *testing.T) { tmpDir := t.TempDir() sourceDir := filepath.Join(tmpDir, "src") From 1a9d1b97219981150ee72ef74f15e0c465c0bd23 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Tue, 23 Jun 2026 20:59:03 +0000 Subject: [PATCH 12/13] Added archive overlay validation + tests --- internal/projectconfig/overlay.go | 25 ++++++++++++-- internal/projectconfig/overlay_test.go | 48 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index f85a9d50..b5ab3404 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -147,12 +147,21 @@ func (c *ComponentOverlay) ArchiveTarget() (archiveName, innerPath string, ok bo return before, after, true } +// SupportsArchiveScope reports whether the overlay type can target files inside a source +// archive via an archive-scoped [ComponentOverlay.Filename] (see [ComponentOverlay.ArchiveTarget]). +// Only file-remove and file-search-replace extract and repack archives; all other types treat +// the path as loose files in the sources tree. +func (c *ComponentOverlay) SupportsArchiveScope() bool { + return c.Type == ComponentOverlayRemoveFile || c.Type == ComponentOverlaySearchAndReplaceInFile +} + // ModifiesArchive returns true if the overlay modifies files inside a source archive. // These overlays require extraction and repacking of the archive. Only file-remove and -// file-search-replace support archive scoping, and only when their [ComponentOverlay.Filename] -// is an archive-scoped path (see [ComponentOverlay.ArchiveTarget]). +// file-search-replace support archive scoping (see [ComponentOverlay.SupportsArchiveScope]), +// and only when their [ComponentOverlay.Filename] is an archive-scoped path +// (see [ComponentOverlay.ArchiveTarget]). func (c *ComponentOverlay) ModifiesArchive() bool { - if c.Type != ComponentOverlayRemoveFile && c.Type != ComponentOverlaySearchAndReplaceInFile { + if !c.SupportsArchiveScope() { return false } @@ -284,6 +293,16 @@ func (c *ComponentOverlay) Validate() error { return nil } + // Archive-scoped file paths (e.g. "pkg-1.0.tar.gz/vendor/**") are only meaningful for overlay + // types that extract and repack archives. Reject them early for every other type so a stray + // archive prefix isn't silently misinterpreted as a directory name in the loose sources tree. + if _, _, ok := c.ArchiveTarget(); ok && !c.SupportsArchiveScope() { + return fmt.Errorf( + "overlay type %#q does not support archive-scoped file paths; found %#q", + c.Type, c.Filename, + ) + } + switch c.Type { case ComponentOverlayAddSpecTag, ComponentOverlayInsertSpecTag, ComponentOverlaySetSpecTag, ComponentOverlayUpdateSpecTag: diff --git a/internal/projectconfig/overlay_test.go b/internal/projectconfig/overlay_test.go index 0fb19f7d..287f04cd 100644 --- a/internal/projectconfig/overlay_test.go +++ b/internal/projectconfig/overlay_test.go @@ -262,6 +262,54 @@ func TestComponentOverlay_Validate(t *testing.T) { errorExpected: true, errorContains: "source", }, + // Archive-scoped path validation: only file-remove and file-search-replace support it. + { + name: "file-remove archive-scoped path accepted", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, + Filename: "pkg-1.0.tar.gz/vendor/**", + }, + errorExpected: false, + }, + { + name: "file-search-replace archive-scoped path accepted", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Filename: "pkg-1.0.tar.gz/vendor/config.h", + Regex: "old", + Replacement: "new", + }, + errorExpected: false, + }, + { + name: "file-add archive-scoped path rejected", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayAddFile, + Filename: "pkg-1.0.tar.gz/vendor/new.txt", + Source: "/path/to/source.txt", + }, + errorExpected: true, + errorContains: "archive-scoped", + }, + { + name: "file-prepend-lines archive-scoped path rejected", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayPrependLinesToFile, + Filename: "pkg-1.0.tar.gz/vendor/config.h", + Lines: []string{"// header"}, + }, + errorExpected: true, + errorContains: "archive-scoped", + }, + { + name: "patch-remove archive-scoped path rejected", + overlay: projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemovePatch, + Filename: "pkg-1.0.tar.gz/fix.patch", + }, + errorExpected: true, + errorContains: "archive-scoped", + }, // Description included in error { name: "error includes description", From 4cbdf67b0fa77af5f7d1557565b76dd7352690bf Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Sun, 28 Jun 2026 19:12:24 +0000 Subject: [PATCH 13/13] fix: archive-overlays require source-file artifact --- docs/user/reference/config/components.md | 42 +++++++++++- .../app/azldev/core/sources/sourceprep.go | 65 +++++++++++++++++++ internal/projectconfig/component.go | 7 +- internal/projectconfig/configfile.go | 44 +++++++++++++ .../sourceproviders/sourcemanager.go | 13 ++++ schemas/azldev.schema.json | 3 +- 6 files changed, 170 insertions(+), 4 deletions(-) diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 35344a8d..94b184d7 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -319,8 +319,21 @@ The `origin` field specifies how to obtain the source file. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| -| Type | `type` | string | **Yes** | Origin type. Currently only `"download"` is supported. | -| URI | `uri` | string | No | URI to download the file from (required when type is `"download"`) | +| Type | `type` | string | **Yes** | Origin type: `"download"` or `"overlay"` (see below) | +| URI | `uri` | string | No | URI to download the file from (required when type is `"download"`; must be omitted when type is `"overlay"`) | + +#### `"download"` origin + +Downloads the file from the given `uri`. Tries the lookaside cache first using `hash`/`hash-type`, then falls back to the URI. + +#### `"overlay"` origin + +Used when archive overlays (see [Archive overlays](overlays.md#archive-overlays)) modify an existing upstream source archive. No download occurs; the archive is already present on disk as a spec source. The `hash` and `hash-type` fields record the **expected hash after all overlays have been applied** to that archive. + +- **`hash` and `hash-type` are required** for `"overlay"` entries — the hash cannot be computed at render time because sources are not downloaded. +- **`replace-upstream = true` is required** — the archive already exists in the upstream `sources` file, and this entry replaces its hash with the post-overlay value. +- During `prep-sources` (full run), azldev verifies that the hash it computed after repacking the archive matches the stated `hash`. A mismatch means the config is stale and must be updated. +- During `render --check-only`, the stated hash is injected directly without downloading or repacking, allowing the check to pass deterministically. ### Example @@ -338,6 +351,31 @@ hash-type = "SHA512" origin = { type = "download", uri = "https://example.com/repo/pkgs/shim/shimaa64.efi/sha512/.../shimaa64.efi" } ``` +### Recording the post-overlay hash for archive overlays + +When you apply archive overlays (e.g. removing vendored files from a tarball) using `file-remove` or `file-search-replace` with an archive-scoped path, the repacked archive has a different hash than the original. Use a `source-files` entry with `origin.type = "overlay"` to record the expected post-overlay hash: + +```toml +[[components.apache-commons-compress.source-files]] +filename = "commons-compress-1.27.1-src.tar.gz" +hash = "c7a2cef26959e687ad19b96b5ba8393d7514095e13bf0f29bd41e6b3c3cb2260d8ff23283ff3d5fd137b2522b843e7f0f50ab46bcf0f66df5383674f35f223ab" +hash-type = "SHA512" +origin = { type = "overlay" } +replace-upstream = true +replace-reason = "Upstream source tarball contains test fixtures flagged as malware by the AZL RPM signing pipeline. These files are not needed at runtime and are removed to allow SRPM publication." +``` + +**Workflow:** + +1. Add the archive overlay(s) in the component's `[[overlays]]` array. +2. Run `prep-sources` once — this repacks the archive and prints the computed hash in the error message. +3. Paste the computed `hash` and `hash-type` into the `source-files` entry above. +4. Run `prep-sources` again to confirm the hash matches, then commit. + +After that, `render --check-only` will pass deterministically without downloading or repacking the archive. + +`replace-upstream = true` and `replace-reason` are required because the archive is already in the upstream `sources` file. The entry replaces its hash with the post-overlay value, regardless of how many overlays target that archive. + ### Replacing an upstream `sources` entry By default, declaring a `source-files` entry whose `filename` matches one already diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index fb16fd4c..b20fd9c6 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -710,6 +710,14 @@ func (p *sourcePreparerImpl) updateSourcesFile( return err } + // In full prep-sources mode, cross-check any 'overlay'-origin source-file entries against + // the hashes that were just computed so stale stated hashes are caught immediately. + if !p.skipLookaside { + if err := validateOverlayResultHashes(existingLines, sourceFiles, modifiedArchives, component.GetName()); err != nil { + return err + } + } + mergedLines, err := p.buildSourceEntries(sourceFiles, existingLines, component.GetName(), outputDir) if err != nil { return err @@ -798,6 +806,63 @@ func (p *sourcePreparerImpl) rehashModifiedEntries( return nil } +// validateOverlayResultHashes cross-checks [projectconfig.OriginTypeOverlay] source-file entries +// against the post-overlay hashes that [rehashModifiedEntries] just computed in existingLines. +// A mismatch means the stated hash in the config is stale and must be updated. +// It is a no-op when no archives were repacked in this run. +func validateOverlayResultHashes( + lines []fedorasource.SourcesFileLine, + sourceFiles []projectconfig.SourceFileReference, + modifiedArchives []string, + componentName string, +) error { + if len(modifiedArchives) == 0 { + return nil + } + + // Only validate archives that were actually repacked in this run. + repacked := make(map[string]bool, len(modifiedArchives)) + for _, name := range modifiedArchives { + repacked[name] = true + } + + // Build a filename → computed (post-overlay) entry map from the updated lines. + computedByName := make(map[string]fedorasource.SourcesFileEntry, len(lines)) + for _, line := range lines { + if line.Entry != nil { + computedByName[line.Entry.Filename] = *line.Entry + } + } + + for _, ref := range sourceFiles { + if ref.Origin.Type != projectconfig.OriginTypeOverlay { + continue + } + + if !repacked[ref.Filename] { + continue + } + + computed, ok := computedByName[ref.Filename] + if !ok { + // Missing upstream entry will be reported by processSourceRef. + continue + } + + if computed.HashType != ref.HashType || computed.Hash != ref.Hash { + return fmt.Errorf( + "component %#q: archive %#q 'source-files' hash does not match the hash computed "+ + "after applying overlays; update the 'hash' and 'hash-type' fields in the "+ + "'source-files' entry:\n stated: %s %s\n computed: %s %s", + componentName, ref.Filename, + ref.HashType, ref.Hash, + computed.HashType, computed.Hash) + } + } + + return nil +} + // readSourcesFileIfExists reads the 'sources' file content if it exists, returning empty string if not. func (p *sourcePreparerImpl) readSourcesFileIfExists(sourcesFilePath string) (string, error) { exists, err := fileutils.Exists(p.fs, sourcesFilePath) diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index b481b35b..828ba725 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -43,13 +43,18 @@ type OriginType string const ( // OriginTypeURI indicates that the source file is fetched from a URI. OriginTypeURI OriginType = "download" + // OriginTypeOverlay indicates that the source file's hash was changed by archive overlays. + // No download occurs; the file is already present as a spec source. The 'hash' and 'hash-type' + // fields record the expected post-overlay hash, which is injected into the 'sources' file + // during render (skip-lookaside) and validated against the computed hash during 'prep-sources'. + OriginTypeOverlay OriginType = "overlay" ) // Origin describes where a source file comes from and how to retrieve it. // When omitted from a source file reference, the file will be resolved via the lookaside cache. type Origin struct { // Type indicates how the source file should be acquired. - Type OriginType `toml:"type" json:"type" jsonschema:"required,enum=download,title=Origin type,description=Type of origin for this source file"` + Type OriginType `toml:"type" json:"type" jsonschema:"required,enum=download,enum=overlay,title=Origin type,description=Type of origin for this source file"` // Uri to download the source file from if origin type is 'download'. Ignored for other origin types. Uri string `toml:"uri,omitempty" json:"uri,omitempty" jsonschema:"title=URI,description=URI to download the source file from if origin type is 'download',example=https://example.com/source.tar.gz"` } diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index e16b79b7..379e3469 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -156,6 +156,7 @@ func (f ConfigFile) Validate() error { // - Hash value without a hash type is not allowed. // - Origin must be present and valid for each source file. // - 'replace-upstream' and 'replace-reason' must be set together. +// - [OriginTypeOverlay] entries additionally require 'hash', 'hash-type', and 'replace-upstream = true'. func validateSourceFiles(sourceFiles []SourceFileReference, componentName string) error { seen := make(map[string]bool, len(sourceFiles)) @@ -192,6 +193,41 @@ func validateSourceFiles(sourceFiles []SourceFileReference, componentName string if err := validateOrigin(ref.Origin, ref.Filename, componentName); err != nil { return err } + + if ref.Origin.Type == OriginTypeOverlay { + if err := validateOverlayOriginRef(ref, componentName); err != nil { + return err + } + } + } + + return nil +} + +// validateOverlayOriginRef enforces additional constraints on [SourceFileReference] entries +// with [OriginTypeOverlay]: 'hash', 'hash-type', and 'replace-upstream = true' are all required. +// These entries record the post-overlay hash of an archive that is already present as a spec +// source, so a download is never performed and the hash cannot be computed on the fly. +func validateOverlayOriginRef(ref SourceFileReference, componentName string) error { + if ref.Hash == "" { + return fmt.Errorf( + "source file %#q in component %#q has 'origin.type = overlay' but no 'hash'; "+ + "'hash' is required to record the expected post-overlay archive hash", + ref.Filename, componentName) + } + + if ref.HashType == "" { + return fmt.Errorf( + "source file %#q in component %#q has 'origin.type = overlay' but no 'hash-type'; "+ + "'hash-type' is required when 'hash' is provided", + ref.Filename, componentName) + } + + if !ref.ReplaceUpstream { + return fmt.Errorf( + "source file %#q in component %#q has 'origin.type = overlay' but 'replace-upstream' is not true; "+ + "'replace-upstream = true' is required because the archive already exists in the upstream 'sources' file", + ref.Filename, componentName) } return nil @@ -225,6 +261,7 @@ func validateReplaceUpstream(ref SourceFileReference, componentName string) erro // validateOrigin checks that a source file [Origin] is present and valid for its type. // For [OriginTypeURI] ('download'), the [Origin.Uri] field must be a valid URI with a scheme. +// For [OriginTypeOverlay] ('overlay'), no URI is used; the archive is already on disk. func validateOrigin(origin Origin, filename string, componentName string) error { if origin.Type == "" { return fmt.Errorf( @@ -255,6 +292,13 @@ func validateOrigin(origin Origin, filename string, componentName string) error "URI %#q is missing a scheme (e.g. 'https://')", filename, componentName, origin.Uri) } + case OriginTypeOverlay: + if origin.Uri != "" { + return fmt.Errorf( + "unexpected 'uri' for source file %#q, component %#q; "+ + "'uri' must not be set when 'origin' type is 'overlay'", + filename, componentName) + } default: return fmt.Errorf( "unsupported 'origin' type %#q for source file %#q, component %#q", diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index b5431d84..c46fe614 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -305,6 +305,13 @@ func (m *sourceManager) fetchSourceFile( destPath := filepath.Join(destDirPath, fileRef.Filename) + // Overlay-origin entries declare the post-overlay hash of an archive that is already + // present as a spec source. No download is needed; the hash is used only to update + // the 'sources' file during render and to validate the output of 'prep-sources'. + if fileRef.Origin.Type == projectconfig.OriginTypeOverlay { + return nil + } + sourceExists, err := fileutils.Exists(m.fs, destPath) if err != nil { return fmt.Errorf("failed to check existence of destination file %#q:\n%w", destPath, err) @@ -403,6 +410,12 @@ func (m *sourceManager) downloadFromOrigin( return nil + case projectconfig.OriginTypeOverlay: + // Overlay-origin files are skipped before reaching this point in fetchSourceFile. + // This case should never be reached. + return fmt.Errorf("internal error: download attempted for 'overlay'-origin source file %#q", + fileRef.Filename) + default: return fmt.Errorf("unsupported origin type %#q for source file %#q", fileRef.Origin.Type, fileRef.Filename) diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7b824f4e..b022ea91 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -720,7 +720,8 @@ "type": { "type": "string", "enum": [ - "download" + "download", + "overlay" ], "title": "Origin type", "description": "Type of origin for this source file"