From f2e9c217610e1ed801921d7db955c5b6c07d0b2a Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 8 Jul 2026 23:05:18 +0000 Subject: [PATCH 1/7] feat(source-files): generative source dependencies --- docs/user/reference/config/components.md | 6 +- .../cmds/component/history_customizations.go | 7 + .../cmds/component/history_internal_test.go | 3 +- .../app/azldev/core/sources/sourceprep.go | 21 ++- .../azldev/core/sources/sourceprep_test.go | 5 + internal/projectconfig/component.go | 15 ++- internal/projectconfig/configfile.go | 21 ++- internal/projectconfig/configfile_test.go | 66 ++++++++++ .../sourceproviders/customsourceprovider.go | 124 +++++++++++++++--- .../customsourceprovider_internal_test.go | 40 ++++++ .../sourceproviders/sourcemanager.go | 18 ++- ...ainer_config_generate-schema_stdout_1.snap | 8 ++ ...shots_config_generate-schema_stdout_1.snap | 8 ++ schemas/azldev.schema.json | 8 ++ 14 files changed, 312 insertions(+), 38 deletions(-) diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index b202a786..101760e0 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -333,12 +333,13 @@ origin = { type = "download", uri = "https://example.com/repo/.../shimx64.efi Use `origin.type = "custom"` when a source archive must be assembled or modified (e.g. stripping sensitive test fixtures from an upstream tarball). azldev runs the script inside a fresh mock chroot — the script **must write all output to `/azldev-gen/output/`**, which azldev packages into the archive named by `filename`. Network access is always enabled; the mock config comes from the project's default distro. -The `script` and `mock-packages` fields are nested under `[origin]`: +The `script`, `mock-packages`, and `inputs` fields are nested under `[origin]`: | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Script | `origin.script` | string | **Yes** | Script filename (relative to the component's spec dir) to run in mock. Required for `origin.type = "custom"`. | | Mock packages | `origin.mock-packages` | array of string | No | Extra RPM packages to install in the mock chroot before the script runs. | +| Inputs | `origin.inputs` | array of string | No | Filenames to make available in the mock chroot before the script runs. Each file must already be present in the fetched source output directory — upstream source tarballs, sidecar files (patches, scripts), and any earlier `source-files` entries are all placed there by the upstream fetch before custom scripts run. | On first use, omit `hash` and run `prep-sources --allow-no-hashes` to generate the archive and print its hash, then copy it into the TOML. @@ -350,8 +351,11 @@ hash = "abc123..." # from: prep-sources --allow-no-hashes origin.type = "custom" origin.script = "gen-yara-stripped.sh" # relative to the component's spec directory origin.mock-packages = ["cmake"] # omit if not needed +origin.inputs = ["yara-4.5.4.tar.gz"] # available to the script as ./yara-4.5.4.tar.gz ``` +> **Note:** Upstream source tarballs are automatically available as inputs before running custom generation scripts. There is no need to re-declare an upstream file in `source-files` to use it as an input. + ### Replacing an upstream `sources` entry A `source-files` entry whose `filename` collides with an upstream `sources` entry is an error by default. Set `replace-upstream = true` (with a non-empty `replace-reason`) to intentionally substitute it: diff --git a/internal/app/azldev/cmds/component/history_customizations.go b/internal/app/azldev/cmds/component/history_customizations.go index ad5de706..1e7f5be7 100644 --- a/internal/app/azldev/cmds/component/history_customizations.go +++ b/internal/app/azldev/cmds/component/history_customizations.go @@ -236,6 +236,13 @@ func appendSourceFileItems( Value: pkg, }) } + + for _, input := range sourceFile.Origin.Inputs { + items = append(items, CustomizationItem{ + Kind: "source-files.inputs", + Value: input, + }) + } } return items diff --git a/internal/app/azldev/cmds/component/history_internal_test.go b/internal/app/azldev/cmds/component/history_internal_test.go index 33e39a24..5a7a9e50 100644 --- a/internal/app/azldev/cmds/component/history_internal_test.go +++ b/internal/app/azldev/cmds/component/history_internal_test.go @@ -142,10 +142,11 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { "SourceFileReference.ReplaceUpstream": "source-files.replace-upstream", "SourceFileReference.Origin": "delegates to Origin walk", - // Origin -- Script and MockPackages are fingerprint-relevant; Type and Uri + // Origin -- Script, MockPackages, and Inputs are fingerprint-relevant; Type and Uri // are tagged fingerprint:"-" (download location, not build input). "Origin.Script": "source-files.script", "Origin.MockPackages": "source-files.mock-packages", + "Origin.Inputs": "source-files.inputs", } actualFields := make(map[string]bool) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 617dcbd8..c391ae61 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -215,17 +215,6 @@ func NewPreparer( func (p *sourcePreparerImpl) PrepareSources( ctx context.Context, component components.Component, outputDir string, applyOverlays bool, ) error { - // Use the source manager to fetch source files (archives, patches, etc.) - // Skip this step when skipLookaside is set — source tarballs are not needed - // for rendering and are the most expensive download. - if !p.skipLookaside { - err := p.sourceManager.FetchFiles(ctx, component, outputDir) - if err != nil { - return fmt.Errorf("failed to fetch source files for component %#q:\n%w", - component.GetName(), err) - } - } - // Preserve the upstream .git directory only when dist-git creation is // requested via --with-git. This is required so that overlay commits can be // appended on top of the upstream commit log during synthetic history generation. @@ -238,13 +227,21 @@ func (p *sourcePreparerImpl) PrepareSources( fetchOpts = append(fetchOpts, sourceproviders.WithSkipLookaside()) } - // Use the source manager to fetch the component (spec file and sidecar files). + // Fetch the component first (spec, sidecar files, and upstream source tarballs). err := p.sourceManager.FetchComponent(ctx, component, outputDir, fetchOpts...) if err != nil { return fmt.Errorf("failed to fetch sources for component %#q:\n%w", component.GetName(), err) } + // Fetch custom and downloaded source files. + if !p.skipLookaside { + if err := p.sourceManager.FetchFiles(ctx, component, outputDir); err != nil { + return fmt.Errorf("failed to fetch source files for component %#q:\n%w", + component.GetName(), err) + } + } + if applyOverlays { err := p.applyOverlaysToSources(ctx, component, outputDir) if err != nil { diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 45a9d286..a0a6f319 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -108,6 +108,8 @@ func TestPrepareSources_SourceManagerError(t *testing.T) { expectedErr := errors.New("failed to fetch files") component.EXPECT().GetName().AnyTimes().Return("test-component") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).Return(nil) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(expectedErr) preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) @@ -633,8 +635,11 @@ func TestDiffSources_FetchError(t *testing.T) { require.NoError(t, fileutils.MkdirAll(ctx.FS(), baseDir)) component.EXPECT().GetName().AnyTimes().Return("test-component") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) expectedErr := errors.New("network failure") + + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, gomock.Any(), gomock.Any()).Return(nil) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, gomock.Any()).Return(expectedErr) preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 7b5519ed..c260f2dd 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -66,17 +66,24 @@ type Origin struct { // MockPackages is a list of RPM package names to install in the mock chroot before // running [Origin.Script]. Only valid when [Origin.Type] is 'custom'. MockPackages []string `toml:"mock-packages,omitempty" json:"mockPackages,omitempty" jsonschema:"title=Mock packages,description=RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'."` + + // Inputs is a list of source-output filenames to copy next to [Origin.Script] + // before it runs. Each entry must be a plain filename. Only valid when + // [Origin.Type] is 'custom'. + Inputs []string `toml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"title=Inputs,description=Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'."` } // HashInclude implements the hashstructure [Includable] interface so that -// [Origin.Script] and [Origin.MockPackages] are omitted from the component -// fingerprint when they hold their zero values. +// [Origin.Script], [Origin.MockPackages], and [Origin.Inputs] are omitted from +// the component fingerprint when they hold their zero values. func (o Origin) HashInclude(field string, _ any) (bool, error) { switch field { case "Script": return o.Script != "", nil case "MockPackages": return len(o.MockPackages) > 0, nil + case "Inputs": + return len(o.Inputs) > 0, nil } return true, nil @@ -114,10 +121,10 @@ type SourceFileReference struct { // HashInclude implements the hashstructure [Includable] interface so that // [SourceFileReference.Origin] is omitted from the component fingerprint when -// neither [Origin.Script] nor [Origin.MockPackages] are set. +// none of [Origin.Script], [Origin.MockPackages], or [Origin.Inputs] are set. func (r SourceFileReference) HashInclude(field string, _ any) (bool, error) { if field == "Origin" { - return r.Origin.Script != "" || len(r.Origin.MockPackages) > 0, nil + return r.Origin.Script != "" || len(r.Origin.MockPackages) > 0 || len(r.Origin.Inputs) > 0, nil } return true, nil diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index da1b5bae..860ca0b5 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -248,11 +248,13 @@ func validateReplaceUpstream(ref SourceFileReference, componentName string) erro return nil } -// validateCustomSourceRef enforces the pairing rules for the 'script' and 'mock-packages' -// fields on a [SourceFileReference]: +// validateCustomSourceRef enforces the pairing rules for the 'script', 'mock-packages', +// and 'inputs' fields on a [SourceFileReference]: // - 'script' is required when 'origin.type' is 'custom'. // - 'script' must be empty when 'origin.type' is not 'custom'. // - 'mock-packages' must be empty when 'origin.type' is not 'custom'. +// - 'inputs' must be empty when 'origin.type' is not 'custom'. +// - each 'inputs' entry must be a valid filename (no path separators). func validateCustomSourceRef(ref SourceFileReference, componentName string) error { if ref.Origin.Type == OriginTypeCustom { if ref.Origin.Script == "" { @@ -268,6 +270,14 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro ref.Origin.Script, ref.Filename, componentName, err) } + for _, input := range ref.Origin.Inputs { + if err := fileutils.ValidateFilename(input); err != nil { + return fmt.Errorf( + "invalid 'inputs' entry %#q for source file %#q in component %#q:\n%w", + input, ref.Filename, componentName, err) + } + } + return nil } @@ -285,6 +295,13 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro ref.Filename, componentName, string(ref.Origin.Type)) } + if len(ref.Origin.Inputs) > 0 { + return fmt.Errorf( + "source file %#q in component %#q has 'inputs' set but origin type is %#q; "+ + "'inputs' is only valid when origin type is 'custom'", + ref.Filename, componentName, string(ref.Origin.Type)) + } + return nil } diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index b6d8f56f..9474ac9d 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -584,3 +584,69 @@ func TestValidateCustomSourceRef_InvalidScriptFilename(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "script") } + +func TestValidateCustomSourceRef_ValidInputs(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "gen.sh", + Inputs: []string{"upstream.tar.gz", "fix.patch"}, + }, + }, + }, + }, + }, + } + assert.NoError(t, file.Validate()) +} + +func TestValidateCustomSourceRef_InputsOnDownloadOrigin(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "src.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeURI, + Uri: "https://example.com/src.tar.gz", + Inputs: []string{"other.tar.gz"}, + }, + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'inputs'") + assert.Contains(t, err.Error(), "custom") +} + +func TestValidateCustomSourceRef_InvalidInputFilename(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "gen.sh", + Inputs: []string{"../escape.tar.gz"}, // path traversal attempt + }, + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'inputs'") + assert.Contains(t, err.Error(), "escape.tar.gz") +} diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index 0f2efcfb..e31de4d8 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -4,10 +4,14 @@ package sourceproviders import ( + "bytes" "context" "fmt" + "io" "log/slog" + "os" "path/filepath" + "strings" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" @@ -30,8 +34,9 @@ const customGenOutputDir = "/azldev-gen/output" // [projectconfig.OriginTypeCustom]. It executes a user-supplied script inside a // fresh mock chroot and packages the output directory as a deterministic archive. type customFileSourceProvider struct { - fs opctx.FS - runner *mock.Runner + fs opctx.FS + runner *mock.Runner + verbose bool } var _ FileSourceProvider = (*customFileSourceProvider)(nil) @@ -51,7 +56,7 @@ func (p *customFileSourceProvider) GetFile( destPath := filepath.Join(destDirPath, ref.Filename) - return generateCustomSourceFile(ctx, p.fs, p.runner, component, &ref, destPath) + return generateCustomSourceFile(ctx, p.fs, p.runner, p.verbose, component, &ref, destPath) } // generateCustomSourceFile runs a single [projectconfig.SourceFileReference] through a @@ -63,6 +68,7 @@ func generateCustomSourceFile( ctx context.Context, fs opctx.FS, baseRunner *mock.Runner, + verbose bool, component components.Component, ref *projectconfig.SourceFileReference, destPath string, @@ -93,6 +99,19 @@ func generateCustomSourceFile( defer cleanup() + // Stage declared input files next to the script so script authors can refer to + // them by filename without knowing azldev's internal mount paths. + destDirPath := filepath.Dir(destPath) + + if len(ref.Origin.Inputs) > 0 { + if inputsErr := stageInputFiles( + fs, ref.Origin.Inputs, destDirPath, scriptTmpDir, ref.Origin.Script, + ); inputsErr != nil { + return fmt.Errorf("failed to resolve inputs for custom source %#q:\n%w", + ref.Filename, inputsErr) + } + } + // Package the output directory as a deterministic archive whose format is // inferred from the filename extension (e.g., .tar.gz, .tar.xz). comp, compErr := archive.DetectCompression(ref.Filename) @@ -104,18 +123,13 @@ func generateCustomSourceFile( // Clone the base runner so bind mounts added here don't persist to other calls. // Network access is always enabled — custom source scripts commonly need to // download upstream tarballs or toolchain artifacts. - runner := baseRunner.Clone() - runner.EnableNetwork() - runner.WithUnprivileged() - runner.AddBindMount(scriptTmpDir, customGenScriptDir) - runner.AddBindMount(genOutputTmpDir, customGenOutputDir) + runner := buildCustomRunner(baseRunner, scriptTmpDir, genOutputTmpDir) - if err := execScriptInChroot(ctx, runner, ref); err != nil { + if err := execScriptInChroot(ctx, runner, verbose, ref); err != nil { return err } - // Ensure the destination directory exists. FetchFiles runs before FetchComponent, - // so the output directory may not have been created yet when this is called. + // Ensure the destination directory exists before writing the archive. if mkdirErr := fileutils.MkdirAll(fs, filepath.Dir(destPath)); mkdirErr != nil { return fmt.Errorf("failed to create destination directory for %#q:\n%w", ref.Filename, mkdirErr) @@ -207,11 +221,25 @@ func prepareStagingDirs( return scriptDir, outputDir, cleanup, nil } +// buildCustomRunner returns a clone of baseRunner configured with the standard bind mounts +// for custom source generation. Network access is always enabled; the unprivileged flag is +// set so the script runs as the 'mockbuild' user rather than root. +func buildCustomRunner(baseRunner *mock.Runner, scriptTmpDir, genOutputTmpDir string) *mock.Runner { + runner := baseRunner.Clone() + runner.EnableNetwork() + runner.WithUnprivileged() + runner.AddBindMount(scriptTmpDir, customGenScriptDir) + runner.AddBindMount(genOutputTmpDir, customGenOutputDir) + + return runner +} + // execScriptInChroot initialises a fresh mock root, optionally installs extra packages, // runs the generation script, and scrubs the root on return. func execScriptInChroot( ctx context.Context, runner *mock.Runner, + verbose bool, ref *projectconfig.SourceFileReference, ) error { if initErr := runner.InitRoot(ctx); initErr != nil { @@ -235,17 +263,83 @@ func execScriptInChroot( } } - scriptChrootPath := filepath.Join(customGenScriptDir, ref.Origin.Script) + scriptCommand := fmt.Sprintf("cd %s && ./%s", customGenScriptDir, ref.Origin.Script) - cmd, cmdErr := runner.CmdInChroot(ctx, []string{scriptChrootPath}, false /* interactive */) + cmd, cmdErr := runner.CmdInChroot(ctx, []string{"bash", "-c", scriptCommand}, false /* interactive */) if cmdErr != nil { return fmt.Errorf("failed to create chroot command for generating %#q:\n%w", ref.Filename, cmdErr) } + var ( + stdout bytes.Buffer + stderr bytes.Buffer + ) + + if verbose { + cmd.SetStdout(io.MultiWriter(os.Stdout, &stdout)) + cmd.SetStderr(io.MultiWriter(os.Stderr, &stderr)) + } else { + cmd.SetStdout(&stdout) + cmd.SetStderr(&stderr) + } + if runErr := cmd.Run(ctx); runErr != nil { - return fmt.Errorf("generation script %#q failed for source %#q:\n%w", - ref.Origin.Script, ref.Filename, runErr) + return fmt.Errorf("generation script %#q failed for source %#q:%s\n%w", + ref.Origin.Script, ref.Filename, formatCustomScriptOutput(stdout.String(), stderr.String()), runErr) + } + + return nil +} + +func formatCustomScriptOutput(stdout, stderr string) string { + var output strings.Builder + + if stdout != "" { + fmt.Fprintf(&output, "\nstdout:\n%s", stdout) + } + + if stderr != "" { + fmt.Fprintf(&output, "\nstderr:\n%s", stderr) + } + + return output.String() +} + +// stageInputFiles copies the files listed in inputs next to the generation script. +// +// All inputs must already be present in destDirPath, which contains the full set of upstream +// source tarballs, sidecar files, and any earlier 'source-files' entries fetched by +// [FetchComponent] and prior [FetchFiles] calls. If a filename is absent, an error is returned. +func stageInputFiles(fs opctx.FS, inputs []string, destDirPath, scriptTmpDir, scriptName string) error { + for _, filename := range inputs { + if filename == scriptName { + return fmt.Errorf("input file %#q conflicts with generation script filename", filename) + } + + sourcePath := filepath.Join(destDirPath, filename) + + exists, statErr := fileutils.Exists(fs, sourcePath) + if statErr != nil { + return fmt.Errorf("failed to stat input file %#q:\n%w", filename, statErr) + } + + if !exists { + return fmt.Errorf( + "input file %#q not found in output directory %#q", + filename, destDirPath) + } + + data, readErr := fileutils.ReadFile(fs, sourcePath) + if readErr != nil { + return fmt.Errorf("failed to read input file %#q:\n%w", filename, readErr) + } + + stagedPath := filepath.Join(scriptTmpDir, filename) + + if writeErr := fileutils.WriteFile(fs, stagedPath, data, fileperms.PublicFile); writeErr != nil { + return fmt.Errorf("failed to stage input file %#q:\n%w", filename, writeErr) + } } return nil diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index e51ba648..76bd2916 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -123,3 +123,43 @@ func TestPrepareStagingDirs_MissingScriptReturnsError(t *testing.T) { assert.Contains(t, err.Error(), "missing.sh") assert.Nil(t, cleanup) } + +func TestStageInputFiles_FoundInDestDir(t *testing.T) { + memFS := afero.NewMemMapFs() + + const fileContent = "upstream tarball data" + + require.NoError(t, afero.WriteFile(memFS, "/output/upstream.tar.gz", []byte(fileContent), fileperms.PublicFile)) + + err := stageInputFiles(memFS, []string{"upstream.tar.gz"}, "/output", "/script", "gen.sh") + require.NoError(t, err) + + data, readErr := afero.ReadFile(memFS, "/script/upstream.tar.gz") + require.NoError(t, readErr) + assert.Equal(t, fileContent, string(data)) +} + +func TestStageInputFiles_NotFoundReturnsError(t *testing.T) { + memFS := afero.NewMemMapFs() + + // No files written — destDirPath is empty. + err := stageInputFiles(memFS, []string{"missing.tar.gz"}, "/output", "/script", "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "missing.tar.gz") + assert.Contains(t, err.Error(), "/output") +} + +func TestStageInputFiles_ScriptNameConflictReturnsError(t *testing.T) { + memFS := afero.NewMemMapFs() + + err := stageInputFiles(memFS, []string{"gen.sh"}, "/output", "/script", "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicts") +} + +func TestFormatCustomScriptOutput(t *testing.T) { + output := formatCustomScriptOutput("stdout line\n", "stderr line\n") + + assert.Contains(t, output, "stdout:\nstdout line") + assert.Contains(t, output, "stderr:\nstderr line") +} diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 4731c61c..39b1f733 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -315,8 +315,9 @@ func (m *sourceManager) createFileProviders(env *azldev.Env) { } m.fileProviders = append(m.fileProviders, &customFileSourceProvider{ - fs: m.fs, - runner: mock.NewRunner(env, mockConfigPath), + fs: m.fs, + runner: mock.NewRunner(env, mockConfigPath), + verbose: env.Verbose(), }) slog.Debug("Registered custom file source provider", "mockConfig", mockConfigPath) @@ -686,7 +687,18 @@ func (m *sourceManager) downloadLookasideSources( packageName := resolvePackageName(component) - err := m.lookasideDownloader.ExtractSourcesFromRepo(ctx, destDirPath, packageName, m.lookasideBaseURI, nil) + // Collect filenames from 'source-files' config so the lookaside extractor skips them. + // These files are managed by FetchFiles (custom generation or explicit download origins) + // and must not be overwritten by a same-named upstream lookaside entry. This mirrors + // the same skip-list built in [FedoraSourcesProviderImpl.GetComponent]. + sourceFiles := component.GetConfig().SourceFiles + + skipFilenames := make([]string, len(sourceFiles)) + for i := range sourceFiles { + skipFilenames[i] = sourceFiles[i].Filename + } + + err := m.lookasideDownloader.ExtractSourcesFromRepo(ctx, destDirPath, packageName, m.lookasideBaseURI, skipFilenames) if err != nil { return fmt.Errorf("failed to extract sources from lookaside cache:\n%w", err) } diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 23778d19..60b29944 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -735,6 +735,14 @@ "type": "array", "title": "Mock packages", "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." + }, + "inputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Inputs", + "description": "Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 23778d19..60b29944 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -735,6 +735,14 @@ "type": "array", "title": "Mock packages", "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." + }, + "inputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Inputs", + "description": "Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'." } }, "additionalProperties": false, diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 23778d19..60b29944 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -735,6 +735,14 @@ "type": "array", "title": "Mock packages", "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." + }, + "inputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Inputs", + "description": "Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'." } }, "additionalProperties": false, From 71c5262855ba1be816df62df04bc7eb4c495ccbe Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 9 Jul 2026 17:28:41 +0000 Subject: [PATCH 2/7] customer source provider fixes --- .../sourceproviders/customsourceprovider.go | 41 ++++++++++++------- .../customsourceprovider_internal_test.go | 33 +++++++++++---- .../sourceproviders/sourcemanager.go | 7 ++-- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index e31de4d8..569b2efa 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -34,9 +34,10 @@ const customGenOutputDir = "/azldev-gen/output" // [projectconfig.OriginTypeCustom]. It executes a user-supplied script inside a // fresh mock chroot and packages the output directory as a deterministic archive. type customFileSourceProvider struct { - fs opctx.FS - runner *mock.Runner - verbose bool + dryRunnable opctx.DryRunnable + fs opctx.FS + runner *mock.Runner + verbose bool } var _ FileSourceProvider = (*customFileSourceProvider)(nil) @@ -56,7 +57,7 @@ func (p *customFileSourceProvider) GetFile( destPath := filepath.Join(destDirPath, ref.Filename) - return generateCustomSourceFile(ctx, p.fs, p.runner, p.verbose, component, &ref, destPath) + return generateCustomSourceFile(ctx, p.dryRunnable, p.fs, p.runner, p.verbose, component, &ref, destPath) } // generateCustomSourceFile runs a single [projectconfig.SourceFileReference] through a @@ -66,6 +67,7 @@ func (p *customFileSourceProvider) GetFile( // is created for each call so bind mounts and package installs do not bleed across invocations. func generateCustomSourceFile( ctx context.Context, + dryRunnable opctx.DryRunnable, fs opctx.FS, baseRunner *mock.Runner, verbose bool, @@ -105,7 +107,7 @@ func generateCustomSourceFile( if len(ref.Origin.Inputs) > 0 { if inputsErr := stageInputFiles( - fs, ref.Origin.Inputs, destDirPath, scriptTmpDir, ref.Origin.Script, + dryRunnable, fs, ref.Origin.Inputs, destDirPath, scriptTmpDir, ref.Origin.Script, ); inputsErr != nil { return fmt.Errorf("failed to resolve inputs for custom source %#q:\n%w", ref.Filename, inputsErr) @@ -263,9 +265,11 @@ func execScriptInChroot( } } - scriptCommand := fmt.Sprintf("cd %s && ./%s", customGenScriptDir, ref.Origin.Script) - - cmd, cmdErr := runner.CmdInChroot(ctx, []string{"bash", "-c", scriptCommand}, false /* interactive */) + // Use positional parameters so the script name is never re-parsed as shell code + // ($1=scriptDir, $2=scriptName; '--' sets $0 and keeps bash from consuming them). + cmd, cmdErr := runner.CmdInChroot(ctx, []string{ + "bash", "-c", `cd "$1" && ./"$2"`, "--", customGenScriptDir, ref.Origin.Script, + }, false /* interactive */) if cmdErr != nil { return fmt.Errorf("failed to create chroot command for generating %#q:\n%w", ref.Filename, cmdErr) @@ -311,8 +315,17 @@ func formatCustomScriptOutput(stdout, stderr string) string { // All inputs must already be present in destDirPath, which contains the full set of upstream // source tarballs, sidecar files, and any earlier 'source-files' entries fetched by // [FetchComponent] and prior [FetchFiles] calls. If a filename is absent, an error is returned. -func stageInputFiles(fs opctx.FS, inputs []string, destDirPath, scriptTmpDir, scriptName string) error { +func stageInputFiles( + dryRunnable opctx.DryRunnable, + fs opctx.FS, + inputs []string, + destDirPath, scriptTmpDir, scriptName string, +) error { for _, filename := range inputs { + if err := fileutils.ValidateFilename(filename); err != nil { + return fmt.Errorf("invalid input filename %#q:\n%w", filename, err) + } + if filename == scriptName { return fmt.Errorf("input file %#q conflicts with generation script filename", filename) } @@ -330,14 +343,12 @@ func stageInputFiles(fs opctx.FS, inputs []string, destDirPath, scriptTmpDir, sc filename, destDirPath) } - data, readErr := fileutils.ReadFile(fs, sourcePath) - if readErr != nil { - return fmt.Errorf("failed to read input file %#q:\n%w", filename, readErr) - } - stagedPath := filepath.Join(scriptTmpDir, filename) - if writeErr := fileutils.WriteFile(fs, stagedPath, data, fileperms.PublicFile); writeErr != nil { + if writeErr := fileutils.CopyFile( + dryRunnable, fs, sourcePath, stagedPath, + fileutils.CopyFileOptions{PreserveFileMode: true}, + ); writeErr != nil { return fmt.Errorf("failed to stage input file %#q:\n%w", filename, writeErr) } } diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index 76bd2916..9b6ed39e 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/spf13/afero" @@ -124,39 +125,55 @@ func TestPrepareStagingDirs_MissingScriptReturnsError(t *testing.T) { assert.Nil(t, cleanup) } -func TestStageInputFiles_FoundInDestDir(t *testing.T) { - memFS := afero.NewMemMapFs() +func TestStageInputFiles_FoundInDestDirPreservesMode(t *testing.T) { + ctx := testctx.NewCtx() + memFS := ctx.FS() const fileContent = "upstream tarball data" - require.NoError(t, afero.WriteFile(memFS, "/output/upstream.tar.gz", []byte(fileContent), fileperms.PublicFile)) + require.NoError(t, afero.WriteFile(memFS, "/output/upstream.tar.gz", []byte(fileContent), fileperms.PublicExecutable)) - err := stageInputFiles(memFS, []string{"upstream.tar.gz"}, "/output", "/script", "gen.sh") + err := stageInputFiles(ctx, memFS, []string{"upstream.tar.gz"}, "/output", "/script", "gen.sh") require.NoError(t, err) data, readErr := afero.ReadFile(memFS, "/script/upstream.tar.gz") require.NoError(t, readErr) assert.Equal(t, fileContent, string(data)) + + info, statErr := memFS.Stat("/script/upstream.tar.gz") + require.NoError(t, statErr) + assert.Equal(t, fileperms.PublicExecutable, info.Mode().Perm()) } func TestStageInputFiles_NotFoundReturnsError(t *testing.T) { - memFS := afero.NewMemMapFs() + ctx := testctx.NewCtx() + memFS := ctx.FS() // No files written — destDirPath is empty. - err := stageInputFiles(memFS, []string{"missing.tar.gz"}, "/output", "/script", "gen.sh") + err := stageInputFiles(ctx, memFS, []string{"missing.tar.gz"}, "/output", "/script", "gen.sh") require.Error(t, err) assert.Contains(t, err.Error(), "missing.tar.gz") assert.Contains(t, err.Error(), "/output") } func TestStageInputFiles_ScriptNameConflictReturnsError(t *testing.T) { - memFS := afero.NewMemMapFs() + ctx := testctx.NewCtx() + memFS := ctx.FS() - err := stageInputFiles(memFS, []string{"gen.sh"}, "/output", "/script", "gen.sh") + err := stageInputFiles(ctx, memFS, []string{"gen.sh"}, "/output", "/script", "gen.sh") require.Error(t, err) assert.Contains(t, err.Error(), "conflicts") } +func TestStageInputFiles_InvalidFilenameReturnsError(t *testing.T) { + ctx := testctx.NewCtx() + memFS := ctx.FS() + + err := stageInputFiles(ctx, memFS, []string{"../escape.tar.gz"}, "/output", "/script", "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid input filename") +} + func TestFormatCustomScriptOutput(t *testing.T) { output := formatCustomScriptOutput("stdout line\n", "stderr line\n") diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 39b1f733..30463741 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -315,9 +315,10 @@ func (m *sourceManager) createFileProviders(env *azldev.Env) { } m.fileProviders = append(m.fileProviders, &customFileSourceProvider{ - fs: m.fs, - runner: mock.NewRunner(env, mockConfigPath), - verbose: env.Verbose(), + dryRunnable: m.dryRunnable, + fs: m.fs, + runner: mock.NewRunner(env, mockConfigPath), + verbose: env.Verbose(), }) slog.Debug("Registered custom file source provider", "mockConfig", mockConfigPath) From 5808349a5dd940de9248dbc061fd198236a20fd4 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 9 Jul 2026 17:41:45 +0000 Subject: [PATCH 3/7] fixed minor mistakes --- internal/projectconfig/configfile.go | 6 ++++++ .../sourceproviders/customsourceprovider.go | 14 +++++++++++--- .../customsourceprovider_internal_test.go | 14 ++++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 860ca0b5..761682dc 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -276,6 +276,12 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro "invalid 'inputs' entry %#q for source file %#q in component %#q:\n%w", input, ref.Filename, componentName, err) } + + if input == ref.Origin.Script { + return fmt.Errorf( + "'inputs' entry %#q for source file %#q in component %#q conflicts with 'script' filename", + input, ref.Filename, componentName) + } } return nil diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index 569b2efa..8970e7a5 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -80,6 +80,12 @@ func generateCustomSourceFile( "script", ref.Origin.Script, "component", component.GetName()) + if dryRunnable.DryRun() { + slog.Info("Dry run; skipping custom source file generation", "filename", ref.Filename) + + return nil + } + specDir, err := resolveComponentSpecDir(component) if err != nil { return fmt.Errorf("failed to resolve spec directory for component %#q:\n%w", @@ -268,7 +274,7 @@ func execScriptInChroot( // Use positional parameters so the script name is never re-parsed as shell code // ($1=scriptDir, $2=scriptName; '--' sets $0 and keeps bash from consuming them). cmd, cmdErr := runner.CmdInChroot(ctx, []string{ - "bash", "-c", `cd "$1" && ./"$2"`, "--", customGenScriptDir, ref.Origin.Script, + "sh", "-c", `cd "$1" && ./"$2"`, "--", customGenScriptDir, ref.Origin.Script, }, false /* interactive */) if cmdErr != nil { return fmt.Errorf("failed to create chroot command for generating %#q:\n%w", @@ -289,8 +295,10 @@ func execScriptInChroot( } if runErr := cmd.Run(ctx); runErr != nil { - return fmt.Errorf("generation script %#q failed for source %#q:%s\n%w", - ref.Origin.Script, ref.Filename, formatCustomScriptOutput(stdout.String(), stderr.String()), runErr) + scriptOutput := formatCustomScriptOutput(stdout.String(), stderr.String()) + + return fmt.Errorf("generation script %#q failed for source %#q%s\n%w", + ref.Origin.Script, ref.Filename, scriptOutput, runErr) } return nil diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index 9b6ed39e..12b3bfca 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -19,9 +19,12 @@ import ( ) func TestCustomFileSourceProvider_GetFile_NonCustomOriginReturnsNotFound(t *testing.T) { + ctx := testctx.NewCtx() + provider := &customFileSourceProvider{ - fs: afero.NewMemMapFs(), - runner: nil, // never reached + dryRunnable: ctx, + fs: ctx.FS(), + runner: nil, // never reached } ctrl := gomock.NewController(t) @@ -37,9 +40,12 @@ func TestCustomFileSourceProvider_GetFile_NonCustomOriginReturnsNotFound(t *test } func TestCustomFileSourceProvider_GetFile_MissingScriptReturnsError(t *testing.T) { + ctx := testctx.NewCtx() + provider := &customFileSourceProvider{ - fs: afero.NewMemMapFs(), - runner: nil, // never reached — script stat check fails first + dryRunnable: ctx, + fs: ctx.FS(), + runner: nil, // never reached — script stat check fails first } ctrl := gomock.NewController(t) From b704d2a3570ac942954b599261cbab0bf9b3723e Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 10 Jul 2026 18:07:51 +0000 Subject: [PATCH 4/7] Add tests to verify retrieval order; duplicate input check --- docs/user/reference/config/components.md | 4 +- internal/projectconfig/configfile.go | 43 ++++-- internal/projectconfig/configfile_test.go | 26 ++++ .../sourceproviders/fedorasourceprovider.go | 6 +- .../fedorasourceprovider_test.go | 25 ++-- .../sourceproviders/sourcemanager.go | 66 ++++----- .../sourcemanager_internal_test.go | 125 ++++++++++++++++++ .../sourceproviders/sourcemanager_test.go | 28 ---- 8 files changed, 234 insertions(+), 89 deletions(-) create mode 100644 internal/providers/sourceproviders/sourcemanager_internal_test.go diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 101760e0..67866951 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -333,13 +333,15 @@ origin = { type = "download", uri = "https://example.com/repo/.../shimx64.efi Use `origin.type = "custom"` when a source archive must be assembled or modified (e.g. stripping sensitive test fixtures from an upstream tarball). azldev runs the script inside a fresh mock chroot — the script **must write all output to `/azldev-gen/output/`**, which azldev packages into the archive named by `filename`. Network access is always enabled; the mock config comes from the project's default distro. +Custom sources are regenerated on every source preparation rather than restored from lookaside. The generated archive is validated against its configured hash, so changes to the script or its inputs fail with a hash mismatch until the hash is intentionally refreshed. + The `script`, `mock-packages`, and `inputs` fields are nested under `[origin]`: | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Script | `origin.script` | string | **Yes** | Script filename (relative to the component's spec dir) to run in mock. Required for `origin.type = "custom"`. | | Mock packages | `origin.mock-packages` | array of string | No | Extra RPM packages to install in the mock chroot before the script runs. | -| Inputs | `origin.inputs` | array of string | No | Filenames to make available in the mock chroot before the script runs. Each file must already be present in the fetched source output directory — upstream source tarballs, sidecar files (patches, scripts), and any earlier `source-files` entries are all placed there by the upstream fetch before custom scripts run. | +| Inputs | `origin.inputs` | array of string | No | Unique filenames to make available in the mock chroot before the script runs. Each file must already be present in the fetched source output directory — upstream source tarballs, sidecar files (patches, scripts), and any earlier `source-files` entries are all placed there by the upstream fetch before custom scripts run. | On first use, omit `hash` and run `prep-sources --allow-no-hashes` to generate the archive and print its hash, then copy it into the TOML. diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 761682dc..851f1607 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -255,6 +255,7 @@ func validateReplaceUpstream(ref SourceFileReference, componentName string) erro // - 'mock-packages' must be empty when 'origin.type' is not 'custom'. // - 'inputs' must be empty when 'origin.type' is not 'custom'. // - each 'inputs' entry must be a valid filename (no path separators). +// - each 'inputs' entry must be unique. func validateCustomSourceRef(ref SourceFileReference, componentName string) error { if ref.Origin.Type == OriginTypeCustom { if ref.Origin.Script == "" { @@ -270,18 +271,8 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro ref.Origin.Script, ref.Filename, componentName, err) } - for _, input := range ref.Origin.Inputs { - if err := fileutils.ValidateFilename(input); err != nil { - return fmt.Errorf( - "invalid 'inputs' entry %#q for source file %#q in component %#q:\n%w", - input, ref.Filename, componentName, err) - } - - if input == ref.Origin.Script { - return fmt.Errorf( - "'inputs' entry %#q for source file %#q in component %#q conflicts with 'script' filename", - input, ref.Filename, componentName) - } + if err := validateCustomSourceInputs(ref, componentName); err != nil { + return err } return nil @@ -311,6 +302,34 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro return nil } +func validateCustomSourceInputs(ref SourceFileReference, componentName string) error { + seen := make(map[string]bool, len(ref.Origin.Inputs)) + + for _, input := range ref.Origin.Inputs { + if err := fileutils.ValidateFilename(input); err != nil { + return fmt.Errorf( + "invalid 'inputs' entry %#q for source file %#q in component %#q:\n%w", + input, ref.Filename, componentName, err) + } + + if seen[input] { + return fmt.Errorf( + "duplicate 'inputs' entry %#q for source file %#q in component %#q; each input filename must be unique", + input, ref.Filename, componentName) + } + + seen[input] = true + + if input == ref.Origin.Script { + return fmt.Errorf( + "'inputs' entry %#q for source file %#q in component %#q conflicts with 'script' filename", + input, ref.Filename, componentName) + } + } + + return nil +} + // validateOrigin checks that a source file [Origin] is present and valid for its type. // For [OriginTypeURI] ('download'), the [Origin.Uri] field must be a valid URI with a scheme. func validateOrigin(origin Origin, filename string, componentName string) error { diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index 9474ac9d..d5c8047e 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -605,6 +605,32 @@ func TestValidateCustomSourceRef_ValidInputs(t *testing.T) { assert.NoError(t, file.Validate()) } +func TestValidateCustomSourceRef_DuplicateInputs(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "gen.sh", + Inputs: []string{"upstream.tar.gz", "upstream.tar.gz"}, + }, + }, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate 'inputs' entry") + assert.Contains(t, err.Error(), "upstream.tar.gz") + assert.Contains(t, err.Error(), "gen.tar.gz") + assert.Contains(t, err.Error(), "comp") +} + func TestValidateCustomSourceRef_InputsOnDownloadOrigin(t *testing.T) { file := projectconfig.ConfigFile{ Components: map[string]projectconfig.ComponentConfig{ diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index 344e90a9..688c66e2 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -141,7 +141,7 @@ func (g *FedoraSourcesProviderImpl) GetComponent( } // Collect filenames from source-files config so the lookaside extractor can skip them. - // These files were already fetched by FetchFiles and take precedence over upstream versions. + // [SourceManager.FetchFiles] acquires the configured versions after component fetching. sourceFiles := component.GetConfig().SourceFiles skipFileNames := make([]string, len(sourceFiles)) @@ -178,7 +178,8 @@ func (g *FedoraSourcesProviderImpl) processClonedRepo( } // Extract sources from repo (downloads lookaside files into the temp dir). - // Files in skipFilenames are not downloaded — they were already fetched by FetchFiles. + // Files in skipFilenames are not downloaded because [SourceManager.FetchFiles] + // provides the configured versions after component fetching. // Skip this step entirely when SkipLookaside is set (e.g., during rendering). if !opts.SkipLookaside { err := g.downloader.ExtractSourcesFromRepo( @@ -195,7 +196,6 @@ func (g *FedoraSourcesProviderImpl) processClonedRepo( } // Copy files from temp dir to destination, skipping files that already exist. - // This preserves any files downloaded by FetchFiles, giving them precedence. copyOptions := fileutils.CopyDirOptions{ CopyFileOptions: fileutils.CopyFileOptions{ PreserveFileMode: true, diff --git a/internal/providers/sourceproviders/fedorasourceprovider_test.go b/internal/providers/sourceproviders/fedorasourceprovider_test.go index 0d45e8fe..13968bda 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider_test.go +++ b/internal/providers/sourceproviders/fedorasourceprovider_test.go @@ -220,7 +220,7 @@ func TestGetComponentFromGit(t *testing.T) { assert.Equal(t, "tarball content", string(tarballContent)) }) - t.Run("existing file in destination skips lookaside download", func(t *testing.T) { + t.Run("configured replacement skips upstream lookaside download", func(t *testing.T) { ctrl := gomock.NewController(t) httpDownloader, err := downloader.NewHTTPDownloader( @@ -288,26 +288,23 @@ func TestGetComponentFromGit(t *testing.T) { mockComponent.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ Name: testPackageName, SourceFiles: []projectconfig.SourceFileReference{ - {Filename: testFileName}, + { + Filename: testFileName, + ReplaceUpstream: true, + ReplaceReason: "use configured replacement", + }, }, }) - // Pre-populate destination with the file — simulating a prior FetchFiles download - err = fileutils.MkdirAll(env.FS(), testDestDir) - require.NoError(t, err) - - preExistingContent := []byte("azure-linux-signed-binary") - err = fileutils.WriteFile(env.FS(), testDestDir+"/"+testFileName, preExistingContent, fileperms.PublicFile) - require.NoError(t, err) - - // Should succeed — the file is in skipFilenames so the 404 lookaside download is skipped + // The upstream hash would return 404 from the test server. Success proves the + // configured filename suppressed that upstream lookaside download. err = provider.GetComponent(context.Background(), mockComponent, testDestDir) require.NoError(t, err) - // Verify the pre-existing file was preserved (not overwritten by git repo version) - content, err := fileutils.ReadFile(env.FS(), testDestDir+"/"+testFileName) + // The skipped upstream lookaside artifact was not materialized. + exists, err := fileutils.Exists(env.FS(), testDestDir+"/"+testFileName) require.NoError(t, err) - assert.Equal(t, preExistingContent, content) + assert.False(t, exists) }) t.Run("successful extraction without lookaside sources", func(t *testing.T) { diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 30463741..20941709 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -368,13 +368,13 @@ func (m *sourceManager) FetchFiles( } // fetchSourceFile acquires a single source file using the following priority order: -// 1. Lookaside cache — if hash info is available, attempt a cached download first. -// This applies to all origin types, including 'custom', so a previously generated -// and cached archive avoids a full mock regeneration. -// 2. File providers — handles origin types that require local generation (e.g. 'custom'). -// 3. Configured download origin — final fallback for 'download' origin types. +// 1. Custom file provider — custom sources are always regenerated so their configured +// hashes validate the current script and inputs. +// 2. Lookaside cache — non-custom files with hash info attempt a cached download first. +// 3. File providers — handles other provider-supported origin types. +// 4. Configured download origin — final fallback for 'download' origin types. // -// When disable-origins is set, step 3 is skipped and only lookaside and file providers apply. +// When disable-origins is set, step 4 is skipped and only lookaside and file providers apply. func (m *sourceManager) fetchSourceFile( ctx context.Context, httpDownloader downloader.Downloader, @@ -389,34 +389,12 @@ func (m *sourceManager) fetchSourceFile( destPath := filepath.Join(destDirPath, fileRef.Filename) - sourceExists, err := fileutils.Exists(m.fs, destPath) - if err != nil { - return fmt.Errorf("failed to check existence of destination file %#q:\n%w", destPath, err) - } - - if sourceExists { - slog.Debug("Source file already exists, skipping download", - "filename", fileRef.Filename, - "path", destPath) - + // Try the lookaside cache first for non-custom files if hash info is available. + // Custom files are always regenerated so stale configured hashes are detected. + if m.trySourceFileLookaside(ctx, httpDownloader, component, fileRef, destPath) { return nil } - // Try the lookaside cache first if hash info is available. This applies to - // all origin types, including 'custom' — if the generated archive is already - // cached in the lookaside (as it will be after the first run), we skip the - // expensive mock generation entirely. - if fileRef.Hash != "" && fileRef.HashType != "" { - lookasideErr := m.tryLookasideDownload(ctx, httpDownloader, component, fileRef, destPath) - if lookasideErr == nil { - return nil - } - - slog.Debug("Lookaside cache download failed", - "filename", fileRef.Filename, - "error", lookasideErr) - } - // Try each registered file provider. Providers return [ErrNotFound] to signal // they don't handle this reference; any other error is fatal. for _, provider := range m.fileProviders { @@ -454,6 +432,32 @@ func (m *sourceManager) fetchSourceFile( return m.fetchFromDownloadOrigin(ctx, httpDownloader, fileRef, destPath) } +// trySourceFileLookaside attempts a cached download for non-custom files with hash information. +// Custom files deliberately bypass lookaside so they are regenerated and validated each time. +func (m *sourceManager) trySourceFileLookaside( + ctx context.Context, + httpDownloader downloader.Downloader, + component components.Component, + fileRef *projectconfig.SourceFileReference, + destPath string, +) bool { + if fileRef.Origin.Type == projectconfig.OriginTypeCustom || + fileRef.Hash == "" || fileRef.HashType == "" { + return false + } + + lookasideErr := m.tryLookasideDownload(ctx, httpDownloader, component, fileRef, destPath) + if lookasideErr == nil { + return true + } + + slog.Debug("Lookaside cache download failed", + "filename", fileRef.Filename, + "error", lookasideErr) + + return false +} + // tryLookasideDownload attempts to download a source file from the lookaside cache. // Returns nil on success, or an error if the download fails. func (m *sourceManager) tryLookasideDownload( diff --git a/internal/providers/sourceproviders/sourcemanager_internal_test.go b/internal/providers/sourceproviders/sourcemanager_internal_test.go new file mode 100644 index 00000000..0dbc6e11 --- /dev/null +++ b/internal/providers/sourceproviders/sourcemanager_internal_test.go @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sourceproviders + +import ( + "context" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/downloader/downloader_test" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +type regeneratingFileProvider struct { + fs opctx.FS + called bool +} + +func (p *regeneratingFileProvider) GetFile( + _ context.Context, + _ components.Component, + ref projectconfig.SourceFileReference, + destDirPath string, +) error { + p.called = true + + return fileutils.WriteFile(p.fs, filepath.Join(destDirPath, ref.Filename), nil, fileperms.PublicFile) +} + +func TestFetchSourceFile_ConfiguredSourceReplacesExistingFile(t *testing.T) { + const ( + destDir = "/output" + filename = "source.tar.gz" + // SHA-256 of the empty replacement produced by each test acquisition path. + emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) + + tests := []struct { + name string + origin projectconfig.Origin + useProvider bool + downloadURL string + }{ + { + name: "custom source is always regenerated", + origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + useProvider: true, + }, + { + name: "download source replaces upstream file", + origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeURI, + Uri: "https://origin.example.com/source.tar.gz", + }, + downloadURL: "https://example.com/test-component/source.tar.gz/sha256/" + + emptyHash + "/source.tar.gz", + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + ctx := testctx.NewCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return("test-component") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) + + httpDownloader := downloader_test.NewMockDownloader(ctrl) + if testCase.downloadURL != "" { + httpDownloader.EXPECT(). + Download(gomock.Any(), testCase.downloadURL, filepath.Join(destDir, filename)). + DoAndReturn(func(_ context.Context, _, destPath string) error { + return fileutils.WriteFile(ctx.FS(), destPath, nil, fileperms.PublicFile) + }) + } + + provider := ®eneratingFileProvider{fs: ctx.FS()} + + var fileProviders []FileSourceProvider + + if testCase.useProvider { + fileProviders = []FileSourceProvider{provider} + } + + manager := &sourceManager{ + dryRunnable: ctx, + fs: ctx.FS(), + lookasideBaseURI: "https://example.com/$pkg/$filename/$hashtype/$hash/$filename", + retryConfig: retry.Disabled(), + fileProviders: fileProviders, + } + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), destDir)) + require.NoError(t, fileutils.WriteFile( + ctx.FS(), filepath.Join(destDir, filename), []byte("upstream content"), fileperms.PublicFile)) + + ref := &projectconfig.SourceFileReference{ + Filename: filename, + Hash: emptyHash, + HashType: fileutils.HashTypeSHA256, + Origin: testCase.origin, + ReplaceUpstream: true, + ReplaceReason: "use configured replacement", + } + + require.NoError(t, manager.fetchSourceFile(t.Context(), httpDownloader, component, ref, destDir)) + assert.Equal(t, testCase.useProvider, provider.called) + + content, err := fileutils.ReadFile(ctx.FS(), filepath.Join(destDir, filename)) + require.NoError(t, err) + assert.Empty(t, content) + }) + } +} diff --git a/internal/providers/sourceproviders/sourcemanager_test.go b/internal/providers/sourceproviders/sourcemanager_test.go index 6297b683..d7bf67d6 100644 --- a/internal/providers/sourceproviders/sourcemanager_test.go +++ b/internal/providers/sourceproviders/sourcemanager_test.go @@ -267,34 +267,6 @@ func TestSourceManager_FetchFiles_NoSourceFiles(t *testing.T) { require.NoError(t, err) } -func TestSourceManager_FetchFiles_ExistingFile(t *testing.T) { - env := testutils.NewTestEnv(t) - ctrl := gomock.NewController(t) - component := components_testutils.NewMockComponent(ctrl) - - require.NoError(t, env.TestFS.MkdirAll(testDestDir, fileperms.PrivateDir)) - - destPath := filepath.Join(testDestDir, testSourceTarball) - require.NoError(t, fileutils.WriteFile(env.TestFS, destPath, []byte("existing content"), fileperms.PrivateFile)) - - componentConfig := &projectconfig.ComponentConfig{ - SourceFiles: []projectconfig.SourceFileReference{{ - Filename: testSourceTarball, - Hash: "abc123", - HashType: fileutils.HashTypeSHA256, - }}, - } - - component.EXPECT().GetName().AnyTimes().Return("test-component") - component.EXPECT().GetConfig().AnyTimes().Return(componentConfig) - - sourceManager, err := sourceproviders.NewSourceManager(env.Env, testDefaultDistro()) - require.NoError(t, err) - - err = sourceManager.FetchFiles(t.Context(), component, testDestDir) - require.NoError(t, err) -} - func TestSourceManager_FetchFiles_Errors(t *testing.T) { tests := []struct { name string From 4377f806da5e0578abd7bef4855bcf6189ba52ba Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 10 Jul 2026 21:09:47 +0000 Subject: [PATCH 5/7] Added custom source gen scenario test --- .../sourcemanager_internal_test.go | 23 +++- scenario/custom_source_test.go | 106 ++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 scenario/custom_source_test.go diff --git a/internal/providers/sourceproviders/sourcemanager_internal_test.go b/internal/providers/sourceproviders/sourcemanager_internal_test.go index 0dbc6e11..3abcf66b 100644 --- a/internal/providers/sourceproviders/sourcemanager_internal_test.go +++ b/internal/providers/sourceproviders/sourcemanager_internal_test.go @@ -51,11 +51,21 @@ func TestFetchSourceFile_ConfiguredSourceReplacesExistingFile(t *testing.T) { origin projectconfig.Origin useProvider bool downloadURL string + hash string + expectError string }{ { name: "custom source is always regenerated", origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, useProvider: true, + hash: emptyHash, + }, + { + name: "custom source with wrong hash fails validation", + origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + useProvider: true, + hash: "0000000000000000000000000000000000000000000000000000000000000000", + expectError: "hash validation failed", }, { name: "download source replaces upstream file", @@ -65,6 +75,7 @@ func TestFetchSourceFile_ConfiguredSourceReplacesExistingFile(t *testing.T) { }, downloadURL: "https://example.com/test-component/source.tar.gz/sha256/" + emptyHash + "/source.tar.gz", + hash: emptyHash, }, } @@ -107,16 +118,24 @@ func TestFetchSourceFile_ConfiguredSourceReplacesExistingFile(t *testing.T) { ref := &projectconfig.SourceFileReference{ Filename: filename, - Hash: emptyHash, + Hash: testCase.hash, HashType: fileutils.HashTypeSHA256, Origin: testCase.origin, ReplaceUpstream: true, ReplaceReason: "use configured replacement", } - require.NoError(t, manager.fetchSourceFile(t.Context(), httpDownloader, component, ref, destDir)) + err := manager.fetchSourceFile(t.Context(), httpDownloader, component, ref, destDir) assert.Equal(t, testCase.useProvider, provider.called) + if testCase.expectError != "" { + require.ErrorContains(t, err, testCase.expectError) + + return + } + + require.NoError(t, err) + content, err := fileutils.ReadFile(ctx.FS(), filepath.Join(destDir, filename)) require.NoError(t, err) assert.Empty(t, content) diff --git a/scenario/custom_source_test.go b/scenario/custom_source_test.go new file mode 100644 index 00000000..d2c4a007 --- /dev/null +++ b/scenario/custom_source_test.go @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build scenario + +package scenario_tests + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/cmdtest" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/projecttest" + "github.com/stretchr/testify/require" +) + +func TestCustomSourceRegenerationDetectsStaleHash(t *testing.T) { + if testing.Short() { + t.Skip("skipping long test") + } + + const ( + componentName = "custom-source" + scriptName = "generate.sh" + archiveName = "generated.tar.gz" + ) + + spec := projecttest.NewSpec(projecttest.WithName(componentName)) + project := projecttest.NewDynamicTestProject( + projecttest.AddSpec(spec), + projecttest.AddComponent(&projectconfig.ComponentConfig{ + Name: componentName, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: filepath.Join("specs", componentName, componentName+".spec"), + }, + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: archiveName, + HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: scriptName, + }, + }, + }, + }), + projecttest.AddFile(filepath.Join("specs", componentName, scriptName), `#!/bin/sh +set -eu +printf 'version-one\n' > /azldev-gen/output/payload.txt +`), + projecttest.UseTestDefaultConfigs(), + ) + + projectStagingDir := t.TempDir() + project.Serialize(t, projectStagingDir) + + testScript := ` +set -eux + +azldev -C project -v component prep-sources -p custom-source \ + -o prepared --without-git --allow-no-hashes + +test "$(tar -xOf prepared/generated.tar.gz payload.txt)" = "version-one" + +GENERATED_HASH="$(sha512sum prepared/generated.tar.gz | awk '{print $1}')" +sed -i "/hash-type =/a hash = \"$GENERATED_HASH\"" project/azldev.toml +sed -i 's/version-one/version-two/' project/specs/custom-source/generate.sh + +if azldev -C project -v component prep-sources -p custom-source \ + -o prepared --without-git --force 2>second-run.stderr; then + echo "expected regenerated custom source to fail hash validation" >&2 + exit 1 +fi + +grep -q "hash validation failed" second-run.stderr +test "$(tar -xOf prepared/generated.tar.gz payload.txt)" = "version-two" +` + + scenarioTest := cmdtest.NewScenarioTest(). + WithScript(strings.NewReader(testScript)). + AddDirRecursive(t, "project", projectStagingDir). + AddDirRecursive(t, projecttest.TestDefaultConfigsSubdir, projecttest.TestDefaultConfigsDir()) + + results, err := scenarioTest. + InContainer(). + WithPrivilege(). + WithNetwork(). + Run(t) + require.NoError(t, err) + + t.Logf("Standard output:\n%s", results.Stdout) + t.Logf("Standard error:\n%s", results.Stderr) + results.AssertZeroExitCode(t) + + stderrPath := filepath.Join(results.Workdir, "second-run.stderr") + require.FileExists(t, stderrPath) + + stderr, err := os.ReadFile(stderrPath) + require.NoError(t, err) + require.Contains(t, string(stderr), "hash validation failed") +} From cf0b7c45a1a79a7e89bbe97035b03e928d95492c Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 10 Jul 2026 21:34:09 +0000 Subject: [PATCH 6/7] Limit bytes written to stdout by script --- .../sourceproviders/customsourceprovider.go | 59 ++++++++++++++++--- .../customsourceprovider_internal_test.go | 42 +++++++++++++ 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index 8970e7a5..9cc0fca6 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -4,7 +4,6 @@ package sourceproviders import ( - "bytes" "context" "fmt" "io" @@ -30,6 +29,11 @@ const customGenScriptDir = "/azldev-gen/script" // must write its output. const customGenOutputDir = "/azldev-gen/output" +// maxCustomScriptOutputBytes limits the retained stdout and stderr from a custom +// generation script. The tail is retained because failures are typically reported +// near the end of command output. +const maxCustomScriptOutputBytes = 64 * 1024 + // customFileSourceProvider implements [FileSourceProvider] for source files with // [projectconfig.OriginTypeCustom]. It executes a user-supplied script inside a // fresh mock chroot and packages the output directory as a deterministic archive. @@ -281,17 +285,15 @@ func execScriptInChroot( ref.Filename, cmdErr) } - var ( - stdout bytes.Buffer - stderr bytes.Buffer - ) + stdout := newOutputTail(maxCustomScriptOutputBytes) + stderr := newOutputTail(maxCustomScriptOutputBytes) if verbose { - cmd.SetStdout(io.MultiWriter(os.Stdout, &stdout)) - cmd.SetStderr(io.MultiWriter(os.Stderr, &stderr)) + cmd.SetStdout(io.MultiWriter(os.Stdout, stdout)) + cmd.SetStderr(io.MultiWriter(os.Stderr, stderr)) } else { - cmd.SetStdout(&stdout) - cmd.SetStderr(&stderr) + cmd.SetStdout(stdout) + cmd.SetStderr(stderr) } if runErr := cmd.Run(ctx); runErr != nil { @@ -304,6 +306,45 @@ func execScriptInChroot( return nil } +// outputTail retains the last limit bytes written to it. It implements [io.Writer] +// so command output can be captured without unbounded memory growth. +type outputTail struct { + buf []byte + limit int + truncated bool +} + +func newOutputTail(limit int) *outputTail { + return &outputTail{buf: make([]byte, 0, limit), limit: limit} +} + +func (b *outputTail) Write(data []byte) (n int, err error) { + n = len(data) + if n >= b.limit { + b.truncated = b.truncated || n > b.limit || len(b.buf) > 0 + b.buf = append(b.buf[:0], data[n-b.limit:]...) + + return n, nil + } + + if overflow := len(b.buf) + n - b.limit; overflow > 0 { + b.buf = append(b.buf[:0], b.buf[overflow:]...) + b.truncated = true + } + + b.buf = append(b.buf, data...) + + return n, nil +} + +func (b *outputTail) String() string { + if b.truncated { + return fmt.Sprintf("[output truncated; showing last %d bytes]\n%s", b.limit, b.buf) + } + + return string(b.buf) +} + func formatCustomScriptOutput(stdout, stderr string) string { var output strings.Builder diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index 12b3bfca..940022ee 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -39,6 +39,48 @@ func TestCustomFileSourceProvider_GetFile_NonCustomOriginReturnsNotFound(t *test assert.ErrorIs(t, err, ErrNotFound) } +func TestOutputTail(t *testing.T) { + tests := []struct { + name string + limit int + writes []string + expected string + }{ + { + name: "retains complete output within limit", + limit: 5, + writes: []string{"abc", "de"}, + expected: "abcde", + }, + { + name: "retains tail after multiple writes", + limit: 5, + writes: []string{"abc", "defg"}, + expected: "[output truncated; showing last 5 bytes]\ncdefg", + }, + { + name: "retains tail of a single oversized write", + limit: 4, + writes: []string{"abcdef"}, + expected: "[output truncated; showing last 4 bytes]\ncdef", + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + tail := newOutputTail(testCase.limit) + + for _, write := range testCase.writes { + written, err := tail.Write([]byte(write)) + require.NoError(t, err) + assert.Equal(t, len(write), written) + } + + assert.Equal(t, testCase.expected, tail.String()) + }) + } +} + func TestCustomFileSourceProvider_GetFile_MissingScriptReturnsError(t *testing.T) { ctx := testctx.NewCtx() From 11afa3a162c9b50b164cbccfa827a7d7a264d882 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 10 Jul 2026 21:57:47 +0000 Subject: [PATCH 7/7] Checks to prevent symlink inputs --- .../sourceproviders/customsourceprovider.go | 12 ++++++++++++ .../customsourceprovider_internal_test.go | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index 9cc0fca6..f60c7622 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -19,6 +19,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" ) // customGenScriptDir is the path inside the mock chroot where the generation script @@ -381,6 +382,17 @@ func stageInputFiles( sourcePath := filepath.Join(destDirPath, filename) + if lstater, ok := fs.(afero.Lstater); ok { + fileInfo, lstatCalled, lstatErr := lstater.LstatIfPossible(sourcePath) + if lstatErr != nil { + return fmt.Errorf("failed to inspect input file %#q:\n%w", filename, lstatErr) + } + + if lstatCalled && fileInfo.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("input file %#q must not be a symbolic link", filename) + } + } + exists, statErr := fileutils.Exists(fs, sourcePath) if statErr != nil { return fmt.Errorf("failed to stat input file %#q:\n%w", filename, statErr) diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index 940022ee..547d66e5 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -5,6 +5,7 @@ package sourceproviders import ( "context" + "os" "path/filepath" "testing" @@ -204,6 +205,22 @@ func TestStageInputFiles_NotFoundReturnsError(t *testing.T) { assert.Contains(t, err.Error(), "/output") } +func TestStageInputFiles_SymlinkRejected(t *testing.T) { + fileSystem := afero.NewOsFs() + dir := t.TempDir() + targetPath := filepath.Join(dir, "target.tar.gz") + inputPath := filepath.Join(dir, "input.tar.gz") + + require.NoError(t, os.WriteFile(targetPath, []byte("input"), fileperms.PublicFile)) + require.NoError(t, os.Symlink(targetPath, inputPath)) + + err := stageInputFiles( + testctx.NewCtx(), fileSystem, []string{"input.tar.gz"}, dir, filepath.Join(dir, "script"), "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "input.tar.gz") + assert.Contains(t, err.Error(), "symbolic link") +} + func TestStageInputFiles_ScriptNameConflictReturnsError(t *testing.T) { ctx := testctx.NewCtx() memFS := ctx.FS()