diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index b202a786..67866951 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -333,12 +333,15 @@ origin = { type = "download", uri = "https://example.com/repo/.../shimx64.efi Use `origin.type = "custom"` when a source archive must be assembled or modified (e.g. stripping sensitive test fixtures from an upstream tarball). azldev runs the script inside a fresh mock chroot — the script **must write all output to `/azldev-gen/output/`**, which azldev packages into the archive named by `filename`. Network access is always enabled; the mock config comes from the project's default distro. -The `script` and `mock-packages` fields are nested under `[origin]`: +Custom sources are regenerated on every source preparation rather than restored from lookaside. The generated archive is validated against its configured hash, so changes to the script or its inputs fail with a hash mismatch until the hash is intentionally refreshed. + +The `script`, `mock-packages`, and `inputs` fields are nested under `[origin]`: | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Script | `origin.script` | string | **Yes** | Script filename (relative to the component's spec dir) to run in mock. Required for `origin.type = "custom"`. | | Mock packages | `origin.mock-packages` | array of string | No | Extra RPM packages to install in the mock chroot before the script runs. | +| Inputs | `origin.inputs` | array of string | No | Unique filenames to make available in the mock chroot before the script runs. Each file must already be present in the fetched source output directory — upstream source tarballs, sidecar files (patches, scripts), and any earlier `source-files` entries are all placed there by the upstream fetch before custom scripts run. | On first use, omit `hash` and run `prep-sources --allow-no-hashes` to generate the archive and print its hash, then copy it into the TOML. @@ -350,8 +353,11 @@ hash = "abc123..." # from: prep-sources --allow-no-hashes origin.type = "custom" origin.script = "gen-yara-stripped.sh" # relative to the component's spec directory origin.mock-packages = ["cmake"] # omit if not needed +origin.inputs = ["yara-4.5.4.tar.gz"] # available to the script as ./yara-4.5.4.tar.gz ``` +> **Note:** Upstream source tarballs are automatically available as inputs before running custom generation scripts. There is no need to re-declare an upstream file in `source-files` to use it as an input. + ### Replacing an upstream `sources` entry A `source-files` entry whose `filename` collides with an upstream `sources` entry is an error by default. Set `replace-upstream = true` (with a non-empty `replace-reason`) to intentionally substitute it: diff --git a/internal/app/azldev/cmds/component/history_customizations.go b/internal/app/azldev/cmds/component/history_customizations.go index ad5de706..1e7f5be7 100644 --- a/internal/app/azldev/cmds/component/history_customizations.go +++ b/internal/app/azldev/cmds/component/history_customizations.go @@ -236,6 +236,13 @@ func appendSourceFileItems( Value: pkg, }) } + + for _, input := range sourceFile.Origin.Inputs { + items = append(items, CustomizationItem{ + Kind: "source-files.inputs", + Value: input, + }) + } } return items diff --git a/internal/app/azldev/cmds/component/history_internal_test.go b/internal/app/azldev/cmds/component/history_internal_test.go index 33e39a24..5a7a9e50 100644 --- a/internal/app/azldev/cmds/component/history_internal_test.go +++ b/internal/app/azldev/cmds/component/history_internal_test.go @@ -142,10 +142,11 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { "SourceFileReference.ReplaceUpstream": "source-files.replace-upstream", "SourceFileReference.Origin": "delegates to Origin walk", - // Origin -- Script and MockPackages are fingerprint-relevant; Type and Uri + // Origin -- Script, MockPackages, and Inputs are fingerprint-relevant; Type and Uri // are tagged fingerprint:"-" (download location, not build input). "Origin.Script": "source-files.script", "Origin.MockPackages": "source-files.mock-packages", + "Origin.Inputs": "source-files.inputs", } actualFields := make(map[string]bool) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 617dcbd8..c391ae61 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -215,17 +215,6 @@ func NewPreparer( func (p *sourcePreparerImpl) PrepareSources( ctx context.Context, component components.Component, outputDir string, applyOverlays bool, ) error { - // Use the source manager to fetch source files (archives, patches, etc.) - // Skip this step when skipLookaside is set — source tarballs are not needed - // for rendering and are the most expensive download. - if !p.skipLookaside { - err := p.sourceManager.FetchFiles(ctx, component, outputDir) - if err != nil { - return fmt.Errorf("failed to fetch source files for component %#q:\n%w", - component.GetName(), err) - } - } - // Preserve the upstream .git directory only when dist-git creation is // requested via --with-git. This is required so that overlay commits can be // appended on top of the upstream commit log during synthetic history generation. @@ -238,13 +227,21 @@ func (p *sourcePreparerImpl) PrepareSources( fetchOpts = append(fetchOpts, sourceproviders.WithSkipLookaside()) } - // Use the source manager to fetch the component (spec file and sidecar files). + // Fetch the component first (spec, sidecar files, and upstream source tarballs). err := p.sourceManager.FetchComponent(ctx, component, outputDir, fetchOpts...) if err != nil { return fmt.Errorf("failed to fetch sources for component %#q:\n%w", component.GetName(), err) } + // Fetch custom and downloaded source files. + if !p.skipLookaside { + if err := p.sourceManager.FetchFiles(ctx, component, outputDir); err != nil { + return fmt.Errorf("failed to fetch source files for component %#q:\n%w", + component.GetName(), err) + } + } + if applyOverlays { err := p.applyOverlaysToSources(ctx, component, outputDir) if err != nil { diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 45a9d286..a0a6f319 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -108,6 +108,8 @@ func TestPrepareSources_SourceManagerError(t *testing.T) { expectedErr := errors.New("failed to fetch files") component.EXPECT().GetName().AnyTimes().Return("test-component") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).Return(nil) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(expectedErr) preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) @@ -633,8 +635,11 @@ func TestDiffSources_FetchError(t *testing.T) { require.NoError(t, fileutils.MkdirAll(ctx.FS(), baseDir)) component.EXPECT().GetName().AnyTimes().Return("test-component") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) expectedErr := errors.New("network failure") + + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, gomock.Any(), gomock.Any()).Return(nil) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, gomock.Any()).Return(expectedErr) preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx) diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 7b5519ed..c260f2dd 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -66,17 +66,24 @@ type Origin struct { // MockPackages is a list of RPM package names to install in the mock chroot before // running [Origin.Script]. Only valid when [Origin.Type] is 'custom'. MockPackages []string `toml:"mock-packages,omitempty" json:"mockPackages,omitempty" jsonschema:"title=Mock packages,description=RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'."` + + // Inputs is a list of source-output filenames to copy next to [Origin.Script] + // before it runs. Each entry must be a plain filename. Only valid when + // [Origin.Type] is 'custom'. + Inputs []string `toml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"title=Inputs,description=Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'."` } // HashInclude implements the hashstructure [Includable] interface so that -// [Origin.Script] and [Origin.MockPackages] are omitted from the component -// fingerprint when they hold their zero values. +// [Origin.Script], [Origin.MockPackages], and [Origin.Inputs] are omitted from +// the component fingerprint when they hold their zero values. func (o Origin) HashInclude(field string, _ any) (bool, error) { switch field { case "Script": return o.Script != "", nil case "MockPackages": return len(o.MockPackages) > 0, nil + case "Inputs": + return len(o.Inputs) > 0, nil } return true, nil @@ -114,10 +121,10 @@ type SourceFileReference struct { // HashInclude implements the hashstructure [Includable] interface so that // [SourceFileReference.Origin] is omitted from the component fingerprint when -// neither [Origin.Script] nor [Origin.MockPackages] are set. +// none of [Origin.Script], [Origin.MockPackages], or [Origin.Inputs] are set. func (r SourceFileReference) HashInclude(field string, _ any) (bool, error) { if field == "Origin" { - return r.Origin.Script != "" || len(r.Origin.MockPackages) > 0, nil + return r.Origin.Script != "" || len(r.Origin.MockPackages) > 0 || len(r.Origin.Inputs) > 0, nil } return true, nil diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index da1b5bae..851f1607 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -248,11 +248,14 @@ func validateReplaceUpstream(ref SourceFileReference, componentName string) erro return nil } -// validateCustomSourceRef enforces the pairing rules for the 'script' and 'mock-packages' -// fields on a [SourceFileReference]: +// validateCustomSourceRef enforces the pairing rules for the 'script', 'mock-packages', +// and 'inputs' fields on a [SourceFileReference]: // - 'script' is required when 'origin.type' is 'custom'. // - 'script' must be empty when 'origin.type' is not 'custom'. // - 'mock-packages' must be empty when 'origin.type' is not 'custom'. +// - 'inputs' must be empty when 'origin.type' is not 'custom'. +// - each 'inputs' entry must be a valid filename (no path separators). +// - each 'inputs' entry must be unique. func validateCustomSourceRef(ref SourceFileReference, componentName string) error { if ref.Origin.Type == OriginTypeCustom { if ref.Origin.Script == "" { @@ -268,6 +271,10 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro ref.Origin.Script, ref.Filename, componentName, err) } + if err := validateCustomSourceInputs(ref, componentName); err != nil { + return err + } + return nil } @@ -285,6 +292,41 @@ func validateCustomSourceRef(ref SourceFileReference, componentName string) erro ref.Filename, componentName, string(ref.Origin.Type)) } + if len(ref.Origin.Inputs) > 0 { + return fmt.Errorf( + "source file %#q in component %#q has 'inputs' set but origin type is %#q; "+ + "'inputs' is only valid when origin type is 'custom'", + ref.Filename, componentName, string(ref.Origin.Type)) + } + + return nil +} + +func validateCustomSourceInputs(ref SourceFileReference, componentName string) error { + seen := make(map[string]bool, len(ref.Origin.Inputs)) + + for _, input := range ref.Origin.Inputs { + if err := fileutils.ValidateFilename(input); err != nil { + return fmt.Errorf( + "invalid 'inputs' entry %#q for source file %#q in component %#q:\n%w", + input, ref.Filename, componentName, err) + } + + if seen[input] { + return fmt.Errorf( + "duplicate 'inputs' entry %#q for source file %#q in component %#q; each input filename must be unique", + input, ref.Filename, componentName) + } + + seen[input] = true + + if input == ref.Origin.Script { + return fmt.Errorf( + "'inputs' entry %#q for source file %#q in component %#q conflicts with 'script' filename", + input, ref.Filename, componentName) + } + } + return nil } diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index b6d8f56f..d5c8047e 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -584,3 +584,95 @@ func TestValidateCustomSourceRef_InvalidScriptFilename(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "script") } + +func TestValidateCustomSourceRef_ValidInputs(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "gen.sh", + Inputs: []string{"upstream.tar.gz", "fix.patch"}, + }, + }, + }, + }, + }, + } + assert.NoError(t, file.Validate()) +} + +func TestValidateCustomSourceRef_DuplicateInputs(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "gen.sh", + Inputs: []string{"upstream.tar.gz", "upstream.tar.gz"}, + }, + }, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate 'inputs' entry") + assert.Contains(t, err.Error(), "upstream.tar.gz") + assert.Contains(t, err.Error(), "gen.tar.gz") + assert.Contains(t, err.Error(), "comp") +} + +func TestValidateCustomSourceRef_InputsOnDownloadOrigin(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "src.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeURI, + Uri: "https://example.com/src.tar.gz", + Inputs: []string{"other.tar.gz"}, + }, + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'inputs'") + assert.Contains(t, err.Error(), "custom") +} + +func TestValidateCustomSourceRef_InvalidInputFilename(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "gen.sh", + Inputs: []string{"../escape.tar.gz"}, // path traversal attempt + }, + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'inputs'") + assert.Contains(t, err.Error(), "escape.tar.gz") +} diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index 0f2efcfb..f60c7622 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -6,8 +6,11 @@ package sourceproviders import ( "context" "fmt" + "io" "log/slog" + "os" "path/filepath" + "strings" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" @@ -16,6 +19,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" ) // customGenScriptDir is the path inside the mock chroot where the generation script @@ -26,12 +30,19 @@ const customGenScriptDir = "/azldev-gen/script" // must write its output. const customGenOutputDir = "/azldev-gen/output" +// maxCustomScriptOutputBytes limits the retained stdout and stderr from a custom +// generation script. The tail is retained because failures are typically reported +// near the end of command output. +const maxCustomScriptOutputBytes = 64 * 1024 + // customFileSourceProvider implements [FileSourceProvider] for source files with // [projectconfig.OriginTypeCustom]. It executes a user-supplied script inside a // fresh mock chroot and packages the output directory as a deterministic archive. type customFileSourceProvider struct { - fs opctx.FS - runner *mock.Runner + dryRunnable opctx.DryRunnable + fs opctx.FS + runner *mock.Runner + verbose bool } var _ FileSourceProvider = (*customFileSourceProvider)(nil) @@ -51,7 +62,7 @@ func (p *customFileSourceProvider) GetFile( destPath := filepath.Join(destDirPath, ref.Filename) - return generateCustomSourceFile(ctx, p.fs, p.runner, component, &ref, destPath) + return generateCustomSourceFile(ctx, p.dryRunnable, p.fs, p.runner, p.verbose, component, &ref, destPath) } // generateCustomSourceFile runs a single [projectconfig.SourceFileReference] through a @@ -61,8 +72,10 @@ func (p *customFileSourceProvider) GetFile( // is created for each call so bind mounts and package installs do not bleed across invocations. func generateCustomSourceFile( ctx context.Context, + dryRunnable opctx.DryRunnable, fs opctx.FS, baseRunner *mock.Runner, + verbose bool, component components.Component, ref *projectconfig.SourceFileReference, destPath string, @@ -72,6 +85,12 @@ func generateCustomSourceFile( "script", ref.Origin.Script, "component", component.GetName()) + if dryRunnable.DryRun() { + slog.Info("Dry run; skipping custom source file generation", "filename", ref.Filename) + + return nil + } + specDir, err := resolveComponentSpecDir(component) if err != nil { return fmt.Errorf("failed to resolve spec directory for component %#q:\n%w", @@ -93,6 +112,19 @@ func generateCustomSourceFile( defer cleanup() + // Stage declared input files next to the script so script authors can refer to + // them by filename without knowing azldev's internal mount paths. + destDirPath := filepath.Dir(destPath) + + if len(ref.Origin.Inputs) > 0 { + if inputsErr := stageInputFiles( + dryRunnable, fs, ref.Origin.Inputs, destDirPath, scriptTmpDir, ref.Origin.Script, + ); inputsErr != nil { + return fmt.Errorf("failed to resolve inputs for custom source %#q:\n%w", + ref.Filename, inputsErr) + } + } + // Package the output directory as a deterministic archive whose format is // inferred from the filename extension (e.g., .tar.gz, .tar.xz). comp, compErr := archive.DetectCompression(ref.Filename) @@ -104,18 +136,13 @@ func generateCustomSourceFile( // Clone the base runner so bind mounts added here don't persist to other calls. // Network access is always enabled — custom source scripts commonly need to // download upstream tarballs or toolchain artifacts. - runner := baseRunner.Clone() - runner.EnableNetwork() - runner.WithUnprivileged() - runner.AddBindMount(scriptTmpDir, customGenScriptDir) - runner.AddBindMount(genOutputTmpDir, customGenOutputDir) + runner := buildCustomRunner(baseRunner, scriptTmpDir, genOutputTmpDir) - if err := execScriptInChroot(ctx, runner, ref); err != nil { + if err := execScriptInChroot(ctx, runner, verbose, ref); err != nil { return err } - // Ensure the destination directory exists. FetchFiles runs before FetchComponent, - // so the output directory may not have been created yet when this is called. + // Ensure the destination directory exists before writing the archive. if mkdirErr := fileutils.MkdirAll(fs, filepath.Dir(destPath)); mkdirErr != nil { return fmt.Errorf("failed to create destination directory for %#q:\n%w", ref.Filename, mkdirErr) @@ -207,11 +234,25 @@ func prepareStagingDirs( return scriptDir, outputDir, cleanup, nil } +// buildCustomRunner returns a clone of baseRunner configured with the standard bind mounts +// for custom source generation. Network access is always enabled; the unprivileged flag is +// set so the script runs as the 'mockbuild' user rather than root. +func buildCustomRunner(baseRunner *mock.Runner, scriptTmpDir, genOutputTmpDir string) *mock.Runner { + runner := baseRunner.Clone() + runner.EnableNetwork() + runner.WithUnprivileged() + runner.AddBindMount(scriptTmpDir, customGenScriptDir) + runner.AddBindMount(genOutputTmpDir, customGenOutputDir) + + return runner +} + // execScriptInChroot initialises a fresh mock root, optionally installs extra packages, // runs the generation script, and scrubs the root on return. func execScriptInChroot( ctx context.Context, runner *mock.Runner, + verbose bool, ref *projectconfig.SourceFileReference, ) error { if initErr := runner.InitRoot(ctx); initErr != nil { @@ -235,17 +276,142 @@ func execScriptInChroot( } } - scriptChrootPath := filepath.Join(customGenScriptDir, ref.Origin.Script) - - cmd, cmdErr := runner.CmdInChroot(ctx, []string{scriptChrootPath}, false /* interactive */) + // Use positional parameters so the script name is never re-parsed as shell code + // ($1=scriptDir, $2=scriptName; '--' sets $0 and keeps bash from consuming them). + cmd, cmdErr := runner.CmdInChroot(ctx, []string{ + "sh", "-c", `cd "$1" && ./"$2"`, "--", customGenScriptDir, ref.Origin.Script, + }, false /* interactive */) if cmdErr != nil { return fmt.Errorf("failed to create chroot command for generating %#q:\n%w", ref.Filename, cmdErr) } + stdout := newOutputTail(maxCustomScriptOutputBytes) + stderr := newOutputTail(maxCustomScriptOutputBytes) + + if verbose { + cmd.SetStdout(io.MultiWriter(os.Stdout, stdout)) + cmd.SetStderr(io.MultiWriter(os.Stderr, stderr)) + } else { + cmd.SetStdout(stdout) + cmd.SetStderr(stderr) + } + if runErr := cmd.Run(ctx); runErr != nil { - return fmt.Errorf("generation script %#q failed for source %#q:\n%w", - ref.Origin.Script, ref.Filename, runErr) + scriptOutput := formatCustomScriptOutput(stdout.String(), stderr.String()) + + return fmt.Errorf("generation script %#q failed for source %#q%s\n%w", + ref.Origin.Script, ref.Filename, scriptOutput, runErr) + } + + return nil +} + +// outputTail retains the last limit bytes written to it. It implements [io.Writer] +// so command output can be captured without unbounded memory growth. +type outputTail struct { + buf []byte + limit int + truncated bool +} + +func newOutputTail(limit int) *outputTail { + return &outputTail{buf: make([]byte, 0, limit), limit: limit} +} + +func (b *outputTail) Write(data []byte) (n int, err error) { + n = len(data) + if n >= b.limit { + b.truncated = b.truncated || n > b.limit || len(b.buf) > 0 + b.buf = append(b.buf[:0], data[n-b.limit:]...) + + return n, nil + } + + if overflow := len(b.buf) + n - b.limit; overflow > 0 { + b.buf = append(b.buf[:0], b.buf[overflow:]...) + b.truncated = true + } + + b.buf = append(b.buf, data...) + + return n, nil +} + +func (b *outputTail) String() string { + if b.truncated { + return fmt.Sprintf("[output truncated; showing last %d bytes]\n%s", b.limit, b.buf) + } + + return string(b.buf) +} + +func formatCustomScriptOutput(stdout, stderr string) string { + var output strings.Builder + + if stdout != "" { + fmt.Fprintf(&output, "\nstdout:\n%s", stdout) + } + + if stderr != "" { + fmt.Fprintf(&output, "\nstderr:\n%s", stderr) + } + + return output.String() +} + +// stageInputFiles copies the files listed in inputs next to the generation script. +// +// All inputs must already be present in destDirPath, which contains the full set of upstream +// source tarballs, sidecar files, and any earlier 'source-files' entries fetched by +// [FetchComponent] and prior [FetchFiles] calls. If a filename is absent, an error is returned. +func stageInputFiles( + dryRunnable opctx.DryRunnable, + fs opctx.FS, + inputs []string, + destDirPath, scriptTmpDir, scriptName string, +) error { + for _, filename := range inputs { + if err := fileutils.ValidateFilename(filename); err != nil { + return fmt.Errorf("invalid input filename %#q:\n%w", filename, err) + } + + if filename == scriptName { + return fmt.Errorf("input file %#q conflicts with generation script filename", filename) + } + + sourcePath := filepath.Join(destDirPath, filename) + + if lstater, ok := fs.(afero.Lstater); ok { + fileInfo, lstatCalled, lstatErr := lstater.LstatIfPossible(sourcePath) + if lstatErr != nil { + return fmt.Errorf("failed to inspect input file %#q:\n%w", filename, lstatErr) + } + + if lstatCalled && fileInfo.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("input file %#q must not be a symbolic link", filename) + } + } + + exists, statErr := fileutils.Exists(fs, sourcePath) + if statErr != nil { + return fmt.Errorf("failed to stat input file %#q:\n%w", filename, statErr) + } + + if !exists { + return fmt.Errorf( + "input file %#q not found in output directory %#q", + filename, destDirPath) + } + + stagedPath := filepath.Join(scriptTmpDir, filename) + + if writeErr := fileutils.CopyFile( + dryRunnable, fs, sourcePath, stagedPath, + fileutils.CopyFileOptions{PreserveFileMode: true}, + ); writeErr != nil { + return fmt.Errorf("failed to stage input file %#q:\n%w", filename, writeErr) + } } return nil diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index e51ba648..547d66e5 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -5,10 +5,12 @@ package sourceproviders import ( "context" + "os" "path/filepath" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/spf13/afero" @@ -18,9 +20,12 @@ import ( ) func TestCustomFileSourceProvider_GetFile_NonCustomOriginReturnsNotFound(t *testing.T) { + ctx := testctx.NewCtx() + provider := &customFileSourceProvider{ - fs: afero.NewMemMapFs(), - runner: nil, // never reached + dryRunnable: ctx, + fs: ctx.FS(), + runner: nil, // never reached } ctrl := gomock.NewController(t) @@ -35,10 +40,55 @@ func TestCustomFileSourceProvider_GetFile_NonCustomOriginReturnsNotFound(t *test assert.ErrorIs(t, err, ErrNotFound) } +func TestOutputTail(t *testing.T) { + tests := []struct { + name string + limit int + writes []string + expected string + }{ + { + name: "retains complete output within limit", + limit: 5, + writes: []string{"abc", "de"}, + expected: "abcde", + }, + { + name: "retains tail after multiple writes", + limit: 5, + writes: []string{"abc", "defg"}, + expected: "[output truncated; showing last 5 bytes]\ncdefg", + }, + { + name: "retains tail of a single oversized write", + limit: 4, + writes: []string{"abcdef"}, + expected: "[output truncated; showing last 4 bytes]\ncdef", + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + tail := newOutputTail(testCase.limit) + + for _, write := range testCase.writes { + written, err := tail.Write([]byte(write)) + require.NoError(t, err) + assert.Equal(t, len(write), written) + } + + assert.Equal(t, testCase.expected, tail.String()) + }) + } +} + func TestCustomFileSourceProvider_GetFile_MissingScriptReturnsError(t *testing.T) { + ctx := testctx.NewCtx() + provider := &customFileSourceProvider{ - fs: afero.NewMemMapFs(), - runner: nil, // never reached — script stat check fails first + dryRunnable: ctx, + fs: ctx.FS(), + runner: nil, // never reached — script stat check fails first } ctrl := gomock.NewController(t) @@ -123,3 +173,75 @@ func TestPrepareStagingDirs_MissingScriptReturnsError(t *testing.T) { assert.Contains(t, err.Error(), "missing.sh") assert.Nil(t, cleanup) } + +func TestStageInputFiles_FoundInDestDirPreservesMode(t *testing.T) { + ctx := testctx.NewCtx() + memFS := ctx.FS() + + const fileContent = "upstream tarball data" + + require.NoError(t, afero.WriteFile(memFS, "/output/upstream.tar.gz", []byte(fileContent), fileperms.PublicExecutable)) + + err := stageInputFiles(ctx, memFS, []string{"upstream.tar.gz"}, "/output", "/script", "gen.sh") + require.NoError(t, err) + + data, readErr := afero.ReadFile(memFS, "/script/upstream.tar.gz") + require.NoError(t, readErr) + assert.Equal(t, fileContent, string(data)) + + info, statErr := memFS.Stat("/script/upstream.tar.gz") + require.NoError(t, statErr) + assert.Equal(t, fileperms.PublicExecutable, info.Mode().Perm()) +} + +func TestStageInputFiles_NotFoundReturnsError(t *testing.T) { + ctx := testctx.NewCtx() + memFS := ctx.FS() + + // No files written — destDirPath is empty. + err := stageInputFiles(ctx, memFS, []string{"missing.tar.gz"}, "/output", "/script", "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "missing.tar.gz") + assert.Contains(t, err.Error(), "/output") +} + +func TestStageInputFiles_SymlinkRejected(t *testing.T) { + fileSystem := afero.NewOsFs() + dir := t.TempDir() + targetPath := filepath.Join(dir, "target.tar.gz") + inputPath := filepath.Join(dir, "input.tar.gz") + + require.NoError(t, os.WriteFile(targetPath, []byte("input"), fileperms.PublicFile)) + require.NoError(t, os.Symlink(targetPath, inputPath)) + + err := stageInputFiles( + testctx.NewCtx(), fileSystem, []string{"input.tar.gz"}, dir, filepath.Join(dir, "script"), "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "input.tar.gz") + assert.Contains(t, err.Error(), "symbolic link") +} + +func TestStageInputFiles_ScriptNameConflictReturnsError(t *testing.T) { + ctx := testctx.NewCtx() + memFS := ctx.FS() + + err := stageInputFiles(ctx, memFS, []string{"gen.sh"}, "/output", "/script", "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicts") +} + +func TestStageInputFiles_InvalidFilenameReturnsError(t *testing.T) { + ctx := testctx.NewCtx() + memFS := ctx.FS() + + err := stageInputFiles(ctx, memFS, []string{"../escape.tar.gz"}, "/output", "/script", "gen.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid input filename") +} + +func TestFormatCustomScriptOutput(t *testing.T) { + output := formatCustomScriptOutput("stdout line\n", "stderr line\n") + + assert.Contains(t, output, "stdout:\nstdout line") + assert.Contains(t, output, "stderr:\nstderr line") +} diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index 344e90a9..688c66e2 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -141,7 +141,7 @@ func (g *FedoraSourcesProviderImpl) GetComponent( } // Collect filenames from source-files config so the lookaside extractor can skip them. - // These files were already fetched by FetchFiles and take precedence over upstream versions. + // [SourceManager.FetchFiles] acquires the configured versions after component fetching. sourceFiles := component.GetConfig().SourceFiles skipFileNames := make([]string, len(sourceFiles)) @@ -178,7 +178,8 @@ func (g *FedoraSourcesProviderImpl) processClonedRepo( } // Extract sources from repo (downloads lookaside files into the temp dir). - // Files in skipFilenames are not downloaded — they were already fetched by FetchFiles. + // Files in skipFilenames are not downloaded because [SourceManager.FetchFiles] + // provides the configured versions after component fetching. // Skip this step entirely when SkipLookaside is set (e.g., during rendering). if !opts.SkipLookaside { err := g.downloader.ExtractSourcesFromRepo( @@ -195,7 +196,6 @@ func (g *FedoraSourcesProviderImpl) processClonedRepo( } // Copy files from temp dir to destination, skipping files that already exist. - // This preserves any files downloaded by FetchFiles, giving them precedence. copyOptions := fileutils.CopyDirOptions{ CopyFileOptions: fileutils.CopyFileOptions{ PreserveFileMode: true, diff --git a/internal/providers/sourceproviders/fedorasourceprovider_test.go b/internal/providers/sourceproviders/fedorasourceprovider_test.go index 0d45e8fe..13968bda 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider_test.go +++ b/internal/providers/sourceproviders/fedorasourceprovider_test.go @@ -220,7 +220,7 @@ func TestGetComponentFromGit(t *testing.T) { assert.Equal(t, "tarball content", string(tarballContent)) }) - t.Run("existing file in destination skips lookaside download", func(t *testing.T) { + t.Run("configured replacement skips upstream lookaside download", func(t *testing.T) { ctrl := gomock.NewController(t) httpDownloader, err := downloader.NewHTTPDownloader( @@ -288,26 +288,23 @@ func TestGetComponentFromGit(t *testing.T) { mockComponent.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ Name: testPackageName, SourceFiles: []projectconfig.SourceFileReference{ - {Filename: testFileName}, + { + Filename: testFileName, + ReplaceUpstream: true, + ReplaceReason: "use configured replacement", + }, }, }) - // Pre-populate destination with the file — simulating a prior FetchFiles download - err = fileutils.MkdirAll(env.FS(), testDestDir) - require.NoError(t, err) - - preExistingContent := []byte("azure-linux-signed-binary") - err = fileutils.WriteFile(env.FS(), testDestDir+"/"+testFileName, preExistingContent, fileperms.PublicFile) - require.NoError(t, err) - - // Should succeed — the file is in skipFilenames so the 404 lookaside download is skipped + // The upstream hash would return 404 from the test server. Success proves the + // configured filename suppressed that upstream lookaside download. err = provider.GetComponent(context.Background(), mockComponent, testDestDir) require.NoError(t, err) - // Verify the pre-existing file was preserved (not overwritten by git repo version) - content, err := fileutils.ReadFile(env.FS(), testDestDir+"/"+testFileName) + // The skipped upstream lookaside artifact was not materialized. + exists, err := fileutils.Exists(env.FS(), testDestDir+"/"+testFileName) require.NoError(t, err) - assert.Equal(t, preExistingContent, content) + assert.False(t, exists) }) t.Run("successful extraction without lookaside sources", func(t *testing.T) { diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 4731c61c..20941709 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -315,8 +315,10 @@ func (m *sourceManager) createFileProviders(env *azldev.Env) { } m.fileProviders = append(m.fileProviders, &customFileSourceProvider{ - fs: m.fs, - runner: mock.NewRunner(env, mockConfigPath), + dryRunnable: m.dryRunnable, + fs: m.fs, + runner: mock.NewRunner(env, mockConfigPath), + verbose: env.Verbose(), }) slog.Debug("Registered custom file source provider", "mockConfig", mockConfigPath) @@ -366,13 +368,13 @@ func (m *sourceManager) FetchFiles( } // fetchSourceFile acquires a single source file using the following priority order: -// 1. Lookaside cache — if hash info is available, attempt a cached download first. -// This applies to all origin types, including 'custom', so a previously generated -// and cached archive avoids a full mock regeneration. -// 2. File providers — handles origin types that require local generation (e.g. 'custom'). -// 3. Configured download origin — final fallback for 'download' origin types. +// 1. Custom file provider — custom sources are always regenerated so their configured +// hashes validate the current script and inputs. +// 2. Lookaside cache — non-custom files with hash info attempt a cached download first. +// 3. File providers — handles other provider-supported origin types. +// 4. Configured download origin — final fallback for 'download' origin types. // -// When disable-origins is set, step 3 is skipped and only lookaside and file providers apply. +// When disable-origins is set, step 4 is skipped and only lookaside and file providers apply. func (m *sourceManager) fetchSourceFile( ctx context.Context, httpDownloader downloader.Downloader, @@ -387,34 +389,12 @@ func (m *sourceManager) fetchSourceFile( destPath := filepath.Join(destDirPath, fileRef.Filename) - sourceExists, err := fileutils.Exists(m.fs, destPath) - if err != nil { - return fmt.Errorf("failed to check existence of destination file %#q:\n%w", destPath, err) - } - - if sourceExists { - slog.Debug("Source file already exists, skipping download", - "filename", fileRef.Filename, - "path", destPath) - + // Try the lookaside cache first for non-custom files if hash info is available. + // Custom files are always regenerated so stale configured hashes are detected. + if m.trySourceFileLookaside(ctx, httpDownloader, component, fileRef, destPath) { return nil } - // Try the lookaside cache first if hash info is available. This applies to - // all origin types, including 'custom' — if the generated archive is already - // cached in the lookaside (as it will be after the first run), we skip the - // expensive mock generation entirely. - if fileRef.Hash != "" && fileRef.HashType != "" { - lookasideErr := m.tryLookasideDownload(ctx, httpDownloader, component, fileRef, destPath) - if lookasideErr == nil { - return nil - } - - slog.Debug("Lookaside cache download failed", - "filename", fileRef.Filename, - "error", lookasideErr) - } - // Try each registered file provider. Providers return [ErrNotFound] to signal // they don't handle this reference; any other error is fatal. for _, provider := range m.fileProviders { @@ -452,6 +432,32 @@ func (m *sourceManager) fetchSourceFile( return m.fetchFromDownloadOrigin(ctx, httpDownloader, fileRef, destPath) } +// trySourceFileLookaside attempts a cached download for non-custom files with hash information. +// Custom files deliberately bypass lookaside so they are regenerated and validated each time. +func (m *sourceManager) trySourceFileLookaside( + ctx context.Context, + httpDownloader downloader.Downloader, + component components.Component, + fileRef *projectconfig.SourceFileReference, + destPath string, +) bool { + if fileRef.Origin.Type == projectconfig.OriginTypeCustom || + fileRef.Hash == "" || fileRef.HashType == "" { + return false + } + + lookasideErr := m.tryLookasideDownload(ctx, httpDownloader, component, fileRef, destPath) + if lookasideErr == nil { + return true + } + + slog.Debug("Lookaside cache download failed", + "filename", fileRef.Filename, + "error", lookasideErr) + + return false +} + // tryLookasideDownload attempts to download a source file from the lookaside cache. // Returns nil on success, or an error if the download fails. func (m *sourceManager) tryLookasideDownload( @@ -686,7 +692,18 @@ func (m *sourceManager) downloadLookasideSources( packageName := resolvePackageName(component) - err := m.lookasideDownloader.ExtractSourcesFromRepo(ctx, destDirPath, packageName, m.lookasideBaseURI, nil) + // Collect filenames from 'source-files' config so the lookaside extractor skips them. + // These files are managed by FetchFiles (custom generation or explicit download origins) + // and must not be overwritten by a same-named upstream lookaside entry. This mirrors + // the same skip-list built in [FedoraSourcesProviderImpl.GetComponent]. + sourceFiles := component.GetConfig().SourceFiles + + skipFilenames := make([]string, len(sourceFiles)) + for i := range sourceFiles { + skipFilenames[i] = sourceFiles[i].Filename + } + + err := m.lookasideDownloader.ExtractSourcesFromRepo(ctx, destDirPath, packageName, m.lookasideBaseURI, skipFilenames) if err != nil { return fmt.Errorf("failed to extract sources from lookaside cache:\n%w", err) } diff --git a/internal/providers/sourceproviders/sourcemanager_internal_test.go b/internal/providers/sourceproviders/sourcemanager_internal_test.go new file mode 100644 index 00000000..3abcf66b --- /dev/null +++ b/internal/providers/sourceproviders/sourcemanager_internal_test.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sourceproviders + +import ( + "context" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/downloader/downloader_test" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +type regeneratingFileProvider struct { + fs opctx.FS + called bool +} + +func (p *regeneratingFileProvider) GetFile( + _ context.Context, + _ components.Component, + ref projectconfig.SourceFileReference, + destDirPath string, +) error { + p.called = true + + return fileutils.WriteFile(p.fs, filepath.Join(destDirPath, ref.Filename), nil, fileperms.PublicFile) +} + +func TestFetchSourceFile_ConfiguredSourceReplacesExistingFile(t *testing.T) { + const ( + destDir = "/output" + filename = "source.tar.gz" + // SHA-256 of the empty replacement produced by each test acquisition path. + emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) + + tests := []struct { + name string + origin projectconfig.Origin + useProvider bool + downloadURL string + hash string + expectError string + }{ + { + name: "custom source is always regenerated", + origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + useProvider: true, + hash: emptyHash, + }, + { + name: "custom source with wrong hash fails validation", + origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + useProvider: true, + hash: "0000000000000000000000000000000000000000000000000000000000000000", + expectError: "hash validation failed", + }, + { + name: "download source replaces upstream file", + origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeURI, + Uri: "https://origin.example.com/source.tar.gz", + }, + downloadURL: "https://example.com/test-component/source.tar.gz/sha256/" + + emptyHash + "/source.tar.gz", + hash: emptyHash, + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + ctx := testctx.NewCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return("test-component") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) + + httpDownloader := downloader_test.NewMockDownloader(ctrl) + if testCase.downloadURL != "" { + httpDownloader.EXPECT(). + Download(gomock.Any(), testCase.downloadURL, filepath.Join(destDir, filename)). + DoAndReturn(func(_ context.Context, _, destPath string) error { + return fileutils.WriteFile(ctx.FS(), destPath, nil, fileperms.PublicFile) + }) + } + + provider := ®eneratingFileProvider{fs: ctx.FS()} + + var fileProviders []FileSourceProvider + + if testCase.useProvider { + fileProviders = []FileSourceProvider{provider} + } + + manager := &sourceManager{ + dryRunnable: ctx, + fs: ctx.FS(), + lookasideBaseURI: "https://example.com/$pkg/$filename/$hashtype/$hash/$filename", + retryConfig: retry.Disabled(), + fileProviders: fileProviders, + } + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), destDir)) + require.NoError(t, fileutils.WriteFile( + ctx.FS(), filepath.Join(destDir, filename), []byte("upstream content"), fileperms.PublicFile)) + + ref := &projectconfig.SourceFileReference{ + Filename: filename, + Hash: testCase.hash, + HashType: fileutils.HashTypeSHA256, + Origin: testCase.origin, + ReplaceUpstream: true, + ReplaceReason: "use configured replacement", + } + + err := manager.fetchSourceFile(t.Context(), httpDownloader, component, ref, destDir) + assert.Equal(t, testCase.useProvider, provider.called) + + if testCase.expectError != "" { + require.ErrorContains(t, err, testCase.expectError) + + return + } + + require.NoError(t, err) + + content, err := fileutils.ReadFile(ctx.FS(), filepath.Join(destDir, filename)) + require.NoError(t, err) + assert.Empty(t, content) + }) + } +} diff --git a/internal/providers/sourceproviders/sourcemanager_test.go b/internal/providers/sourceproviders/sourcemanager_test.go index 6297b683..d7bf67d6 100644 --- a/internal/providers/sourceproviders/sourcemanager_test.go +++ b/internal/providers/sourceproviders/sourcemanager_test.go @@ -267,34 +267,6 @@ func TestSourceManager_FetchFiles_NoSourceFiles(t *testing.T) { require.NoError(t, err) } -func TestSourceManager_FetchFiles_ExistingFile(t *testing.T) { - env := testutils.NewTestEnv(t) - ctrl := gomock.NewController(t) - component := components_testutils.NewMockComponent(ctrl) - - require.NoError(t, env.TestFS.MkdirAll(testDestDir, fileperms.PrivateDir)) - - destPath := filepath.Join(testDestDir, testSourceTarball) - require.NoError(t, fileutils.WriteFile(env.TestFS, destPath, []byte("existing content"), fileperms.PrivateFile)) - - componentConfig := &projectconfig.ComponentConfig{ - SourceFiles: []projectconfig.SourceFileReference{{ - Filename: testSourceTarball, - Hash: "abc123", - HashType: fileutils.HashTypeSHA256, - }}, - } - - component.EXPECT().GetName().AnyTimes().Return("test-component") - component.EXPECT().GetConfig().AnyTimes().Return(componentConfig) - - sourceManager, err := sourceproviders.NewSourceManager(env.Env, testDefaultDistro()) - require.NoError(t, err) - - err = sourceManager.FetchFiles(t.Context(), component, testDestDir) - require.NoError(t, err) -} - func TestSourceManager_FetchFiles_Errors(t *testing.T) { tests := []struct { name string diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 23778d19..60b29944 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -735,6 +735,14 @@ "type": "array", "title": "Mock packages", "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." + }, + "inputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Inputs", + "description": "Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 23778d19..60b29944 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -735,6 +735,14 @@ "type": "array", "title": "Mock packages", "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." + }, + "inputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Inputs", + "description": "Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'." } }, "additionalProperties": false, diff --git a/scenario/custom_source_test.go b/scenario/custom_source_test.go new file mode 100644 index 00000000..d2c4a007 --- /dev/null +++ b/scenario/custom_source_test.go @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build scenario + +package scenario_tests + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/cmdtest" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/projecttest" + "github.com/stretchr/testify/require" +) + +func TestCustomSourceRegenerationDetectsStaleHash(t *testing.T) { + if testing.Short() { + t.Skip("skipping long test") + } + + const ( + componentName = "custom-source" + scriptName = "generate.sh" + archiveName = "generated.tar.gz" + ) + + spec := projecttest.NewSpec(projecttest.WithName(componentName)) + project := projecttest.NewDynamicTestProject( + projecttest.AddSpec(spec), + projecttest.AddComponent(&projectconfig.ComponentConfig{ + Name: componentName, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: filepath.Join("specs", componentName, componentName+".spec"), + }, + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: archiveName, + HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: scriptName, + }, + }, + }, + }), + projecttest.AddFile(filepath.Join("specs", componentName, scriptName), `#!/bin/sh +set -eu +printf 'version-one\n' > /azldev-gen/output/payload.txt +`), + projecttest.UseTestDefaultConfigs(), + ) + + projectStagingDir := t.TempDir() + project.Serialize(t, projectStagingDir) + + testScript := ` +set -eux + +azldev -C project -v component prep-sources -p custom-source \ + -o prepared --without-git --allow-no-hashes + +test "$(tar -xOf prepared/generated.tar.gz payload.txt)" = "version-one" + +GENERATED_HASH="$(sha512sum prepared/generated.tar.gz | awk '{print $1}')" +sed -i "/hash-type =/a hash = \"$GENERATED_HASH\"" project/azldev.toml +sed -i 's/version-one/version-two/' project/specs/custom-source/generate.sh + +if azldev -C project -v component prep-sources -p custom-source \ + -o prepared --without-git --force 2>second-run.stderr; then + echo "expected regenerated custom source to fail hash validation" >&2 + exit 1 +fi + +grep -q "hash validation failed" second-run.stderr +test "$(tar -xOf prepared/generated.tar.gz payload.txt)" = "version-two" +` + + scenarioTest := cmdtest.NewScenarioTest(). + WithScript(strings.NewReader(testScript)). + AddDirRecursive(t, "project", projectStagingDir). + AddDirRecursive(t, projecttest.TestDefaultConfigsSubdir, projecttest.TestDefaultConfigsDir()) + + results, err := scenarioTest. + InContainer(). + WithPrivilege(). + WithNetwork(). + Run(t) + require.NoError(t, err) + + t.Logf("Standard output:\n%s", results.Stdout) + t.Logf("Standard error:\n%s", results.Stderr) + results.AssertZeroExitCode(t) + + stderrPath := filepath.Join(results.Workdir, "second-run.stderr") + require.FileExists(t, stderrPath) + + stderr, err := os.ReadFile(stderrPath) + require.NoError(t, err) + require.Contains(t, string(stderr), "hash validation failed") +} diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 23778d19..60b29944 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -735,6 +735,14 @@ "type": "array", "title": "Mock packages", "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." + }, + "inputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Inputs", + "description": "Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'." } }, "additionalProperties": false,