diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index c6f76712..35344a8d 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -11,6 +11,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Spec source | `spec` | [SpecSource](#spec-source) | No | Where to find the spec file for this component. Inherited from distro defaults if not specified. | | Release config | `release` | [ReleaseConfig](#release-configuration) | No | Controls how the Release tag is managed during rendering | | Overlays | `overlays` | array of [Overlay](overlays.md) | No | Modifications to apply to the spec and/or source files | +| Overlay files | `overlay-files` | array of string | No | Glob patterns that load per-file overlay documents. See [Per-file overlay format](overlays.md#per-file-overlay-format). | | Build config | `build` | [BuildConfig](#build-configuration) | No | Build-time options (macros, conditionals, check config) | | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | diff --git a/docs/user/reference/config/overlays.md b/docs/user/reference/config/overlays.md index 55061118..9a3abc9f 100644 --- a/docs/user/reference/config/overlays.md +++ b/docs/user/reference/config/overlays.md @@ -61,10 +61,153 @@ successfully makes a replacement to at least one matching file. | Replacement | `replacement` | Literal replacement text; capture group references like `$1` are **not** expanded. Omit or leave empty to delete matched text. | `spec-search-replace`, `file-search-replace`, `file-rename` | | Lines | `lines` | Array of text lines to insert | `spec-prepend-lines`, `spec-append-lines`, `file-prepend-lines` | | File | `file` | The name of the non-spec file to modify or add | `file-prepend-lines`, `file-search-replace`, `file-add`, `file-remove`, `file-rename`, `patch-add` (optional), `patch-remove` | -| Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file | `file-add`, `patch-add` | +| Source | `source` | Path to source file for `file-add` and `patch-add`; relative paths are relative to the config file that defines the overlay (the overlay file if loaded via [`overlay-files`](#per-file-overlay-format), otherwise the component config) | `file-add`, `patch-add` | +| Metadata | `metadata` | Documentation table describing intent and provenance — see [Overlay Metadata](#overlay-metadata). Not allowed inside an overlay file loaded via `overlay-files` (the file-level `[metadata]` block applies to every overlay in the file). | All (optional) | > **Note:** For `file-rename`, the `replacement` field is a **filename only** (not a path). The file is renamed within its current directory. +### Component-level fields for overlays + +In addition to per-overlay fields, the following fields are set directly on the component: + +| Field | TOML Key | Description | +|-------|----------|-------------| +| Overlay files | `overlay-files` | List of glob patterns (relative to the component's config file) matched against the filesystem at load time to locate per-file overlay documents. Patterns support `**` (globstar). Matches are concatenated in declaration order; within a single pattern, matches are applied in filename (lexicographic) order, with full path as a tie-breaker for duplicate filenames. Each pattern must match at least one file. Duplicate matches across patterns are de-duplicated. See [Per-file overlay format](#per-file-overlay-format). | + +## Overlay Metadata + +Overlays can carry an optional `metadata` table that documents *why* the overlay exists and *when* it can be removed. Metadata is reviewed by humans and surfaced in tooling; it does **not** affect how the overlay is applied and is excluded from component fingerprints (so editing metadata never invalidates build caches). + +### `metadata` fields + +| Field | TOML Key | Description | +|-------|----------|-------------| +| Category | `category` | **Required.** Classification of the overlay's intent. See the table below. | +| Commits | `commits` | List of upstream commit URLs (Fedora dist-git or upstream project) that this overlay backports or implements. Each entry must be an absolute http(s) URL. | +| Bugs | `bugs` | List of bug-tracker references. Each entry is a table with a required `url`. See [Bug references](#bug-references). | +| Upstreamable | `upstreamable` | Boolean indicating whether this change can be upstreamed: `true` or `false`. Omit the field when upstreamability has not yet been assessed. | + +### Categories + +| Category | When to use | +|----------|-------------| +| `backport-dist-git` | Fix backported from (or being upstreamed to) a dist-git or upstream project. Self-resolves when AZL bumps past it. Requires at least one entry in `commits`. | +| `azl-pruning` | Removing content from a component for AZL: dependencies that are not shipped, unneeded features, subpackages, or files. | +| `azl-compatibility` | Making a component work in the AZL build/runtime environment: toolchain and mock adjustments, and similar compatibility fixes that are not themselves backports. | +| `azl-dep-missing-workaround` | Working around a runtime or build dependency that has not yet been imported into AZL (or is unavailable on a given target). Drop the overlay once the dependency lands. | +| `azl-branding-policy` | Fedora→Azure Linux name/path changes; RHEL/enterprise convention alignment. | +| `azl-disable-flaky-tests` | Skipping tests that fail intermittently or due to environmental flakiness rather than a real problem with the component. | +| `azl-disable-unsupported-tests` | Skipping tests that cannot meaningfully run in AZL's build/runtime environment (e.g. tests that require network access, root, or hardware that is unavailable in mock). | +| `azl-security-compliance` | FIPS or crypto-policy changes. | +| `azl-release-management` | Release-tag and changelog mechanics. | +| `azl-platform-adaptation` | Architecture-specific adjustments. | + +### Bug references + +The `bugs` field is a list of references to issue-tracker entries. Each entry is a table with a single required field: + +| Field | TOML Key | Description | +|-------|----------|-------------| +| URL | `url` | **Required.** HTTP(S) link to the bug entry. | + +Example: + +```toml +[[components.mypackage.overlays.metadata.bugs]] +url = "https://bugzilla.redhat.com/show_bug.cgi?id=2234567" + +[[components.mypackage.overlays.metadata.bugs]] +url = "https://github.com/example/repo/issues/42" +``` + +The inline-table form is more compact for short lists: + +```toml +bugs = [ + { url = "https://bugzilla.redhat.com/show_bug.cgi?id=2234567" }, + { url = "https://github.com/example/repo/issues/42" }, +] +``` + +### Inline metadata example + +TOML inline tables (`metadata = { ... }`) must fit on a single line. When the metadata has more than one or two fields, use a sub-table (`[components..overlays.metadata]`) so each field gets its own line: + +```toml +[[components.xclock.overlays]] +type = "spec-search-replace" +description = "Pass --force to autoreconf" +regex = "autoreconf -i" +replacement = "autoreconf -fi" + + [components.xclock.overlays.metadata] + category = "backport-dist-git" + commits = ["https://src.fedoraproject.org/rpms/xclock/c/1e407488"] +``` + +For short metadata, the single-line inline form is also valid: + +```toml +[[components.xclock.overlays]] +type = "spec-set-tag" +tag = "Vendor" +value = "Microsoft" +metadata = { category = "azl-branding-policy" } +``` + +## Per-file overlay format + +When a single logical change (a CVE backport, a feature disablement, a Fedora cherry-pick…) needs **several overlays** that all share the same provenance, declaring them inline in the component config gets noisy and makes the boundary between unrelated changes hard to see. Use the per-file format instead. + +### Layout + +Set `overlay-files` on the component to one or more globs (relative to the component config) and drop one overlay file per logical change into a directory of your choosing. The conventional layout uses a sibling `overlays/` directory and a `*.overlay.toml` filename suffix, but neither is required — `overlay-files` is just a glob, so any layout you can describe with `**`/`*` patterns works. + +``` +base/comps/mypackage/ +├── mypackage.comp.toml +└── overlays/ + ├── 0001-cve-2024-1234.overlay.toml + ├── 0002-disable-broken-tests.overlay.toml + └── 0003-azl-branding.overlay.toml +``` + +```toml +# mypackage.comp.toml +[components.mypackage] +overlay-files = ["overlays/*.overlay.toml"] +``` + +Files are loaded in **filename (lexicographic) order** within each glob, using the full path as a tie-breaker when multiple matches have the same filename. Globs are concatenated in declaration order, so prefix each file with a numeric ordinal (`0001-`, `0002-`, …) to make the apply order obvious and stable. Files that don't match any of your globs are ignored, so you can keep `README.md` or other notes alongside without naming them out explicitly. Each declared glob must match at least one file; an empty match is treated as a misconfiguration and surfaced as an error. + +Overlays loaded via `overlay-files` are **appended after** any inline overlays declared directly on the component. + +### File structure + +Each overlay file represents one logical change. It has: + +* exactly one top-level `[metadata]` table (uses the same fields documented in [Overlay Metadata](#overlay-metadata)); and +* one or more `[[overlays]]` entries, applied in declaration order. + +Per-overlay `metadata` is **not allowed** inside an overlay file — the file-level `[metadata]` is the single source of truth and is stamped onto every overlay in the file. Relative `source` paths are resolved against the directory of the overlay file (not the component config). + +```toml +# overlays/0001-cve-2024-1234.overlay.toml +[metadata] +category = "backport-dist-git" +commits = ["https://src.fedoraproject.org/rpms/mypackage/c/abc123def456"] +bugs = [{ url = "https://bugzilla.redhat.com/show_bug.cgi?id=12345" }] + +[[overlays]] +type = "patch-add" +source = "patches/CVE-2024-1234.patch" + +[[overlays]] +type = "spec-append-lines" +section = "%changelog" +lines = ["- Fix CVE-2024-1234"] +``` + ## Examples ### Adding a Build Dependency diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 9d45a2b4..b481b35b 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -273,6 +273,19 @@ type ComponentConfig struct { // Overlays to apply to sources after they've been acquired. May mutate the spec as well as sources. Overlays []ComponentOverlay `toml:"overlays,omitempty" json:"overlays,omitempty" table:"-" jsonschema:"title=Overlays,description=Overlays to apply to this component's spec and/or sources"` + // OverlayFiles, if set, lists glob patterns (relative to this component config file) + // matched against the filesystem at load time to locate per-file overlay documents. + // Each matched file is parsed as an [OverlayFile]: one logical change consisting of a + // file-level `[metadata]` block plus an ordered list of `[[overlays]]`. The per-file + // metadata is stamped onto every overlay in the file. Matches are concatenated in the + // order patterns are declared; within a single pattern, matches are applied in + // filename (lexicographic) order, using the full path as a tie-breaker when + // filenames match. Duplicate matches are de-duplicated, preserving first + // occurrence. The resulting overlays are appended to [ComponentConfig.Overlays] + // after any inline overlays. Excluded from the fingerprint because the value affects + // only where overlays are sourced from, not their content. + OverlayFiles []string `toml:"overlay-files,omitempty" json:"overlayFiles,omitempty" table:"-" validate:"dive,required" jsonschema:"title=Overlay files,description=Glob patterns (relative to the component config file) matched against the filesystem to locate per-file overlay documents at load time" fingerprint:"-"` + // Configuration for building the component. Build ComponentBuildConfig `toml:"build,omitempty" json:"build,omitempty" table:"-" jsonschema:"title=Build configuration,description=Configuration for building the component"` @@ -383,6 +396,10 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi SourceFiles: deep.MustCopy(c.SourceFiles), Packages: deep.MustCopy(c.Packages), Publish: deep.MustCopy(c.Publish), + // OverlayFiles is consumed at load time (see applyOverlayFiles) before paths are + // absolutized; nothing reads it afterward, so preserve the original value verbatim + // rather than redundantly re-rooting each glob. + OverlayFiles: slices.Clone(c.OverlayFiles), } // Fix up paths. diff --git a/internal/projectconfig/component_test.go b/internal/projectconfig/component_test.go index 9f5f3d87..2acc9a7e 100644 --- a/internal/projectconfig/component_test.go +++ b/internal/projectconfig/component_test.go @@ -406,6 +406,49 @@ func TestComponentPublishConfig_Validate(t *testing.T) { } } +func TestComponentConfig_OverlayFilesValidate(t *testing.T) { + t.Parallel() + + validate := validator.New() + + validCases := []struct { + name string + overlayFiles []string + }{ + {name: "unset"}, + {name: "empty slice", overlayFiles: []string{}}, + {name: "non-empty pattern", overlayFiles: []string{"overlays/*.overlay.toml"}}, + } + + for _, testCase := range validCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + cfg := projectconfig.ComponentConfig{OverlayFiles: testCase.overlayFiles} + require.NoError(t, validate.Struct(&cfg)) + }) + } + + invalidCases := []struct { + name string + overlayFiles []string + }{ + {name: "empty pattern", overlayFiles: []string{""}}, + {name: "empty pattern among valid patterns", overlayFiles: []string{"overlays/*.overlay.toml", ""}}, + } + + for _, testCase := range invalidCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + cfg := projectconfig.ComponentConfig{OverlayFiles: testCase.overlayFiles} + err := validate.Struct(&cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "OverlayFiles") + }) + } +} + func TestIsDebugInfoPackage(t *testing.T) { t.Parallel() diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 3bd14eff..9fe56870 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -70,6 +70,12 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // ComponentOverlay.Source — absolute path that varies by checkout location. // Overlay content is hashed separately by ComputeIdentity. "ComponentOverlay.Source": true, + // ComponentOverlay.Metadata — documentation describing overlay intent and provenance. + "ComponentOverlay.Metadata": true, + + // ComponentConfig.OverlayFiles — affects only where overlays are sourced from, not + // their content; the resulting overlays are fingerprinted normally. + "ComponentConfig.OverlayFiles": true, // SourceFileReference.Component — back-reference to parent, not a build input. "SourceFileReference.Component": true, diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 31971f90..ab6227b5 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -433,6 +433,13 @@ func loadProjectConfigFile( cfg.sourcePath = absFilePath cfg.dir = filepath.Dir(absFilePath) + // Resolve per-component overlay file globs, stamping each + // file's [metadata] onto its overlays and appending them to the component before + // validation runs. + if err := applyOverlayFiles(fs, cfg, permissiveConfigParsing); err != nil { + return nil, err + } + // Make sure that the read data is internally consistent. err = cfg.Validate() if err != nil { diff --git a/internal/projectconfig/overlay.go b/internal/projectconfig/overlay.go index 7ec0b50e..d6a13c76 100644 --- a/internal/projectconfig/overlay.go +++ b/internal/projectconfig/overlay.go @@ -43,6 +43,13 @@ type ComponentOverlay struct { // Excluded from fingerprint because it contains an absolute path that varies by checkout // location. Overlay content is hashed separately by [fingerprint.ComputeIdentity]. Source string `toml:"source,omitempty" json:"source,omitempty" jsonschema:"title=Source,description=For overlays that require a source file as input, indicates a path to that file; relative paths are relative to the config file that defines the overlay" fingerprint:"-"` + + // Metadata describes the intent and provenance of the overlay (category, related + // commits, bug links, upstreamability, etc.). Optional. Populated either inline + // in the component config file or by the [ComponentConfig.OverlayFiles] loader, + // which stamps the per-file metadata onto every overlay declared in that file. + // Excluded from the fingerprint because it is documentation only. + Metadata *OverlayMetadata `toml:"metadata,omitempty" json:"metadata,omitempty" jsonschema:"title=Metadata,description=Optional documentation metadata describing the overlay's intent and provenance" fingerprint:"-"` } // EffectiveSourceName returns the checkout-independent identity of the overlay's @@ -333,6 +340,12 @@ func (c *ComponentOverlay) Validate() error { return fmt.Errorf("unknown overlay type %#q: %#q", c.Type, desc) } + if c.Metadata != nil { + if err := c.Metadata.Validate(); err != nil { + return fmt.Errorf("invalid metadata for overlay %q:\n%w", desc, err) + } + } + return nil } diff --git a/internal/projectconfig/overlay_file.go b/internal/projectconfig/overlay_file.go new file mode 100644 index 00000000..eecb2d22 --- /dev/null +++ b/internal/projectconfig/overlay_file.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "cmp" + "errors" + "fmt" + "log/slog" + "path" + "path/filepath" + "slices" + + "github.com/bmatcuk/doublestar/v4" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" +) + +// ErrOverlayFilePerOverlayMetadata is returned when an overlay file declares +// `metadata` on an individual `[[overlays]]` entry. Each file represents one logical +// change; the file-level `[metadata]` block is the single source of truth and is +// stamped onto every overlay in the file at load time. +var ErrOverlayFilePerOverlayMetadata = errors.New( + "per-overlay metadata is not allowed inside an overlay file; declare metadata once at the file level", +) + +// ErrOverlayFileEmpty is returned when an overlay file decodes to zero overlays. +// Such a file is almost certainly a typo (e.g. `[overlays]` instead of `[[overlays]]`) +// and would otherwise silently contribute nothing. +var ErrOverlayFileEmpty = errors.New("overlay file declares no overlays") + +// ErrOverlayFilesNoMatches is returned when one of a component's +// [ComponentConfig.OverlayFiles] glob patterns matches no files on disk. A configured +// glob that matches nothing is almost always a misconfiguration (wrong path or +// missing files) and is surfaced as an error rather than silently contributing +// nothing. +var ErrOverlayFilesNoMatches = errors.New("overlay-files pattern matched no files") + +// ErrOverlayFilesInDefaultConfig is returned when `overlay-files` is set on a +// default component config. Overlay file globs are resolved per concrete component +// before default configs are merged in, so a value set on a default would be +// silently ignored. Until overlay-files is wired through the default-merge path, +// declaring it on a default config is rejected. +var ErrOverlayFilesInDefaultConfig = errors.New( + "overlay-files is not supported on default-component-config; set it on individual components", +) + +// OverlayFile is the on-disk representation of a single overlay document. Each file +// represents one logical change: the file-level [OverlayFile.Metadata] is applied +// to every overlay in [OverlayFile.Overlays] at load time, after which the resulting +// overlays are appended to the owning component's [ComponentConfig.Overlays] slice. +type OverlayFile struct { + // Metadata is the shared metadata for every overlay in this file. Required. + Metadata OverlayMetadata `toml:"metadata"` + + // Overlays is the ordered list of overlays applied by this file. Must be non-empty. + // Per-overlay `metadata` is not allowed — the file-level [OverlayFile.Metadata] is + // the single source of truth. + Overlays []ComponentOverlay `toml:"overlays"` +} + +// applyOverlayFiles resolves each component's [ComponentConfig.OverlayFiles] glob +// patterns (when set), parses every matched file as an [OverlayFile] in deterministic +// order, stamps the per-file metadata onto each overlay, and appends the resulting +// overlays to the component's [ComponentConfig.Overlays] slice. Inline overlays +// declared directly on the component come first; file-sourced overlays are appended +// after them. +// +// Called from [loadProjectConfigFile] after TOML decode but before [ConfigFile.Validate], +// so all per-overlay validation rules apply uniformly regardless of declaration site. +func applyOverlayFiles(fs opctx.FS, cfg *ConfigFile, permissiveConfigParsing bool) error { + if err := rejectOverlayFilesInDefaults(cfg); err != nil { + return err + } + + for componentName, component := range cfg.Components { + if len(component.OverlayFiles) == 0 { + continue + } + + loaded, err := loadOverlayFiles(fs, cfg.dir, component.OverlayFiles, permissiveConfigParsing) + if err != nil { + return fmt.Errorf("component %#q overlay-files:\n%w", componentName, err) + } + + component.Overlays = append(component.Overlays, loaded...) + cfg.Components[componentName] = component + } + + return nil +} + +// rejectOverlayFilesInDefaults returns [ErrOverlayFilesInDefaultConfig] if any default +// component config in cfg sets `overlay-files`. Overlay file globs are resolved per +// concrete component (see [applyOverlayFiles]) before default configs are merged, so a +// value declared on a default would be silently dropped. This stopgap surfaces the +// misconfiguration until overlay-files is plumbed through the default-merge path. +func rejectOverlayFilesInDefaults(cfg *ConfigFile) error { + if cfg.DefaultComponentConfig != nil && len(cfg.DefaultComponentConfig.OverlayFiles) > 0 { + return fmt.Errorf("%w (project-level default-component-config)", ErrOverlayFilesInDefaultConfig) + } + + for name, group := range cfg.ComponentGroups { + if len(group.DefaultComponentConfig.OverlayFiles) > 0 { + return fmt.Errorf("%w (component-group %#q)", ErrOverlayFilesInDefaultConfig, name) + } + } + + for name, distro := range cfg.Distros { + for version, versionDef := range distro.Versions { + if len(versionDef.DefaultComponentConfig.OverlayFiles) > 0 { + return fmt.Errorf("%w (distro %#q version %#q)", ErrOverlayFilesInDefaultConfig, name, version) + } + } + } + + return nil +} + +// loadOverlayFiles resolves each glob pattern (relative to referenceDir if not +// already absolute), parses every matched overlay file, validates each file's +// metadata, stamps the file metadata onto each overlay, and resolves overlay +// `source` paths relative to the overlay file. +// +// Matches are concatenated in the order patterns are declared; within a single +// pattern, matches are applied in filename (lexicographic) order, using the full +// path as a tie-breaker when filenames match. Duplicate matches across patterns +// are de-duplicated, preserving first occurrence. Each pattern is required to +// match at least one file. +func loadOverlayFiles( + fs opctx.FS, referenceDir string, patterns []string, permissiveConfigParsing bool, +) ([]ComponentOverlay, error) { + var ( + ordered []string + seen = make(map[string]bool) + ) + + for _, pattern := range patterns { + absPattern := pattern + if !filepath.IsAbs(absPattern) { + absPattern = path.Join(referenceDir, absPattern) + } + + matches, err := fileutils.Glob( + fs, absPattern, + doublestar.WithFailOnIOErrors(), + doublestar.WithFailOnPatternNotExist(), + doublestar.WithFilesOnly(), + ) + + switch { + case errors.Is(err, doublestar.ErrPatternNotExist): + return nil, fmt.Errorf("%w: %q", ErrOverlayFilesNoMatches, pattern) + case err != nil: + return nil, fmt.Errorf("failed to scan for overlay files matching %q:\n%w", pattern, err) + case len(matches) == 0: + return nil, fmt.Errorf("%w: %q", ErrOverlayFilesNoMatches, pattern) + } + + slices.SortFunc(matches, func(left, right string) int { + if result := cmp.Compare(filepath.Base(left), filepath.Base(right)); result != 0 { + return result + } + + return cmp.Compare(left, right) + }) + + for _, match := range matches { + if seen[match] { + continue + } + + seen[match] = true + ordered = append(ordered, match) + } + } + + var result []ComponentOverlay + + for _, overlayPath := range ordered { + fromFile, err := loadOverlayFile(fs, overlayPath, permissiveConfigParsing) + if err != nil { + return nil, err + } + + result = append(result, fromFile...) + } + + return result, nil +} + +// loadOverlayFile parses a single overlay file, validates its metadata, stamps the +// metadata onto each overlay, and absolutizes per-overlay `source` paths relative +// to the file's directory. +func loadOverlayFile( + fs opctx.FS, overlayPath string, permissiveConfigParsing bool, +) ([]ComponentOverlay, error) { + file, err := fs.Open(overlayPath) + if err != nil { + return nil, fmt.Errorf("failed to open overlay file %q:\n%w", overlayPath, err) + } + defer file.Close() + + decoder := toml.NewDecoder(file) + + if !permissiveConfigParsing { + decoder.DisallowUnknownFields() + } + + var ofile OverlayFile + if err := decoder.Decode(&ofile); err != nil { + return nil, fmt.Errorf("failed to parse overlay file %q:\n%w", overlayPath, err) + } + + // In permissive mode, drop invalid file-level metadata so overlays still load + // cleanly; otherwise stamping the bad metadata would cause overlay-level + // validation to re-fail later with a misleading location. + stampedMetadata := &ofile.Metadata + if err := ofile.Metadata.Validate(); err != nil { + if !permissiveConfigParsing { + return nil, fmt.Errorf("invalid [metadata] in overlay file %q:\n%w", overlayPath, err) + } + + slog.Warn( + "Overlay file metadata validation failed; dropping metadata and continuing due to '--permissive-config'", + "overlayFile", overlayPath, + "error", err, + ) + + stampedMetadata = nil + } + + if len(ofile.Overlays) == 0 { + return nil, fmt.Errorf("%w: %q", ErrOverlayFileEmpty, overlayPath) + } + + overlayDir := filepath.Dir(overlayPath) + + for idx := range ofile.Overlays { + overlay := &ofile.Overlays[idx] + + if overlay.Metadata != nil { + return nil, fmt.Errorf("%w (overlay %d in %q)", ErrOverlayFilePerOverlayMetadata, idx+1, overlayPath) + } + + overlay.Metadata = stampedMetadata.clone() + overlay.Source = makeAbsolute(overlayDir, overlay.Source) + } + + return ofile.Overlays, nil +} diff --git a/internal/projectconfig/overlay_file_test.go b/internal/projectconfig/overlay_file_test.go new file mode 100644 index 00000000..d714dc00 --- /dev/null +++ b/internal/projectconfig/overlay_file_test.go @@ -0,0 +1,517 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//nolint:testpackage // Allow to test private functions (i.e., applyOverlayFiles). +package projectconfig + +import ( + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const validBackportOverlayFile = ` +[metadata] +category = "backport-dist-git" +commits = ["https://src.fedoraproject.org/rpms/ant/c/4ca7a3b"] + +[[overlays]] +description = "Remove openjdk21 binding" +type = "spec-search-replace" +section = "%install" +regex = ".*openjdk21.*" + +[[overlays]] +type = "spec-remove-subpackage" +package = "openjdk21" +` + +const ( + badOverlayTestDir = "/project/comps/x/overlays" + antOverlayDir = "/project/comps/ant/overlays" +) + +// overlayGlob returns the conventional `*.overlay.toml` glob beneath dir; used by +// tests to exercise the loader the same way the docs recommend. +func overlayGlob(dir string) string { + return filepath.Join(dir, "*.overlay.toml") +} + +func TestLoadOverlayFiles_StampsMetadataAndPreservesOrder(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := antOverlayDir + + // Files are intentionally out of glob order to confirm filename-sort. + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0002-azl-branding.overlay.toml"), + []byte(` +[metadata] +category = "azl-branding-policy" + +[[overlays]] +type = "spec-set-tag" +tag = "Vendor" +value = "Microsoft" +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-backport-drop-openjdk21.overlay.toml"), + []byte(validBackportOverlayFile), fileperms.PrivateFile)) + + loaded, err := loadOverlayFiles(ctx.FS(), "/project", []string{overlayGlob(overlayDir)}, false) + require.NoError(t, err) + require.Len(t, loaded, 3, "two overlays from 0001 + one from 0002") + + // 0001-* contributes first. + assert.Equal(t, ComponentOverlaySearchAndReplaceInSpec, loaded[0].Type) + require.NotNil(t, loaded[0].Metadata) + assert.Equal(t, OverlayCategoryBackportDistGit, loaded[0].Metadata.Category) + + assert.Equal(t, ComponentOverlayRemoveSubpackage, loaded[1].Type) + require.NotNil(t, loaded[1].Metadata) + assert.Equal(t, OverlayCategoryBackportDistGit, loaded[1].Metadata.Category) + + // Mutating one overlay's metadata must not affect another (each must own a copy). + loaded[0].Metadata.Commits = append(loaded[0].Metadata.Commits, "mutated") + assert.Len(t, loaded[1].Metadata.Commits, 1, "each overlay must own its metadata copy") + + // 0002-* comes second. + assert.Equal(t, ComponentOverlaySetSpecTag, loaded[2].Type) + require.NotNil(t, loaded[2].Metadata) + assert.Equal(t, OverlayCategoryAZLBrandingPolicy, loaded[2].Metadata.Category) +} + +func TestLoadOverlayFiles_SortsMatchesByBasename(t *testing.T) { + ctx := testctx.NewCtx() + + // If matches were sorted by full path, "a/0002" would load before "b/0001". + // The documented order is by filename within each glob, using full path as a + // tie-breaker for matching filenames. + relativeToFullPathFirstDir := "/project/comps/ant/overlays/a" + filenameFirstDir := "/project/comps/ant/overlays/b" + + relativeToFullPathFirstOverlay := filepath.Join(relativeToFullPathFirstDir, "0002-second.overlay.toml") + filenameFirstOverlay := filepath.Join(filenameFirstDir, "0001-first.overlay.toml") + tiedBasenameFirstOverlay := filepath.Join(relativeToFullPathFirstDir, "0003-tied.overlay.toml") + tiedBasenameSecondOverlay := filepath.Join(filenameFirstDir, "0003-tied.overlay.toml") + + require.NoError(t, fileutils.WriteFile(ctx.FS(), relativeToFullPathFirstOverlay, []byte(` +[metadata] +category = "azl-branding-policy" + +[[overlays]] +type = "spec-set-tag" +tag = "Order" +value = "second" +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), filenameFirstOverlay, []byte(` +[metadata] +category = "azl-branding-policy" + +[[overlays]] +type = "spec-set-tag" +tag = "Order" +value = "first" +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), tiedBasenameFirstOverlay, []byte(` +[metadata] +category = "azl-branding-policy" + +[[overlays]] +type = "spec-set-tag" +tag = "Order" +value = "third" +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), tiedBasenameSecondOverlay, []byte(` +[metadata] +category = "azl-branding-policy" + +[[overlays]] +type = "spec-set-tag" +tag = "Order" +value = "fourth" +`), fileperms.PrivateFile)) + + loaded, err := loadOverlayFiles(ctx.FS(), "/project", []string{"/project/comps/ant/overlays/**/*.overlay.toml"}, false) + require.NoError(t, err) + require.Len(t, loaded, 4) + assert.Equal(t, "first", loaded[0].Value) + assert.Equal(t, "second", loaded[1].Value) + assert.Equal(t, "third", loaded[2].Value) + assert.Equal(t, "fourth", loaded[3].Value) +} + +func TestLoadOverlayFiles_NoMatchesIsError(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := "/project/comps/empty/overlays" + + // A non-overlay file alongside must not be picked up; a glob that matches no + // files is a misconfiguration and must error. + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "README.md"), + []byte("not an overlay"), fileperms.PrivateFile)) + + _, err := loadOverlayFiles(ctx.FS(), "/project", []string{overlayGlob(overlayDir)}, false) + require.ErrorIs(t, err, ErrOverlayFilesNoMatches) +} + +func TestLoadOverlayFiles_MissingDirIsError(t *testing.T) { + ctx := testctx.NewCtx() + + _, err := loadOverlayFiles( + ctx.FS(), "/project", + []string{overlayGlob("/project/comps/does-not-exist/overlays")}, false, + ) + require.ErrorIs(t, err, ErrOverlayFilesNoMatches) +} + +func TestLoadOverlayFiles_RejectsPerOverlayMetadata(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := badOverlayTestDir + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-bad.overlay.toml"), + []byte(` +[metadata] +category = "azl-compatibility" + +[[overlays]] +type = "spec-set-tag" +tag = "Vendor" +value = "Microsoft" +metadata = { category = "azl-branding-policy" } +`), fileperms.PrivateFile)) + + _, err := loadOverlayFiles(ctx.FS(), "/project", []string{overlayGlob(overlayDir)}, false) + require.ErrorIs(t, err, ErrOverlayFilePerOverlayMetadata) +} + +func TestLoadOverlayFiles_RejectsEmptyFile(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := badOverlayTestDir + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-empty.overlay.toml"), + []byte(` +[metadata] +category = "azl-compatibility" +`), fileperms.PrivateFile)) + + _, err := loadOverlayFiles(ctx.FS(), "/project", []string{overlayGlob(overlayDir)}, false) + require.ErrorIs(t, err, ErrOverlayFileEmpty) +} + +func TestLoadOverlayFiles_RejectsInvalidMetadata(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := badOverlayTestDir + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-bad.overlay.toml"), + []byte(` +[metadata] +# Missing category. +commits = ["abc"] + +[[overlays]] +type = "spec-set-tag" +tag = "Vendor" +value = "Microsoft" +`), fileperms.PrivateFile)) + + _, err := loadOverlayFiles(ctx.FS(), "/project", []string{overlayGlob(overlayDir)}, false) + require.Error(t, err) + assert.Contains(t, err.Error(), "category") +} + +func TestLoadOverlayFile_PermissiveTolerates_InvalidMetadata(t *testing.T) { + ctx := testctx.NewCtx() + overlayPath := filepath.Join(badOverlayTestDir, "0001-bad.overlay.toml") + + // Same fixture as the strict counterpart above: category is missing, which + // fails OverlayMetadata.Validate(). Under permissive parsing the load must + // still succeed so older configs targeting a stricter newer schema can be + // inspected. + require.NoError(t, fileutils.WriteFile(ctx.FS(), + overlayPath, + []byte(` +[metadata] +# Missing category. +commits = ["abc"] + +[[overlays]] +type = "spec-set-tag" +tag = "Vendor" +value = "Microsoft" +`), fileperms.PrivateFile)) + + loaded, err := loadOverlayFile(ctx.FS(), overlayPath, true) + require.NoError(t, err) + require.Len(t, loaded, 1) + assert.Equal(t, ComponentOverlaySetSpecTag, loaded[0].Type) + assert.Nil(t, loaded[0].Metadata, + "invalid file-level metadata must be dropped under permissive parsing so overlay validation does not re-fail") +} + +func TestLoadOverlayFiles_ResolvesSourceRelativeToOverlayFile(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := "/project/comps/foo/overlays" + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-patch.overlay.toml"), + []byte(` +[metadata] +category = "backport-dist-git" +commits = ["https://src.fedoraproject.org/rpms/foo/c/abc"] + +[[overlays]] +type = "patch-add" +source = "patches/fix.patch" +`), fileperms.PrivateFile)) + + loaded, err := loadOverlayFiles(ctx.FS(), "/project", []string{overlayGlob(overlayDir)}, false) + require.NoError(t, err) + require.Len(t, loaded, 1) + assert.Equal(t, + filepath.Join(overlayDir, "patches/fix.patch"), + loaded[0].Source, + "source paths in overlay files resolve relative to the file itself") +} + +func TestLoadOverlayFiles_DedupsAcrossPatterns(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := antOverlayDir + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-backport.overlay.toml"), + []byte(validBackportOverlayFile), fileperms.PrivateFile)) + + // Two overlapping globs both match the same file. The file must contribute + // its overlays only once. + loaded, err := loadOverlayFiles(ctx.FS(), "/project", []string{ + overlayGlob(overlayDir), + filepath.Join(overlayDir, "0001-*.overlay.toml"), + }, false) + require.NoError(t, err) + require.Len(t, loaded, 2, "overlapping globs must not double-apply matched files") +} + +func TestLoadOverlayFiles_DoubleStarMatchesNested(t *testing.T) { + ctx := testctx.NewCtx() + root := "/project/comps/ant/overlays" + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(root, "security", "0001-backport.overlay.toml"), + []byte(validBackportOverlayFile), fileperms.PrivateFile)) + + loaded, err := loadOverlayFiles( + ctx.FS(), "/project", + []string{filepath.Join(root, "**", "*.overlay.toml")}, false, + ) + require.NoError(t, err) + require.Len(t, loaded, 2, "**-style glob must descend into subdirectories") +} + +func TestApplyOverlayFiles_AppendsAfterInlineOverlays(t *testing.T) { + ctx := testctx.NewCtx() + overlayDir := antOverlayDir + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(overlayDir, "0001-backport.overlay.toml"), + []byte(validBackportOverlayFile), fileperms.PrivateFile)) + + cfg := &ConfigFile{ + dir: "/project", + Components: map[string]ComponentConfig{ + "ant": { + OverlayFiles: []string{"comps/ant/overlays/*.overlay.toml"}, + Overlays: []ComponentOverlay{ + { + Type: ComponentOverlaySetSpecTag, + Tag: "Vendor", + Value: "Microsoft", + Metadata: &OverlayMetadata{ + Category: OverlayCategoryAZLBrandingPolicy, + }, + }, + }, + }, + }, + } + + require.NoError(t, applyOverlayFiles(ctx.FS(), cfg, false)) + + ant := cfg.Components["ant"] + require.Len(t, ant.Overlays, 3, "inline overlay + two file-sourced overlays") + + // Inline first. + assert.Equal(t, ComponentOverlaySetSpecTag, ant.Overlays[0].Type) + assert.Equal(t, OverlayCategoryAZLBrandingPolicy, ant.Overlays[0].Metadata.Category) + + // File-sourced overlays appended in file/declaration order, with the file's + // metadata stamped onto each. + assert.Equal(t, ComponentOverlaySearchAndReplaceInSpec, ant.Overlays[1].Type) + require.NotNil(t, ant.Overlays[1].Metadata) + assert.Equal(t, OverlayCategoryBackportDistGit, ant.Overlays[1].Metadata.Category) + + assert.Equal(t, ComponentOverlayRemoveSubpackage, ant.Overlays[2].Type) + require.NotNil(t, ant.Overlays[2].Metadata) + assert.Equal(t, OverlayCategoryBackportDistGit, ant.Overlays[2].Metadata.Category) +} + +func TestApplyOverlayFiles_NoopWhenUnset(t *testing.T) { + ctx := testctx.NewCtx() + + cfg := &ConfigFile{ + dir: "/project", + Components: map[string]ComponentConfig{ + "ant": { + Overlays: []ComponentOverlay{ + {Type: ComponentOverlayAddSpecTag, Tag: "Vendor", Value: "Microsoft"}, + }, + }, + }, + } + + require.NoError(t, applyOverlayFiles(ctx.FS(), cfg, false)) + + ant := cfg.Components["ant"] + require.Len(t, ant.Overlays, 1, "no overlay-files, no merging") +} + +func TestRejectOverlayFilesInDefaults(t *testing.T) { + const overlayFilePattern = "overlays/*.overlay.toml" + + testCases := []struct { + name string + cfg ConfigFile + errorContains string + }{ + { + name: "project-level default-component-config", + cfg: ConfigFile{ + DefaultComponentConfig: &ComponentConfig{ + OverlayFiles: []string{overlayFilePattern}, + }, + }, + errorContains: "project-level default-component-config", + }, + { + name: "component-group default-component-config", + cfg: ConfigFile{ + ComponentGroups: map[string]ComponentGroupConfig{ + "core": { + DefaultComponentConfig: ComponentConfig{ + OverlayFiles: []string{overlayFilePattern}, + }, + }, + }, + }, + errorContains: "component-group `core`", + }, + { + name: "distro version default-component-config", + cfg: ConfigFile{ + Distros: map[string]DistroDefinition{ + "azl": { + Versions: map[string]DistroVersionDefinition{ + "3.0": { + DefaultComponentConfig: ComponentConfig{ + OverlayFiles: []string{overlayFilePattern}, + }, + }, + }, + }, + }, + }, + errorContains: "distro `azl` version `3.0`", + }, + { + name: "unset defaults", + cfg: ConfigFile{ + DefaultComponentConfig: &ComponentConfig{}, + ComponentGroups: map[string]ComponentGroupConfig{ + "core": {}, + }, + Distros: map[string]DistroDefinition{ + "azl": { + Versions: map[string]DistroVersionDefinition{ + "3.0": {}, + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + err := rejectOverlayFilesInDefaults(&testCase.cfg) + + if testCase.errorContains == "" { + require.NoError(t, err) + + return + } + + require.ErrorIs(t, err, ErrOverlayFilesInDefaultConfig) + assert.Contains(t, err.Error(), testCase.errorContains) + }) + } +} + +func TestApplyOverlayFiles_AcceptsAbsoluteGlob(t *testing.T) { + ctx := testctx.NewCtx() + absDir := "/elsewhere/overlays" + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(absDir, "0001-backport.overlay.toml"), + []byte(validBackportOverlayFile), fileperms.PrivateFile)) + + cfg := &ConfigFile{ + dir: "/project", + Components: map[string]ComponentConfig{ + "ant": {OverlayFiles: []string{overlayGlob(absDir)}}, + }, + } + + require.NoError(t, applyOverlayFiles(ctx.FS(), cfg, false)) + + ant := cfg.Components["ant"] + require.Len(t, ant.Overlays, 2, "absolute glob is not re-rooted under cfg.dir") + require.NotNil(t, ant.Overlays[0].Metadata) + assert.Equal(t, OverlayCategoryBackportDistGit, ant.Overlays[0].Metadata.Category) +} + +// TestLoadAndResolveProjectConfig_OverlayFiles exercises the full loader pipeline +// (loadAndResolveProjectConfig -> loadProjectConfigFile -> applyOverlayFiles) and +// guards against regressions that drop the overlay-files hook from the loader. +func TestLoadAndResolveProjectConfig_OverlayFiles(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(` +[components.ant] +overlay-files = ["comps/ant/overlays/*.overlay.toml"] +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + "/project/comps/ant/overlays/0001-backport.overlay.toml", + []byte(validBackportOverlayFile), fileperms.PrivateFile)) + + cfg, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.NoError(t, err) + require.NotNil(t, cfg) + + ant, ok := cfg.Components["ant"] + require.True(t, ok, "ant component should be present") + require.Len(t, ant.Overlays, 2) + require.NotNil(t, ant.Overlays[0].Metadata) + assert.Equal(t, OverlayCategoryBackportDistGit, ant.Overlays[0].Metadata.Category) +} diff --git a/internal/projectconfig/overlay_metadata.go b/internal/projectconfig/overlay_metadata.go new file mode 100644 index 00000000..1aa81685 --- /dev/null +++ b/internal/projectconfig/overlay_metadata.go @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + "slices" + + "github.com/go-playground/validator/v10" +) + +// OverlayCategory is a classification label for an overlay's intent. Categories help +// reviewers and tooling reason about why an overlay exists and when it can be removed. +type OverlayCategory string + +const ( + // OverlayCategoryBackportDistGit applies a fix backported from (or being upstreamed to) + // a dist-git or upstream project. The overlay can be dropped once AZL bumps its upstream + // pin past the fix. + OverlayCategoryBackportDistGit OverlayCategory = "backport-dist-git" + // OverlayCategoryAZLPruning removes content from a component for AZL: dependencies that + // are not shipped, unneeded features, subpackages, or files. Replaces the more granular + // historical 'azl-dependency-pruning' and 'azl-feature-disablement' categories. + OverlayCategoryAZLPruning OverlayCategory = "azl-pruning" + // OverlayCategoryAZLCompatibility makes a component work in the AZL build/runtime + // environment: toolchain and mock adjustments and similar compatibility fixes that + // are not themselves backports. See [OverlayCategoryAZLDepMissingWorkaround] for + // workarounds for dependencies that have not yet been imported into AZL. + OverlayCategoryAZLCompatibility OverlayCategory = "azl-compatibility" + // OverlayCategoryAZLDepMissingWorkaround works around a runtime or build dependency + // that has not yet been imported into AZL (or is unavailable on a given target). Drop + // the overlay once the dependency lands. + OverlayCategoryAZLDepMissingWorkaround OverlayCategory = "azl-dep-missing-workaround" + // OverlayCategoryAZLBrandingPolicy applies Fedora→AzureLinux name/path changes or + // RHEL/enterprise convention alignment. + OverlayCategoryAZLBrandingPolicy OverlayCategory = "azl-branding-policy" + // OverlayCategoryAZLDisableFlakyTests skips tests that fail intermittently or due to + // environmental flakiness rather than a real problem with the component, + // may require retry logic / improve the tests. + OverlayCategoryAZLDisableFlakyTests OverlayCategory = "azl-disable-flaky-tests" + // OverlayCategoryAZLDisableUnsupportedTests skips tests that cannot meaningfully run + // in AZL's build/runtime environment (e.g. tests that require network access, root, + // or hardware that is unavailable in mock), may require figuring out how to extend support. + OverlayCategoryAZLDisableUnsupportedTests OverlayCategory = "azl-disable-unsupported-tests" + // OverlayCategoryAZLSecurityCompliance applies FIPS or crypto policy changes. + OverlayCategoryAZLSecurityCompliance OverlayCategory = "azl-security-compliance" + // OverlayCategoryAZLReleaseManagement adjusts release tag and changelog mechanics. + OverlayCategoryAZLReleaseManagement OverlayCategory = "azl-release-management" + // OverlayCategoryAZLPlatformAdaptation applies architecture-specific adjustments. + OverlayCategoryAZLPlatformAdaptation OverlayCategory = "azl-platform-adaptation" +) + +// allOverlayCategories lists every recognized [OverlayCategory] value. +// +//nolint:gochecknoglobals // effectively a constant; Go doesn't allow const slices. +var allOverlayCategories = []OverlayCategory{ + OverlayCategoryBackportDistGit, + OverlayCategoryAZLPruning, + OverlayCategoryAZLCompatibility, + OverlayCategoryAZLDepMissingWorkaround, + OverlayCategoryAZLBrandingPolicy, + OverlayCategoryAZLDisableFlakyTests, + OverlayCategoryAZLDisableUnsupportedTests, + OverlayCategoryAZLSecurityCompliance, + OverlayCategoryAZLReleaseManagement, + OverlayCategoryAZLPlatformAdaptation, +} + +// IsValid reports whether c is one of the recognized [OverlayCategory] values. +func (c OverlayCategory) IsValid() bool { + return slices.Contains(allOverlayCategories, c) +} + +// BugRef is a typed reference to an issue-tracker entry related to an overlay. Today +// it carries only a URL; the struct form leaves room for tracker-specific metadata to +// be added later without breaking the on-disk schema. +type BugRef struct { + // URL is the http(s) link to the bug entry. Required. + URL string `toml:"url" json:"url" validate:"required,http_url" jsonschema:"required,format=uri,pattern=^https?://,title=URL,description=HTTP(S) link to the bug entry"` +} + +// OverlayMetadata describes the intent and provenance of an overlay. It is documentation +// only — it does not affect how the overlay is applied and is excluded from component +// fingerprints. When present, it must declare a [OverlayMetadata.Category]; other fields +// are optional but constrained by category-specific rules (see [OverlayMetadata.Validate]). +type OverlayMetadata struct { + // Category classifies the overlay's intent. Required. + Category OverlayCategory `toml:"category" json:"category" jsonschema:"required,enum=backport-dist-git,enum=azl-pruning,enum=azl-compatibility,enum=azl-dep-missing-workaround,enum=azl-branding-policy,enum=azl-disable-flaky-tests,enum=azl-disable-unsupported-tests,enum=azl-security-compliance,enum=azl-release-management,enum=azl-platform-adaptation,title=Category,description=Classification of the overlay's intent"` + + // Commits lists URLs of upstream commits (typically Fedora dist-git or upstream-project + // commits) that this overlay backports or references. + Commits []string `toml:"commits,omitempty" json:"commits,omitempty" validate:"omitempty,dive,http_url" jsonschema:"title=Commits,description=URLs of upstream commits this overlay backports or references"` + + // Bugs holds references to issue-tracker entries related to this overlay. Each entry + // must carry an http(s) URL (see [BugRef]). + Bugs []BugRef `toml:"bugs,omitempty" json:"bugs,omitempty" validate:"omitempty,dive" jsonschema:"title=Bug references,description=References to issue-tracker entries related to this overlay"` + + // Upstreamable records whether this overlay's change can be upstreamed. It is omitted + // (nil) when upstreamability has not yet been assessed. + Upstreamable *bool `toml:"upstreamable,omitempty" json:"upstreamable,omitempty" jsonschema:"title=Upstreamable,description=Whether this overlay's change can be upstreamed; omit if not yet assessed"` +} + +// clone returns a deep copy of the metadata. It is used to stamp a single file-level +// [OverlayMetadata] onto every overlay in an overlay file without aliasing the +// slice fields. A manual copy is preferred over a reflection-based deep copy so that a +// future struct change can never panic during config load. +func (m *OverlayMetadata) clone() *OverlayMetadata { + if m == nil { + return nil + } + + cloned := *m + cloned.Commits = slices.Clone(m.Commits) + cloned.Bugs = slices.Clone(m.Bugs) + + if m.Upstreamable != nil { + upstreamable := *m.Upstreamable + cloned.Upstreamable = &upstreamable + } + + return &cloned +} + +// metadataValidator is a shared, concurrency-safe validator instance reused across all +// [OverlayMetadata.Validate] calls to avoid per-call allocation. +// +//nolint:gochecknoglobals // validator instances are safe for concurrent use once configured. +var metadataValidator = validator.New() + +// Validate checks that the metadata is internally consistent: the category is recognized, +// category-specific required fields are present, and URL-shaped fields parse as http(s) +// URLs. +func (m *OverlayMetadata) Validate() error { + if m.Category == "" { + return errors.New("'metadata' requires 'category'") + } + + if !m.Category.IsValid() { + return fmt.Errorf("unknown overlay category %#q", string(m.Category)) + } + + // 'backport-dist-git' is the only category that imposes an extra required field; all + // other categories require nothing beyond a valid 'category'. + if m.Category == OverlayCategoryBackportDistGit && len(m.Commits) == 0 { + return fmt.Errorf( + "overlay category %#q requires at least one entry in 'commits'", + string(m.Category), + ) + } + + if err := metadataValidator.Struct(m); err != nil { + return fmt.Errorf("invalid overlay metadata:\n%w", err) + } + + return nil +} diff --git a/internal/projectconfig/overlay_metadata_test.go b/internal/projectconfig/overlay_metadata_test.go new file mode 100644 index 00000000..22e52ac4 --- /dev/null +++ b/internal/projectconfig/overlay_metadata_test.go @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOverlayMetadata_Validate(t *testing.T) { + testCases := []struct { + name string + metadata projectconfig.OverlayMetadata + errorContains string + }{ + { + name: "backport-dist-git with commits is valid", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryBackportDistGit, + Commits: []string{"https://src.fedoraproject.org/rpms/xclock/c/abc123"}, + }, + }, + { + name: "backport-dist-git requires commits", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryBackportDistGit, + }, + errorContains: "commits", + }, + { + name: "backport-dist-git with bug refs valid", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryBackportDistGit, + Commits: []string{"https://src.fedoraproject.org/rpms/xclock/c/abc123"}, + Bugs: []projectconfig.BugRef{ + {URL: "https://github.com/example/repo/issues/1"}, + }, + }, + }, + { + name: "azl-branding-policy needs no extras", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLBrandingPolicy, + }, + }, + { + name: "azl-dep-missing-workaround needs no extras", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLDepMissingWorkaround, + }, + }, + { + name: "azl-branding-policy may carry bugs and commits", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLBrandingPolicy, + Bugs: []projectconfig.BugRef{ + {URL: "https://bugzilla.redhat.com/show_bug.cgi?id=2234567"}, + {URL: "https://github.com/example/repo/issues/2"}, + }, + Commits: []string{"https://github.com/example/repo/commit/deadbeef"}, + Upstreamable: lo.ToPtr(true), + }, + }, + { + name: "upstreamable false is valid", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLCompatibility, + Upstreamable: lo.ToPtr(false), + }, + }, + { + name: "missing category", + metadata: projectconfig.OverlayMetadata{ + Commits: []string{"https://example.com/commit/abc"}, + }, + errorContains: "category", + }, + { + name: "unknown category", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategory("bogus"), + }, + errorContains: "unknown overlay category", + }, + { + name: "bug requires url", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLCompatibility, + Bugs: []projectconfig.BugRef{{}}, + }, + errorContains: "URL", + }, + { + name: "bug rejects non-http url", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLCompatibility, + Bugs: []projectconfig.BugRef{ + {URL: "ftp://example.com/bug/1"}, + }, + }, + errorContains: "URL", + }, + { + name: "commit url must be http", + metadata: projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryBackportDistGit, + Commits: []string{"not-a-url"}, + }, + errorContains: "Commits", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + err := testCase.metadata.Validate() + + if testCase.errorContains == "" { + require.NoError(t, err) + + return + } + + require.Error(t, err) + assert.Contains(t, err.Error(), testCase.errorContains) + }) + } +} + +func TestComponentOverlay_Validate_Metadata(t *testing.T) { + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayAddSpecTag, + Tag: "Vendor", + Value: "Microsoft", + Metadata: &projectconfig.OverlayMetadata{ + Category: projectconfig.OverlayCategoryAZLBrandingPolicy, + }, + } + require.NoError(t, overlay.Validate()) + + overlay.Metadata.Category = projectconfig.OverlayCategory("bogus") + + err := overlay.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown overlay category") +} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 7aa38d7e..7b824f4e 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -3,6 +3,22 @@ "$id": "https://github.com/microsoft/azure-linux-dev-tools/internal/projectconfig/config-file", "$ref": "#/$defs/ConfigFile", "$defs": { + "BugRef": { + "properties": { + "url": { + "type": "string", + "pattern": "^https?://", + "format": "uri", + "title": "URL", + "description": "HTTP(S) link to the bug entry" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "url" + ] + }, "CheckConfig": { "properties": { "skip": { @@ -119,6 +135,14 @@ "title": "Overlays", "description": "Overlays to apply to this component's spec and/or sources" }, + "overlay-files": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Overlay files", + "description": "Glob patterns (relative to the component config file) matched against the filesystem to locate per-file overlay documents at load time" + }, "build": { "$ref": "#/$defs/ComponentBuildConfig", "title": "Build configuration", @@ -278,6 +302,11 @@ "type": "string", "title": "Source", "description": "For overlays that require a source file as input" + }, + "metadata": { + "$ref": "#/$defs/OverlayMetadata", + "title": "Metadata", + "description": "Optional documentation metadata describing the overlay's intent and provenance" } }, "additionalProperties": false, @@ -711,6 +740,53 @@ "type" ] }, + "OverlayMetadata": { + "properties": { + "category": { + "type": "string", + "enum": [ + "backport-dist-git", + "azl-pruning", + "azl-compatibility", + "azl-dep-missing-workaround", + "azl-branding-policy", + "azl-disable-flaky-tests", + "azl-disable-unsupported-tests", + "azl-security-compliance", + "azl-release-management", + "azl-platform-adaptation" + ], + "title": "Category", + "description": "Classification of the overlay's intent" + }, + "commits": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Commits", + "description": "URLs of upstream commits this overlay backports or references" + }, + "bugs": { + "items": { + "$ref": "#/$defs/BugRef" + }, + "type": "array", + "title": "Bug references", + "description": "References to issue-tracker entries related to this overlay" + }, + "upstreamable": { + "type": "boolean", + "title": "Upstreamable", + "description": "Whether this overlay's change can be upstreamed; omit if not yet assessed" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "category" + ] + }, "PackageConfig": { "properties": { "publish": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 7aa38d7e..7b824f4e 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -3,6 +3,22 @@ "$id": "https://github.com/microsoft/azure-linux-dev-tools/internal/projectconfig/config-file", "$ref": "#/$defs/ConfigFile", "$defs": { + "BugRef": { + "properties": { + "url": { + "type": "string", + "pattern": "^https?://", + "format": "uri", + "title": "URL", + "description": "HTTP(S) link to the bug entry" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "url" + ] + }, "CheckConfig": { "properties": { "skip": { @@ -119,6 +135,14 @@ "title": "Overlays", "description": "Overlays to apply to this component's spec and/or sources" }, + "overlay-files": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Overlay files", + "description": "Glob patterns (relative to the component config file) matched against the filesystem to locate per-file overlay documents at load time" + }, "build": { "$ref": "#/$defs/ComponentBuildConfig", "title": "Build configuration", @@ -278,6 +302,11 @@ "type": "string", "title": "Source", "description": "For overlays that require a source file as input" + }, + "metadata": { + "$ref": "#/$defs/OverlayMetadata", + "title": "Metadata", + "description": "Optional documentation metadata describing the overlay's intent and provenance" } }, "additionalProperties": false, @@ -711,6 +740,53 @@ "type" ] }, + "OverlayMetadata": { + "properties": { + "category": { + "type": "string", + "enum": [ + "backport-dist-git", + "azl-pruning", + "azl-compatibility", + "azl-dep-missing-workaround", + "azl-branding-policy", + "azl-disable-flaky-tests", + "azl-disable-unsupported-tests", + "azl-security-compliance", + "azl-release-management", + "azl-platform-adaptation" + ], + "title": "Category", + "description": "Classification of the overlay's intent" + }, + "commits": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Commits", + "description": "URLs of upstream commits this overlay backports or references" + }, + "bugs": { + "items": { + "$ref": "#/$defs/BugRef" + }, + "type": "array", + "title": "Bug references", + "description": "References to issue-tracker entries related to this overlay" + }, + "upstreamable": { + "type": "boolean", + "title": "Upstreamable", + "description": "Whether this overlay's change can be upstreamed; omit if not yet assessed" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "category" + ] + }, "PackageConfig": { "properties": { "publish": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7aa38d7e..7b824f4e 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -3,6 +3,22 @@ "$id": "https://github.com/microsoft/azure-linux-dev-tools/internal/projectconfig/config-file", "$ref": "#/$defs/ConfigFile", "$defs": { + "BugRef": { + "properties": { + "url": { + "type": "string", + "pattern": "^https?://", + "format": "uri", + "title": "URL", + "description": "HTTP(S) link to the bug entry" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "url" + ] + }, "CheckConfig": { "properties": { "skip": { @@ -119,6 +135,14 @@ "title": "Overlays", "description": "Overlays to apply to this component's spec and/or sources" }, + "overlay-files": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Overlay files", + "description": "Glob patterns (relative to the component config file) matched against the filesystem to locate per-file overlay documents at load time" + }, "build": { "$ref": "#/$defs/ComponentBuildConfig", "title": "Build configuration", @@ -278,6 +302,11 @@ "type": "string", "title": "Source", "description": "For overlays that require a source file as input" + }, + "metadata": { + "$ref": "#/$defs/OverlayMetadata", + "title": "Metadata", + "description": "Optional documentation metadata describing the overlay's intent and provenance" } }, "additionalProperties": false, @@ -711,6 +740,53 @@ "type" ] }, + "OverlayMetadata": { + "properties": { + "category": { + "type": "string", + "enum": [ + "backport-dist-git", + "azl-pruning", + "azl-compatibility", + "azl-dep-missing-workaround", + "azl-branding-policy", + "azl-disable-flaky-tests", + "azl-disable-unsupported-tests", + "azl-security-compliance", + "azl-release-management", + "azl-platform-adaptation" + ], + "title": "Category", + "description": "Classification of the overlay's intent" + }, + "commits": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Commits", + "description": "URLs of upstream commits this overlay backports or references" + }, + "bugs": { + "items": { + "$ref": "#/$defs/BugRef" + }, + "type": "array", + "title": "Bug references", + "description": "References to issue-tracker entries related to this overlay" + }, + "upstreamable": { + "type": "boolean", + "title": "Upstreamable", + "description": "Whether this overlay's change can be upstreamed; omit if not yet assessed" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "category" + ] + }, "PackageConfig": { "properties": { "publish": {