diff --git a/eng/common/pipelines/templates/steps/check-spelling.yml b/eng/common/pipelines/templates/steps/check-spelling.yml index 3963e785fd96..2a2a91017912 100644 --- a/eng/common/pipelines/templates/steps/check-spelling.yml +++ b/eng/common/pipelines/templates/steps/check-spelling.yml @@ -46,10 +46,14 @@ steps: arguments: >- -CspellConfigPath ${{ parameters.CspellConfigPath }} -ExitWithError:(!$${{ parameters.ContinueOnError }}) + -SourceCommittish 'HEAD' + -TargetCommittish 'HEAD^1' pwsh: true - ${{ if ne('', parameters.ScriptToValidateUpgrade) }}: - pwsh: | - $changedFiles = ./eng/common/scripts/get-changedfiles.ps1 + $changedFiles = ./eng/common/scripts/get-changedfiles.ps1 ` + -SourceCommittish 'HEAD' ` + -TargetCommittish 'HEAD^1' if ($changedFiles -notcontains 'eng/common/spelling/package-lock.json') { Write-Host "No changes to cspell package-lock.json detected." diff --git a/eng/common/pipelines/templates/steps/detect-api-changes.yml b/eng/common/pipelines/templates/steps/detect-api-changes.yml index fadeaacc6791..0b26d45195fa 100644 --- a/eng/common/pipelines/templates/steps/detect-api-changes.yml +++ b/eng/common/pipelines/templates/steps/detect-api-changes.yml @@ -23,6 +23,8 @@ steps: -RepoFullName $(Build.Repository.Name) -APIViewUri $(ApiChangeDetectRequestUrl) -ArtifactName ${{ parameters.ArtifactName }} + -SourceCommittish 'HEAD' + -TargetBranch 'HEAD^1' -DevopsProject $(System.TeamProject) pwsh: true displayName: Create APIView if API has changes diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 90581dbe74cc..43f1e8fc1642 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -37,6 +37,8 @@ steps: -TargetPath '${{ parameters.TargetPath }}' -ArtifactPath '${{ parameters.DiffDirectory }}' -ExcludePaths ('${{ convertToJson(parameters.ExcludePaths) }}' | ConvertFrom-Json) + -SourceCommittish 'HEAD' + -TargetCommittish 'HEAD^1' pwsh: true workingDirectory: '${{ parameters.WorkingDirectory }}' diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1 index c8e00935cbe0..e568494f7725 100644 --- a/eng/common/scripts/Detect-Api-Changes.ps1 +++ b/eng/common/scripts/Detect-Api-Changes.ps1 @@ -13,6 +13,7 @@ Param ( [string] $APIViewUri, [string] $RepoFullName = "", [string] $ArtifactName = "packages", + [string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}", [string] $TargetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"), [string] $DevopsProject = "internal" ) @@ -98,7 +99,11 @@ function Submit-Request($filePath, $packageInfo) function Should-Process-Package($packageInfo) { $packagePath = $packageInfo.DirectoryPath - $modifiedFiles = @(Get-ChangedFiles -DiffPath "$packagePath/*" -DiffFilterType '') + $modifiedFiles = @(Get-ChangedFiles ` + -SourceCommittish $SourceCommittish ` + -TargetCommittish $TargetBranch ` + -DiffPath "$packagePath/*" ` + -DiffFilterType '') $filteredFileCount = $modifiedFiles.Count LogInfo "Number of modified files for package: $filteredFileCount" return ($filteredFileCount -gt 0 -and $packageInfo.IsNewSdk) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 50f6ec2b2d1a..18d2d9e6a4e6 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -10,6 +10,12 @@ The folder in which the result will be written. .PARAMETER TargetPath The path under which changes will be detected. + +.PARAMETER SourceCommittish +Commit or ref containing the proposed changes. + +.PARAMETER TargetCommittish +Commit or ref used as the comparison base. #> [CmdletBinding()] Param ( @@ -19,7 +25,11 @@ Param ( [string] $TargetPath, [Parameter(Mandatory=$false)] [AllowEmptyCollection()] - [array] $ExcludePaths + [array] $ExcludePaths, + [Parameter(Mandatory=$false)] + [string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}", + [Parameter(Mandatory=$false)] + [string] $TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/") ) . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") @@ -47,8 +57,15 @@ $ArtifactName = Join-Path $ArtifactPath "diff.json" $changedFiles = @() $changedServices = @() -$changedFiles = Get-ChangedFiles -DiffPath $TargetPath -$deletedFiles = Get-ChangedFiles -DiffPath $TargetPath -DiffFilterType "D" +$changedFiles = Get-ChangedFiles ` + -SourceCommittish $SourceCommittish ` + -TargetCommittish $TargetCommittish ` + -DiffPath $TargetPath +$deletedFiles = Get-ChangedFiles ` + -SourceCommittish $SourceCommittish ` + -TargetCommittish $TargetCommittish ` + -DiffPath $TargetPath ` + -DiffFilterType "D" if ($changedFiles) { $changedServices = Get-ChangedServices -ChangedFiles $changedFiles diff --git a/eng/common/scripts/Helpers/git-helpers.ps1 b/eng/common/scripts/Helpers/git-helpers.ps1 index 846c65a2b451..048e28f30a61 100644 --- a/eng/common/scripts/Helpers/git-helpers.ps1 +++ b/eng/common/scripts/Helpers/git-helpers.ps1 @@ -52,6 +52,9 @@ function Get-ChangedFiles { } Write-Host $command $changedFiles = Invoke-Expression -Command $command + if ($LASTEXITCODE -ne 0) { + throw "git diff failed with exit code $LASTEXITCODE for '$TargetCommittish...$SourceCommittish'" + } if (!$changedFiles) { Write-Host "No changed files in git diff between $TargetCommittish and $SourceCommittish" } diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 551a561c20c5..ee53793ec6a6 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -31,16 +31,11 @@ extends: os: linux steps: - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' - - 'sdk/tools/linting-extensions' + - checkout: self + fetchDepth: 2 + fetchTags: false - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -66,28 +61,21 @@ extends: # Authenticate with Azure Artifacts - template: /eng/pipelines/templates/steps/maven-authenticate.yml - # The only time generate_from_source_pom.py should be used to set the SparseCheckoutDirectories - # is for FromSource runs or, in the case of code quality reports, a run that needs to build - # everything using the latest source. It'll greedily set any service directories as it figures - # out what libraries, their dependents and so on, that need to be in ClientFromSourcePom.xml + # Generate the source dependency graph and identify projects that can skip linting. - task: PythonScript@0 - displayName: 'Generate FromSource POM and directories for sparse checkout' + displayName: 'Generate FromSource POM and linting project list' inputs: scriptPath: 'eng/scripts/generate_from_source_pom.py' arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Paths: $(SparseCheckoutDirectories) - - task: PowerShell@2 inputs: filePath: $(System.DefaultWorkingDirectory)/eng/pipelines/scripts/Get-Linting-Commands.ps1 arguments: > -BuildReason $(Build.Reason) -SourceBranch "HEAD" + -TargetBranch "HEAD^1" -LintingPipelineVariable "LintingGoals" pwsh: true displayName: 'Generate Linting Commands' diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index bddc3dd58215..2fd358da178e 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -24,14 +24,9 @@ jobs: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 2 + fetchTags: false - script: | echo "##vso[build.addbuildtag]Scheduled" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 713cf0b689b0..d20d7de7c079 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -108,17 +108,11 @@ jobs: exit 1 displayName: 'Validate: at least one package is selected for release' - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/ci*.yml' # necessary for save-package-properties.yml - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 2 + fetchTags: false - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -134,25 +128,6 @@ jobs: inputs: versionSpec: $(PythonVersion) - # This step needs to run before we update to using dev versions otherwise dependency - # check in the script will not fully work because the versions will mismatch - - task: PowerShell@2 - displayName: Generate directories variable for sparse checkout - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1 - arguments: > - -SourcesDirectory $(Build.SourcesDirectory) - -ProjectList $(ProjectList) - - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. - - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml - parameters: - Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - - ${{ parameters.PreBuildSteps }} - script: | @@ -318,18 +293,11 @@ jobs: inputs: versionSpec: $(PythonVersion) - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' - - '.vscode/cspell.json' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 2 + fetchTags: false - task: PowerShell@2 displayName: 'Verify versions in POM files' @@ -350,23 +318,6 @@ jobs: ServiceDirectory: ${{parameters.ServiceDirectory}} ExcludePaths: ${{parameters.ExcludePaths}} - - task: PowerShell@2 - displayName: Generate directories variable for sparse checkout - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1 - arguments: > - -SourcesDirectory $(Build.SourcesDirectory) - -ProjectList $(ProjectList) - - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. - - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml - parameters: - Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - - task: UseNode@1 inputs: version: 22.x diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 27e2f04cdd7f..8d8fae9e9519 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -13,17 +13,11 @@ parameters: default: [] steps: - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/ci*.yml' # necessary for save-package-properties.yml - - ${{ if not(parameters.CheckoutRecordings) }}: - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 2 + fetchTags: false - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -54,33 +48,11 @@ steps: ServiceDirectory: ${{parameters.ServiceDirectory}} ExcludePaths: ${{parameters.ExcludePaths}} - # This call is used generate the sparse checkout variables for non-FromSource runs. - - task: PowerShell@2 - displayName: Generate directories variable for sparse checkout - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1 - arguments: > - -SourcesDirectory $(Build.SourcesDirectory) - -ProjectList $(ProjectList) - condition: and(succeeded(), ne(variables['TestFromSource'], 'true')) - - # The only time generate_from_source_pom.py should be used to set the SparseCheckoutDirectories - # is for FromSource runs or, in the case of code quality reports, a run that needs to build - # everything using the latest source. It'll greedily set any service directories as it figures - # out what libraries, their dependents and so on, that need to be in ClientFromSourcePom.xml + # FromSource runs use ClientFromSourcePom.xml to build the complete source dependency graph. - task: PythonScript@0 - displayName: 'Generate FromSource POM and directories for sparse checkout' + displayName: 'Generate FromSource POM' inputs: scriptPath: 'eng/scripts/generate_from_source_pom.py' arguments: '--artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' condition: and(succeeded(), eq(variables['TestFromSource'], 'true')) - - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. - - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml - parameters: - SkipCheckoutNone: true - Paths: $(SparseCheckoutDirectories)