From 3f2fc9ed9aa218d407938729d1b8809367558b3e Mon Sep 17 00:00:00 2001 From: Victor Colin Amador Date: Wed, 19 Aug 2026 10:41:27 -0700 Subject: [PATCH 1/3] Removed sparse checkout in favor of shallow clone --- eng/pipelines/code-quality-reports.yml | 17 +++++--------- .../templates/jobs/build-validate-pom.yml | 11 +++------ eng/pipelines/templates/jobs/ci.yml | 23 +++++-------------- .../steps/initialize-test-environment.yml | 11 +++------ 4 files changed, 18 insertions(+), 44 deletions(-) diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 551a561c20c5..5d4184e8975d 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -34,13 +34,9 @@ extends: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' - - 'sdk/tools/linting-extensions' + - checkout: self + fetchDepth: 1 + fetchTags: false - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -77,10 +73,9 @@ extends: arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Paths: $(SparseCheckoutDirectories) + - checkout: self + fetchDepth: 1 + fetchTags: false - task: PowerShell@2 inputs: diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index bddc3dd58215..1eeccc347034 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -24,14 +24,9 @@ jobs: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 1 + fetchTags: false - script: | echo "##vso[build.addbuildtag]Scheduled" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 713cf0b689b0..502e7be8f5d9 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -111,14 +111,9 @@ jobs: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/ci*.yml' # necessary for save-package-properties.yml - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 1 + fetchTags: false - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -321,15 +316,9 @@ jobs: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' - - '.vscode/cspell.json' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 1 + fetchTags: false - task: PowerShell@2 displayName: 'Verify versions in POM files' diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 27e2f04cdd7f..c8f35285b850 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -16,14 +16,9 @@ steps: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/ci*.yml' # necessary for save-package-properties.yml - - ${{ if not(parameters.CheckoutRecordings) }}: - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchDepth: 1 + fetchTags: false - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' From 93263eba9c4cec2a8e81df9651833e057ef72f08 Mon Sep 17 00:00:00 2001 From: Victor Colin Amador Date: Wed, 19 Aug 2026 17:12:19 -0700 Subject: [PATCH 2/3] Updated depth to use the correct commits for comparison --- eng/pipelines/code-quality-reports.yml | 6 +----- eng/pipelines/templates/jobs/build-validate-pom.yml | 2 +- eng/pipelines/templates/jobs/ci.yml | 4 ++-- .../templates/steps/initialize-test-environment.yml | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 5d4184e8975d..67d8e6fd3664 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -35,7 +35,7 @@ extends: # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self - fetchDepth: 1 + fetchDepth: 2 fetchTags: false - task: UsePythonVersion@0 @@ -73,10 +73,6 @@ extends: arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' - - checkout: self - fetchDepth: 1 - fetchTags: false - - task: PowerShell@2 inputs: filePath: $(System.DefaultWorkingDirectory)/eng/pipelines/scripts/Get-Linting-Commands.ps1 diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index 1eeccc347034..2fd358da178e 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -25,7 +25,7 @@ jobs: steps: - checkout: self - fetchDepth: 1 + fetchDepth: 2 fetchTags: false - script: | diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 502e7be8f5d9..e63cb37f0c7c 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -112,7 +112,7 @@ jobs: # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self - fetchDepth: 1 + fetchDepth: 2 fetchTags: false - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml @@ -317,7 +317,7 @@ jobs: # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self - fetchDepth: 1 + fetchDepth: 2 fetchTags: false - task: PowerShell@2 diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index c8f35285b850..016fc5e143ed 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -17,7 +17,7 @@ steps: # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self - fetchDepth: 1 + fetchDepth: 2 fetchTags: false - task: UsePythonVersion@0 From 634c5d742592d61a763b891198626f3df3432dfe Mon Sep 17 00:00:00 2001 From: Victor Colin Amador Date: Wed, 19 Aug 2026 17:29:10 -0700 Subject: [PATCH 3/3] Updated PR diff script and docs --- .../templates/steps/check-spelling.yml | 6 ++- .../templates/steps/detect-api-changes.yml | 2 + .../steps/save-package-properties.yml | 2 + eng/common/scripts/Detect-Api-Changes.ps1 | 7 +++- eng/common/scripts/Generate-PR-Diff.ps1 | 23 ++++++++-- eng/common/scripts/Helpers/git-helpers.ps1 | 3 ++ eng/pipelines/code-quality-reports.yml | 11 ++--- eng/pipelines/templates/jobs/ci.yml | 42 +------------------ .../steps/initialize-test-environment.yml | 29 ++----------- 9 files changed, 47 insertions(+), 78 deletions(-) diff --git a/eng/common/pipelines/templates/steps/check-spelling.yml b/eng/common/pipelines/templates/steps/check-spelling.yml index 3963e785fd96..2a2a91017912 100644 --- a/eng/common/pipelines/templates/steps/check-spelling.yml +++ b/eng/common/pipelines/templates/steps/check-spelling.yml @@ -46,10 +46,14 @@ steps: arguments: >- -CspellConfigPath ${{ parameters.CspellConfigPath }} -ExitWithError:(!$${{ parameters.ContinueOnError }}) + -SourceCommittish 'HEAD' + -TargetCommittish 'HEAD^1' pwsh: true - ${{ if ne('', parameters.ScriptToValidateUpgrade) }}: - pwsh: | - $changedFiles = ./eng/common/scripts/get-changedfiles.ps1 + $changedFiles = ./eng/common/scripts/get-changedfiles.ps1 ` + -SourceCommittish 'HEAD' ` + -TargetCommittish 'HEAD^1' if ($changedFiles -notcontains 'eng/common/spelling/package-lock.json') { Write-Host "No changes to cspell package-lock.json detected." diff --git a/eng/common/pipelines/templates/steps/detect-api-changes.yml b/eng/common/pipelines/templates/steps/detect-api-changes.yml index fadeaacc6791..0b26d45195fa 100644 --- a/eng/common/pipelines/templates/steps/detect-api-changes.yml +++ b/eng/common/pipelines/templates/steps/detect-api-changes.yml @@ -23,6 +23,8 @@ steps: -RepoFullName $(Build.Repository.Name) -APIViewUri $(ApiChangeDetectRequestUrl) -ArtifactName ${{ parameters.ArtifactName }} + -SourceCommittish 'HEAD' + -TargetBranch 'HEAD^1' -DevopsProject $(System.TeamProject) pwsh: true displayName: Create APIView if API has changes diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 90581dbe74cc..43f1e8fc1642 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -37,6 +37,8 @@ steps: -TargetPath '${{ parameters.TargetPath }}' -ArtifactPath '${{ parameters.DiffDirectory }}' -ExcludePaths ('${{ convertToJson(parameters.ExcludePaths) }}' | ConvertFrom-Json) + -SourceCommittish 'HEAD' + -TargetCommittish 'HEAD^1' pwsh: true workingDirectory: '${{ parameters.WorkingDirectory }}' diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1 index c8e00935cbe0..e568494f7725 100644 --- a/eng/common/scripts/Detect-Api-Changes.ps1 +++ b/eng/common/scripts/Detect-Api-Changes.ps1 @@ -13,6 +13,7 @@ Param ( [string] $APIViewUri, [string] $RepoFullName = "", [string] $ArtifactName = "packages", + [string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}", [string] $TargetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"), [string] $DevopsProject = "internal" ) @@ -98,7 +99,11 @@ function Submit-Request($filePath, $packageInfo) function Should-Process-Package($packageInfo) { $packagePath = $packageInfo.DirectoryPath - $modifiedFiles = @(Get-ChangedFiles -DiffPath "$packagePath/*" -DiffFilterType '') + $modifiedFiles = @(Get-ChangedFiles ` + -SourceCommittish $SourceCommittish ` + -TargetCommittish $TargetBranch ` + -DiffPath "$packagePath/*" ` + -DiffFilterType '') $filteredFileCount = $modifiedFiles.Count LogInfo "Number of modified files for package: $filteredFileCount" return ($filteredFileCount -gt 0 -and $packageInfo.IsNewSdk) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 50f6ec2b2d1a..18d2d9e6a4e6 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -10,6 +10,12 @@ The folder in which the result will be written. .PARAMETER TargetPath The path under which changes will be detected. + +.PARAMETER SourceCommittish +Commit or ref containing the proposed changes. + +.PARAMETER TargetCommittish +Commit or ref used as the comparison base. #> [CmdletBinding()] Param ( @@ -19,7 +25,11 @@ Param ( [string] $TargetPath, [Parameter(Mandatory=$false)] [AllowEmptyCollection()] - [array] $ExcludePaths + [array] $ExcludePaths, + [Parameter(Mandatory=$false)] + [string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}", + [Parameter(Mandatory=$false)] + [string] $TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/") ) . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") @@ -47,8 +57,15 @@ $ArtifactName = Join-Path $ArtifactPath "diff.json" $changedFiles = @() $changedServices = @() -$changedFiles = Get-ChangedFiles -DiffPath $TargetPath -$deletedFiles = Get-ChangedFiles -DiffPath $TargetPath -DiffFilterType "D" +$changedFiles = Get-ChangedFiles ` + -SourceCommittish $SourceCommittish ` + -TargetCommittish $TargetCommittish ` + -DiffPath $TargetPath +$deletedFiles = Get-ChangedFiles ` + -SourceCommittish $SourceCommittish ` + -TargetCommittish $TargetCommittish ` + -DiffPath $TargetPath ` + -DiffFilterType "D" if ($changedFiles) { $changedServices = Get-ChangedServices -ChangedFiles $changedFiles diff --git a/eng/common/scripts/Helpers/git-helpers.ps1 b/eng/common/scripts/Helpers/git-helpers.ps1 index 846c65a2b451..048e28f30a61 100644 --- a/eng/common/scripts/Helpers/git-helpers.ps1 +++ b/eng/common/scripts/Helpers/git-helpers.ps1 @@ -52,6 +52,9 @@ function Get-ChangedFiles { } Write-Host $command $changedFiles = Invoke-Expression -Command $command + if ($LASTEXITCODE -ne 0) { + throw "git diff failed with exit code $LASTEXITCODE for '$TargetCommittish...$SourceCommittish'" + } if (!$changedFiles) { Write-Host "No changed files in git diff between $TargetCommittish and $SourceCommittish" } diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 67d8e6fd3664..ee53793ec6a6 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -31,8 +31,7 @@ extends: os: linux steps: - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self fetchDepth: 2 @@ -62,12 +61,9 @@ extends: # Authenticate with Azure Artifacts - template: /eng/pipelines/templates/steps/maven-authenticate.yml - # The only time generate_from_source_pom.py should be used to set the SparseCheckoutDirectories - # is for FromSource runs or, in the case of code quality reports, a run that needs to build - # everything using the latest source. It'll greedily set any service directories as it figures - # out what libraries, their dependents and so on, that need to be in ClientFromSourcePom.xml + # Generate the source dependency graph and identify projects that can skip linting. - task: PythonScript@0 - displayName: 'Generate FromSource POM and directories for sparse checkout' + displayName: 'Generate FromSource POM and linting project list' inputs: scriptPath: 'eng/scripts/generate_from_source_pom.py' arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' @@ -79,6 +75,7 @@ extends: arguments: > -BuildReason $(Build.Reason) -SourceBranch "HEAD" + -TargetBranch "HEAD^1" -LintingPipelineVariable "LintingGoals" pwsh: true displayName: 'Generate Linting Commands' diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index e63cb37f0c7c..d20d7de7c079 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -108,8 +108,7 @@ jobs: exit 1 displayName: 'Validate: at least one package is selected for release' - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self fetchDepth: 2 @@ -129,25 +128,6 @@ jobs: inputs: versionSpec: $(PythonVersion) - # This step needs to run before we update to using dev versions otherwise dependency - # check in the script will not fully work because the versions will mismatch - - task: PowerShell@2 - displayName: Generate directories variable for sparse checkout - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1 - arguments: > - -SourcesDirectory $(Build.SourcesDirectory) - -ProjectList $(ProjectList) - - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. - - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml - parameters: - Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - - ${{ parameters.PreBuildSteps }} - script: | @@ -313,8 +293,7 @@ jobs: inputs: versionSpec: $(PythonVersion) - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self fetchDepth: 2 @@ -339,23 +318,6 @@ jobs: ServiceDirectory: ${{parameters.ServiceDirectory}} ExcludePaths: ${{parameters.ExcludePaths}} - - task: PowerShell@2 - displayName: Generate directories variable for sparse checkout - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1 - arguments: > - -SourcesDirectory $(Build.SourcesDirectory) - -ProjectList $(ProjectList) - - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. - - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml - parameters: - Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - - task: UseNode@1 inputs: version: 22.x diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 016fc5e143ed..8d8fae9e9519 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -13,8 +13,7 @@ parameters: default: [] steps: - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. + # Use the default service-connection checkout for private mirrored repositories. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - checkout: self fetchDepth: 2 @@ -49,33 +48,11 @@ steps: ServiceDirectory: ${{parameters.ServiceDirectory}} ExcludePaths: ${{parameters.ExcludePaths}} - # This call is used generate the sparse checkout variables for non-FromSource runs. - - task: PowerShell@2 - displayName: Generate directories variable for sparse checkout - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1 - arguments: > - -SourcesDirectory $(Build.SourcesDirectory) - -ProjectList $(ProjectList) - condition: and(succeeded(), ne(variables['TestFromSource'], 'true')) - - # The only time generate_from_source_pom.py should be used to set the SparseCheckoutDirectories - # is for FromSource runs or, in the case of code quality reports, a run that needs to build - # everything using the latest source. It'll greedily set any service directories as it figures - # out what libraries, their dependents and so on, that need to be in ClientFromSourcePom.xml + # FromSource runs use ClientFromSourcePom.xml to build the complete source dependency graph. - task: PythonScript@0 - displayName: 'Generate FromSource POM and directories for sparse checkout' + displayName: 'Generate FromSource POM' inputs: scriptPath: 'eng/scripts/generate_from_source_pom.py' arguments: '--artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' condition: and(succeeded(), eq(variables['TestFromSource'], 'true')) - - # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories - # as we require the GitHub service connection to be loaded. - - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml - parameters: - SkipCheckoutNone: true - Paths: $(SparseCheckoutDirectories)