[Prototype] Removed sparse checkout in favor of shallow clone

eng/common/pipelines/templates/steps/check-spelling.yml

@@ -46,10 +46,14 @@ steps:
         arguments: >-
           -CspellConfigPath ${{ parameters.CspellConfigPath }}
           -ExitWithError:(!$${{ parameters.ContinueOnError }})
+          -SourceCommittish 'HEAD'
+          -TargetCommittish 'HEAD^1'
         pwsh: true
     - ${{ if ne('', parameters.ScriptToValidateUpgrade) }}:
       - pwsh: |
-          $changedFiles = ./eng/common/scripts/get-changedfiles.ps1
+          $changedFiles = ./eng/common/scripts/get-changedfiles.ps1 `
+            -SourceCommittish 'HEAD' `
+            -TargetCommittish 'HEAD^1'
 
           if ($changedFiles -notcontains 'eng/common/spelling/package-lock.json') {
             Write-Host "No changes to cspell package-lock.json detected."

eng/common/pipelines/templates/steps/detect-api-changes.yml

@@ -23,6 +23,8 @@ steps:
           -RepoFullName $(Build.Repository.Name)
           -APIViewUri $(ApiChangeDetectRequestUrl)
           -ArtifactName ${{ parameters.ArtifactName }}
+          -SourceCommittish 'HEAD'
+          -TargetBranch 'HEAD^1'
           -DevopsProject $(System.TeamProject)
         pwsh: true
       displayName: Create APIView if API has changes

eng/common/pipelines/templates/steps/save-package-properties.yml

@@ -37,6 +37,8 @@ steps:
             -TargetPath '${{ parameters.TargetPath }}'
             -ArtifactPath '${{ parameters.DiffDirectory }}'
             -ExcludePaths ('${{ convertToJson(parameters.ExcludePaths) }}' | ConvertFrom-Json)
+            -SourceCommittish 'HEAD'
+            -TargetCommittish 'HEAD^1'
           pwsh: true
           workingDirectory: '${{ parameters.WorkingDirectory }}'
 

eng/common/scripts/Detect-Api-Changes.ps1

@@ -13,6 +13,7 @@ Param (
   [string] $APIViewUri,
   [string] $RepoFullName = "",
   [string] $ArtifactName = "packages",
+  [string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}",
   [string] $TargetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"),
   [string] $DevopsProject = "internal"
 )
@@ -98,7 +99,11 @@ function Submit-Request($filePath, $packageInfo)
 function Should-Process-Package($packageInfo)
 {
     $packagePath = $packageInfo.DirectoryPath
-    $modifiedFiles  = @(Get-ChangedFiles -DiffPath "$packagePath/*" -DiffFilterType '')
+    $modifiedFiles = @(Get-ChangedFiles `
+        -SourceCommittish $SourceCommittish `
+        -TargetCommittish $TargetBranch `
+        -DiffPath "$packagePath/*" `
+        -DiffFilterType '')
     $filteredFileCount = $modifiedFiles.Count
     LogInfo "Number of modified files for package: $filteredFileCount"
     return ($filteredFileCount -gt 0 -and $packageInfo.IsNewSdk)

eng/common/scripts/Generate-PR-Diff.ps1

@@ -10,6 +10,12 @@ The folder in which the result will be written.
 
 .PARAMETER TargetPath
 The path under which changes will be detected.
+
+.PARAMETER SourceCommittish
+Commit or ref containing the proposed changes.
+
+.PARAMETER TargetCommittish
+Commit or ref used as the comparison base.
 #>
 [CmdletBinding()]
 Param (
@@ -19,7 +25,11 @@ Param (
   [string] $TargetPath,
   [Parameter(Mandatory=$false)]
   [AllowEmptyCollection()]
-  [array] $ExcludePaths
+  [array] $ExcludePaths,
+  [Parameter(Mandatory=$false)]
+  [string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}",
+  [Parameter(Mandatory=$false)]
+  [string] $TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/")
 )
 
 . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1")
@@ -47,8 +57,15 @@ $ArtifactName = Join-Path $ArtifactPath "diff.json"
 $changedFiles = @()
 $changedServices = @()
 
-$changedFiles = Get-ChangedFiles -DiffPath $TargetPath
-$deletedFiles = Get-ChangedFiles -DiffPath $TargetPath -DiffFilterType "D"
+$changedFiles = Get-ChangedFiles `
+  -SourceCommittish $SourceCommittish `
+  -TargetCommittish $TargetCommittish `
+  -DiffPath $TargetPath
+$deletedFiles = Get-ChangedFiles `
+  -SourceCommittish $SourceCommittish `
+  -TargetCommittish $TargetCommittish `
+  -DiffPath $TargetPath `
+  -DiffFilterType "D"
 
 if ($changedFiles) {
   $changedServices = Get-ChangedServices -ChangedFiles $changedFiles

eng/common/scripts/Helpers/git-helpers.ps1

@@ -52,6 +52,9 @@ function Get-ChangedFiles {
   }
   Write-Host $command
   $changedFiles = Invoke-Expression -Command $command
+  if ($LASTEXITCODE -ne 0) {
+    throw "git diff failed with exit code $LASTEXITCODE for '$TargetCommittish...$SourceCommittish'"
+  }
   if (!$changedFiles) {
     Write-Host "No changed files in git diff between $TargetCommittish and $SourceCommittish"
   }

eng/pipelines/code-quality-reports.yml

@@ -31,16 +31,11 @@ extends:
               os: linux
 
             steps:
-              # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-              # as we require the GitHub service connection to be loaded.
+              # Use the default service-connection checkout for private mirrored repositories.
               - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-                  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-                    parameters:
-                      Paths:
-                        - '**/*.xml'
-                        - '!sdk/**/test-recordings'
-                        - '!sdk/**/session-records'
-                        - 'sdk/tools/linting-extensions'
+                - checkout: self
+                  fetchDepth: 2
+                  fetchTags: false
 
               - task: UsePythonVersion@0
                 displayName: 'Use Python $(PythonVersion)'
@@ -66,28 +61,21 @@ extends:
               # Authenticate with Azure Artifacts
               - template: /eng/pipelines/templates/steps/maven-authenticate.yml
 
-              # The only time generate_from_source_pom.py should be used to set the SparseCheckoutDirectories
-              # is for FromSource runs or, in the case of code quality reports, a run that needs to build
-              # everything using the latest source. It'll greedily set any service directories as it figures
-              # out what libraries, their dependents and so on, that need to be in ClientFromSourcePom.xml
+              # Generate the source dependency graph and identify projects that can skip linting.
               - task: PythonScript@0
-                displayName: 'Generate FromSource POM and directories for sparse checkout'
+                displayName: 'Generate FromSource POM and linting project list'
                 inputs:
                   scriptPath: 'eng/scripts/generate_from_source_pom.py'
                   arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)'
                   workingDirectory: '$(System.DefaultWorkingDirectory)'
 
-              - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-                parameters:
-                  SkipCheckoutNone: true
-                  Paths: $(SparseCheckoutDirectories)
-
               - task: PowerShell@2
                 inputs:
                   filePath: $(System.DefaultWorkingDirectory)/eng/pipelines/scripts/Get-Linting-Commands.ps1
                   arguments: >
                     -BuildReason $(Build.Reason)
                     -SourceBranch "HEAD"
+                    -TargetBranch "HEAD^1"
                     -LintingPipelineVariable "LintingGoals"
                   pwsh: true
                 displayName: 'Generate Linting Commands'

eng/pipelines/templates/jobs/build-validate-pom.yml

@@ -24,14 +24,9 @@ jobs:
       os: linux
 
     steps:
-      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-        parameters:
-          Paths:
-            - 'sdk/${{ parameters.ServiceDirectory }}'
-            - '**/*.xml'
-            - '**/*.md'
-            - '!sdk/**/test-recordings'
-            - '!sdk/**/session-records'
+      - checkout: self
+        fetchDepth: 2
+        fetchTags: false
 
       - script: |
           echo "##vso[build.addbuildtag]Scheduled"

eng/pipelines/templates/jobs/ci.yml

@@ -108,17 +108,11 @@ jobs:
             exit 1
           displayName: 'Validate: at least one package is selected for release'
 
-      # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-      # as we require the GitHub service connection to be loaded.
+      # Use the default service-connection checkout for private mirrored repositories.
       - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-        - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-          parameters:
-            Paths:
-              - '**/*.xml'
-              - '**/*.md'
-              - '**/ci*.yml' # necessary for save-package-properties.yml
-              - '!sdk/**/test-recordings'
-              - '!sdk/**/session-records'
+        - checkout: self
+          fetchDepth: 2
+          fetchTags: false
 
       - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
         parameters:
@@ -134,25 +128,6 @@ jobs:
         inputs:
           versionSpec: $(PythonVersion)
 
-      # This step needs to run before we update to using dev versions otherwise dependency
-      # check in the script will not fully work because the versions will mismatch
-      - task: PowerShell@2
-        displayName: Generate directories variable for sparse checkout
-        inputs:
-          pwsh: true
-          filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1
-          arguments: >
-            -SourcesDirectory $(Build.SourcesDirectory)
-            -ProjectList $(ProjectList)
-
-      # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-      # as we require the GitHub service connection to be loaded.
-      - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-        - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml
-          parameters:
-            Paths: $(SparseCheckoutDirectories)
-            SkipCheckoutNone: true
-
       - ${{ parameters.PreBuildSteps }}
 
       - script: |
@@ -318,18 +293,11 @@ jobs:
         inputs:
           versionSpec: $(PythonVersion)
 
-      # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-      # as we require the GitHub service connection to be loaded.
+      # Use the default service-connection checkout for private mirrored repositories.
       - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-        - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-          parameters:
-            Paths:
-              - '**/*.xml'
-              - '**/*.md'
-              - '**/*.yml'
-              - '.vscode/cspell.json'
-              - '!sdk/**/test-recordings'
-              - '!sdk/**/session-records'
+        - checkout: self
+          fetchDepth: 2
+          fetchTags: false
 
       - task: PowerShell@2
         displayName: 'Verify versions in POM files'
@@ -350,23 +318,6 @@ jobs:
           ServiceDirectory: ${{parameters.ServiceDirectory}}
           ExcludePaths: ${{parameters.ExcludePaths}}
 
-      - task: PowerShell@2
-        displayName: Generate directories variable for sparse checkout
-        inputs:
-          pwsh: true
-          filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1
-          arguments: >
-            -SourcesDirectory $(Build.SourcesDirectory)
-            -ProjectList $(ProjectList)
-
-      # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-      # as we require the GitHub service connection to be loaded.
-      - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-        - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml
-          parameters:
-            Paths: $(SparseCheckoutDirectories)
-            SkipCheckoutNone: true
-
       - task: UseNode@1
         inputs:
           version: 22.x

eng/pipelines/templates/steps/initialize-test-environment.yml

@@ -13,17 +13,11 @@ parameters:
     default: []
 
 steps:
-  # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-  # as we require the GitHub service connection to be loaded.
+  # Use the default service-connection checkout for private mirrored repositories.
   - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-      parameters:
-        Paths:
-          - '**/*.xml'
-          - '**/ci*.yml' # necessary for save-package-properties.yml
-          - ${{ if not(parameters.CheckoutRecordings) }}:
-            - '!sdk/**/test-recordings'
-            - '!sdk/**/session-records'
+    - checkout: self
+      fetchDepth: 2
+      fetchTags: false
 
   - task: UsePythonVersion@0
     displayName: 'Use Python $(PythonVersion)'
@@ -54,33 +48,11 @@ steps:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
       ExcludePaths: ${{parameters.ExcludePaths}}
 
-  # This call is used generate the sparse checkout variables for non-FromSource runs.
-  - task: PowerShell@2
-    displayName: Generate directories variable for sparse checkout
-    inputs:
-      pwsh: true
-      filePath: $(Build.SourcesDirectory)/eng/scripts/Generate-ServiceDirectories-From-Project-List.ps1
-      arguments: >
-        -SourcesDirectory $(Build.SourcesDirectory)
-        -ProjectList $(ProjectList)
-    condition: and(succeeded(), ne(variables['TestFromSource'], 'true'))
-
-  # The only time generate_from_source_pom.py should be used to set the SparseCheckoutDirectories
-  # is for FromSource runs or, in the case of code quality reports, a run that needs to build
-  # everything using the latest source. It'll greedily set any service directories as it figures
-  # out what libraries, their dependents and so on, that need to be in ClientFromSourcePom.xml
+  # FromSource runs use ClientFromSourcePom.xml to build the complete source dependency graph.
   - task: PythonScript@0
-    displayName: 'Generate FromSource POM and directories for sparse checkout'
+    displayName: 'Generate FromSource POM'
     inputs:
       scriptPath: 'eng/scripts/generate_from_source_pom.py'
       arguments: '--artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)'
       workingDirectory: '$(System.DefaultWorkingDirectory)'
     condition: and(succeeded(), eq(variables['TestFromSource'], 'true'))
-
-  # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-  # as we require the GitHub service connection to be loaded.
-  - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-    - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml
-      parameters:
-        SkipCheckoutNone: true
-        Paths: $(SparseCheckoutDirectories)