diff --git a/azure-devops/asan-pipeline.yml b/azure-devops/asan-pipeline.yml index 4879cc48d4..985520ff41 100644 --- a/azure-devops/asan-pipeline.yml +++ b/azure-devops/asan-pipeline.yml @@ -24,7 +24,7 @@ stages: demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64SlowPoolWorkRoot)' readonly: true jobs: - template: build-and-test.yml @@ -43,7 +43,7 @@ stages: demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64SlowPoolWorkRoot)' readonly: true jobs: - template: build-and-test.yml diff --git a/azure-devops/config.yml b/azure-devops/config.yml index 4cce273f81..b32d6469d4 100644 --- a/azure-devops/config.yml +++ b/azure-devops/config.yml @@ -5,13 +5,16 @@ variables: - name: x64SlowPoolName - value: 'Stl-2026-07-15T0800-x64-Fasv6-Pool' + value: 'Stl-2026-07-18T1628-x64-Fasv6-Pool' + readonly: true +- name: x64MediumPoolName + value: 'Stl-2026-07-18T1628-x64-Fasv7-Pool' readonly: true - name: x64FastPoolName - value: 'Stl-2026-07-15T0800-x64-Fasv7-Pool' + value: 'Stl-2026-07-18T1628-x64-Fadsv7-Pool' readonly: true - name: arm64PoolName - value: 'Stl-2026-07-15T0800-arm64-Dpdsv6-Pool' + value: 'Stl-2026-07-18T1628-arm64-Dpdsv6-Pool' readonly: true - name: poolDemands value: 'EnableSpotVM -equals false' @@ -19,11 +22,18 @@ variables: - name: launchVsDevShell value: 'C:\Program Files\Microsoft Visual Studio\18\Insiders\Common7\Tools\Launch-VsDevShell.ps1' readonly: true -# The x64 SKUs Fasv6 and Fasv7 don't have local temporary storage, so they must use 'C:'. -# When we can use Fadsv7, local temporary storage should be available as 'N:'. -- name: x64PoolWorkRoot +# The x64 SKU Fasv6 doesn't have local temporary storage, so it must use 'C:'. +- name: x64SlowPoolWorkRoot + value: 'C:' + readonly: true +# The x64 SKU Fasv7 doesn't have local temporary storage, so it must use 'C:'. +- name: x64MediumPoolWorkRoot value: 'C:' readonly: true +# The x64 SKU Fadsv7 has local temporary storage, available as 'N:' (for NVMe). +- name: x64FastPoolWorkRoot + value: 'N:' + readonly: true # The arm64 SKU Dpdsv6 has local temporary storage, available as 'E:'. - name: arm64PoolWorkRoot value: 'E:' diff --git a/azure-devops/create-1es-hosted-pool.ps1 b/azure-devops/create-1es-hosted-pool.ps1 index 037a33ea65..445e8f318b 100644 --- a/azure-devops/create-1es-hosted-pool.ps1 +++ b/azure-devops/create-1es-hosted-pool.ps1 @@ -9,11 +9,11 @@ Creates a 1ES Hosted Pool, set up for the STL's CI. See https://github.com/microsoft/STL/wiki/Checklist-for-Toolset-Updates for more information. .PARAMETER VMSku -The VM SKU can be Fasv6, Fasv7, or Dpdsv6. +The VM SKU can be Fasv6, Fasv7, Fadsv7, or Dpdsv6. #> [CmdletBinding(PositionalBinding=$false)] Param( - [Parameter(Mandatory)][ValidateSet('Fasv6', 'Fasv7', 'Dpdsv6')][String]$VMSku + [Parameter(Mandatory)][ValidateSet('Fasv6', 'Fasv7', 'Fadsv7', 'Dpdsv6')][String]$VMSku ) $ErrorActionPreference = 'Stop' @@ -21,33 +21,51 @@ $ErrorActionPreference = 'Stop' $CurrentDate = Get-Date $Timestamp = $CurrentDate.ToString('yyyy-MM-ddTHHmm') -# | SKU | Location | Cores | Notes | -# |--------|----------------|------:|--------------------| -# | Fasv6 | eastus2 | 4096 | | -# | Fasv7 | australiaeast | 740 | | -# | Fasv7 | northeurope | 640 | | -# | Fasv7 | southeastasia | 640 | | -# | Dpdsv6 | australiaeast | 2048 | | -# | Dpdsv6 | southcentralus | 2048 | | +# We use 16 cores for the prototype VM ($ProtoVMSize) because we don't need a ton of cores to run provision-image.ps1, +# and this allows us to stay below our total regional vCPU quota when running create-1es-hosted-pool.ps1 for several +# SKUs simultaneously. (Our regional vCPU quota is 100, "enforced across all VM series in a given region" as +# https://learn.microsoft.com/azure/quotas/regional-quota-requests explains. We're creating individual prototype VMs, +# outside of the 1ES Hosted Pools we prepare later, so the prototypes are subject to the limit of 100.) +# As long as everything else matches, we can use the image captured from the prototype VM +# to create a 1ES Hosted Pool with larger VMs ($PoolSkuName) without any issues. + +# | SKU | Location | Cores | Notes +# |--------|----------------|------:|------- +# | Fasv6 | eastus2 | 4096 | +# | Fasv7 | australiaeast | 740 | +# | Fasv7 | northeurope | 640 | +# | Fasv7 | southeastasia | 640 | +# | Fadsv7 | australiaeast | 2048 | +# | Dpdsv6 | australiaeast | 2048 | +# | Dpdsv6 | southcentralus | 2048 | if ($VMSku -ieq 'Fasv6') { $Arch = 'x64' - $VMSize = 'Standard_F32as_v6' - $PoolSize = 64 # We have quota for 4096 cores (128 VMs), so we can have old and new pools of 64 VMs each. + $ProtoVMSize = 'Standard_F16as_v6' + $PoolSkuName = 'Standard_F64as_v6' + $PoolSize = 32 # We have quota for 4096 cores (64 VMs), so we can have old and new pools of 32 VMs each. $AvailableLocations = @('eastus2') } elseif ($VMSku -ieq 'Fasv7') { $Arch = 'x64' - $VMSize = 'Standard_F32as_v7' - $PoolSize = 20 # Locations where we have quota for at least 640 cores (20 VMs): + $ProtoVMSize = 'Standard_F16as_v7' + $PoolSkuName = 'Standard_F48as_v7' + $PoolSize = 13 # Locations where we have quota for at least 640 cores (13 VMs): $AvailableLocations = @('australiaeast', 'northeurope', 'southeastasia') +} elseif ($VMSku -ieq 'Fadsv7') { + $Arch = 'x64' + $ProtoVMSize = 'Standard_F16ads_v7' + $PoolSkuName = 'Standard_F48ads_v7' + $PoolSize = 21 # We have quota for 2048 cores (42 VMs), so we can have old and new pools of 21 VMs each. + $AvailableLocations = @('australiaeast') } elseif ($VMSku -ieq 'Dpdsv6') { $Arch = 'arm64' - $VMSize = 'Standard_D32pds_v6' - $PoolSize = 64 # Locations where we have quota for at least 2048 cores (64 VMs): + $ProtoVMSize = 'Standard_D16pds_v6' + $PoolSkuName = 'Standard_D64pds_v6' + $PoolSize = 32 # Locations where we have quota for at least 2048 cores (32 VMs): $AvailableLocations = @('australiaeast', 'southcentralus') } -$AvailableLocationIdx = 11 # Increment for each new set of pools, to cycle through the available locations. +$AvailableLocationIdx = 12 # Increment for each new set of pools, to cycle through the available locations. $Location = $AvailableLocations[$AvailableLocationIdx % $AvailableLocations.Length] if ($Arch -ieq 'x64') { @@ -213,13 +231,13 @@ Display-ProgressBar -Status 'Creating prototype VM config' if ($Arch -ieq 'x64') { $VM = New-AzVMConfig ` -VMName $ProtoVMName ` - -VMSize $VMSize ` + -VMSize $ProtoVMSize ` -DiskControllerType 'NVMe' ` -Priority 'Regular' } else { $VM = New-AzVMConfig ` -VMName $ProtoVMName ` - -VMSize $VMSize ` + -VMSize $ProtoVMSize ` -DiskControllerType 'SCSI' ` -Priority 'Regular' ` -SecurityType 'TrustedLaunch' ` @@ -426,7 +444,7 @@ $PoolName = "$ResourceGroupName-Pool" $PoolProperties = @{ 'organization' = 'https://dev.azure.com/vclibs' 'projects' = @('STL') - 'sku' = @{ 'name' = $VMSize; 'tier' = 'StandardSSD'; 'enableSpot' = $false; } + 'sku' = @{ 'name' = $PoolSkuName; 'tier' = 'StandardSSD'; 'enableSpot' = $false; } 'images' = @(@{ 'imageName' = $ImageName; 'poolBufferPercentage' = '100'; }) 'maxPoolSize' = $PoolSize 'agentProfile' = @{ 'type' = 'Stateless'; } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9edfc85a9b..1979b1b2cd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,15 +10,16 @@ pr: drafts: false stages: +##### Phase 1: Initial Builds ##### - stage: Code_Format dependsOn: [] displayName: 'Code Format' pool: - name: ${{ variables.x64SlowPoolName }} + name: ${{ variables.x64MediumPoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64MediumPoolWorkRoot)' readonly: true jobs: - template: azure-devops/format-validation.yml @@ -27,11 +28,11 @@ stages: dependsOn: [] displayName: 'Build x64' pool: - name: ${{ variables.x64SlowPoolName }} + name: ${{ variables.x64MediumPoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64MediumPoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml @@ -49,11 +50,11 @@ stages: dependsOn: [] displayName: 'Build x86' pool: - name: ${{ variables.x64SlowPoolName }} + name: ${{ variables.x64MediumPoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64MediumPoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml @@ -67,20 +68,20 @@ stages: configureTesting: false runTesting: false - - stage: Build_ARM64_Cross + - stage: Build_ARM64 dependsOn: [] - displayName: 'Build ARM64 (Cross)' + displayName: 'Build ARM64' pool: - name: ${{ variables.x64SlowPoolName }} + name: ${{ variables.arm64PoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(arm64PoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml parameters: - hostArch: x64 + hostArch: arm64 targetArch: arm64 targetPlatform: arm64 analyzeBuild: true @@ -89,33 +90,9 @@ stages: configureTesting: false runTesting: false - - stage: Build_ARM64EC_Cross + - stage: Build_ARM64EC dependsOn: [] - displayName: 'Build ARM64EC (Cross)' - pool: - name: ${{ variables.x64SlowPoolName }} - demands: ${{ variables.poolDemands }} - variables: - - name: workRoot - value: '$(x64PoolWorkRoot)' - readonly: true - jobs: - - template: azure-devops/build-and-test.yml - parameters: - hostArch: x64 - targetArch: arm64 - targetPlatform: arm64ec - analyzeBuild: true - buildBenchmarks: true - numShards: 1 - configureTesting: false - runTesting: false - - # This ARM64-native build will detect problems with the ARM64 pool as early as possible. - # The stage dependencies are structured to optimize the critical path. - - stage: Build_ARM64_Native - dependsOn: [] - displayName: 'Build ARM64 (Native)' + displayName: 'Build ARM64EC' pool: name: ${{ variables.arm64PoolName }} demands: ${{ variables.poolDemands }} @@ -128,7 +105,7 @@ stages: parameters: hostArch: arm64 targetArch: arm64 - targetPlatform: arm64 + targetPlatform: arm64ec analyzeBuild: true buildBenchmarks: true numShards: 1 @@ -139,11 +116,11 @@ stages: dependsOn: [] displayName: 'Configure Tests' pool: - name: ${{ variables.x64SlowPoolName }} + name: ${{ variables.x64MediumPoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64MediumPoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml @@ -155,13 +132,14 @@ stages: buildStl: false runTesting: false +##### Phase 2: Primary Test Architectures ##### - stage: Test_x64 dependsOn: - Code_Format - Build_x64 - Build_x86 - - Build_ARM64_Cross - - Build_ARM64EC_Cross + - Build_ARM64 + - Build_ARM64EC - Configure_Tests displayName: 'Test x64' pool: @@ -169,7 +147,7 @@ stages: demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(x64FastPoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml @@ -178,46 +156,53 @@ stages: targetArch: x64 targetPlatform: x64 - - stage: Test_x86 - dependsOn: Test_x64 - displayName: 'Test x86' + - stage: Test_ARM64 + dependsOn: + - Code_Format + - Build_x64 + - Build_x86 + - Build_ARM64 + - Build_ARM64EC + - Configure_Tests + displayName: 'Test ARM64' pool: - name: ${{ variables.x64SlowPoolName }} + name: ${{ variables.arm64PoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(x64PoolWorkRoot)' + value: '$(arm64PoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml parameters: - hostArch: x86 - targetArch: x86 - targetPlatform: x86 + hostArch: arm64 + targetArch: arm64 + targetPlatform: arm64 - - stage: Test_ARM64 +##### Phase 3: Secondary Test Architectures ##### + - stage: Test_x86 dependsOn: - - Build_ARM64_Native - Test_x64 - displayName: 'Test ARM64' + - Test_ARM64 + displayName: 'Test x86' pool: - name: ${{ variables.arm64PoolName }} + name: ${{ variables.x64SlowPoolName }} demands: ${{ variables.poolDemands }} variables: - name: workRoot - value: '$(arm64PoolWorkRoot)' + value: '$(x64SlowPoolWorkRoot)' readonly: true jobs: - template: azure-devops/build-and-test.yml parameters: - hostArch: arm64 - targetArch: arm64 - targetPlatform: arm64 + hostArch: x86 + targetArch: x86 + targetPlatform: x86 - stage: Test_ARM64EC dependsOn: - - Build_ARM64_Native - Test_x64 + - Test_ARM64 displayName: 'Test ARM64EC' pool: name: ${{ variables.arm64PoolName }}