apache · snuyanzin · Aug 5, 2026 · Aug 5, 2026 · Aug 6, 2026 · Aug 6, 2026
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -40,8 +40,9 @@ resources:
   # see https://github.com/apache/flink-connector-shared-utils/tree/ci_utils
   - container: flink-build-container
     image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy
-    # On AZP provided machines, set this flag to allow writing coredumps in docker
-    options: --privileged
+    # On AZP provided machines, set this flag to allow writing coredumps in docker.
+    # --tmpfs provides RAM-backed scratch; the Machine info step reports whether the agent honours it.
+    options: --privileged --tmpfs /mnt/ram:size=24g,exec,mode=1777
 
 # Define variables:
 # - See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
@@ -84,6 +85,8 @@ stages:
         pool:
           vmImage: 'ubuntu-24.04'
         steps:
+          - script: ./tools/ci/print_machine_info.sh
+            displayName: Machine info
           - task: GoTool@0
             inputs:
               version: '1.18.1'

diff --git a/flink-filesystems/flink-azure-fs-hadoop/pom.xml b/flink-filesystems/flink-azure-fs-hadoop/pom.xml
@@ -134,6 +134,8 @@ under the License.
 							<goal>shade</goal>
 						</goals>
 						<configuration>
+							<!-- Skippable so the fast CI compile can drop this FS-plugin shade off the critical path -->
+							<skip>${flink.fsShade.skip}</skip>
 							<artifactSet>
 								<includes>
 									<include>*:*</include>

diff --git a/flink-filesystems/flink-gs-fs-hadoop/pom.xml b/flink-filesystems/flink-gs-fs-hadoop/pom.xml
@@ -222,6 +222,8 @@ under the License.
 							<goal>shade</goal>
 						</goals>
 						<configuration>
+							<!-- Skippable so the fast CI compile can drop this FS-plugin shade off the critical path -->
+							<skip>${flink.fsShade.skip}</skip>
 							<artifactSet>
 								<includes>
 									<include>*:*</include>

diff --git a/flink-filesystems/flink-oss-fs-hadoop/pom.xml b/flink-filesystems/flink-oss-fs-hadoop/pom.xml
@@ -151,6 +151,8 @@ under the License.
 							<goal>shade</goal>
 						</goals>
 						<configuration>
+							<!-- Skippable so the fast CI compile can drop this FS-plugin shade off the critical path -->
+							<skip>${flink.fsShade.skip}</skip>
 							<artifactSet>
 								<includes>
 									<include>*:*</include>

diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
@@ -246,6 +246,8 @@ under the License.
 							<goal>shade</goal>
 						</goals>
 						<configuration>
+							<!-- Skippable so the fast CI compile can drop this FS-plugin shade off the critical path -->
+							<skip>${flink.fsShade.skip}</skip>
 							<artifactSet>
 								<includes>
 									<include>*:*</include>

diff --git a/flink-filesystems/flink-s3-fs-native/pom.xml b/flink-filesystems/flink-s3-fs-native/pom.xml
@@ -179,6 +179,8 @@ under the License.
 							<goal>shade</goal>
 						</goals>
 						<configuration>
+							<!-- Skippable so the fast CI compile can drop this FS-plugin shade off the critical path -->
+							<skip>${flink.fsShade.skip}</skip>
 							<artifactSet>
 								<includes>
 									<include>software.amazon.awssdk:*</include>

diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml
@@ -641,6 +641,8 @@ under the License.
 							<goal>shade</goal>
 						</goals>
 						<configuration>
+							<!-- Skippable so the fast CI compile can drop this FS-plugin shade off the critical path -->
+							<skip>${flink.fsShade.skip}</skip>
 							<artifactSet>
 								<includes>
 									<include>*:*</include>

diff --git a/pom.xml b/pom.xml
@@ -131,6 +131,8 @@ under the License.
 		<flink.shaded.jackson.version>2.20.1</flink.shaded.jackson.version>
 		<flink.shaded.jsonpath.version>2.10.0</flink.shaded.jsonpath.version>
 		<flink.markBundledAsOptional>true</flink.markBundledAsOptional>
+		<!-- Default false = normal shade; the fast CI compile sets it true to skip the FS-plugin shade off the critical path (see tools/ci/compile.sh) -->
+		<flink.fsShade.skip>false</flink.fsShade.skip>
 		<source.java.version>11</source.java.version>
 		<target.java.version>17</target.java.version>
 		<slf4j.version>1.7.36</slf4j.version>

diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml
@@ -40,6 +40,8 @@ resources:
   # see https://github.com/apache/flink-connector-shared-utils/tree/ci_utils
   - container: flink-build-container
     image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy
+    # RAM-backed scratch for the build; the Machine info step reports whether the agent honours it.
+    options: --tmpfs /mnt/ram:size=24g,exec,mode=1777
 
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
@@ -77,6 +79,8 @@ stages:
         pool:
           vmImage: 'ubuntu-24.04'
         steps:
+          - script: ./tools/ci/print_machine_info.sh
+            displayName: Machine info
           # Skip docs check if this is a pull request that doesn't contain a documentation change
           - task: GoTool@0
             inputs:
@@ -167,6 +171,8 @@ stages:
         pool:
           vmImage: 'ubuntu-24.04'
         steps:
+          - script: ./tools/ci/print_machine_info.sh
+            displayName: Machine info
           - task: GoTool@0
             inputs:
               version: '1.18.1'

diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml
@@ -24,6 +24,8 @@ jobs:
     workspace:
       clean: all
     steps:
+      - script: ./tools/ci/print_machine_info.sh
+        displayName: Machine info
       # free up disk space on the hosted agent; the full dist build plus the
       # Maven cache does not fit in the ~16 GB the image leaves free
       - script: ./tools/azure-pipelines/free_disk_space.sh
@@ -82,6 +84,8 @@ jobs:
     workspace:
       clean: all
     steps:
+      - script: ./tools/ci/print_machine_info.sh
+        displayName: Machine info
       # free up disk space on the hosted agent; the snapshot deploy build plus
       # the Maven cache does not fit in the ~16 GB the image leaves free
       - script: ./tools/azure-pipelines/free_disk_space.sh

diff --git a/tools/azure-pipelines/build-python-wheels.yml b/tools/azure-pipelines/build-python-wheels.yml
@@ -18,6 +18,8 @@ jobs:
     pool:
       vmImage: 'ubuntu-24.04'
     steps:
+      - script: ./tools/ci/print_machine_info.sh
+        displayName: Machine info
       - task: UsePythonVersion@0
         inputs:
           versionSpec: '3.12'
@@ -34,6 +36,8 @@ jobs:
     pool:
       vmImage: 'macOS-latest'
     steps:
+      - script: ./tools/ci/print_machine_info.sh
+        displayName: Machine info
       - task: UsePythonVersion@0
         inputs:
           versionSpec: '3.12'

diff --git a/tools/azure-pipelines/e2e-template.yml b/tools/azure-pipelines/e2e-template.yml
@@ -44,6 +44,8 @@ jobs:
       .gitattributes
       .github/**
   steps:
+    - script: ./tools/ci/print_machine_info.sh
+      displayName: Machine info
     # Skip e2e test execution if this is a documentation only pull request (master / release builds will still be checked regularly)
     - bash: |
         source ./tools/azure-pipelines/build_properties.sh
@@ -140,7 +142,8 @@ jobs:
         sudo apt install ./libssl1.0.0_*.deb
       displayName: Prepare E2E run
       condition: not(eq(variables['SKIP'], '1'))
-    - script: ${{parameters.environment}} PROFILE="$PROFILE -Dfast -Pskip-webui-build" ./tools/ci/compile_ci.sh
+    # e2e only needs the build; skip all QA checks (they run in the qa_check job) so it can build with -T1C
+    - script: SKIP_QA_CHECKS=true MVN_COMPILE_THREADS=1C ${{parameters.environment}} PROFILE="$PROFILE -Pskip-webui-build" ./tools/ci/compile_ci.sh
       displayName: Build Flink
       condition: not(eq(variables['SKIP'], '1'))
     - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/ci/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}}

diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
@@ -36,6 +36,8 @@ jobs:
     # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
 
   steps:
+  - script: ./tools/ci/print_machine_info.sh
+    displayName: Machine info
   # if on Azure, free up disk space
   - script: ./tools/ci/free_disk_space.sh
     target: host
@@ -67,8 +69,11 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
     displayName: "Set JDK"
   # Compile
+  # SKIP_QA_CHECKS skips all QA checks (javadoc/scala/shaded/bundled/license); they run in the
+  # dedicated qa_check job. This lets the compile build run fast and multi-threaded (-T1C), off the
+  # test critical path.
   - script: |
-      ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $?
+      SKIP_QA_CHECKS=true MVN_COMPILE_THREADS=1C ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $?
       ./tools/ci/create_build_artifact.sh
     displayName: Compile
 
@@ -78,6 +83,59 @@ jobs:
       targetPath: $(FLINK_ARTIFACT_DIR)
       artifact: FlinkCompileArtifact-${{parameters.stage_name}}
 
+  # upload the installed Flink jars (run-scoped, NOT cached) so test jobs can reuse them and skip rebuild
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: $(MAVEN_CACHE_FOLDER)/org/apache/flink
+      artifact: FlinkMavenRepoArtifact-${{parameters.stage_name}}
+
+  # build outputs that reuse stages read from target/ dirs (not .m2): the assembled dist (build-target)
+  # and flink-python/target (pyflink's test classpath: test-dependencies, *-tests.jar, testDataStream.jar)
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: flink-dist/target
+      artifact: FlinkDistArtifact-${{parameters.stage_name}}
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: flink-python/target
+      artifact: FlinkPythonTargetArtifact-${{parameters.stage_name}}
+
+# All QA checks (javadoc/scala/shaded/bundled/license) run here: this job builds Flink from scratch
+# single-threaded and validates it. Runs as an independent job (in parallel with compile/test) so it
+# is off the test critical path.
+- job: qa_check_${{parameters.stage_name}}
+  # succeeded() is needed to allow job cancellation
+  condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
+  pool: ${{parameters.test_pool_definition}}
+  container: ${{parameters.container}}
+  timeoutInMinutes: 240
+  cancelTimeoutInMinutes: 1
+  workspace:
+    clean: all
+  steps:
+  - script: ./tools/ci/print_machine_info.sh
+    displayName: Machine info
+  # if on Azure, free up disk space
+  - script: ./tools/ci/free_disk_space.sh
+    target: host
+    condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
+    displayName: Free up disk space
+  - task: Cache@2
+    inputs:
+      key: $(PIPELINE_START_YEAR) | $(CACHE_KEY)
+      restoreKeys: $(PIPELINE_START_YEAR) | $(CACHE_FALLBACK_KEY)
+      path: $(MAVEN_CACHE_FOLDER)
+    continueOnError: true # continue the build even if the cache fails.
+    condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
+    displayName: Cache Maven local repo
+  - script: |
+      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
+      echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
+    displayName: "Set JDK"
+  # Build from scratch single-threaded and run all QA checks (no SKIP_QA_CHECKS).
+  - script: ${{parameters.environment}} ./tools/ci/compile_ci.sh
+    displayName: QA checks
+
 - job: test_${{parameters.stage_name}}
   dependsOn: compile_${{parameters.stage_name}}
   condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
@@ -88,20 +146,31 @@ jobs:
   workspace:
     clean: all
   strategy:
+    # reuseArtifacts: reuse the compile job's installed jars (skip Step 1) instead of a full rebuild.
+    # Only stages that resolve everything from .m2 can reuse; stages needing build outputs under module
+    # target/ dirs (python's test classpath, the assembled dist, cluster ITs) do a full rebuild.
     matrix:
       core:
         module: core
+        reuseArtifacts: 'true'
       python:
         module: python
+        reuseArtifacts: 'true'
       table:
         module: table
+        reuseArtifacts: 'true'
       connect:
         module: connect
+        reuseArtifacts: 'true'
       tests:
         module: tests
+        reuseArtifacts: 'false'
       misc:
         module: misc
+        reuseArtifacts: 'false'
   steps:
+  - script: ./tools/ci/print_machine_info.sh
+    displayName: Machine info
   # if on Azure, free up disk space
   - script: ./tools/ci/free_disk_space.sh
     target: host
@@ -133,6 +202,30 @@ jobs:
     condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
     displayName: Cache Maven local repo
 
+  # overlay the compile job's freshly-installed Flink jars (run-scoped, overrides the cache) when this
+  # stage reuses them. continueOnError so a missing/expired artifact falls back to a rebuild.
+  - task: DownloadPipelineArtifact@2
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['reuseArtifacts'], 'true'))
+    inputs:
+      path: $(MAVEN_CACHE_FOLDER)/org/apache/flink
+      artifact: FlinkMavenRepoArtifact-${{parameters.stage_name}}
+
+  # python reuse also needs the assembled dist (build-target) and flink-python/target test artifacts,
+  # which pyflink reads from target/ dirs. continueOnError so a missing artifact falls back to a rebuild.
+  - task: DownloadPipelineArtifact@2
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['reuseArtifacts'], 'true'), eq(variables['module'], 'python'))
+    inputs:
+      path: flink-dist/target
+      artifact: FlinkDistArtifact-${{parameters.stage_name}}
+  - task: DownloadPipelineArtifact@2
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['reuseArtifacts'], 'true'), eq(variables['module'], 'python'))
+    inputs:
+      path: flink-python/target
+      artifact: FlinkPythonTargetArtifact-${{parameters.stage_name}}
+
   - task: Cache@2
     inputs:
       key: '"$(module)" | $(DOCKER_IMAGES_CACHE_KEY)'
@@ -155,8 +248,9 @@ jobs:
   - script: sudo sysctl -w kernel.core_pattern=core.%p
     displayName: Set coredump pattern
 
-  # Test
-  - script: ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
+  # Test — reuse the handed-off jars on stages that can (see the matrix); test_controller.sh falls back
+  # to a full rebuild when reuse is off or the jars aren't present.
+  - script: REUSE_INSTALLED_ARTIFACTS=$(reuseArtifacts) ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
     displayName: Test - $(module)
     env:
       IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)