ci: extend Maven setup and retry dependency downloads

.github/actions/java-test/action.yaml

@@ -76,45 +76,8 @@ runs:
         restore-keys: |
           ${{ runner.os }}-java-maven-
 
-    # Maven itself is stored outside the dependency repository. Keep its cache
-    # independent of pom.xml changes and preserve the macOS cache workaround.
-    - name: Restore Maven distribution
-      id: maven-distribution
-      if: ${{ runner.os != 'macOS' }}
-      uses: actions/cache/restore@v5
-      with:
-        path: |
-          ~/.m2/wrapper/dists
-          /root/.m2/wrapper/dists
-        key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }}
-
-    # Retry only the wrapper download, never compilation or test execution.
-    # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter.
-    - name: Bootstrap Maven
-      shell: bash
-      run: |
-        for attempt in 1 2 3 4; do
-          if ./mvnw -B --version; then
-            break
-          fi
-          if [ "$attempt" -eq 4 ]; then
-            echo "::error::Maven bootstrap failed after $attempt attempts; tests were not started."
-            exit 1
-          fi
-          delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5))
-          echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s."
-          sleep "$delay"
-        done
-
-    # Save a successful bootstrap even when the subsequent tests fail.
-    - name: Save Maven distribution
-      if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }}
-      uses: actions/cache/save@v5
-      with:
-        path: |
-          ~/.m2/wrapper/dists
-          /root/.m2/wrapper/dists
-        key: ${{ steps.maven-distribution.outputs.cache-primary-key }}
+    - name: Setup Maven
+      uses: ./.github/actions/setup-maven
 
     - name: Run all tests
       shell: bash
@@ -123,7 +86,7 @@ runs:
         SPARK_LOCAL_HOSTNAME: "localhost"
         SPARK_LOCAL_IP: "127.0.0.1"
       run: |
-        MAVEN_OPTS="-Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
+        MAVEN_OPTS="${MAVEN_OPTS:-} -Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
     - name: Run specified tests
       shell: bash
       if: ${{ inputs.suites != '' }}
@@ -133,7 +96,7 @@ runs:
       run: |
         MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)"
         echo "Running with MAVEN_SUITES=$MAVEN_SUITES"
-        MAVEN_OPTS="-Xmx4G -Xms2G -DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
+        MAVEN_OPTS="${MAVEN_OPTS:-} -Xmx4G -Xms2G -DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
     - name: Upload crash logs
       if: failure()
       uses: actions/upload-artifact@v6

.github/actions/rust-test/action.yaml

@@ -51,6 +51,9 @@ runs:
         restore-keys: |
           ${{ runner.os }}-rust-maven-
 
+    - name: Setup Maven
+      uses: ./.github/actions/setup-maven
+
     - name: Build common module (pre-requisite for Rust tests)
       shell: bash
       run: |

.github/actions/setup-maven/action.yaml

@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Maven
+description: 'Restore and bootstrap the Maven wrapper before building or testing'
+runs:
+  using: "composite"
+  steps:
+    # Maven 3.9.6 uses Resolver's native HTTP transport. Retry individual
+    # transfers rather than rerunning Maven goals (and potentially their tests).
+    # MAVEN_ARGS is not read by our wrapper; carry these JVM properties through
+    # MAVEN_OPTS, including callers that also configure their heap there.
+    - name: Configure Maven transfer retries
+      shell: bash
+      run: |
+        retry_options="-Daether.connector.http.retryHandler.count=3"
+        retry_options+=" -Daether.connector.http.retryHandler.interval=5000"
+        retry_options+=" -Daether.connector.http.retryHandler.intervalMax=60000"
+        retry_options+=" -Daether.connector.http.retryHandler.serviceUnavailable=429,500,502,503,504"
+        # JVM options are whitespace-separated. Normalize multiline YAML env
+        # values before writing GitHub's single-line environment-file format.
+        maven_options="${MAVEN_OPTS:-}"
+        maven_options="${maven_options//$'\r'/ }"
+        maven_options="${maven_options//$'\n'/ }"
+        printf 'MAVEN_OPTS=%s %s\n' "$maven_options" "$retry_options" >> "$GITHUB_ENV"
+
+    # Maven itself is stored outside the dependency repository. Keep its cache
+    # independent of pom.xml changes and preserve the macOS cache workaround.
+    # Globbing /root on a host runner aborts cache save with EACCES, so select
+    # only the directory this JVM's wrapper actually uses.
+    - name: Locate Maven distribution cache
+      id: maven-cache-path
+      if: ${{ runner.os != 'macOS' }}
+      shell: bash
+      run: |
+        cache_path=$(bash "$GITHUB_ACTION_PATH/cache-path.sh")
+        printf 'path=%s\n' "$cache_path" >> "$GITHUB_OUTPUT"
+
+    - name: Restore Maven distribution
+      id: maven-distribution
+      if: ${{ runner.os != 'macOS' }}
+      uses: actions/cache/restore@v5
+      with:
+        path: ${{ steps.maven-cache-path.outputs.path }}
+        key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }}
+
+    # Retry only the wrapper download, never compilation or test execution.
+    # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter.
+    - name: Bootstrap Maven
+      shell: bash
+      run: |
+        for attempt in 1 2 3 4; do
+          if ./mvnw -B --version; then
+            break
+          fi
+          if [ "$attempt" -eq 4 ]; then
+            echo "::error::Maven bootstrap failed after $attempt attempts; tests were not started."
+            exit 1
+          fi
+          delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5))
+          echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s."
+          sleep "$delay"
+        done
+
+    # Save a successful bootstrap even when the subsequent tests fail.
+    - name: Save Maven distribution
+      if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }}
+      uses: actions/cache/save@v5
+      with:
+        path: ${{ steps.maven-cache-path.outputs.path }}
+        key: ${{ steps.maven-distribution.outputs.cache-primary-key }}

.github/actions/setup-maven/cache-path.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Match mvnw's JVM and option order. In job containers, shell HOME is
+# /github/home while the JVM's user.home (and the wrapper cache) is /root.
+java_command="${JAVACMD:-${JAVA_HOME:+$JAVA_HOME/bin/}java}"
+java_options="${MAVEN_OPTS:-}"
+if [ -f .mvn/jvm.config ]; then
+  java_options="$(tr '\n' ' ' < .mvn/jvm.config) $java_options"
+fi
+java_options="${java_options//$'\r'/ }"
+java_options="${java_options//$'\n'/ }"
+read -r -a java_arguments <<< "$java_options"
+properties=$("$java_command" "${java_arguments[@]}" -XshowSettings:properties -version 2>&1) || {
+  status=$?
+  printf '%s\n' "$properties" >&2
+  exit "$status"
+}
+
+# Wrapper 3.2.0 prefers the JVM property, then the environment override,
+# then user.home/.m2. Both its distribution and ZIP paths use wrapper/dists.
+maven_user_home=$(sed -n 's/^[[:space:]]*maven\.user\.home = //p' <<< "$properties")
+maven_user_home="${maven_user_home:-${MAVEN_USER_HOME:-}}"
+if [ -z "$maven_user_home" ]; then
+  jvm_user_home=$(sed -n 's/^[[:space:]]*user\.home = //p' <<< "$properties")
+  if [ -z "$jvm_user_home" ]; then
+    echo "Could not determine the Maven wrapper cache directory from JVM properties." >&2
+    exit 1
+  fi
+  maven_user_home="$jvm_user_home/.m2"
+fi
+printf '%s/wrapper/dists\n' "${maven_user_home%/}"

.github/actions/setup-spark-builder/action.yaml

@@ -32,6 +32,10 @@ inputs:
     description: 'Skip cloning Spark and applying patches (when apache-spark/ is pre-staged from a build-jvm artifact)'
     required: false
     default: 'false'
+  sbt-projects:
+    description: 'Space-separated Spark projects whose test dependencies are needed'
+    required: false
+    default: 'catalyst sql hive'
 runs:
   using: "composite"
   steps:
@@ -61,6 +65,9 @@ runs:
         restore-keys: |
           ${{ runner.os }}-spark-sql-
 
+    - name: Setup Maven
+      uses: ./.github/actions/setup-maven
+
     - name: Build Comet (with native)
       if: ${{ inputs.skip-native-build != 'true' }}
       shell: bash
@@ -93,3 +100,52 @@ runs:
               "$(dirname "$pom")/_remote.repositories"
           done
         done
+
+    - name: Remove incomplete Parquet test dependencies
+      shell: bash
+      run: |
+        # Maven installs Parquet POMs and main JARs, but not all test classifiers.
+        # Coursier treats those entries as local and will not fetch the missing
+        # tests.jar remotely. Do this before dependency resolution, not tests.
+        rm -rf "$HOME/.m2/repository/org/apache/parquet" /root/.m2/repository/org/apache/parquet
+
+    - name: Restore SBT dependency cache
+      id: sbt-dependencies
+      uses: actions/cache/restore@v6
+      with:
+        path: |
+          ~/.cache/coursier
+          /root/.cache/coursier
+          ~/.sbt/boot
+          /root/.sbt/boot
+          ~/.ivy2/cache
+          /root/.ivy2/cache
+        key: ${{ runner.os }}-${{ runner.arch }}-spark-sbt-v1-${{ inputs.spark-version }}-${{ hashFiles('**/pom.xml', '!**/target/**', 'apache-spark/project/build.properties', 'apache-spark/project/*.sbt', 'apache-spark/project/*.scala') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ runner.arch }}-spark-sbt-v1-${{ inputs.spark-version }}-
+
+    - name: Resolve Spark dependencies
+      shell: bash
+      env:
+        SBT_PROJECTS: ${{ inputs.sbt-projects }}
+      run: |
+        cd apache-spark
+        read -r -a projects <<< "$SBT_PROJECTS"
+        ../dev/ci/resolve-spark-dependencies.sh "${projects[@]}"
+
+    # Keep a successful download even if a later compilation or test fails.
+    # Only the build job, which resolves all three projects, may publish this
+    # shared immutable key. A writer-only or shard job must not fill it with
+    # partial coverage and prevent the full build from saving its downloads.
+    - name: Save SBT dependency cache
+      if: inputs.sbt-projects == 'catalyst sql hive' && steps.sbt-dependencies.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v6
+      with:
+        path: |
+          ~/.cache/coursier
+          /root/.cache/coursier
+          ~/.sbt/boot
+          /root/.sbt/boot
+          ~/.ivy2/cache
+          /root/.ivy2/cache
+        key: ${{ steps.sbt-dependencies.outputs.cache-primary-key }}

.github/workflows/ci.yml

@@ -68,6 +68,9 @@ jobs:
           distribution: temurin
           java-version: 11
 
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
+
       - name: Apache RAT license check
         run: ./mvnw -B -N apache-rat:check
 
@@ -88,6 +91,13 @@ jobs:
       - name: Check micro benchmark runner
         run: python3 dev/ci/check-benchmark-runner.py
 
+      - name: Check CI download handling
+        env:
+          COMET_TEST_MAVEN_DOWNLOADS: "1"
+        run: |
+          python3 dev/ci/test-download-retry.py
+          python3 dev/ci/test-delta-gate.py
+
       - name: Install actionlint
         run: |
           curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash

.github/workflows/delta_build_gate.yml

@@ -87,6 +87,32 @@ jobs:
           rust-version: ${{ env.RUST_VERSION }}
           jdk-version: 17
 
+      - name: Compute Delta Maven cache key
+        id: delta-maven-cache-key
+        run: echo "hash=${{ hashFiles('**/pom.xml', '.mvn/wrapper/maven-wrapper.properties') }}" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Delta Maven dependencies
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.m2/repository
+            /root/.m2/repository
+          key: ${{ runner.os }}-${{ runner.arch }}-delta-maven-jdk17-${{ steps.delta-maven-cache-key.outputs.hash }}
+          restore-keys: |
+            ${{ runner.os }}-${{ runner.arch }}-delta-maven-jdk17-
+
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
+
       - name: Run dev/verify-contrib-delta-gate.sh
         run: |
           dev/verify-contrib-delta-gate.sh
+
+      - name: Upload Delta gate logs
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: delta-build-gate-logs
+          path: artifactlog/delta-build-gate/
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/docs.yaml

@@ -49,6 +49,9 @@ jobs:
           java-version: '17'
           cache: 'maven'
 
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
+
       - name: Install dependencies
         run: |
           set -x

.github/workflows/iceberg_spark_test_reusable.yml

@@ -129,6 +129,8 @@ jobs:
         with:
           name: native-lib-iceberg
           path: native/target/release/
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Build Comet
         run: |
           ./mvnw install -Prelease -DskipTests -Pspark-${{ inputs.spark-short }} -Pscala-${{ inputs.scala }}
@@ -164,6 +166,8 @@ jobs:
         with:
           name: native-lib-iceberg
           path: native/target/release/
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Build Comet
         run: |
           ./mvnw install -Prelease -DskipTests -Pspark-${{ inputs.spark-short }} -Pscala-${{ inputs.scala }}
@@ -199,6 +203,8 @@ jobs:
         with:
           name: native-lib-iceberg
           path: native/target/release/
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Build Comet
         run: |
           ./mvnw install -Prelease -DskipTests -Pspark-${{ inputs.spark-short }} -Pscala-${{ inputs.scala }}

.github/workflows/pr_benchmark_check.yml

@@ -69,6 +69,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-benchmark-maven-
 
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
+
       - name: Check Scala compilation and linting
         # Pin to spark-4.0 (Scala 2.13.16) because the default profile is now
         # spark-4.1 / Scala 2.13.17, and semanticdb-scalac_2.13.17 is not yet