HDDS-16036. SCM Support allocate Container by StoragePolicy

.asf.yaml

@@ -33,7 +33,22 @@ github:
     - HDFS
     - RATIS
   enabled_merge_buttons:
-    squash:  true
+    squash: true
     squash_commit_message: PR_TITLE
-    merge:   false
-    rebase:  false
+    merge: false
+    rebase: false
+  copilot_code_review:
+    enabled: true
+    review_drafts: true
+  rulesets:
+    - name: "Default Branch Protection"
+      type: branch
+      branches:
+        includes:
+          - "~DEFAULT_BRANCH"
+          - "ozone-*"
+        excludes: []
+      bypass_teams:
+        - root
+      restrict_deletion: true
+      restrict_force_push: true

.github/dependabot.yml

@@ -15,23 +15,69 @@
 version: 2
 updates:
   - package-ecosystem: maven
+    open-pull-requests-limit: 5
     directory: "/"
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
+      - dependency-name: "com.fasterxml.jackson:*"
+        versions:
+          - "2.22" # not LTS
+      - dependency-name: "com.github.ekryd.sortpom:sortpom-maven-plugin"
+        versions:
+          - ">=3.1.0" # requires Java 11
+      - dependency-name: "com.github.spotbugs:spotbugs-maven-plugin"
+        versions:
+          - ">=4.0.0.0" # too many new warnings
+          - ">=4.9.0.0" # requires Java 11
+      - dependency-name: "com.googlecode.maven-download-plugin:download-maven-plugin"
+        versions:
+          - ">=1.10.0" # requires Java 11
+      - dependency-name: "dev.langchain4j:*"
+        versions:
+          - ">=0.36.0" # requires Java 17
+      - dependency-name: "info.picocli:*"
+        versions:
+          - "4.7.6" # bug https://github.com/remkop/picocli/issues/2309
+          - "4.7.7" # bug https://github.com/remkop/picocli/issues/2407
+      - dependency-name: "io.netty:*"
+        update-types: ["version-update:semver-minor"]
+      - dependency-name: "org.apache.derby:derby"
+        versions:
+          - "<10.17.1.0" # CVE-2022-46337 https://issues.apache.org/jira/browse/DERBY-7147
+          - ">=10.17.1.0" # requires Java 21
+      - dependency-name: "org.apache.hadoop:*" # using multiple versions
+      - dependency-name: "org.apache.iceberg:*"
+        versions:
+          - ">=1.11.0" # requires Java 17
+      - dependency-name: "org.apache.parquet:*" # update in sync with iceberg
+      - dependency-name: "org.apache.rat:apache-rat-plugin"
+        versions:
+          - "0.17" # bug https://issues.apache.org/jira/browse/RAT-476
+          - ">=0.18" # requires Java 17
+      - dependency-name: "org.aspectj:*" # using multiple versions
+      - dependency-name: "org.jgrapht"
+        versions:
+          - ">=1.5.0" # requires Java 11
+      - dependency-name: "org.jooq:*"
+        update-types: ["version-update:semver-minor"]
+      - dependency-name: "org.mockito:mockito-core"
+        versions:
+          - ">=5.0.0" # requires Java 11
+      - dependency-name: "org.rocksdb:*"
+        update-types: ["version-update:semver-minor"]
     schedule:
-      interval: "weekly"
-      day: "saturday"
-      time: "07:00" # UTC
+      interval: "cron"
+      cronjob: "0 5 * * 0,6"
     cooldown:
       default-days: 7
     pull-request-branch-name:
       separator: "-"
   - package-ecosystem: "github-actions"
+    open-pull-requests-limit: 5
     directory: "/"
     schedule:
-      # 'daily' only runs on weekdays
       interval: "cron"
-      cronjob: "15 6 * * *"
+      cronjob: "0 6 * * 0,6"
     cooldown:
       default-days: 7

.github/workflows/asf-allowlist-check.yaml

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "ASF Allowlist Check"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - ".github/workflows/**"
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+    tags:
+      - '**'
+    paths:
+      - ".github/workflows/**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || case(github.repository_owner == 'apache', github.sha, github.ref_name) }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || github.repository_owner != 'apache' }}
+
+jobs:
+  asf-allowlist-check:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Check actions
+        uses: apache/infrastructure-actions/allowlist-check@775350a154e610e84c460cb1bbe2d2ab26c15cb3
+        with:
+          scan-glob: ".github/workflows/*.y*ml"

.github/workflows/build-ratis.yml

@@ -54,6 +54,9 @@ on:
       protobuf-version:
         description: "Protobuf Version"
         value: ${{ jobs.ratis-thirdparty.outputs.protobuf-version }}
+      build-args:
+        description: "Arguments for building with Ratis"
+        value: ${{ jobs.summary.outputs.build-args }}
 env:
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
 permissions: { }
@@ -66,23 +69,23 @@ jobs:
       thirdparty-version: ${{ steps.versions.outputs.thirdparty }}
     steps:
       - name: Checkout project
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           repository: ${{ inputs.repo }}
           ref: ${{ inputs.ref }}
       - name: Cache for maven dependencies
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.m2/repository
             !~/.m2/repository/org/apache/ratis
           key: ratis-dependencies-${{ hashFiles('**/pom.xml') }}
       - name: Setup java
-        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
         with:
           distribution: 'temurin'
-          java-version: 8
+          java-version: 25
       - name: Get component versions
         id: versions
         run: |
@@ -115,7 +118,7 @@ jobs:
       protobuf-version: ${{ steps.versions.outputs.protobuf }}
     steps:
       - name: Checkout project
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           repository: apache/ratis-thirdparty
@@ -126,19 +129,30 @@ jobs:
           echo "grpc=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT
           echo "netty=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT
           echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT
-  debug:
+  summary:
     runs-on: ubuntu-slim
     needs:
       - ratis
       - ratis-thirdparty
+    outputs:
+      build-args: ${{ steps.versions.outputs.build-args }}
     steps:
       - name: Print versions
+        id: versions
         run: |
+          build_args="-Dratis.version=$ratis_version"
+          build_args="$build_args -Dratis.thirdparty.version=$ratis_thirdparty_version"
+          build_args="$build_args -Dratis-thirdparty.grpc.version=$grpc_version"
+          build_args="$build_args -Dratis-thirdparty.netty.version=$netty_version"
+          build_args="$build_args -Dratis-thirdparty.protobuf.version=$protobuf_version"
+          echo "build-args=$build_args" >> "$GITHUB_OUTPUT"
+
           echo "Ratis: $ratis_version"
           echo "Thirdparty: $ratis_thirdparty_version"
           echo "Grpc: $grpc_version"
           echo "Netty: $netty_version"
           echo "Protobuf: $protobuf_version"
+          echo "Build args for Ozone: $build_args"
         env:
           ratis_version: ${{ needs.ratis.outputs.ratis-version }}
           ratis_thirdparty_version: ${{ needs.ratis.outputs.thirdparty-version }}

.github/workflows/check.yml

@@ -153,7 +153,7 @@ jobs:
     steps:
       - name: Checkout project
         if: ${{ !inputs.needs-ozone-source-tarball }}
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           ref: ${{ inputs.sha }}
@@ -172,7 +172,7 @@ jobs:
 
       - name: Cache for NPM dependencies
         if: ${{ inputs.needs-npm-cache }}
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.pnpm-store
@@ -182,7 +182,7 @@ jobs:
 
       - name: Cache for Maven dependencies
         if: ${{ inputs.needs-maven-cache }}
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.m2/repository/*/*/*
@@ -223,7 +223,7 @@ jobs:
 
       - name: Setup java ${{ inputs.java-version }}
         if: ${{ inputs.java-version }}
-        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
         with:
           distribution: 'temurin'
           java-version: ${{ inputs.java-version }}

.github/workflows/ci-with-ratis.yml

@@ -54,5 +54,5 @@ jobs:
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
     with:
-      ratis_args: "-Dratis.version=${{ needs.ratis.outputs.ratis-version }} -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }} -Dratis-thirdparty.grpc.version=${{ needs.ratis.outputs.grpc-version }} -Dratis-thirdparty.netty.version=${{ needs.ratis.outputs.netty-version }} -Dratis-thirdparty.protobuf.version=${{ needs.ratis.outputs.protobuf-version }}"
+      ratis_args: ${{ needs.ratis.outputs.build-args }}
       ref: ${{ github.event.inputs.ref }}

.github/workflows/ci.yml

@@ -38,9 +38,10 @@ on:
         required: false
 
 env:
+  # BUILD_ARGS and TEST_JAVA_VERSION are duplicated in populate-cache.yml, please keep in sync
   BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true -Drocks_tools_native"
-  TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
-  # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml, please keep in sync
+  TEST_JAVA_VERSION: 25 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
+  # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml and populate-cache.yml, please keep in sync
   MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
   OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
@@ -69,19 +70,19 @@ jobs:
       with-coverage: ${{ env.OZONE_WITH_COVERAGE }}
     steps:
       - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)"
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
         if: github.event_name  == 'push'
       - name: "Checkout ${{ github.sha }} with its parent (pull request)"
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.sha }}
           fetch-depth: 2
           persist-credentials: false
         if: github.event_name  == 'pull_request'
       - name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)"
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ inputs.ref }}
           persist-credentials: false
@@ -172,7 +173,7 @@ jobs:
     if: needs.build-info.outputs.needs-compile == 'true'
     strategy:
       matrix:
-        java: [ 8, 11, 17 ]
+        java: [ 8, 17, 25 ]
         include:
           - os: ubuntu-24.04
           - java: 21
@@ -339,7 +340,7 @@ jobs:
       pre-script: sudo hostname localhost
       ratis-args: ${{ inputs.ratis_args }}
       script: integration
-      script-args: -Ptest-${{ matrix.profile }} -Drocks_tools_native
+      script-args: -Ptest-${{ matrix.profile }} -Phadoop-native-lib -Drocks_tools_native
       sha: ${{ needs.build-info.outputs.sha }}
       split: ${{ matrix.profile }}
       timeout-minutes: 90
@@ -359,13 +360,13 @@ jobs:
       - integration
     steps:
       - name: Checkout project
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
           ref: ${{ needs.build-info.outputs.sha }}
       - name: Cache for maven dependencies
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.m2/repository/*/*/*
@@ -382,7 +383,7 @@ jobs:
           mkdir -p hadoop-ozone/dist/target
           tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
       - name: Setup java ${{ env.TEST_JAVA_VERSION }}
-        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
         with:
           distribution: 'temurin'
           java-version: ${{ env.TEST_JAVA_VERSION }}

.github/workflows/close-stale-prs.yaml

@@ -27,9 +27,10 @@ jobs:
     runs-on: ubuntu-slim
     steps:
       - name: Close Stale PRs
-        uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
+        uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
         with:
           stale-pr-label: 'stale'
+          exempt-pr-labels: 'design'
           exempt-draft-pr: false
           days-before-issue-stale: -1
           days-before-pr-stale: 21

.github/workflows/generate-config-doc.yml

@@ -28,13 +28,13 @@ jobs:
     runs-on: ubuntu-slim
     steps:
       - name: Checkout project
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           ref: ${{ inputs.sha }}
 
       - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.x'