diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a0cdd50d6fa36..66703c3596b95 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,8 +40,9 @@ resources: # see https://github.com/apache/flink-connector-shared-utils/tree/ci_utils - container: flink-build-container image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy - # On AZP provided machines, set this flag to allow writing coredumps in docker - options: --privileged + # On AZP provided machines, set this flag to allow writing coredumps in docker. + # --tmpfs provides RAM-backed scratch; the Machine info step reports whether the agent honours it. + options: --privileged --tmpfs /mnt/ram:size=24g,exec,mode=1777 # Define variables: # - See tools/azure-pipelines/jobs-template.yml for a short summary of the caching @@ -84,6 +85,8 @@ stages: pool: vmImage: 'ubuntu-24.04' steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info - task: GoTool@0 inputs: version: '1.18.1' diff --git a/flink-filesystems/flink-azure-fs-hadoop/pom.xml b/flink-filesystems/flink-azure-fs-hadoop/pom.xml index 9794679b181b6..21436521ab027 100644 --- a/flink-filesystems/flink-azure-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-azure-fs-hadoop/pom.xml @@ -134,6 +134,8 @@ under the License. shade + + ${flink.fsShade.skip} *:* diff --git a/flink-filesystems/flink-gs-fs-hadoop/pom.xml b/flink-filesystems/flink-gs-fs-hadoop/pom.xml index ec92c44dfae04..0b80e2aa1c01f 100644 --- a/flink-filesystems/flink-gs-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-gs-fs-hadoop/pom.xml @@ -222,6 +222,8 @@ under the License. shade + + ${flink.fsShade.skip} *:* diff --git a/flink-filesystems/flink-oss-fs-hadoop/pom.xml b/flink-filesystems/flink-oss-fs-hadoop/pom.xml index d6bb8c51a3ca3..f202f061d3db2 100644 --- a/flink-filesystems/flink-oss-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-oss-fs-hadoop/pom.xml @@ -151,6 +151,8 @@ under the License. shade + + ${flink.fsShade.skip} *:* diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml index 098296e49561a..0220b54f23978 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml @@ -246,6 +246,8 @@ under the License. shade + + ${flink.fsShade.skip} *:* diff --git a/flink-filesystems/flink-s3-fs-native/pom.xml b/flink-filesystems/flink-s3-fs-native/pom.xml index b0403112df036..12ad537f649dc 100644 --- a/flink-filesystems/flink-s3-fs-native/pom.xml +++ b/flink-filesystems/flink-s3-fs-native/pom.xml @@ -179,6 +179,8 @@ under the License. shade + + ${flink.fsShade.skip} software.amazon.awssdk:* diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml index cb3236ce1a67c..a6c6fdbe7b15c 100644 --- a/flink-filesystems/flink-s3-fs-presto/pom.xml +++ b/flink-filesystems/flink-s3-fs-presto/pom.xml @@ -641,6 +641,8 @@ under the License. shade + + ${flink.fsShade.skip} *:* diff --git a/pom.xml b/pom.xml index b67166326c562..cd6c14f0c9c81 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,8 @@ under the License. 2.20.1 2.10.0 true + + false 11 17 1.7.36 diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml index 5befcf292c69f..07e5d03fec448 100644 --- a/tools/azure-pipelines/build-apache-repo.yml +++ b/tools/azure-pipelines/build-apache-repo.yml @@ -40,6 +40,8 @@ resources: # see https://github.com/apache/flink-connector-shared-utils/tree/ci_utils - container: flink-build-container image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy + # RAM-backed scratch for the build; the Machine info step reports whether the agent honours it. + options: --tmpfs /mnt/ram:size=24g,exec,mode=1777 variables: MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository @@ -77,6 +79,8 @@ stages: pool: vmImage: 'ubuntu-24.04' steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info # Skip docs check if this is a pull request that doesn't contain a documentation change - task: GoTool@0 inputs: @@ -167,6 +171,8 @@ stages: pool: vmImage: 'ubuntu-24.04' steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info - task: GoTool@0 inputs: version: '1.18.1' diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml index c3c7849a70ac2..775cc84b17bde 100644 --- a/tools/azure-pipelines/build-nightly-dist.yml +++ b/tools/azure-pipelines/build-nightly-dist.yml @@ -24,6 +24,8 @@ jobs: workspace: clean: all steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info # free up disk space on the hosted agent; the full dist build plus the # Maven cache does not fit in the ~16 GB the image leaves free - script: ./tools/azure-pipelines/free_disk_space.sh @@ -82,6 +84,8 @@ jobs: workspace: clean: all steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info # free up disk space on the hosted agent; the snapshot deploy build plus # the Maven cache does not fit in the ~16 GB the image leaves free - script: ./tools/azure-pipelines/free_disk_space.sh diff --git a/tools/azure-pipelines/build-python-wheels.yml b/tools/azure-pipelines/build-python-wheels.yml index 418bdeb88c2ea..a0fdc78b961ad 100644 --- a/tools/azure-pipelines/build-python-wheels.yml +++ b/tools/azure-pipelines/build-python-wheels.yml @@ -18,6 +18,8 @@ jobs: pool: vmImage: 'ubuntu-24.04' steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info - task: UsePythonVersion@0 inputs: versionSpec: '3.12' @@ -34,6 +36,8 @@ jobs: pool: vmImage: 'macOS-latest' steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info - task: UsePythonVersion@0 inputs: versionSpec: '3.12' diff --git a/tools/azure-pipelines/e2e-template.yml b/tools/azure-pipelines/e2e-template.yml index dcffa9b36a072..9898720d52215 100644 --- a/tools/azure-pipelines/e2e-template.yml +++ b/tools/azure-pipelines/e2e-template.yml @@ -44,6 +44,8 @@ jobs: .gitattributes .github/** steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info # Skip e2e test execution if this is a documentation only pull request (master / release builds will still be checked regularly) - bash: | source ./tools/azure-pipelines/build_properties.sh @@ -140,7 +142,8 @@ jobs: sudo apt install ./libssl1.0.0_*.deb displayName: Prepare E2E run condition: not(eq(variables['SKIP'], '1')) - - script: ${{parameters.environment}} PROFILE="$PROFILE -Dfast -Pskip-webui-build" ./tools/ci/compile_ci.sh + # e2e only needs the build; skip all QA checks (they run in the qa_check job) so it can build with -T1C + - script: SKIP_QA_CHECKS=true MVN_COMPILE_THREADS=1C ${{parameters.environment}} PROFILE="$PROFILE -Pskip-webui-build" ./tools/ci/compile_ci.sh displayName: Build Flink condition: not(eq(variables['SKIP'], '1')) - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/ci/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}} diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index d86b5210465ed..bdba9e4b2a282 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -36,6 +36,8 @@ jobs: # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info # if on Azure, free up disk space - script: ./tools/ci/free_disk_space.sh target: host @@ -67,8 +69,11 @@ jobs: echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" displayName: "Set JDK" # Compile + # SKIP_QA_CHECKS skips all QA checks (javadoc/scala/shaded/bundled/license); they run in the + # dedicated qa_check job. This lets the compile build run fast and multi-threaded (-T1C), off the + # test critical path. - script: | - ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $? + SKIP_QA_CHECKS=true MVN_COMPILE_THREADS=1C ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $? ./tools/ci/create_build_artifact.sh displayName: Compile @@ -78,6 +83,59 @@ jobs: targetPath: $(FLINK_ARTIFACT_DIR) artifact: FlinkCompileArtifact-${{parameters.stage_name}} + # upload the installed Flink jars (run-scoped, NOT cached) so test jobs can reuse them and skip rebuild + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(MAVEN_CACHE_FOLDER)/org/apache/flink + artifact: FlinkMavenRepoArtifact-${{parameters.stage_name}} + + # build outputs that reuse stages read from target/ dirs (not .m2): the assembled dist (build-target) + # and flink-python/target (pyflink's test classpath: test-dependencies, *-tests.jar, testDataStream.jar) + - task: PublishPipelineArtifact@1 + inputs: + targetPath: flink-dist/target + artifact: FlinkDistArtifact-${{parameters.stage_name}} + - task: PublishPipelineArtifact@1 + inputs: + targetPath: flink-python/target + artifact: FlinkPythonTargetArtifact-${{parameters.stage_name}} + +# All QA checks (javadoc/scala/shaded/bundled/license) run here: this job builds Flink from scratch +# single-threaded and validates it. Runs as an independent job (in parallel with compile/test) so it +# is off the test critical path. +- job: qa_check_${{parameters.stage_name}} + # succeeded() is needed to allow job cancellation + condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) + pool: ${{parameters.test_pool_definition}} + container: ${{parameters.container}} + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all + steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info + # if on Azure, free up disk space + - script: ./tools/ci/free_disk_space.sh + target: host + condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) + displayName: Free up disk space + - task: Cache@2 + inputs: + key: $(PIPELINE_START_YEAR) | $(CACHE_KEY) + restoreKeys: $(PIPELINE_START_YEAR) | $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + continueOnError: true # continue the build even if the cache fails. + condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) + displayName: Cache Maven local repo + - script: | + echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" + echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" + displayName: "Set JDK" + # Build from scratch single-threaded and run all QA checks (no SKIP_QA_CHECKS). + - script: ${{parameters.environment}} ./tools/ci/compile_ci.sh + displayName: QA checks + - job: test_${{parameters.stage_name}} dependsOn: compile_${{parameters.stage_name}} condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) @@ -88,20 +146,31 @@ jobs: workspace: clean: all strategy: + # reuseArtifacts: reuse the compile job's installed jars (skip Step 1) instead of a full rebuild. + # Only stages that resolve everything from .m2 can reuse; stages needing build outputs under module + # target/ dirs (python's test classpath, the assembled dist, cluster ITs) do a full rebuild. matrix: core: module: core + reuseArtifacts: 'true' python: module: python + reuseArtifacts: 'true' table: module: table + reuseArtifacts: 'true' connect: module: connect + reuseArtifacts: 'true' tests: module: tests + reuseArtifacts: 'false' misc: module: misc + reuseArtifacts: 'false' steps: + - script: ./tools/ci/print_machine_info.sh + displayName: Machine info # if on Azure, free up disk space - script: ./tools/ci/free_disk_space.sh target: host @@ -133,6 +202,30 @@ jobs: condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) displayName: Cache Maven local repo + # overlay the compile job's freshly-installed Flink jars (run-scoped, overrides the cache) when this + # stage reuses them. continueOnError so a missing/expired artifact falls back to a rebuild. + - task: DownloadPipelineArtifact@2 + continueOnError: true + condition: and(succeeded(), eq(variables['reuseArtifacts'], 'true')) + inputs: + path: $(MAVEN_CACHE_FOLDER)/org/apache/flink + artifact: FlinkMavenRepoArtifact-${{parameters.stage_name}} + + # python reuse also needs the assembled dist (build-target) and flink-python/target test artifacts, + # which pyflink reads from target/ dirs. continueOnError so a missing artifact falls back to a rebuild. + - task: DownloadPipelineArtifact@2 + continueOnError: true + condition: and(succeeded(), eq(variables['reuseArtifacts'], 'true'), eq(variables['module'], 'python')) + inputs: + path: flink-dist/target + artifact: FlinkDistArtifact-${{parameters.stage_name}} + - task: DownloadPipelineArtifact@2 + continueOnError: true + condition: and(succeeded(), eq(variables['reuseArtifacts'], 'true'), eq(variables['module'], 'python')) + inputs: + path: flink-python/target + artifact: FlinkPythonTargetArtifact-${{parameters.stage_name}} + - task: Cache@2 inputs: key: '"$(module)" | $(DOCKER_IMAGES_CACHE_KEY)' @@ -155,8 +248,9 @@ jobs: - script: sudo sysctl -w kernel.core_pattern=core.%p displayName: Set coredump pattern - # Test - - script: ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) + # Test — reuse the handed-off jars on stages that can (see the matrix); test_controller.sh falls back + # to a full rebuild when reuse is off or the jars aren't present. + - script: REUSE_INSTALLED_ARTIFACTS=$(reuseArtifacts) ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) displayName: Test - $(module) env: IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET) diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh index bdc53428fa6cb..cabfd45d4d035 100755 --- a/tools/ci/compile.sh +++ b/tools/ci/compile.sh @@ -49,6 +49,13 @@ rm -rf ${MVN_VALIDATION_DIR} source "${CI_DIR}/stage.sh" source "${CI_DIR}/shade.sh" +# Sample host load (CPU %steal / disk util) during the build to detect agent contention; killed on exit. +# Written to stdout (the job log) because compile/qa/e2e don't publish a debug-files artifact. +chmod +x "${CI_DIR}/sample_load.sh" 2>/dev/null || true +"${CI_DIR}/sample_load.sh" 5 /dev/stdout & +LOAD_SAMPLER_PID=$! +trap 'kill "$LOAD_SAMPLER_PID" 2>/dev/null' EXIT + echo "Maven version:" $MVN -version @@ -58,9 +65,24 @@ echo "========================================================================== EXIT_CODE=0 -# run with -T1 because our maven output parsers don't support multi-threaded builds -$MVN clean deploy -DaltDeploymentRepository=validation_repository::default::file:$MVN_VALIDATION_DIR -Dflink.convergence.phase=install -Pcheck-convergence \ - -Dmaven.javadoc.skip=true -U -DskipTests -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=DEBUG "${@}" -T1 | tee $MVN_CLEAN_COMPILE_OUT +# QA builds add shade DEBUG output and deploy to a local repo for the license check; other builds build fast and only install +if [[ "${SKIP_QA_CHECKS:-false}" != "true" ]]; then + BUILD_MODE_ARGS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=DEBUG" + DEPLOY_ARGS="deploy -DaltDeploymentRepository=validation_repository::default::file:$MVN_VALIDATION_DIR -Dflink.convergence.phase=install -Pcheck-convergence" +else + # -Dflink.fsShade.skip=true drops the FS-plugin shade (gs/azure/s3/oss) off the compile critical + # path; those plugins aren't needed by the reuse consumers (connect re-shades, e2e rebuilds). + BUILD_MODE_ARGS="-Pfast -Dflink.fsShade.skip=true" + DEPLOY_ARGS="install" +fi +# Only force snapshot updates (-U) unless the global Maven options already disable them (--no-snapshot-updates) +UPDATE_SNAPSHOTS_ARG="-U" +if [[ "${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR}" == *"--no-snapshot-updates"* ]]; then + UPDATE_SNAPSHOTS_ARG="" +fi +# The bundled/license QA checks parse single-threaded output (-T1); QA-skipping builds may set MVN_COMPILE_THREADS (e.g. 1C) +$MVN clean ${DEPLOY_ARGS} \ + -Dmaven.javadoc.skip=true ${UPDATE_SNAPSHOTS_ARG} -DskipTests ${BUILD_MODE_ARGS} "${@}" -T${MVN_COMPILE_THREADS:-1} | tee $MVN_CLEAN_COMPILE_OUT EXIT_CODE=${PIPESTATUS[0]} @@ -79,6 +101,9 @@ if [ $EXIT_CODE != 0 ]; then exit $EXIT_CODE fi +# All QA checks run only in the dedicated QA job (and local runs); compile/e2e skip them via SKIP_QA_CHECKS +if [[ "${SKIP_QA_CHECKS:-false}" != "true" ]]; then + echo "============ Checking Javadocs ============" javadoc_output=/tmp/javadoc.out @@ -94,10 +119,6 @@ if [ $EXIT_CODE != 0 ] ; then exit $EXIT_CODE fi -echo "============ Checking bundled dependencies marked as optional ============" - -MVN=$MVN ${CI_DIR}/verify_bundled_optional.sh $MVN_CLEAN_COMPILE_OUT || exit $? - echo "============ Checking scala suffixes ============" MVN=$MVN ${CI_DIR}/verify_scala_suffixes.sh || exit $? @@ -111,10 +132,16 @@ EXIT_CODE=$(($EXIT_CODE+$?)) check_shaded_artifacts_s3_fs presto EXIT_CODE=$(($EXIT_CODE+$?)) +echo "============ Checking bundled dependencies marked as optional ============" + +MVN=$MVN ${CI_DIR}/verify_bundled_optional.sh $MVN_CLEAN_COMPILE_OUT || exit $? + echo "============ Run license check ============" find $MVN_VALIDATION_DIR MVN=$MVN ${CI_DIR}/license_check.sh $MVN_CLEAN_COMPILE_OUT $MVN_VALIDATION_DIR || exit $? +fi + exit $EXIT_CODE diff --git a/tools/ci/print_machine_info.sh b/tools/ci/print_machine_info.sh new file mode 100755 index 0000000000000..70e2e1ae08683 --- /dev/null +++ b/tools/ci/print_machine_info.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# +# Prints the CPU, core count, memory and disk throughput of the machine running the job. Intended as +# the first step of every CI job so build/test timings can be correlated with the hardware they ran on. +# + +echo "==============================================================================" +echo "Machine information" +echo "==============================================================================" + +echo "---- CPU ----" +if command -v lscpu >/dev/null 2>&1; then + lscpu | grep -E "^(Architecture|Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per socket|Socket\(s\)):" +elif [ -r /proc/cpuinfo ]; then + grep -E "^model name" /proc/cpuinfo | head -n 1 +else + sysctl -n machdep.cpu.brand_string 2>/dev/null || echo "CPU model: n/a" +fi +# getconf is portable (Linux + macOS); nproc additionally honours cgroup limits in containers +echo "Logical cores available: $(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 'n/a')" + +echo "---- Memory ----" +if command -v free >/dev/null 2>&1; then + free -h +elif [ -r /proc/meminfo ]; then + grep -E "^(MemTotal|MemAvailable):" /proc/meminfo +else + mem_bytes=$(sysctl -n hw.memsize 2>/dev/null) && echo "MemTotal: $((mem_bytes / 1024 / 1024)) MB" || echo "Memory: n/a" +fi + +echo "---- Disk (working dir: $(pwd)) ----" +df -h . 2>/dev/null +# filesystem type (Linux df supports -T; ignored elsewhere) +df -T . 2>/dev/null | awk 'NR==2 {print "Filesystem type: " $2}' +if command -v lsblk >/dev/null 2>&1; then + lsblk -d -o NAME,ROTA,SIZE,MODEL 2>/dev/null +fi +# Rough sequential write throughput incl. flush. O_DIRECT is often unsupported on overlayfs (CI runs in +# containers), so use conv=fdatasync; fall back to non-GNU dd (e.g. macOS) which lacks that option. +if command -v dd >/dev/null 2>&1; then + disk_probe="./.machine_info_disktest" + echo "Sequential write (1 GiB, incl. flush):" + probe_out=$(dd if=/dev/zero of="$disk_probe" bs=1M count=1024 conv=fdatasync 2>&1) + if [ $? -ne 0 ]; then + probe_out=$(dd if=/dev/zero of="$disk_probe" bs=1m count=1024 2>&1) + fi + echo "$probe_out" | tail -n 1 + rm -f "$disk_probe" +fi + +# Small-file / metadata I/O -- the workload maven-shade actually generates (unpack/repack thousands of +# tiny class files). Sequential MB/s says nothing about this; a slow-IOPS disk shows up here, not above. +# Zero deps: time creating then deleting many tiny files. GNU date gives ms; falls back to seconds*1000. +now_ms() { local d; d=$(date +%s%3N 2>/dev/null); case "$d" in ''|*[!0-9]*) d=$(( $(date +%s) * 1000 ));; esac; echo "$d"; } +iops_dir="./.machine_info_iops_$$" +mkdir -p "$iops_dir" 2>/dev/null +N=2000 +t0=$(now_ms); i=0 +while [ "$i" -lt "$N" ]; do printf 'x' > "$iops_dir/f$i"; i=$(( i + 1 )); done +sync 2>/dev/null +t1=$(now_ms) +rm -rf "$iops_dir" +t2=$(now_ms) +cr=$(( t1 - t0 )); [ "$cr" -le 0 ] && cr=1 +dl=$(( t2 - t1 )); [ "$dl" -lt 0 ] && dl=0 +echo "Small-file I/O: create ${N} tiny files in ${cr} ms ($(( N * 1000 / cr )) files/s), delete in ${dl} ms" + +# tmpfs availability -- can we put the (IOPS-heavy) build I/O in RAM to dodge a slow disk? +echo "---- tmpfs availability ----" +echo "/dev/shm: $(df -h /dev/shm 2>/dev/null | awk 'NR==2{print $2" total, "$4" free"}' || echo 'n/a')" +RAMDIR="${TMPFS_BUILD_DIR:-/mnt/ram}" +if [ -d "$RAMDIR" ] && : > "$RAMDIR/.mi_wtest" 2>/dev/null; then + rm -f "$RAMDIR/.mi_wtest" + echo "$RAMDIR: PRESENT & writable (fs=$(df -T "$RAMDIR" 2>/dev/null | awk 'NR==2{print $2}'), size=$(df -h "$RAMDIR" 2>/dev/null | awk 'NR==2{print $2}'))" +else + tdir=$(mktemp -d 2>/dev/null || echo /tmp/.mi_mnt); mkdir -p "$tdir" 2>/dev/null + if mount -t tmpfs -o size=8m tmpfs "$tdir" 2>/dev/null; then + echo "$RAMDIR: absent, but in-job 'mount -t tmpfs' is ALLOWED"; umount "$tdir" 2>/dev/null + elif sudo -n mount -t tmpfs -o size=8m tmpfs "$tdir" 2>/dev/null; then + echo "$RAMDIR: absent, but 'sudo mount -t tmpfs' is ALLOWED"; sudo -n umount "$tdir" 2>/dev/null + else + echo "$RAMDIR: absent and no tmpfs-mount privilege -> add container '--tmpfs $RAMDIR', or use /dev/shm" + fi + rmdir "$tdir" 2>/dev/null +fi diff --git a/tools/ci/sample_load.sh b/tools/ci/sample_load.sh new file mode 100755 index 0000000000000..ecfa279e5f07a --- /dev/null +++ b/tools/ci/sample_load.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# +# Samples host load every INTERVAL seconds until killed, to detect contention on shared agents. +# Zero dependencies (reads /proc). The key metric is CPU %steal: time this vCPU was runnable but the +# hypervisor gave the physical core to another tenant -> the direct signal of an oversubscribed host. +# Also reports %busy, %iowait, an approximate primary-disk util%, and loadavg vs core count. +# +# Usage: sample_load.sh [interval_seconds] [output_file] +# + +INTERVAL="${1:-5}" +OUT="${2:-/dev/stdout}" + +# Direct all output at the target. For a real file, reopen fd 1 once (works even when backgrounded). +# For stdout keep the inherited fd: reopening /dev/stdout by path fails with ENXIO when this runs in +# the background and stdout is a pipe. +case "$OUT" in + ""|"-"|/dev/stdout|/dev/fd/1) : ;; + *) exec >> "$OUT" 2>&1 ;; +esac + +if [ ! -r /proc/stat ]; then + echo "sample_load: /proc/stat not readable (not Linux?); skipping." + exit 0 +fi + +# cpu line fields after 'cpu': user nice system idle iowait irq softirq steal +read_cpu() { awk '/^cpu /{print $2,$3,$4,$5,$6,$7,$8,$9}' /proc/stat; } +# sum of io_ticks (ms spent doing I/O, field 13) over whole block devices (not partitions). +# printf "%.0f" (not "print s+0") so large sums are plain integers, not awk scientific notation +# (e.g. 3.431e+09), which bash arithmetic cannot parse. +read_io() { awk '$3 ~ /^(sd[a-z]+|vd[a-z]+|xvd[a-z]+|nvme[0-9]+n[0-9]+)$/ {s+=$13} END{printf "%.0f\n", s+0}' /proc/diskstats; } + +ncpu=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1) +prev_cpu=($(read_cpu)); prev_io=$(read_io); prev_io=${prev_io:-0} +echo "# sample_load interval=${INTERVAL}s cores=${ncpu} -- watch 'steal' (>0 => contended host)" + +while :; do + sleep "$INTERVAL" + cur_cpu=($(read_cpu)); cur_io=$(read_io); cur_io=${cur_io:-0} + total=0 + for i in 0 1 2 3 4 5 6 7; do + d[$i]=$(( ${cur_cpu[$i]} - ${prev_cpu[$i]} )) + total=$(( total + ${d[$i]} )) + done + [ "$total" -le 0 ] && total=1 + idle=$(( ${d[3]} + ${d[4]} )) # idle + iowait + busy=$(( 100 * (total - idle) / total )) + iowait=$(( 100 * ${d[4]} / total )) + steal=$(( 100 * ${d[7]} / total )) + # io_ticks delta (ms) over the interval (INTERVAL*1000 ms) -> util%. Guard so a bad read can't + # abort the whole sample line. + util=0 + case "$cur_io$prev_io" in + *[!0-9]*) : ;; + *) util=$(( (cur_io - prev_io) / (INTERVAL * 10) )) ;; + esac + [ "$util" -gt 100 ] && util=100 + load=$(cut -d' ' -f1 /proc/loadavg) + printf '%s busy=%3d%% iowait=%3d%% steal=%3d%% disk_util=%3d%% load=%s/%s\n' \ + "$(date +%T)" "$busy" "$iowait" "$steal" "$util" "$load" "$ncpu" + prev_cpu=("${cur_cpu[@]}"); prev_io=$cur_io +done diff --git a/tools/ci/test_controller.sh b/tools/ci/test_controller.sh index 0a620fb6f5cc9..91bd8f4b59305 100755 --- a/tools/ci/test_controller.sh +++ b/tools/ci/test_controller.sh @@ -73,6 +73,13 @@ export WATCHDOG_ADDITIONAL_MONITORING_FILES="$DEBUG_FILES_OUTPUT_DIR/mvn-*.log" source "${HERE}/watchdog.sh" +# Sample host load (CPU %steal, iowait, disk util) during the whole build+test, to measure whether the +# agent is contended. Output goes to the debug-files artifact; the sampler is killed when this exits. +chmod +x "${HERE}/sample_load.sh" 2>/dev/null || true +"${HERE}/sample_load.sh" 5 "$DEBUG_FILES_OUTPUT_DIR/load-sample.log" & +LOAD_SAMPLER_PID=$! +trap 'kill "$LOAD_SAMPLER_PID" 2>/dev/null' EXIT + # ============================================================================= # Step 1: Rebuild jars and install Flink to local maven repository # ============================================================================= @@ -81,12 +88,34 @@ export LOG4J_PROPERTIES=${HERE}/log4j.properties MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES" MVN_COMMON_OPTIONS="-Dfast -Pskip-webui-build $MVN_LOGGING_OPTIONS" -MVN_COMPILE_OPTIONS="-DskipTests" -MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${STAGE}) - CALLBACK_ON_TIMEOUT="print_stacktraces | tee ${DEBUG_FILES_OUTPUT_DIR}/jps-traces.out" -run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install" $CALLBACK_ON_TIMEOUT -EXIT_CODE=$? + +# Reuse the jars the compile job handed off (REUSE_INSTALLED_ARTIFACTS + jars actually present) and skip +# the rebuild. Only stages that resolve everything from .m2 can do this; stages needing build outputs +# under module target/ dirs (e.g. python's test classpath, the assembled dist) must do a full rebuild. +# Fall back to a full rebuild if no jars are present (e.g. an isolated retry after the artifact expired). +FLINK_INSTALLED_JARS_DIR="${MAVEN_CACHE_FOLDER:-$HOME/.m2/repository}/org/apache/flink" +if [[ "${REUSE_INSTALLED_ARTIFACTS:-false}" == "true" && -d "$FLINK_INSTALLED_JARS_DIR" && -n "$(ls -A "$FLINK_INSTALLED_JARS_DIR" 2>/dev/null)" ]]; then + echo "Reusing jars installed by the compile job; skipping rebuild/install." + # relink build-target when a prebuilt distribution was handed off (e.g. the python stage) + if [ ! -e build-target ]; then + dist_dir=$(ls -d flink-dist/target/flink-*-bin/flink-* 2>/dev/null | head -n 1) + if [ -n "$dist_dir" ]; then + ln -sfn "$dist_dir" build-target + echo "Linked build-target -> $dist_dir" + fi + fi + EXIT_CODE=0 +else + if [[ "${REUSE_INSTALLED_ARTIFACTS:-false}" == "true" ]]; then + echo "REUSE_INSTALLED_ARTIFACTS set but no handed-off jars in ${FLINK_INSTALLED_JARS_DIR}; doing a full rebuild." + fi + MVN_COMPILE_OPTIONS="-DskipTests" + MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${STAGE}) + # the install step is a -DskipTests build (no test-output log parsing here), so it can run multi-threaded + run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES -T${MVN_COMPILE_THREADS:-1C} install" $CALLBACK_ON_TIMEOUT + EXIT_CODE=$? +fi if [ $EXIT_CODE != 0 ]; then echo "=============================================================================="